* Re: [Xenomai-core] [PATCH] ADEOS IPIPE PPC patches upgrade
2006-12-18 13:06 ` Wolfgang Grandegger
@ 2006-12-19 12:44 ` Benjamin Zores
2006-12-19 16:10 ` Wolfgang Grandegger
0 siblings, 1 reply; 5+ messages in thread
From: Benjamin Zores @ 2006-12-19 12:44 UTC (permalink / raw)
To: xenomai
[-- Attachment #1: Type: text/plain, Size: 1435 bytes --]
On Mon, 18 Dec 2006 14:06:15 +0100
Wolfgang Grandegger <wg@domain.hid> wrote:
> I don't understand your questions. These patches are for the PPC tree.
> The POWERPC is still not supported. Do you already have a working port
> for the PowerPC tree 2.6.18?
I send one a month ago but which wasn't 100% complete.
Now, here's an updated one for both ppc and powerpc arch.
It based on your patch + the equivalent functions ported to merged powerpc architecture.
This time it should compile fine but i haven't tested if ppc arch still works
(i can't do so).
I just remove the xenomai parts from your patch (which obviously had nothing to do
in adeos patch itself).
With ipipe enabled on my card, the kernels boots up
but still hangs due to IRQ problems has it used to do in 2.6.18.
It seems to hang at SATA driver load (does the same for USB/I2C and so on after all)
but if I had printk() in each function from ipipe-root.c it goes in an endless
loop of:
...
__ipipe_handle_irq:226
__ipipe_do_timer:404
__ipipe_grab_timer:357
__ipipe_handle_irq:226
__ipipe_do_timer:404
__ipipe_grab_timer:357
...
and seems to never go further :-(
Do you have an idea where and why this might come from ?
FYI, the vanilla 2.6.19 kernel boots up my card fine and goes to userspace
but seems to also hang a few time later (but at least it boots up to user space).
Anyhow, please consider this patch for review and if possible test it/apply it.
Ben
[-- Attachment #2: patch-adeos-2.6.19-1.5-01-ppc-powerpc.diff --]
[-- Type: text/plain, Size: 230454 bytes --]
diff -Naur linux-2.6.19.orig/arch/powerpc/Kconfig linux-2.6.19.ipipe/arch/powerpc/Kconfig
--- linux-2.6.19.orig/arch/powerpc/Kconfig 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/Kconfig 2006-12-19 13:02:15.000000000 +0100
@@ -613,6 +613,8 @@
menu "Kernel options"
+source "kernel/ipipe/Kconfig"
+
config HIGHMEM
bool "High memory support"
depends on PPC32
diff -Naur linux-2.6.19.orig/arch/powerpc/kernel/entry_32.S linux-2.6.19.ipipe/arch/powerpc/kernel/entry_32.S
--- linux-2.6.19.orig/arch/powerpc/kernel/entry_32.S 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/kernel/entry_32.S 2006-12-19 13:02:15.000000000 +0100
@@ -132,8 +132,23 @@
* check for stack overflow
*/
lwz r9,THREAD_INFO-THREAD(r12)
+#ifdef CONFIG_IPIPE
+ /* Allow for private kernel-based stacks: those must not cause
+ the stack overflow detection to trigger when some activity has
+ been preempted over them. We just check if the kernel stack is
+ not treading on the memory area ranging from
+ ¤t->thread_info to ¤t->thread, which is coarser
+ than the vanilla implementation, but likely sensitive enough
+ to catch overflows soon enough though.*/
+ addi r12,r9,THREAD
+ cmplw 0,r1,r9
+ cmplw 1,r1,r12
+ crand 1,1,4
+ bgt- stack_ovf /* if r9 < r1 < r9+THREAD */
+#else /* CONFIG_IPIPE */
cmplw r1,r9 /* if r1 <= current->thread_info */
ble- stack_ovf /* then the kernel stack overflowed */
+#endif /* CONFIG_IPIPE */
5:
#ifdef CONFIG_6xx
tophys(r9,r9) /* check local flags */
@@ -198,6 +213,21 @@
lwz r11,_CCR(r1) /* Clear SO bit in CR */
rlwinm r11,r11,0,4,2
stw r11,_CCR(r1)
+#ifdef CONFIG_IPIPE
+ addi r3,r1,GPR0
+ bl __ipipe_syscall_root
+ cmpwi r3,0
+ lwz r3,GPR3(r1)
+ lwz r0,GPR0(r1)
+ lwz r4,GPR4(r1)
+ lwz r5,GPR5(r1)
+ lwz r6,GPR6(r1)
+ lwz r7,GPR7(r1)
+ lwz r8,GPR8(r1)
+ lwz r9,GPR9(r1)
+ bgt .ipipe_end_syscall
+ blt ret_from_syscall
+#endif /* CONFIG_IPIPE */
#ifdef SHOW_SYSCALLS
bl do_show_syscall
#endif /* SHOW_SYSCALLS */
@@ -260,11 +290,34 @@
SYNC
RFI
+#ifdef CONFIG_IPIPE
+.ipipe_end_syscall:
+ LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */
+ SYNC
+ MTMSRD(r10)
+ b syscall_exit_cont
+#endif /* CONFIG_IPIPE */
+
66: li r3,-ENOSYS
b ret_from_syscall
.globl ret_from_fork
ret_from_fork:
+#ifdef CONFIG_IPIPE
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+ stwu r1,-4(r1)
+ stw r3,0(r1)
+ lis r3,(0x80000000)@h
+ ori r3,r3,(0x80000000)@l
+ bl ipipe_trace_end
+ lwz r3,0(r1)
+ addi r1,r1,4
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+ LOAD_MSR_KERNEL(r10,MSR_KERNEL)
+ ori r10,r10,MSR_EE
+ SYNC
+ MTMSRD(r10)
+#endif /* CONFIG_IPIPE */
REST_NVGPRS(r1)
bl schedule_tail
li r3,0
@@ -630,6 +683,12 @@
SYNC /* Some chip revs have problems here... */
MTMSRD(r10) /* disable interrupts */
+#ifdef CONFIG_IPIPE
+ bl __ipipe_check_root
+ cmpwi r3, 0
+ beq- restore
+#endif /* CONFIG_IPIPE */
+
lwz r3,_MSR(r1) /* Returning to user mode? */
andi. r0,r3,MSR_PR
beq resume_kernel
@@ -665,11 +724,37 @@
beq+ restore
andi. r0,r3,MSR_EE /* interrupts off? */
beq restore /* don't schedule if so */
+#ifdef CONFIG_IPIPE
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+ lis r3,(0x80000000)@h
+ ori r3,r3,(0x80000000)@l
+ bl ipipe_trace_end
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+ LOAD_MSR_KERNEL(r10,MSR_KERNEL)
+ ori r10,r10,MSR_EE
+ SYNC
+ MTMSRD(r10)
+ bl __ipipe_fast_stall_root
+#endif /* CONFIG_IPIPE */
1: bl preempt_schedule_irq
rlwinm r9,r1,0,0,(31-THREAD_SHIFT)
lwz r3,TI_FLAGS(r9)
andi. r0,r3,_TIF_NEED_RESCHED
bne- 1b
+#ifdef CONFIG_IPIPE
+ bl __ipipe_fast_unstall_root
+ LOAD_MSR_KERNEL(r10,MSR_KERNEL)
+ SYNC
+ MTMSRD(r10)
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+ lwz r3,_MSR(r1)
+ andi. r0,r3,MSR_EE
+ bne restore
+ lis r3,(0x80000000)@h
+ ori r3,r3,(0x80000000)@l
+ bl ipipe_trace_begin
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+#endif /* CONFIG_IPIPE */
#else
resume_kernel:
#endif /* CONFIG_PREEMPT */
@@ -929,6 +1014,13 @@
.comm ee_restarts,4
+#ifdef CONFIG_IPIPE
+_GLOBAL(__ipipe_ret_from_except)
+ cmpwi r3, 0
+ bne+ ret_from_except
+ b restore
+#endif /* CONFIG_IPIPE */
+
/*
* PROM code for specific machines follows. Put it
* here so it's easy to add arch-specific sections later.
diff -Naur linux-2.6.19.orig/arch/powerpc/kernel/head_32.S linux-2.6.19.ipipe/arch/powerpc/kernel/head_32.S
--- linux-2.6.19.orig/arch/powerpc/kernel/head_32.S 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/kernel/head_32.S 2006-12-19 13:02:15.000000000 +0100
@@ -329,6 +329,12 @@
EXC_XFER_TEMPLATE(n, hdlr, n, NOCOPY, transfer_to_handler_full, \
ret_from_except_full)
+#ifdef CONFIG_IPIPE
+#define EXC_XFER_IPIPE(n, hdlr) \
+ EXC_XFER_TEMPLATE(n, hdlr, n+1, NOCOPY, transfer_to_handler, \
+ __ipipe_ret_from_except)
+#endif /* CONFIG_IPIPE */
+
#define EXC_XFER_LITE(n, hdlr) \
EXC_XFER_TEMPLATE(n, hdlr, n+1, NOCOPY, transfer_to_handler, \
ret_from_except)
@@ -418,8 +424,12 @@
EXC_XFER_EE_LITE(0x400, handle_page_fault)
/* External interrupt */
+#ifdef CONFIG_IPIPE
+ EXCEPTION(0x500, HardwareInterrupt, __ipipe_grab_irq, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
-
+#endif /* CONFIG_IPIPE */
+
/* Alignment exception */
. = 0x600
Alignment:
@@ -443,7 +453,11 @@
EXC_XFER_EE_LITE(0x800, kernel_fp_unavailable_exception)
/* Decrementer */
+#ifdef CONFIG_IPIPE
+ EXCEPTION(0x900, Decrementer, __ipipe_grab_timer, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)
+#endif /* CONFIG_IPIPE */
EXCEPTION(0xa00, Trap_0a, unknown_exception, EXC_XFER_EE)
EXCEPTION(0xb00, Trap_0b, unknown_exception, EXC_XFER_EE)
diff -Naur linux-2.6.19.orig/arch/powerpc/kernel/head_44x.S linux-2.6.19.ipipe/arch/powerpc/kernel/head_44x.S
--- linux-2.6.19.orig/arch/powerpc/kernel/head_44x.S 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/kernel/head_44x.S 2006-12-19 13:02:15.000000000 +0100
@@ -427,8 +427,12 @@
INSTRUCTION_STORAGE_EXCEPTION
/* External Input Interrupt */
+#ifdef CONFIG_IPIPE
+ EXCEPTION(0x0500, ExternalInput, __ipipe_grab_irq, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
EXCEPTION(0x0500, ExternalInput, do_IRQ, EXC_XFER_LITE)
-
+#endif /* CONFIG_IPIPE */
+
/* Alignment Interrupt */
ALIGNMENT_EXCEPTION
diff -Naur linux-2.6.19.orig/arch/powerpc/kernel/head_4xx.S linux-2.6.19.ipipe/arch/powerpc/kernel/head_4xx.S
--- linux-2.6.19.orig/arch/powerpc/kernel/head_4xx.S 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/kernel/head_4xx.S 2006-12-19 13:02:15.000000000 +0100
@@ -228,6 +228,12 @@
EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, NOCOPY, transfer_to_handler_full, \
ret_from_except_full)
+#ifdef CONFIG_IPIPE
+#define EXC_XFER_IPIPE(n, hdlr) \
+ EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, NOCOPY, transfer_to_handler, \
+ __ipipe_ret_from_except)
+#endif /* CONFIG_IPIPE */
+
#define EXC_XFER_LITE(n, hdlr) \
EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, NOCOPY, transfer_to_handler, \
ret_from_except)
@@ -396,8 +402,12 @@
EXC_XFER_EE_LITE(0x400, handle_page_fault)
/* 0x0500 - External Interrupt Exception */
+#ifdef CONFIG_IPIPE
+ EXCEPTION(0x0500, HardwareInterrupt, __ipipe_grab_irq, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
EXCEPTION(0x0500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
-
+#endif /* CONFIG_IPIPE */
+
/* 0x0600 - Alignment Exception */
START_EXCEPTION(0x0600, Alignment)
NORMAL_EXCEPTION_PROLOG
@@ -434,8 +444,12 @@
lis r0,TSR_PIS@h
mtspr SPRN_TSR,r0 /* Clear the PIT exception */
addi r3,r1,STACK_FRAME_OVERHEAD
+#ifdef CONFIG_IPIPE
+ EXC_XFER_IPIPE(0x1000, __ipipe_grab_timer)
+#else /* !CONFIG_IPIPE */
EXC_XFER_LITE(0x1000, timer_interrupt)
-
+#endif /* CONFIG_IPIPE */
+
#if 0
/* NOTE:
* FIT and WDT handlers are not implemented yet.
diff -Naur linux-2.6.19.orig/arch/powerpc/kernel/head_8xx.S linux-2.6.19.ipipe/arch/powerpc/kernel/head_8xx.S
--- linux-2.6.19.orig/arch/powerpc/kernel/head_8xx.S 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/kernel/head_8xx.S 2006-12-19 13:02:15.000000000 +0100
@@ -187,6 +187,12 @@
EXC_XFER_TEMPLATE(n, hdlr, n, NOCOPY, transfer_to_handler_full, \
ret_from_except_full)
+#ifdef CONFIG_IPIPE
+#define EXC_XFER_IPIPE(n, hdlr) \
+ EXC_XFER_TEMPLATE(n, hdlr, n+1, NOCOPY, transfer_to_handler, \
+ __ipipe_ret_from_except)
+#endif /* CONFIG_IPIPE */
+
#define EXC_XFER_LITE(n, hdlr) \
EXC_XFER_TEMPLATE(n, hdlr, n+1, NOCOPY, transfer_to_handler, \
ret_from_except)
@@ -238,8 +244,12 @@
EXC_XFER_EE_LITE(0x400, handle_page_fault)
/* External interrupt */
+#ifdef CONFIG_IPIPE
+ EXCEPTION(0x500, HardwareInterrupt, __ipipe_grab_irq, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
-
+#endif /* CONFIG_IPIPE */
+
/* Alignment exception */
. = 0x600
Alignment:
@@ -259,8 +269,12 @@
EXCEPTION(0x800, FPUnavailable, unknown_exception, EXC_XFER_STD)
/* Decrementer */
+#ifdef CONFIG_IPIPE
+ EXCEPTION(0x900, Decrementer, __ipipe_grab_timer, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)
-
+#endif /* CONFIG_IPIPE */
+
EXCEPTION(0xa00, Trap_0a, unknown_exception, EXC_XFER_EE)
EXCEPTION(0xb00, Trap_0b, unknown_exception, EXC_XFER_EE)
diff -Naur linux-2.6.19.orig/arch/powerpc/kernel/head_booke.h linux-2.6.19.ipipe/arch/powerpc/kernel/head_booke.h
--- linux-2.6.19.orig/arch/powerpc/kernel/head_booke.h 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/kernel/head_booke.h 2006-12-19 13:02:15.000000000 +0100
@@ -187,6 +187,12 @@
EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, NOCOPY, transfer_to_handler_full, \
ret_from_except_full)
+#ifdef CONFIG_IPIPE
+#define EXC_XFER_IPIPE(n, hdlr) \
+ EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, NOCOPY, transfer_to_handler, \
+ __ipipe_ret_from_except)
+#endif /* CONFIG_IPIPE */
+
#define EXC_XFER_LITE(n, hdlr) \
EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, NOCOPY, transfer_to_handler, \
ret_from_except)
@@ -345,6 +351,15 @@
addi r3,r1,STACK_FRAME_OVERHEAD; \
EXC_XFER_STD(0x0700, program_check_exception)
+#ifdef CONFIG_IPIPE
+#define DECREMENTER_EXCEPTION \
+ START_EXCEPTION(Decrementer) \
+ NORMAL_EXCEPTION_PROLOG; \
+ lis r0,TSR_DIS@h; /* Setup the DEC interrupt mask */ \
+ mtspr SPRN_TSR,r0; /* Clear the DEC interrupt */ \
+ addi r3,r1,STACK_FRAME_OVERHEAD; \
+ EXC_XFER_IPIPE(0x0900, __ipipe_grab_timer)
+#else /* !CONFIG_IPIPE */
#define DECREMENTER_EXCEPTION \
START_EXCEPTION(Decrementer) \
NORMAL_EXCEPTION_PROLOG; \
@@ -352,6 +367,7 @@
mtspr SPRN_TSR,r0; /* Clear the DEC interrupt */ \
addi r3,r1,STACK_FRAME_OVERHEAD; \
EXC_XFER_LITE(0x0900, timer_interrupt)
+#endif /* CONFIG_IPIPE */
#define FP_UNAVAILABLE_EXCEPTION \
START_EXCEPTION(FloatingPointUnavailable) \
diff -Naur linux-2.6.19.orig/arch/powerpc/kernel/head_fsl_booke.S linux-2.6.19.ipipe/arch/powerpc/kernel/head_fsl_booke.S
--- linux-2.6.19.orig/arch/powerpc/kernel/head_fsl_booke.S 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/kernel/head_fsl_booke.S 2006-12-19 13:02:15.000000000 +0100
@@ -526,8 +526,12 @@
INSTRUCTION_STORAGE_EXCEPTION
/* External Input Interrupt */
+#ifdef CONFIG_IPIPE
+ EXCEPTION(0x0500, ExternalInput, __ipipe_grab_irq, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
EXCEPTION(0x0500, ExternalInput, do_IRQ, EXC_XFER_LITE)
-
+#endif /* CONFIG_IPIPE */
+
/* Alignment Interrupt */
ALIGNMENT_EXCEPTION
diff -Naur linux-2.6.19.orig/arch/powerpc/kernel/idle.c linux-2.6.19.ipipe/arch/powerpc/kernel/idle.c
--- linux-2.6.19.orig/arch/powerpc/kernel/idle.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/kernel/idle.c 2006-12-19 13:02:15.000000000 +0100
@@ -51,6 +51,7 @@
while (1) {
while (!need_resched() && !cpu_should_die()) {
ppc64_runlatch_off();
+ ipipe_suspend_domain();
if (ppc_md.power_save) {
clear_thread_flag(TIF_POLLING_NRFLAG);
@@ -59,13 +60,21 @@
* is ordered w.r.t. need_resched() test.
*/
smp_mb();
+#ifdef CONFIG_IPIPE
+ local_irq_disable_hw();
+#else /* !CONFIG_IPIPE */
local_irq_disable();
+#endif /* CONFIG_IPIPE */
/* check again after disabling irqs */
if (!need_resched() && !cpu_should_die())
ppc_md.power_save();
+#ifdef CONFIG_IPIPE
+ local_irq_enable_hw();
+#else /* !CONFIG_IPIPE */
local_irq_enable();
+#endif /* CONFIG_IPIPE */
set_thread_flag(TIF_POLLING_NRFLAG);
} else {
diff -Naur linux-2.6.19.orig/arch/powerpc/kernel/ipipe-core.c linux-2.6.19.ipipe/arch/powerpc/kernel/ipipe-core.c
--- linux-2.6.19.orig/arch/powerpc/kernel/ipipe-core.c 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/kernel/ipipe-core.c 2006-12-19 13:02:15.000000000 +0100
@@ -0,0 +1,263 @@
+/* -*- linux-c -*-
+ * linux/arch/powerpc/kernel/ipipe-core.c
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ * Copyright (C) 2004 Wolfgang Grandegger (Adeos/ppc port over 2.4).
+ * Copyright (C) 2005 Heikki Lindholm (PowerPC 970 fixes).
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-dependent I-PIPE core support for PowerPC.
+ */
+
+#include <linux/kernel.h>
+#include <linux/smp.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/bitops.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/module.h>
+#include <asm/system.h>
+#include <asm/atomic.h>
+#include <asm/hardirq.h>
+#include <asm/io.h>
+#include <asm/time.h>
+
+/* Current reload value for the decrementer. */
+unsigned long __ipipe_decr_ticks;
+
+/* Next tick date (timebase value). */
+unsigned long long __ipipe_decr_next[IPIPE_NR_CPUS];
+
+struct pt_regs __ipipe_tick_regs[IPIPE_NR_CPUS];
+
+#ifdef CONFIG_POWER4
+extern struct irqaction k2u3_cascade_action;
+extern int openpic2_get_irq(struct pt_regs *regs);
+#endif
+
+#ifdef CONFIG_SMP
+
+static cpumask_t __ipipe_cpu_sync_map;
+
+static cpumask_t __ipipe_cpu_lock_map;
+
+static ipipe_spinlock_t __ipipe_cpu_barrier = IPIPE_SPIN_LOCK_UNLOCKED;
+
+static atomic_t __ipipe_critical_count = ATOMIC_INIT(0);
+
+static void (*__ipipe_cpu_sync) (void);
+
+/* Always called with hw interrupts off. */
+
+void __ipipe_do_critical_sync(unsigned irq)
+{
+ ipipe_declare_cpuid;
+
+ ipipe_load_cpuid();
+
+ cpu_set(cpuid, __ipipe_cpu_sync_map);
+
+ /*
+ * Now we are in sync with the lock requestor running on another
+ * CPU. Enter a spinning wait until he releases the global
+ * lock.
+ */
+ spin_lock_hw(&__ipipe_cpu_barrier);
+
+ /* Got it. Now get out. */
+
+ if (__ipipe_cpu_sync)
+ /* Call the sync routine if any. */
+ __ipipe_cpu_sync();
+
+ spin_unlock_hw(&__ipipe_cpu_barrier);
+
+ cpu_clear(cpuid, __ipipe_cpu_sync_map);
+}
+
+#endif /* CONFIG_SMP */
+
+/*
+ * ipipe_critical_enter() -- Grab the superlock excluding all CPUs
+ * but the current one from a critical section. This lock is used when
+ * we must enforce a global critical section for a single CPU in a
+ * possibly SMP system whichever context the CPUs are running.
+ */
+unsigned long ipipe_critical_enter(void (*syncfn) (void))
+{
+ unsigned long flags;
+
+ local_irq_save_hw(flags);
+
+#ifdef CONFIG_SMP
+ if (num_online_cpus() > 1) { /* We might be running a SMP-kernel on a UP box... */
+ ipipe_declare_cpuid;
+ cpumask_t lock_map;
+
+ ipipe_load_cpuid();
+
+ if (!cpu_test_and_set(cpuid, __ipipe_cpu_lock_map)) {
+ while (cpu_test_and_set(BITS_PER_LONG - 1,
+ __ipipe_cpu_lock_map)) {
+ int n = 0;
+ do {
+ cpu_relax();
+ } while (++n < cpuid);
+ }
+
+ spin_lock_hw(&__ipipe_cpu_barrier);
+
+ __ipipe_cpu_sync = syncfn;
+
+ /* Send the sync IPI to all processors but the current one. */
+ send_IPI_allbutself(IPIPE_CRITICAL_VECTOR);
+
+ cpus_andnot(lock_map, cpu_online_map,
+ __ipipe_cpu_lock_map);
+
+ while (!cpus_equal(__ipipe_cpu_sync_map, lock_map))
+ cpu_relax();
+ }
+
+ atomic_inc(&__ipipe_critical_count);
+ }
+#endif /* CONFIG_SMP */
+
+ return flags;
+}
+
+/* ipipe_critical_exit() -- Release the superlock. */
+
+void ipipe_critical_exit(unsigned long flags)
+{
+#ifdef CONFIG_SMP
+ if (num_online_cpus() > 1) { /* We might be running a SMP-kernel on a UP box... */
+ ipipe_declare_cpuid;
+
+ ipipe_load_cpuid();
+
+ if (atomic_dec_and_test(&__ipipe_critical_count)) {
+ spin_unlock_hw(&__ipipe_cpu_barrier);
+
+ while (!cpus_empty(__ipipe_cpu_sync_map))
+ cpu_relax();
+
+ cpu_clear(cpuid, __ipipe_cpu_lock_map);
+ cpu_clear(BITS_PER_LONG - 1, __ipipe_cpu_lock_map);
+ }
+ }
+#endif /* CONFIG_SMP */
+
+ local_irq_restore_hw(flags);
+}
+
+void __ipipe_init_platform(void)
+{
+ unsigned timer_virq;
+
+ /*
+ * Allocate a virtual IRQ for the decrementer trap early to
+ * get it mapped to IPIPE_VIRQ_BASE
+ */
+
+ timer_virq = ipipe_alloc_virq();
+
+ if (timer_virq != IPIPE_TIMER_VIRQ)
+ panic("I-pipe: cannot reserve timer virq #%d (got #%d)",
+ IPIPE_TIMER_VIRQ, timer_virq);
+
+ __ipipe_decr_ticks = tb_ticks_per_jiffy;
+}
+
+int ipipe_get_sysinfo(struct ipipe_sysinfo *info)
+{
+ info->ncpus = num_online_cpus();
+ info->cpufreq = ipipe_cpu_freq();
+ info->archdep.tmirq = IPIPE_TIMER_VIRQ;
+ info->archdep.tmfreq = info->cpufreq;
+
+ return 0;
+}
+
+/*
+ * ipipe_trigger_irq() -- Push the interrupt at front of the pipeline
+ * just like if it has been actually received from a hw source. Also
+ * works for virtual interrupts.
+ */
+int ipipe_trigger_irq(unsigned irq)
+{
+ unsigned long flags;
+
+ if (irq >= IPIPE_NR_IRQS ||
+ (ipipe_virtual_irq_p(irq)
+ && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)))
+ return -EINVAL;
+
+ local_irq_save_hw(flags);
+
+ __ipipe_handle_irq(irq, NULL);
+
+ local_irq_restore_hw(flags);
+
+ return 1;
+}
+
+static void __ipipe_set_decr(void)
+{
+ ipipe_declare_cpuid;
+
+ ipipe_load_cpuid();
+
+#ifdef CONFIG_40x
+ /* Enable and set auto-reload. */
+ mtspr(SPRN_TCR, mfspr(SPRN_TCR) | TCR_ARE);
+ mtspr(SPRN_PIT, __ipipe_decr_ticks);
+#else /* !CONFIG_40x */
+ __ipipe_decr_next[cpuid] = __ipipe_read_timebase() + __ipipe_decr_ticks;
+ set_dec(__ipipe_decr_ticks);
+#endif /* CONFIG_40x */
+}
+
+int ipipe_tune_timer(unsigned long ns, int flags)
+{
+ unsigned long x, ticks;
+
+ if (flags & IPIPE_RESET_TIMER)
+ ticks = tb_ticks_per_jiffy;
+ else {
+ ticks = (ns / 1000) * tb_ticks_per_jiffy / (1000000 / HZ);
+
+ if (ticks > tb_ticks_per_jiffy)
+ return -EINVAL;
+ }
+
+ x = ipipe_critical_enter(&__ipipe_set_decr); /* Sync with all CPUs */
+ __ipipe_decr_ticks = ticks;
+ __ipipe_set_decr();
+ ipipe_critical_exit(x);
+
+ return 0;
+}
+
+EXPORT_SYMBOL(__ipipe_decr_ticks);
+EXPORT_SYMBOL(__ipipe_decr_next);
+EXPORT_SYMBOL(ipipe_critical_enter);
+EXPORT_SYMBOL(ipipe_critical_exit);
+EXPORT_SYMBOL(ipipe_trigger_irq);
+EXPORT_SYMBOL(ipipe_get_sysinfo);
+EXPORT_SYMBOL(ipipe_tune_timer);
diff -Naur linux-2.6.19.orig/arch/powerpc/kernel/ipipe-mcount.S linux-2.6.19.ipipe/arch/powerpc/kernel/ipipe-mcount.S
--- linux-2.6.19.orig/arch/powerpc/kernel/ipipe-mcount.S 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/kernel/ipipe-mcount.S 2006-12-19 13:02:15.000000000 +0100
@@ -0,0 +1,76 @@
+/*
+ * linux/arch/powerpc/kernel/ipipe-mcount.S
+ *
+ * Adapted from glibc's ppc32 profiling support --rpm.
+ */
+
+#include <linux/config.h>
+#include <asm/page.h>
+#include <asm/ppc_asm.h>
+#include <asm/reg.h>
+
+/*
+ * Excerpt from sysdeps/powerpc/powerpc32/ppc-mcount.S
+ *
+ * Do profiling as described in the SYSV ELF ABI, _mcount is called
+ * with the address of a data word in r0 (that is different for every
+ * routine, initialised to 0, and otherwise unused). The caller has
+ * put the address the caller will return to in the usual place on the stack,
+ * 4(r1). _mcount is responsible for ensuring that when it returns no
+ * argument-passing registers are disturbed, and that the LR is set back
+ * to (what the caller sees as) 4(r1).
+ *
+ * This is intended so that the following code can be inserted at the
+ * front of any routine without changing the routine:
+ *
+ * mflr r0
+ * lis r12,0b@domain.hid
+ * stw r0,4(r1)
+ * addi r0,r12,0b@domain.hid
+ * bl _mcount
+ */
+
+.globl _mcount
+_mcount:
+ stwu r1,-48(r1)
+/* We need to save the parameter-passing registers. */
+ stw r3, 12(r1)
+ stw r4, 16(r1)
+ stw r5, 20(r1)
+ stw r6, 24(r1)
+ stw r7, 28(r1)
+ stw r8, 32(r1)
+ stw r9, 36(r1)
+ stw r10,40(r1)
+ mflr r4
+ mfcr r5
+ stw r4, 44(r1)
+ stw r5, 8(r1)
+ tophys(r0,0)
+ addis r6,r0,ipipe_trace_enable@domain.hid
+ lwz r3,ipipe_trace_enable@domain.hid)
+ cmpwi r3,0
+ beq 1f
+ li r3,0
+ lwz r5, 52(r1)
+ li r6,0
+ bl __ipipe_trace
+1:
+ /* Restore the registers... */
+ lwz r6, 8(r1)
+ lwz r0, 44(r1)
+ lwz r3, 12(r1)
+ mtctr r0
+ lwz r4, 16(r1)
+ mtcrf 0xff,r6
+ lwz r5, 20(r1)
+ lwz r6, 24(r1)
+ lwz r0, 52(r1)
+ lwz r7, 28(r1)
+ lwz r8, 32(r1)
+ mtlr r0
+ lwz r9, 36(r1)
+ lwz r10,40(r1)
+ /* ...unwind the stack frame, and return to your usual programming. */
+ addi r1,r1,48
+ bctr
diff -Naur linux-2.6.19.orig/arch/powerpc/kernel/ipipe-root.c linux-2.6.19.ipipe/arch/powerpc/kernel/ipipe-root.c
--- linux-2.6.19.orig/arch/powerpc/kernel/ipipe-root.c 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/kernel/ipipe-root.c 2006-12-19 13:02:15.000000000 +0100
@@ -0,0 +1,490 @@
+/* -*- linux-c -*-
+ * linux/arch/powerpc/kernel/ipipe-root.c
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum (Adeos/ppc port over 2.6).
+ * Copyright (C) 2004 Wolfgang Grandegger (Adeos/ppc port over 2.4).
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-dependent I-pipe support for PowerPC.
+ */
+
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/errno.h>
+#include <asm/unistd.h>
+#include <asm/system.h>
+#include <asm/hardirq.h>
+#include <asm/machdep.h>
+#include <asm/atomic.h>
+#include <asm/io.h>
+#include <asm/time.h>
+#include <asm/mmu_context.h>
+
+extern irq_desc_t irq_desc[];
+
+static struct hw_interrupt_type __ipipe_std_irq_dtype[NR_IRQS];
+
+static void __ipipe_override_irq_enable(unsigned irq)
+{
+ unsigned long flags;
+
+ local_irq_save_hw(flags);
+ ipipe_irq_unlock(irq);
+ __ipipe_std_irq_dtype[irq].enable(irq);
+ local_irq_restore_hw(flags);
+}
+
+static void __ipipe_override_irq_disable(unsigned irq)
+{
+ unsigned long flags;
+
+ local_irq_save_hw(flags);
+ ipipe_irq_lock(irq);
+ __ipipe_std_irq_dtype[irq].disable(irq);
+ local_irq_restore_hw(flags);
+}
+
+static void __ipipe_override_irq_end(unsigned irq)
+{
+ unsigned long flags;
+
+ local_irq_save_hw(flags);
+
+ if (!(irq_desc[irq].status & (IRQ_DISABLED | IRQ_INPROGRESS)))
+ ipipe_irq_unlock(irq);
+
+ __ipipe_std_irq_dtype[irq].end(irq);
+
+ local_irq_restore_hw(flags);
+}
+
+static void __ipipe_override_irq_affinity(unsigned irq, cpumask_t mask)
+{
+ unsigned long flags;
+
+ local_irq_save_hw(flags);
+ __ipipe_std_irq_dtype[irq].set_affinity(irq, mask);
+ local_irq_restore_hw(flags);
+}
+
+static void __ipipe_enable_sync(void)
+{
+ __ipipe_decr_next[ipipe_processor_id()] =
+ __ipipe_read_timebase() + get_dec();
+}
+
+void __ipipe_enable_irqdesc(unsigned irq)
+{
+ irq_desc[irq].status &= ~IRQ_DISABLED;
+}
+
+/*
+ * __ipipe_enable_pipeline() -- We are running on the boot CPU, hw
+ * interrupts are off, and secondary CPUs are still lost in space.
+ */
+void __ipipe_enable_pipeline(void)
+{
+ unsigned long flags;
+ unsigned irq;
+
+ flags = ipipe_critical_enter(&__ipipe_enable_sync);
+
+ /* First, virtualize all interrupts from the root domain. */
+
+ for (irq = 0; irq < NR_IRQS; irq++)
+ ipipe_virtualize_irq(ipipe_root_domain,
+ irq,
+ (ipipe_irq_handler_t)&__ipipe_do_IRQ, NULL,
+ &__ipipe_ack_irq,
+ IPIPE_HANDLE_MASK | IPIPE_PASS_MASK);
+
+ /*
+ * We use a virtual IRQ to handle the timer irq (decrementer trap)
+ * which has been allocated early in __ipipe_init_platform().
+ */
+
+ ipipe_virtualize_irq(ipipe_root_domain,
+ IPIPE_TIMER_VIRQ,
+ (ipipe_irq_handler_t)&__ipipe_do_timer, NULL,
+ NULL, IPIPE_HANDLE_MASK | IPIPE_PASS_MASK);
+
+ /*
+ * Interpose on the IRQ control routines so we can make them
+ * atomic using hw masking and prevent the interrupt log from
+ * being untimely flushed.
+ */
+
+ for (irq = 0; irq < NR_IRQS; irq++) {
+ if (irq_desc[irq].chip != NULL)
+ __ipipe_std_irq_dtype[irq] = *irq_desc[irq].chip;
+ }
+
+ /*
+ * The original controller structs are often shared, so we first
+ * save them all before changing any of them. Notice that we don't
+ * override the ack() handler since we will enforce the necessary
+ * setup in __ipipe_ack_irq().
+ */
+
+ for (irq = 0; irq < NR_IRQS; irq++) {
+ struct hw_interrupt_type *handler = irq_desc[irq].chip;
+
+ if (handler == NULL)
+ continue;
+
+ if (handler->enable != NULL)
+ handler->enable = &__ipipe_override_irq_enable;
+
+ if (handler->disable != NULL)
+ handler->disable = &__ipipe_override_irq_disable;
+
+ if (handler->end != NULL)
+ handler->end = &__ipipe_override_irq_end;
+
+ if (handler->set_affinity != NULL)
+ handler->set_affinity = &__ipipe_override_irq_affinity;
+ }
+
+ __ipipe_decr_next[ipipe_processor_id()] =
+ __ipipe_read_timebase() + get_dec();
+
+ ipipe_critical_exit(flags);
+}
+
+int __ipipe_ack_irq(unsigned irq)
+{
+ irq_desc_t *desc = irq_desc + irq;
+ unsigned long flags;
+ ipipe_declare_cpuid;
+
+ if (desc->chip->ack == NULL)
+ return 1;
+
+ /*
+ * No need to mask IRQs at hw level: we are always called from
+ * __ipipe_handle_irq(), so interrupts are already off. We
+ * stall the pipeline so that spin_lock_irq*() ops won't
+ * unintentionally flush it, since this could cause infinite
+ * recursion.
+ */
+
+ ipipe_load_cpuid();
+ flags = ipipe_test_and_stall_pipeline();
+ preempt_disable();
+ desc->chip->ack(irq);
+#ifdef CONFIG_POWER4
+ /* if it is a k2u3 cascaded irq, acknowledge it, also */
+ if (desc->action == &k2u3_cascade_action) {
+ struct pt_regs regs;
+ int irq2 = openpic2_get_irq(®s);
+ if (irq2 != -1) {
+ irq_desc_t *desc2 = irq_desc + irq2;
+ if (desc2->chip->ack)
+ desc2->chip->ack(irq2);
+ }
+ }
+#endif
+ preempt_enable_no_resched();
+ ipipe_restore_pipeline_nosync(per_cpu(ipipe_percpu_domain, cpuid), flags, cpuid);
+
+ return 1;
+}
+
+/*
+ * __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic
+ * interrupt protection log is maintained here for each domain. Hw
+ * interrupts are off on entry.
+ */
+void __ipipe_handle_irq(int irq, struct pt_regs *regs)
+{
+ struct ipipe_domain *this_domain, *next_domain;
+ struct list_head *head, *pos;
+ ipipe_declare_cpuid;
+ int m_ack, s_ack;
+
+ m_ack = (regs == NULL); /* Software-triggered IRQs do not need
+ * any ack. */
+ if (irq >= IPIPE_NR_IRQS) {
+ printk(KERN_ERR "I-pipe: spurious interrupt %d\n", irq);
+ return;
+ }
+
+ ipipe_load_cpuid();
+
+ this_domain = per_cpu(ipipe_percpu_domain, cpuid);
+
+ if (unlikely(test_bit(IPIPE_STICKY_FLAG, &this_domain->irqs[irq].control)))
+ head = &this_domain->p_link;
+ else {
+ head = __ipipe_pipeline.next;
+ next_domain = list_entry(head, struct ipipe_domain, p_link);
+ if (likely(test_bit(IPIPE_WIRED_FLAG, &next_domain->irqs[irq].control))) {
+ if (!m_ack && next_domain->irqs[irq].acknowledge != NULL)
+ next_domain->irqs[irq].acknowledge(irq);
+ if (likely(__ipipe_dispatch_wired(next_domain, irq)))
+ goto finalize;
+ return;
+ }
+ }
+
+ /* Ack the interrupt. */
+
+ s_ack = m_ack;
+ pos = head;
+
+ while (pos != &__ipipe_pipeline) {
+ next_domain = list_entry(pos, struct ipipe_domain, p_link);
+ /*
+ * For each domain handling the incoming IRQ, mark it as
+ * pending in its log.
+ */
+ if (test_bit(IPIPE_HANDLE_FLAG,
+ &next_domain->irqs[irq].control)) {
+ /*
+ * Domains that handle this IRQ are polled for
+ * acknowledging it by decreasing priority order. The
+ * interrupt must be made pending _first_ in the
+ * domain's status flags before the PIC is unlocked.
+ */
+
+ next_domain->cpudata[cpuid].irq_counters[irq].total_hits++;
+ next_domain->cpudata[cpuid].irq_counters[irq].pending_hits++;
+ __ipipe_set_irq_bit(next_domain, cpuid, irq);
+
+ /*
+ * Always get the first master acknowledge available.
+ * Once we've got it, allow slave acknowledge
+ * handlers to run (until one of them stops us).
+ */
+ if (next_domain->irqs[irq].acknowledge != NULL) {
+ if (!m_ack)
+ m_ack = next_domain->irqs[irq].acknowledge(irq);
+ else if (test_bit
+ (IPIPE_SHARED_FLAG,
+ &next_domain->irqs[irq].control) && !s_ack)
+ s_ack = next_domain->irqs[irq].acknowledge(irq);
+ }
+ }
+
+ /*
+ * If the domain does not want the IRQ to be passed down the
+ * interrupt pipe, exit the loop now.
+ */
+
+ if (!test_bit(IPIPE_PASS_FLAG, &next_domain->irqs[irq].control))
+ break;
+
+ pos = next_domain->p_link.next;
+ }
+
+finalize:
+ /*
+ * Now walk the pipeline, yielding control to the highest
+ * priority domain that has pending interrupt(s) or
+ * immediately to the current domain if the interrupt has been
+ * marked as 'sticky'. This search does not go beyond the
+ * current domain in the pipeline.
+ */
+
+ __ipipe_walk_pipeline(head, cpuid);
+}
+
+int __ipipe_grab_irq(struct pt_regs *regs)
+{
+ extern int ppc_spurious_interrupts;
+ ipipe_declare_cpuid;
+ int irq;
+
+ irq = ppc_md.get_irq();
+
+ if (irq != NO_IRQ && irq != NO_IRQ_IGNORE) {
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+ ipipe_trace_begin(irq);
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+ __ipipe_handle_irq(irq, regs);
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+ ipipe_trace_end(irq);
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+ }
+ else if (irq != NO_IRQ_IGNORE)
+ ppc_spurious_interrupts++;
+
+ ipipe_load_cpuid();
+
+ return (per_cpu(ipipe_percpu_domain, cpuid) == ipipe_root_domain &&
+ !test_bit(IPIPE_STALL_FLAG,
+ &ipipe_root_domain->cpudata[cpuid].status));
+}
+
+void __ipipe_do_IRQ(int irq, struct pt_regs *regs)
+{
+ irq_enter();
+ __do_IRQ(irq);
+ irq_exit();
+}
+
+int __ipipe_grab_timer(struct pt_regs *regs)
+{
+ ipipe_declare_cpuid;
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+ ipipe_trace_begin(IPIPE_TIMER_VIRQ);
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+
+#ifdef CONFIG_POWER4
+ /* On 970 CPUs DEC cannot be disabled, and without setting DEC
+ * here, DEC interrupt would be triggered as soon as interrupts
+ * are enabled in __ipipe_sync_stage
+ */
+ set_dec(0x7fffffff);
+#endif
+
+ __ipipe_tick_regs[cpuid].msr = regs->msr; /* for do_timer() */
+
+#ifndef CONFIG_40x
+ if (__ipipe_decr_ticks != tb_ticks_per_jiffy) {
+ unsigned long long next_date, now;
+
+ next_date = __ipipe_decr_next[cpuid];
+
+ while ((now = __ipipe_read_timebase()) >= next_date)
+ next_date += __ipipe_decr_ticks;
+
+ set_dec(next_date - now);
+
+ __ipipe_decr_next[cpuid] = next_date;
+ }
+#endif /* !CONFIG_40x */
+
+ __ipipe_handle_irq(IPIPE_TIMER_VIRQ, regs);
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+ ipipe_trace_end(IPIPE_TIMER_VIRQ);
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+
+ ipipe_load_cpuid();
+
+ return (per_cpu(ipipe_percpu_domain, cpuid) == ipipe_root_domain &&
+ !test_bit(IPIPE_STALL_FLAG,
+ &ipipe_root_domain->cpudata[cpuid].status));
+}
+
+void __ipipe_do_timer(int irq, struct pt_regs *regs)
+{
+ timer_interrupt(regs);
+}
+
+int __ipipe_check_root(struct pt_regs *regs)
+{
+ ipipe_declare_cpuid;
+ /*
+ * This routine is called with hw interrupts off, so no migration
+ * can occur while checking the identity of the current domain.
+ */
+ ipipe_load_cpuid();
+ return per_cpu(ipipe_percpu_domain, cpuid) == ipipe_root_domain;
+}
+
+void __ipipe_fast_stall_root(void)
+{
+ ipipe_declare_cpuid;
+ unsigned long flags;
+
+ ipipe_get_cpu(flags); /* Care for migration. */
+
+ set_bit(IPIPE_STALL_FLAG,
+ &ipipe_root_domain->cpudata[cpuid].status);
+
+ ipipe_put_cpu(flags);
+}
+
+void __ipipe_fast_unstall_root(void)
+{
+ ipipe_declare_cpuid;
+ unsigned long flags;
+
+ ipipe_get_cpu(flags); /* Care for migration. */
+
+ clear_bit(IPIPE_STALL_FLAG,
+ &ipipe_root_domain->cpudata[cpuid].status);
+
+ ipipe_put_cpu(flags);
+}
+
+int __ipipe_syscall_root(struct pt_regs *regs)
+{
+ ipipe_declare_cpuid;
+ unsigned long flags;
+
+ /*
+ * This routine either returns:
+ * 0 -- if the syscall is to be passed to Linux;
+ * >0 -- if the syscall should not be passed to Linux, and no
+ * tail work should be performed;
+ * <0 -- if the syscall should not be passed to Linux but the
+ * tail work has to be performed (for handling signals etc).
+ */
+
+ if (__ipipe_syscall_watched_p(current, regs->gpr[0]) &&
+ __ipipe_event_monitored_p(IPIPE_EVENT_SYSCALL) &&
+ __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL,regs) > 0) {
+ if (ipipe_current_domain == ipipe_root_domain && !in_atomic()) {
+ /*
+ * Sync pending VIRQs before _TIF_NEED_RESCHED
+ * is tested.
+ */
+ ipipe_lock_cpu(flags);
+ if ((ipipe_root_domain->cpudata[cpuid].irq_pending_hi & IPIPE_IRQMASK_VIRT) != 0)
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_VIRT);
+ ipipe_unlock_cpu(flags);
+ return -1;
+ }
+ return 1;
+ }
+
+ return 0;
+}
+
+void atomic_set_mask(unsigned long mask,
+ unsigned long *ptr);
+
+void atomic_clear_mask(unsigned long mask,
+ unsigned long *ptr);
+
+extern unsigned long context_map[];
+
+EXPORT_SYMBOL_GPL(__switch_to);
+EXPORT_SYMBOL_GPL(show_stack);
+EXPORT_SYMBOL_GPL(atomic_set_mask);
+EXPORT_SYMBOL_GPL(atomic_clear_mask);
+EXPORT_SYMBOL_GPL(context_map);
+EXPORT_SYMBOL_GPL(_switch);
+EXPORT_SYMBOL_GPL(last_task_used_math);
+#ifdef FEW_CONTEXTS
+EXPORT_SYMBOL_GPL(nr_free_contexts);
+EXPORT_SYMBOL_GPL(context_mm);
+EXPORT_SYMBOL_GPL(steal_context);
+#endif
+
+#ifdef CONFIG_IPIPE_TRACE_MCOUNT
+void notrace _mcount(void);
+EXPORT_SYMBOL(_mcount);
+#endif /* CONFIG_IPIPE_TRACE_MCOUNT */
diff -Naur linux-2.6.19.orig/arch/powerpc/kernel/irq.c linux-2.6.19.ipipe/arch/powerpc/kernel/irq.c
--- linux-2.6.19.orig/arch/powerpc/kernel/irq.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/kernel/irq.c 2006-12-19 13:02:15.000000000 +0100
@@ -69,7 +69,11 @@
#endif
int __irq_offset_value;
+#ifdef CONFIG_IPIPE
+int ppc_spurious_interrupts;
+#else /* !CONFIG_IPIPE */
static int ppc_spurious_interrupts;
+#endif /* CONFIG_IPIPE */
#ifdef CONFIG_PPC32
EXPORT_SYMBOL(__irq_offset_value);
diff -Naur linux-2.6.19.orig/arch/powerpc/kernel/Makefile linux-2.6.19.ipipe/arch/powerpc/kernel/Makefile
--- linux-2.6.19.orig/arch/powerpc/kernel/Makefile 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/kernel/Makefile 2006-12-19 13:02:15.000000000 +0100
@@ -57,6 +57,8 @@
obj-$(CONFIG_MODULES) += ppc_ksyms.o
obj-$(CONFIG_BOOTX_TEXT) += btext.o
obj-$(CONFIG_SMP) += smp.o
+obj-$(CONFIG_IPIPE) += ipipe-core.o ipipe-root.o
+obj-$(CONFIG_IPIPE_TRACE_MCOUNT) += ipipe-mcount.o
obj-$(CONFIG_KPROBES) += kprobes.o
obj-$(CONFIG_PPC_UDBG_16550) += legacy_serial.o udbg_16550.o
module-$(CONFIG_PPC64) += module_64.o
diff -Naur linux-2.6.19.orig/arch/powerpc/kernel/setup_32.c linux-2.6.19.ipipe/arch/powerpc/kernel/setup_32.c
--- linux-2.6.19.orig/arch/powerpc/kernel/setup_32.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/kernel/setup_32.c 2006-12-19 13:02:15.000000000 +0100
@@ -88,7 +88,7 @@
* from the address that it was linked at, so we must use RELOC/PTRRELOC
* to access static data (including strings). -- paulus
*/
-unsigned long __init early_init(unsigned long dt_ptr)
+notrace unsigned long __init early_init(unsigned long dt_ptr)
{
unsigned long offset = reloc_offset();
struct cpu_spec *spec;
diff -Naur linux-2.6.19.orig/arch/powerpc/kernel/time.c linux-2.6.19.ipipe/arch/powerpc/kernel/time.c
--- linux-2.6.19.orig/arch/powerpc/kernel/time.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/kernel/time.c 2006-12-19 13:02:15.000000000 +0100
@@ -296,7 +296,11 @@
#else /* ! CONFIG_VIRT_CPU_ACCOUNTING */
#define calc_cputime_factors()
+#ifdef CONFIG_IPIPE
+#define account_process_time(regs) update_process_times(user_mode(__ipipe_tick_regs + smp_processor_id()))
+#else
#define account_process_time(regs) update_process_times(user_mode(regs))
+#endif /* CONFIG_IPIPE */
#define calculate_steal_time() do { } while (0)
#endif
@@ -669,10 +673,17 @@
}
write_sequnlock(&xtime_lock);
}
-
+
+#ifdef CONFIG_IPIPE
+ if (__ipipe_decr_ticks == tb_ticks_per_jiffy) {
+ next_dec = tb_ticks_per_jiffy - ticks;
+ set_dec(next_dec);
+ }
+#else /* !CONFIG_IPIPE */
next_dec = tb_ticks_per_jiffy - ticks;
set_dec(next_dec);
-
+#endif /* CONFIG_IPIPE */
+
#ifdef CONFIG_PPC_ISERIES
if (hvlpevent_is_pending())
process_hvlpevents();
diff -Naur linux-2.6.19.orig/arch/powerpc/kernel/traps.c linux-2.6.19.ipipe/arch/powerpc/kernel/traps.c
--- linux-2.6.19.orig/arch/powerpc/kernel/traps.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/kernel/traps.c 2006-12-19 13:02:15.000000000 +0100
@@ -324,6 +324,9 @@
int recover = 0;
unsigned long reason = get_mc_reason(regs);
+ if (ipipe_trap_notify(IPIPE_TRAP_MCE,regs))
+ return;
+
/* See if any machine dependent calls */
if (ppc_md.machine_check_exception)
recover = ppc_md.machine_check_exception(regs);
@@ -490,6 +493,10 @@
void unknown_exception(struct pt_regs *regs)
{
+ if (ipipe_trap_notify(IPIPE_TRAP_IABR,regs))
+ return;
+ if (ipipe_trap_notify(IPIPE_TRAP_UNKNOWN,regs))
+ return;
printk("Bad trap at PC: %lx, SR: %lx, vector=%lx\n",
regs->nip, regs->msr, regs->trap);
@@ -508,13 +515,17 @@
void RunModeException(struct pt_regs *regs)
{
+ if (ipipe_trap_notify(IPIPE_TRAP_RM,regs))
+ return;
_exception(SIGTRAP, regs, 0, 0);
}
void __kprobes single_step_exception(struct pt_regs *regs)
{
regs->msr &= ~(MSR_SE | MSR_BE); /* Turn off 'trace' bits */
-
+ if (ipipe_trap_notify(IPIPE_TRAP_SSTEP,regs))
+ return;
+
if (notify_die(DIE_SSTEP, "single_step", regs, 5,
5, SIGTRAP) == NOTIFY_STOP)
return;
@@ -534,7 +545,9 @@
{
if (single_stepping(regs)) {
clear_single_step(regs);
- _exception(SIGTRAP, regs, TRAP_TRACE, 0);
+ if (ipipe_trap_notify(IPIPE_TRAP_SSTEP,regs))
+ return;
+ _exception(SIGTRAP, regs, TRAP_TRACE, 0);
}
}
@@ -786,6 +799,9 @@
unsigned int reason = get_reason(regs);
extern int do_mathemu(struct pt_regs *regs);
+ if (ipipe_trap_notify(IPIPE_TRAP_PCE,regs))
+ return;
+
#ifdef CONFIG_MATH_EMULATION
/* (reason & REASON_ILLEGAL) would be the obvious thing here,
* but there seems to be a hardware bug on the 405GP (RevD)
@@ -855,6 +871,9 @@
return;
}
+ if (ipipe_trap_notify(IPIPE_TRAP_ALIGNMENT,regs))
+ return;
+
/* Operand address was bad */
if (fixed == -EFAULT) {
sig = SIGSEGV;
@@ -880,6 +899,8 @@
void nonrecoverable_exception(struct pt_regs *regs)
{
+ if (ipipe_trap_notify(IPIPE_TRAP_NREC,regs))
+ return;
printk(KERN_ERR "Non-recoverable exception at PC=%lx MSR=%lx\n",
regs->nip, regs->msr);
debugger(regs);
@@ -902,6 +923,9 @@
void altivec_unavailable_exception(struct pt_regs *regs)
{
+ if (ipipe_trap_notify(IPIPE_TRAP_ALTUNAVAIL,regs))
+ return;
+
if (user_mode(regs)) {
/* A user program has executed an altivec instruction,
but this kernel doesn't support altivec. */
@@ -926,6 +950,9 @@
extern int Soft_emulate_8xx(struct pt_regs *);
int errcode;
+ if (ipipe_trap_notify(IPIPE_TRAP_SOFTEMU,regs))
+ return;
+
CHECK_FULL_REGS(regs);
if (!user_mode(regs)) {
@@ -954,6 +981,9 @@
void DebugException(struct pt_regs *regs, unsigned long debug_status)
{
+ if (ipipe_trap_notify(IPIPE_TRAP_DEBUG,regs))
+ return;
+
if (debug_status & DBSR_IC) { /* instruction completion */
regs->msr &= ~MSR_DE;
if (user_mode(regs)) {
@@ -984,6 +1014,9 @@
{
int err;
+ if (ipipe_trap_notify(IPIPE_TRAP_ALTASSIST,regs))
+ return;
+
if (!user_mode(regs)) {
printk(KERN_EMERG "VMX/Altivec assist exception in kernel mode"
" at %lx\n", regs->nip);
@@ -1034,6 +1067,9 @@
int fpexc_mode;
int code = 0;
+ if (ipipe_trap_notify(IPIPE_TRAP_SPE,regs))
+ return;
+
spefscr = current->thread.spefscr;
fpexc_mode = current->thread.fpexc_mode;
diff -Naur linux-2.6.19.orig/arch/powerpc/mm/fault.c linux-2.6.19.ipipe/arch/powerpc/mm/fault.c
--- linux-2.6.19.orig/arch/powerpc/mm/fault.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/powerpc/mm/fault.c 2006-12-19 13:02:15.000000000 +0100
@@ -175,6 +175,9 @@
is_write = error_code & ESR_DST;
#endif /* CONFIG_4xx || CONFIG_BOOKE */
+ if (ipipe_trap_notify(IPIPE_TRAP_ACCESS,regs))
+ return 0;
+
if (notify_page_fault(DIE_PAGE_FAULT, "page_fault", regs, error_code,
11, SIGSEGV) == NOTIFY_STOP)
return 0;
diff -Naur linux-2.6.19.orig/arch/ppc/boot/Makefile linux-2.6.19.ipipe/arch/ppc/boot/Makefile
--- linux-2.6.19.orig/arch/ppc/boot/Makefile 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/boot/Makefile 2006-12-19 13:02:15.000000000 +0100
@@ -16,6 +16,14 @@
CFLAGS += -fno-builtin -D__BOOTER__ -Iarch/$(ARCH)/boot/include
HOSTCFLAGS += -Iarch/$(ARCH)/boot/include
+ifdef CONFIG_IPIPE_TRACE
+# do not trace the boot loader
+nullstring :=
+space := $(nullstring) # end of the line
+pg_flag = $(nullstring) -pg # end of the line
+CFLAGS := $(subst ${pg_flag},${space},${CFLAGS})
+endif
+
BOOT_TARGETS = zImage zImage.initrd znetboot znetboot.initrd
bootdir-y := simple
diff -Naur linux-2.6.19.orig/arch/ppc/Kconfig linux-2.6.19.ipipe/arch/ppc/Kconfig
--- linux-2.6.19.orig/arch/ppc/Kconfig 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/Kconfig 2006-12-19 13:02:15.000000000 +0100
@@ -950,6 +950,8 @@
depends on SMP
default "4"
+source "kernel/ipipe/Kconfig"
+
config HIGHMEM
bool "High memory support"
diff -Naur linux-2.6.19.orig/arch/ppc/kernel/entry.S linux-2.6.19.ipipe/arch/ppc/kernel/entry.S
--- linux-2.6.19.orig/arch/ppc/kernel/entry.S 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/kernel/entry.S 2006-12-19 13:02:15.000000000 +0100
@@ -132,8 +132,23 @@
* check for stack overflow
*/
lwz r9,THREAD_INFO-THREAD(r12)
+#ifdef CONFIG_IPIPE
+ /* Allow for private kernel-based stacks: those must not cause
+ the stack overflow detection to trigger when some activity has
+ been preempted over them. We just check if the kernel stack is
+ not treading on the memory area ranging from
+ ¤t->thread_info to ¤t->thread, which is coarser
+ than the vanilla implementation, but likely sensitive enough
+ to catch overflows soon enough though.*/
+ addi r12,r9,THREAD
+ cmplw 0,r1,r9
+ cmplw 1,r1,r12
+ crand 1,1,4
+ bgt- stack_ovf /* if r9 < r1 < r9+THREAD */
+#else /* CONFIG_IPIPE */
cmplw r1,r9 /* if r1 <= current->thread_info */
ble- stack_ovf /* then the kernel stack overflowed */
+#endif /* CONFIG_IPIPE */
5:
#ifdef CONFIG_6xx
tophys(r9,r9) /* check local flags */
@@ -198,6 +213,21 @@
lwz r11,_CCR(r1) /* Clear SO bit in CR */
rlwinm r11,r11,0,4,2
stw r11,_CCR(r1)
+#ifdef CONFIG_IPIPE
+ addi r3,r1,GPR0
+ bl __ipipe_syscall_root
+ cmpwi r3,0
+ lwz r3,GPR3(r1)
+ lwz r0,GPR0(r1)
+ lwz r4,GPR4(r1)
+ lwz r5,GPR5(r1)
+ lwz r6,GPR6(r1)
+ lwz r7,GPR7(r1)
+ lwz r8,GPR8(r1)
+ lwz r9,GPR9(r1)
+ bgt .ipipe_end_syscall
+ blt ret_from_syscall
+#endif /* CONFIG_IPIPE */
#ifdef SHOW_SYSCALLS
bl do_show_syscall
#endif /* SHOW_SYSCALLS */
@@ -260,11 +290,34 @@
SYNC
RFI
+#ifdef CONFIG_IPIPE
+.ipipe_end_syscall:
+ LOAD_MSR_KERNEL(r10,MSR_KERNEL) /* doesn't include MSR_EE */
+ SYNC
+ MTMSRD(r10)
+ b syscall_exit_cont
+#endif /* CONFIG_IPIPE */
+
66: li r3,-ENOSYS
b ret_from_syscall
.globl ret_from_fork
ret_from_fork:
+#ifdef CONFIG_IPIPE
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+ stwu r1,-4(r1)
+ stw r3,0(r1)
+ lis r3,(0x80000000)@h
+ ori r3,r3,(0x80000000)@l
+ bl ipipe_trace_end
+ lwz r3,0(r1)
+ addi r1,r1,4
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+ LOAD_MSR_KERNEL(r10,MSR_KERNEL)
+ ori r10,r10,MSR_EE
+ SYNC
+ MTMSRD(r10)
+#endif /* CONFIG_IPIPE */
REST_NVGPRS(r1)
bl schedule_tail
li r3,0
@@ -630,6 +683,12 @@
SYNC /* Some chip revs have problems here... */
MTMSRD(r10) /* disable interrupts */
+#ifdef CONFIG_IPIPE
+ bl __ipipe_check_root
+ cmpwi r3, 0
+ beq- restore
+#endif /* CONFIG_IPIPE */
+
lwz r3,_MSR(r1) /* Returning to user mode? */
andi. r0,r3,MSR_PR
beq resume_kernel
@@ -665,11 +724,37 @@
beq+ restore
andi. r0,r3,MSR_EE /* interrupts off? */
beq restore /* don't schedule if so */
+#ifdef CONFIG_IPIPE
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+ lis r3,(0x80000000)@h
+ ori r3,r3,(0x80000000)@l
+ bl ipipe_trace_end
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+ LOAD_MSR_KERNEL(r10,MSR_KERNEL)
+ ori r10,r10,MSR_EE
+ SYNC
+ MTMSRD(r10)
+ bl __ipipe_fast_stall_root
+#endif /* CONFIG_IPIPE */
1: bl preempt_schedule_irq
rlwinm r9,r1,0,0,18
lwz r3,TI_FLAGS(r9)
andi. r0,r3,_TIF_NEED_RESCHED
bne- 1b
+#ifdef CONFIG_IPIPE
+ bl __ipipe_fast_unstall_root
+ LOAD_MSR_KERNEL(r10,MSR_KERNEL)
+ SYNC
+ MTMSRD(r10)
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+ lwz r3,_MSR(r1)
+ andi. r0,r3,MSR_EE
+ bne restore
+ lis r3,(0x80000000)@h
+ ori r3,r3,(0x80000000)@l
+ bl ipipe_trace_begin
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+#endif /* CONFIG_IPIPE */
#else
resume_kernel:
#endif /* CONFIG_PREEMPT */
@@ -928,3 +1013,10 @@
b 4b
.comm ee_restarts,4
+
+#ifdef CONFIG_IPIPE
+_GLOBAL(__ipipe_ret_from_except)
+ cmpwi r3, 0
+ bne+ ret_from_except
+ b restore
+#endif /* CONFIG_IPIPE */
diff -Naur linux-2.6.19.orig/arch/ppc/kernel/head_44x.S linux-2.6.19.ipipe/arch/ppc/kernel/head_44x.S
--- linux-2.6.19.orig/arch/ppc/kernel/head_44x.S 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/kernel/head_44x.S 2006-12-19 13:02:15.000000000 +0100
@@ -427,7 +427,11 @@
INSTRUCTION_STORAGE_EXCEPTION
/* External Input Interrupt */
- EXCEPTION(0x0500, ExternalInput, do_IRQ, EXC_XFER_LITE)
+#ifdef CONFIG_IPIPE
+ EXCEPTION(0x0500, ExternalInput, __ipipe_grab_irq, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
+ EXCEPTION(0x0500, ExternalInput, do_IRQ, EXC_XFER_LITE)
+#endif /* CONFIG_IPIPE */
/* Alignment Interrupt */
ALIGNMENT_EXCEPTION
diff -Naur linux-2.6.19.orig/arch/ppc/kernel/head_4xx.S linux-2.6.19.ipipe/arch/ppc/kernel/head_4xx.S
--- linux-2.6.19.orig/arch/ppc/kernel/head_4xx.S 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/kernel/head_4xx.S 2006-12-19 13:02:15.000000000 +0100
@@ -228,6 +228,12 @@
EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, NOCOPY, transfer_to_handler_full, \
ret_from_except_full)
+#ifdef CONFIG_IPIPE
+#define EXC_XFER_IPIPE(n, hdlr) \
+ EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, NOCOPY, transfer_to_handler, \
+ __ipipe_ret_from_except)
+#endif /* CONFIG_IPIPE */
+
#define EXC_XFER_LITE(n, hdlr) \
EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, NOCOPY, transfer_to_handler, \
ret_from_except)
@@ -396,7 +402,11 @@
EXC_XFER_EE_LITE(0x400, handle_page_fault)
/* 0x0500 - External Interrupt Exception */
+#ifdef CONFIG_IPIPE
+ EXCEPTION(0x0500, HardwareInterrupt, __ipipe_grab_irq, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
EXCEPTION(0x0500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
+#endif /* CONFIG_IPIPE */
/* 0x0600 - Alignment Exception */
START_EXCEPTION(0x0600, Alignment)
@@ -434,7 +444,11 @@
lis r0,TSR_PIS@h
mtspr SPRN_TSR,r0 /* Clear the PIT exception */
addi r3,r1,STACK_FRAME_OVERHEAD
+#ifdef CONFIG_IPIPE
+ EXC_XFER_IPIPE(0x1000, __ipipe_grab_timer)
+#else /* !CONFIG_IPIPE */
EXC_XFER_LITE(0x1000, timer_interrupt)
+#endif /* CONFIG_IPIPE */
#if 0
/* NOTE:
diff -Naur linux-2.6.19.orig/arch/ppc/kernel/head_8xx.S linux-2.6.19.ipipe/arch/ppc/kernel/head_8xx.S
--- linux-2.6.19.orig/arch/ppc/kernel/head_8xx.S 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/kernel/head_8xx.S 2006-12-19 13:02:15.000000000 +0100
@@ -186,6 +186,11 @@
#define EXC_XFER_STD(n, hdlr) \
EXC_XFER_TEMPLATE(n, hdlr, n, NOCOPY, transfer_to_handler_full, \
ret_from_except_full)
+#ifdef CONFIG_IPIPE
+#define EXC_XFER_IPIPE(n, hdlr) \
+ EXC_XFER_TEMPLATE(n, hdlr, n+1, NOCOPY, transfer_to_handler, \
+ __ipipe_ret_from_except)
+#endif /* CONFIG_IPIPE */
#define EXC_XFER_LITE(n, hdlr) \
EXC_XFER_TEMPLATE(n, hdlr, n+1, NOCOPY, transfer_to_handler, \
@@ -238,7 +243,11 @@
EXC_XFER_EE_LITE(0x400, handle_page_fault)
/* External interrupt */
+#ifdef CONFIG_IPIPE
+ EXCEPTION(0x500, HardwareInterrupt, __ipipe_grab_irq, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
+#endif /* CONFIG_IPIPE */
/* Alignment exception */
. = 0x600
@@ -259,7 +268,11 @@
EXCEPTION(0x800, FPUnavailable, unknown_exception, EXC_XFER_STD)
/* Decrementer */
+#ifdef CONFIG_IPIPE
+ EXCEPTION(0x900, Decrementer, __ipipe_grab_timer, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)
+#endif /* CONFIG_IPIPE */
EXCEPTION(0xa00, Trap_0a, unknown_exception, EXC_XFER_EE)
EXCEPTION(0xb00, Trap_0b, unknown_exception, EXC_XFER_EE)
diff -Naur linux-2.6.19.orig/arch/ppc/kernel/head_booke.h linux-2.6.19.ipipe/arch/ppc/kernel/head_booke.h
--- linux-2.6.19.orig/arch/ppc/kernel/head_booke.h 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/kernel/head_booke.h 2006-12-19 13:02:15.000000000 +0100
@@ -187,6 +187,12 @@
EXC_XFER_TEMPLATE(hdlr, n, MSR_KERNEL, NOCOPY, transfer_to_handler_full, \
ret_from_except_full)
+#ifdef CONFIG_IPIPE
+#define EXC_XFER_IPIPE(n, hdlr) \
+ EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, NOCOPY, transfer_to_handler, \
+ __ipipe_ret_from_except)
+#endif /* CONFIG_IPIPE */
+
#define EXC_XFER_LITE(n, hdlr) \
EXC_XFER_TEMPLATE(hdlr, n+1, MSR_KERNEL, NOCOPY, transfer_to_handler, \
ret_from_except)
@@ -345,6 +351,15 @@
addi r3,r1,STACK_FRAME_OVERHEAD; \
EXC_XFER_STD(0x0700, program_check_exception)
+#ifdef CONFIG_IPIPE
+#define DECREMENTER_EXCEPTION \
+ START_EXCEPTION(Decrementer) \
+ NORMAL_EXCEPTION_PROLOG; \
+ lis r0,TSR_DIS@h; /* Setup the DEC interrupt mask */ \
+ mtspr SPRN_TSR,r0; /* Clear the DEC interrupt */ \
+ addi r3,r1,STACK_FRAME_OVERHEAD; \
+ EXC_XFER_IPIPE(0x0900, __ipipe_grab_timer)
+#else /* !CONFIG_IPIPE */
#define DECREMENTER_EXCEPTION \
START_EXCEPTION(Decrementer) \
NORMAL_EXCEPTION_PROLOG; \
@@ -352,6 +367,7 @@
mtspr SPRN_TSR,r0; /* Clear the DEC interrupt */ \
addi r3,r1,STACK_FRAME_OVERHEAD; \
EXC_XFER_LITE(0x0900, timer_interrupt)
+#endif /* CONFIG_IPIPE */
#define FP_UNAVAILABLE_EXCEPTION \
START_EXCEPTION(FloatingPointUnavailable) \
diff -Naur linux-2.6.19.orig/arch/ppc/kernel/head_fsl_booke.S linux-2.6.19.ipipe/arch/ppc/kernel/head_fsl_booke.S
--- linux-2.6.19.orig/arch/ppc/kernel/head_fsl_booke.S 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/kernel/head_fsl_booke.S 2006-12-19 13:02:15.000000000 +0100
@@ -526,7 +526,11 @@
INSTRUCTION_STORAGE_EXCEPTION
/* External Input Interrupt */
+#ifdef CONFIG_IPIPE
+ EXCEPTION(0x0500, ExternalInput, __ipipe_grab_irq, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
EXCEPTION(0x0500, ExternalInput, do_IRQ, EXC_XFER_LITE)
+#endif /* CONFIG_IPIPE */
/* Alignment Interrupt */
ALIGNMENT_EXCEPTION
diff -Naur linux-2.6.19.orig/arch/ppc/kernel/head.S linux-2.6.19.ipipe/arch/ppc/kernel/head.S
--- linux-2.6.19.orig/arch/ppc/kernel/head.S 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/kernel/head.S 2006-12-19 13:02:15.000000000 +0100
@@ -295,6 +295,12 @@
EXC_XFER_TEMPLATE(n, hdlr, n, NOCOPY, transfer_to_handler_full, \
ret_from_except_full)
+#ifdef CONFIG_IPIPE
+#define EXC_XFER_IPIPE(n, hdlr) \
+ EXC_XFER_TEMPLATE(n, hdlr, n+1, NOCOPY, transfer_to_handler, \
+ __ipipe_ret_from_except)
+#endif /* CONFIG_IPIPE */
+
#define EXC_XFER_LITE(n, hdlr) \
EXC_XFER_TEMPLATE(n, hdlr, n+1, NOCOPY, transfer_to_handler, \
ret_from_except)
@@ -356,7 +362,11 @@
EXC_XFER_EE_LITE(0x400, handle_page_fault)
/* External interrupt */
+#ifdef CONFIG_IPIPE
+ EXCEPTION(0x500, HardwareInterrupt, __ipipe_grab_irq, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
EXCEPTION(0x500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
+#endif /* CONFIG_IPIPE */
/* Alignment exception */
. = 0x600
@@ -381,7 +391,11 @@
EXC_XFER_EE_LITE(0x800, kernel_fp_unavailable_exception)
/* Decrementer */
+#ifdef CONFIG_IPIPE
+ EXCEPTION(0x900, Decrementer, __ipipe_grab_timer, EXC_XFER_IPIPE)
+#else /* !CONFIG_IPIPE */
EXCEPTION(0x900, Decrementer, timer_interrupt, EXC_XFER_LITE)
+#endif /* CONFIG_IPIPE */
EXCEPTION(0xa00, Trap_0a, unknown_exception, EXC_XFER_EE)
EXCEPTION(0xb00, Trap_0b, unknown_exception, EXC_XFER_EE)
diff -Naur linux-2.6.19.orig/arch/ppc/kernel/ipipe-core.c linux-2.6.19.ipipe/arch/ppc/kernel/ipipe-core.c
--- linux-2.6.19.orig/arch/ppc/kernel/ipipe-core.c 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/kernel/ipipe-core.c 2006-12-19 13:02:15.000000000 +0100
@@ -0,0 +1,264 @@
+/* -*- linux-c -*-
+ * linux/arch/ppc/kernel/ipipe-core.c
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ * Copyright (C) 2004 Wolfgang Grandegger (Adeos/ppc port over 2.4).
+ * Copyright (C) 2005 Heikki Lindholm (PowerPC 970 fixes).
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-dependent I-PIPE core support for PowerPC.
+ */
+
+#include <linux/kernel.h>
+#include <linux/smp.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/bitops.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/module.h>
+#include <asm/system.h>
+#include <asm/atomic.h>
+#include <asm/hardirq.h>
+#include <asm/io.h>
+#include <asm/time.h>
+
+/* Current reload value for the decrementer. */
+unsigned long __ipipe_decr_ticks;
+
+/* Next tick date (timebase value). */
+unsigned long long __ipipe_decr_next[IPIPE_NR_CPUS];
+
+struct pt_regs __ipipe_tick_regs[IPIPE_NR_CPUS];
+
+#ifdef CONFIG_POWER4
+extern struct irqaction k2u3_cascade_action;
+extern int openpic2_get_irq(struct pt_regs *regs);
+#endif
+
+#ifdef CONFIG_SMP
+
+static cpumask_t __ipipe_cpu_sync_map;
+
+static cpumask_t __ipipe_cpu_lock_map;
+
+static ipipe_spinlock_t __ipipe_cpu_barrier = IPIPE_SPIN_LOCK_UNLOCKED;
+
+static atomic_t __ipipe_critical_count = ATOMIC_INIT(0);
+
+static void (*__ipipe_cpu_sync) (void);
+
+/* Always called with hw interrupts off. */
+
+void __ipipe_do_critical_sync(unsigned irq)
+{
+ ipipe_declare_cpuid;
+
+ ipipe_load_cpuid();
+
+ cpu_set(cpuid, __ipipe_cpu_sync_map);
+
+ /*
+ * Now we are in sync with the lock requestor running on another
+ * CPU. Enter a spinning wait until he releases the global
+ * lock.
+ */
+ spin_lock_hw(&__ipipe_cpu_barrier);
+
+ /* Got it. Now get out. */
+
+ if (__ipipe_cpu_sync)
+ /* Call the sync routine if any. */
+ __ipipe_cpu_sync();
+
+ spin_unlock_hw(&__ipipe_cpu_barrier);
+
+ cpu_clear(cpuid, __ipipe_cpu_sync_map);
+}
+
+#endif /* CONFIG_SMP */
+
+/*
+ * ipipe_critical_enter() -- Grab the superlock excluding all CPUs
+ * but the current one from a critical section. This lock is used when
+ * we must enforce a global critical section for a single CPU in a
+ * possibly SMP system whichever context the CPUs are running.
+ */
+unsigned long ipipe_critical_enter(void (*syncfn) (void))
+{
+ unsigned long flags;
+
+ local_irq_save_hw(flags);
+
+#ifdef CONFIG_SMP
+ if (num_online_cpus() > 1) { /* We might be running a SMP-kernel on a UP box... */
+ ipipe_declare_cpuid;
+ cpumask_t lock_map;
+
+ ipipe_load_cpuid();
+
+ if (!cpu_test_and_set(cpuid, __ipipe_cpu_lock_map)) {
+ while (cpu_test_and_set(BITS_PER_LONG - 1,
+ __ipipe_cpu_lock_map)) {
+ int n = 0;
+ do {
+ cpu_relax();
+ } while (++n < cpuid);
+ }
+
+ spin_lock_hw(&__ipipe_cpu_barrier);
+
+ __ipipe_cpu_sync = syncfn;
+
+ /* Send the sync IPI to all processors but the current one. */
+ send_IPI_allbutself(IPIPE_CRITICAL_VECTOR);
+
+ cpus_andnot(lock_map, cpu_online_map,
+ __ipipe_cpu_lock_map);
+
+ while (!cpus_equal(__ipipe_cpu_sync_map, lock_map))
+ cpu_relax();
+ }
+
+ atomic_inc(&__ipipe_critical_count);
+ }
+#endif /* CONFIG_SMP */
+
+ return flags;
+}
+
+/* ipipe_critical_exit() -- Release the superlock. */
+
+void ipipe_critical_exit(unsigned long flags)
+{
+#ifdef CONFIG_SMP
+ if (num_online_cpus() > 1) { /* We might be running a SMP-kernel on a UP box... */
+ ipipe_declare_cpuid;
+
+ ipipe_load_cpuid();
+
+ if (atomic_dec_and_test(&__ipipe_critical_count)) {
+ spin_unlock_hw(&__ipipe_cpu_barrier);
+
+ while (!cpus_empty(__ipipe_cpu_sync_map))
+ cpu_relax();
+
+ cpu_clear(cpuid, __ipipe_cpu_lock_map);
+ cpu_clear(BITS_PER_LONG - 1, __ipipe_cpu_lock_map);
+ }
+ }
+#endif /* CONFIG_SMP */
+
+ local_irq_restore_hw(flags);
+}
+
+void __ipipe_init_platform(void)
+{
+ unsigned timer_virq;
+
+ /*
+ * Allocate a virtual IRQ for the decrementer trap early to
+ * get it mapped to IPIPE_VIRQ_BASE
+ */
+
+ timer_virq = ipipe_alloc_virq();
+
+ if (timer_virq != IPIPE_TIMER_VIRQ)
+ panic("I-pipe: cannot reserve timer virq #%d (got #%d)",
+ IPIPE_TIMER_VIRQ, timer_virq);
+
+ __ipipe_decr_ticks = tb_ticks_per_jiffy;
+}
+
+int ipipe_get_sysinfo(struct ipipe_sysinfo *info)
+{
+ info->ncpus = num_online_cpus();
+ info->cpufreq = ipipe_cpu_freq();
+ info->archdep.tmirq = IPIPE_TIMER_VIRQ;
+ info->archdep.tmfreq = info->cpufreq;
+
+ return 0;
+}
+
+/*
+ * ipipe_trigger_irq() -- Push the interrupt at front of the pipeline
+ * just like if it has been actually received from a hw source. Also
+ * works for virtual interrupts.
+ */
+int ipipe_trigger_irq(unsigned irq)
+{
+ unsigned long flags;
+
+ if (irq >= IPIPE_NR_IRQS ||
+ (ipipe_virtual_irq_p(irq)
+ && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)))
+ return -EINVAL;
+
+ local_irq_save_hw(flags);
+
+ __ipipe_handle_irq(irq, NULL);
+
+ local_irq_restore_hw(flags);
+
+ return 1;
+}
+
+static void __ipipe_set_decr(void)
+{
+ ipipe_declare_cpuid;
+
+ ipipe_load_cpuid();
+
+ disarm_decr[cpuid] = (__ipipe_decr_ticks != tb_ticks_per_jiffy);
+#ifdef CONFIG_40x
+ /* Enable and set auto-reload. */
+ mtspr(SPRN_TCR, mfspr(SPRN_TCR) | TCR_ARE);
+ mtspr(SPRN_PIT, __ipipe_decr_ticks);
+#else /* !CONFIG_40x */
+ __ipipe_decr_next[cpuid] = __ipipe_read_timebase() + __ipipe_decr_ticks;
+ set_dec(__ipipe_decr_ticks);
+#endif /* CONFIG_40x */
+}
+
+int ipipe_tune_timer(unsigned long ns, int flags)
+{
+ unsigned long x, ticks;
+
+ if (flags & IPIPE_RESET_TIMER)
+ ticks = tb_ticks_per_jiffy;
+ else {
+ ticks = (ns / 1000) * tb_ticks_per_jiffy / (1000000 / HZ);
+
+ if (ticks > tb_ticks_per_jiffy)
+ return -EINVAL;
+ }
+
+ x = ipipe_critical_enter(&__ipipe_set_decr); /* Sync with all CPUs */
+ __ipipe_decr_ticks = ticks;
+ __ipipe_set_decr();
+ ipipe_critical_exit(x);
+
+ return 0;
+}
+
+EXPORT_SYMBOL(__ipipe_decr_ticks);
+EXPORT_SYMBOL(__ipipe_decr_next);
+EXPORT_SYMBOL(ipipe_critical_enter);
+EXPORT_SYMBOL(ipipe_critical_exit);
+EXPORT_SYMBOL(ipipe_trigger_irq);
+EXPORT_SYMBOL(ipipe_get_sysinfo);
+EXPORT_SYMBOL(ipipe_tune_timer);
diff -Naur linux-2.6.19.orig/arch/ppc/kernel/ipipe-mcount.S linux-2.6.19.ipipe/arch/ppc/kernel/ipipe-mcount.S
--- linux-2.6.19.orig/arch/ppc/kernel/ipipe-mcount.S 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/kernel/ipipe-mcount.S 2006-12-19 13:02:15.000000000 +0100
@@ -0,0 +1,76 @@
+/*
+ * linux/arch/ppc/kernel/ipipe-mcount.S
+ *
+ * Adapted from glibc's ppc32 profiling support --rpm.
+ */
+
+#include <linux/config.h>
+#include <asm/page.h>
+#include <asm/ppc_asm.h>
+#include <asm/reg.h>
+
+/*
+ * Excerpt from sysdeps/powerpc/powerpc32/ppc-mcount.S
+ *
+ * Do profiling as described in the SYSV ELF ABI, _mcount is called
+ * with the address of a data word in r0 (that is different for every
+ * routine, initialised to 0, and otherwise unused). The caller has
+ * put the address the caller will return to in the usual place on the stack,
+ * 4(r1). _mcount is responsible for ensuring that when it returns no
+ * argument-passing registers are disturbed, and that the LR is set back
+ * to (what the caller sees as) 4(r1).
+ *
+ * This is intended so that the following code can be inserted at the
+ * front of any routine without changing the routine:
+ *
+ * mflr r0
+ * lis r12,0b@domain.hid
+ * stw r0,4(r1)
+ * addi r0,r12,0b@domain.hid
+ * bl _mcount
+ */
+
+.globl _mcount
+_mcount:
+ stwu r1,-48(r1)
+/* We need to save the parameter-passing registers. */
+ stw r3, 12(r1)
+ stw r4, 16(r1)
+ stw r5, 20(r1)
+ stw r6, 24(r1)
+ stw r7, 28(r1)
+ stw r8, 32(r1)
+ stw r9, 36(r1)
+ stw r10,40(r1)
+ mflr r4
+ mfcr r5
+ stw r4, 44(r1)
+ stw r5, 8(r1)
+ tophys(r0,0)
+ addis r6,r0,ipipe_trace_enable@domain.hid
+ lwz r3,ipipe_trace_enable@domain.hid)
+ cmpwi r3,0
+ beq 1f
+ li r3,0
+ lwz r5, 52(r1)
+ li r6,0
+ bl __ipipe_trace
+1:
+ /* Restore the registers... */
+ lwz r6, 8(r1)
+ lwz r0, 44(r1)
+ lwz r3, 12(r1)
+ mtctr r0
+ lwz r4, 16(r1)
+ mtcrf 0xff,r6
+ lwz r5, 20(r1)
+ lwz r6, 24(r1)
+ lwz r0, 52(r1)
+ lwz r7, 28(r1)
+ lwz r8, 32(r1)
+ mtlr r0
+ lwz r9, 36(r1)
+ lwz r10,40(r1)
+ /* ...unwind the stack frame, and return to your usual programming. */
+ addi r1,r1,48
+ bctr
diff -Naur linux-2.6.19.orig/arch/ppc/kernel/ipipe-root.c linux-2.6.19.ipipe/arch/ppc/kernel/ipipe-root.c
--- linux-2.6.19.orig/arch/ppc/kernel/ipipe-root.c 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/kernel/ipipe-root.c 2006-12-19 13:02:15.000000000 +0100
@@ -0,0 +1,488 @@
+/* -*- linux-c -*-
+ * linux/arch/ppc/kernel/ipipe-root.c
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum (Adeos/ppc port over 2.6).
+ * Copyright (C) 2004 Wolfgang Grandegger (Adeos/ppc port over 2.4).
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-dependent I-pipe support for PowerPC.
+ */
+
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/errno.h>
+#include <asm/unistd.h>
+#include <asm/system.h>
+#include <asm/hardirq.h>
+#include <asm/machdep.h>
+#include <asm/atomic.h>
+#include <asm/io.h>
+#include <asm/time.h>
+#include <asm/mmu_context.h>
+
+extern irq_desc_t irq_desc[];
+
+static struct hw_interrupt_type __ipipe_std_irq_dtype[NR_IRQS];
+
+static void __ipipe_override_irq_enable(unsigned irq)
+{
+ unsigned long flags;
+
+ local_irq_save_hw(flags);
+ ipipe_irq_unlock(irq);
+ __ipipe_std_irq_dtype[irq].enable(irq);
+ local_irq_restore_hw(flags);
+}
+
+static void __ipipe_override_irq_disable(unsigned irq)
+{
+ unsigned long flags;
+
+ local_irq_save_hw(flags);
+ ipipe_irq_lock(irq);
+ __ipipe_std_irq_dtype[irq].disable(irq);
+ local_irq_restore_hw(flags);
+}
+
+static void __ipipe_override_irq_end(unsigned irq)
+{
+ unsigned long flags;
+
+ local_irq_save_hw(flags);
+
+ if (!(irq_desc[irq].status & (IRQ_DISABLED | IRQ_INPROGRESS)))
+ ipipe_irq_unlock(irq);
+
+ __ipipe_std_irq_dtype[irq].end(irq);
+
+ local_irq_restore_hw(flags);
+}
+
+static void __ipipe_override_irq_affinity(unsigned irq, cpumask_t mask)
+{
+ unsigned long flags;
+
+ local_irq_save_hw(flags);
+ __ipipe_std_irq_dtype[irq].set_affinity(irq, mask);
+ local_irq_restore_hw(flags);
+}
+
+static void __ipipe_enable_sync(void)
+{
+ __ipipe_decr_next[ipipe_processor_id()] =
+ __ipipe_read_timebase() + get_dec();
+}
+
+void __ipipe_enable_irqdesc(unsigned irq)
+{
+ irq_desc[irq].status &= ~IRQ_DISABLED;
+}
+
+/*
+ * __ipipe_enable_pipeline() -- We are running on the boot CPU, hw
+ * interrupts are off, and secondary CPUs are still lost in space.
+ */
+void __ipipe_enable_pipeline(void)
+{
+ unsigned long flags;
+ unsigned irq;
+
+ flags = ipipe_critical_enter(&__ipipe_enable_sync);
+
+ /* First, virtualize all interrupts from the root domain. */
+
+ for (irq = 0; irq < NR_IRQS; irq++)
+ ipipe_virtualize_irq(ipipe_root_domain,
+ irq,
+ (ipipe_irq_handler_t)&__ipipe_do_IRQ, NULL,
+ &__ipipe_ack_irq,
+ IPIPE_HANDLE_MASK | IPIPE_PASS_MASK);
+
+ /*
+ * We use a virtual IRQ to handle the timer irq (decrementer trap)
+ * which has been allocated early in __ipipe_init_platform().
+ */
+
+ ipipe_virtualize_irq(ipipe_root_domain,
+ IPIPE_TIMER_VIRQ,
+ (ipipe_irq_handler_t)&__ipipe_do_timer, NULL,
+ NULL, IPIPE_HANDLE_MASK | IPIPE_PASS_MASK);
+
+ /*
+ * Interpose on the IRQ control routines so we can make them
+ * atomic using hw masking and prevent the interrupt log from
+ * being untimely flushed.
+ */
+
+ for (irq = 0; irq < NR_IRQS; irq++) {
+ if (irq_desc[irq].chip != NULL)
+ __ipipe_std_irq_dtype[irq] = *irq_desc[irq].chip;
+ }
+
+ /*
+ * The original controller structs are often shared, so we first
+ * save them all before changing any of them. Notice that we don't
+ * override the ack() handler since we will enforce the necessary
+ * setup in __ipipe_ack_irq().
+ */
+
+ for (irq = 0; irq < NR_IRQS; irq++) {
+ struct hw_interrupt_type *handler = irq_desc[irq].chip;
+
+ if (handler == NULL)
+ continue;
+
+ if (handler->enable != NULL)
+ handler->enable = &__ipipe_override_irq_enable;
+
+ if (handler->disable != NULL)
+ handler->disable = &__ipipe_override_irq_disable;
+
+ if (handler->end != NULL)
+ handler->end = &__ipipe_override_irq_end;
+
+ if (handler->set_affinity != NULL)
+ handler->set_affinity = &__ipipe_override_irq_affinity;
+ }
+
+ __ipipe_decr_next[ipipe_processor_id()] =
+ __ipipe_read_timebase() + get_dec();
+
+ ipipe_critical_exit(flags);
+}
+
+int __ipipe_ack_irq(unsigned irq)
+{
+ irq_desc_t *desc = irq_desc + irq;
+ unsigned long flags;
+ ipipe_declare_cpuid;
+
+ if (desc->chip->ack == NULL)
+ return 1;
+
+ /*
+ * No need to mask IRQs at hw level: we are always called from
+ * __ipipe_handle_irq(), so interrupts are already off. We
+ * stall the pipeline so that spin_lock_irq*() ops won't
+ * unintentionally flush it, since this could cause infinite
+ * recursion.
+ */
+
+ ipipe_load_cpuid();
+ flags = ipipe_test_and_stall_pipeline();
+ preempt_disable();
+ desc->chip->ack(irq);
+#ifdef CONFIG_POWER4
+ /* if it is a k2u3 cascaded irq, acknowledge it, also */
+ if (desc->action == &k2u3_cascade_action) {
+ struct pt_regs regs;
+ int irq2 = openpic2_get_irq(®s);
+ if (irq2 != -1) {
+ irq_desc_t *desc2 = irq_desc + irq2;
+ if (desc2->chip->ack)
+ desc2->chip->ack(irq2);
+ }
+ }
+#endif
+ preempt_enable_no_resched();
+ ipipe_restore_pipeline_nosync(per_cpu(ipipe_percpu_domain, cpuid), flags, cpuid);
+
+ return 1;
+}
+
+/*
+ * __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic
+ * interrupt protection log is maintained here for each domain. Hw
+ * interrupts are off on entry.
+ */
+void __ipipe_handle_irq(int irq, struct pt_regs *regs)
+{
+ struct ipipe_domain *this_domain, *next_domain;
+ struct list_head *head, *pos;
+ ipipe_declare_cpuid;
+ int m_ack, s_ack;
+
+ m_ack = (regs == NULL); /* Software-triggered IRQs do not need
+ * any ack. */
+ if (irq >= IPIPE_NR_IRQS) {
+ printk(KERN_ERR "I-pipe: spurious interrupt %d\n", irq);
+ return;
+ }
+
+ ipipe_load_cpuid();
+
+ this_domain = per_cpu(ipipe_percpu_domain, cpuid);
+
+ if (unlikely(test_bit(IPIPE_STICKY_FLAG, &this_domain->irqs[irq].control)))
+ head = &this_domain->p_link;
+ else {
+ head = __ipipe_pipeline.next;
+ next_domain = list_entry(head, struct ipipe_domain, p_link);
+ if (likely(test_bit(IPIPE_WIRED_FLAG, &next_domain->irqs[irq].control))) {
+ if (!m_ack && next_domain->irqs[irq].acknowledge != NULL)
+ next_domain->irqs[irq].acknowledge(irq);
+ if (likely(__ipipe_dispatch_wired(next_domain, irq)))
+ goto finalize;
+ return;
+ }
+ }
+
+ /* Ack the interrupt. */
+
+ s_ack = m_ack;
+ pos = head;
+
+ while (pos != &__ipipe_pipeline) {
+ next_domain = list_entry(pos, struct ipipe_domain, p_link);
+ /*
+ * For each domain handling the incoming IRQ, mark it as
+ * pending in its log.
+ */
+ if (test_bit(IPIPE_HANDLE_FLAG,
+ &next_domain->irqs[irq].control)) {
+ /*
+ * Domains that handle this IRQ are polled for
+ * acknowledging it by decreasing priority order. The
+ * interrupt must be made pending _first_ in the
+ * domain's status flags before the PIC is unlocked.
+ */
+
+ next_domain->cpudata[cpuid].irq_counters[irq].total_hits++;
+ next_domain->cpudata[cpuid].irq_counters[irq].pending_hits++;
+ __ipipe_set_irq_bit(next_domain, cpuid, irq);
+
+ /*
+ * Always get the first master acknowledge available.
+ * Once we've got it, allow slave acknowledge
+ * handlers to run (until one of them stops us).
+ */
+ if (next_domain->irqs[irq].acknowledge != NULL) {
+ if (!m_ack)
+ m_ack = next_domain->irqs[irq].acknowledge(irq);
+ else if (test_bit
+ (IPIPE_SHARED_FLAG,
+ &next_domain->irqs[irq].control) && !s_ack)
+ s_ack = next_domain->irqs[irq].acknowledge(irq);
+ }
+ }
+
+ /*
+ * If the domain does not want the IRQ to be passed down the
+ * interrupt pipe, exit the loop now.
+ */
+
+ if (!test_bit(IPIPE_PASS_FLAG, &next_domain->irqs[irq].control))
+ break;
+
+ pos = next_domain->p_link.next;
+ }
+
+finalize:
+ /*
+ * Now walk the pipeline, yielding control to the highest
+ * priority domain that has pending interrupt(s) or
+ * immediately to the current domain if the interrupt has been
+ * marked as 'sticky'. This search does not go beyond the
+ * current domain in the pipeline.
+ */
+
+ __ipipe_walk_pipeline(head, cpuid);
+}
+
+int __ipipe_grab_irq(struct pt_regs *regs)
+{
+ extern int ppc_spurious_interrupts;
+ ipipe_declare_cpuid;
+ int irq;
+
+ if ((irq = ppc_md.get_irq()) >= 0) {
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+ ipipe_trace_begin(irq);
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+ __ipipe_handle_irq(irq, regs);
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+ ipipe_trace_end(irq);
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+ }
+ else if (irq != -2)
+ ppc_spurious_interrupts++;
+
+ ipipe_load_cpuid();
+
+ return (per_cpu(ipipe_percpu_domain, cpuid) == ipipe_root_domain &&
+ !test_bit(IPIPE_STALL_FLAG,
+ &ipipe_root_domain->cpudata[cpuid].status));
+}
+
+void __ipipe_do_IRQ(int irq, struct pt_regs *regs)
+{
+ irq_enter();
+ __do_IRQ(irq);
+ irq_exit();
+}
+
+int __ipipe_grab_timer(struct pt_regs *regs)
+{
+ ipipe_declare_cpuid;
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+ ipipe_trace_begin(IPIPE_TIMER_VIRQ);
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+
+#ifdef CONFIG_POWER4
+ /* On 970 CPUs DEC cannot be disabled, and without setting DEC
+ * here, DEC interrupt would be triggered as soon as interrupts
+ * are enabled in __ipipe_sync_stage
+ */
+ set_dec(0x7fffffff);
+#endif
+
+ __ipipe_tick_regs[cpuid].msr = regs->msr; /* for do_timer() */
+
+#ifndef CONFIG_40x
+ if (__ipipe_decr_ticks != tb_ticks_per_jiffy) {
+ unsigned long long next_date, now;
+
+ next_date = __ipipe_decr_next[cpuid];
+
+ while ((now = __ipipe_read_timebase()) >= next_date)
+ next_date += __ipipe_decr_ticks;
+
+ set_dec(next_date - now);
+
+ __ipipe_decr_next[cpuid] = next_date;
+ }
+#endif /* !CONFIG_40x */
+
+ __ipipe_handle_irq(IPIPE_TIMER_VIRQ, regs);
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+ ipipe_trace_end(IPIPE_TIMER_VIRQ);
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+
+ ipipe_load_cpuid();
+
+ return (per_cpu(ipipe_percpu_domain, cpuid) == ipipe_root_domain &&
+ !test_bit(IPIPE_STALL_FLAG,
+ &ipipe_root_domain->cpudata[cpuid].status));
+}
+
+void __ipipe_do_timer(int irq, struct pt_regs *regs)
+{
+ timer_interrupt(regs);
+}
+
+int __ipipe_check_root(struct pt_regs *regs)
+{
+ ipipe_declare_cpuid;
+ /*
+ * This routine is called with hw interrupts off, so no migration
+ * can occur while checking the identity of the current domain.
+ */
+ ipipe_load_cpuid();
+ return per_cpu(ipipe_percpu_domain, cpuid) == ipipe_root_domain;
+}
+
+void __ipipe_fast_stall_root(void)
+{
+ ipipe_declare_cpuid;
+ unsigned long flags;
+
+ ipipe_get_cpu(flags); /* Care for migration. */
+
+ set_bit(IPIPE_STALL_FLAG,
+ &ipipe_root_domain->cpudata[cpuid].status);
+
+ ipipe_put_cpu(flags);
+}
+
+void __ipipe_fast_unstall_root(void)
+{
+ ipipe_declare_cpuid;
+ unsigned long flags;
+
+ ipipe_get_cpu(flags); /* Care for migration. */
+
+ clear_bit(IPIPE_STALL_FLAG,
+ &ipipe_root_domain->cpudata[cpuid].status);
+
+ ipipe_put_cpu(flags);
+}
+
+int __ipipe_syscall_root(struct pt_regs *regs)
+{
+ ipipe_declare_cpuid;
+ unsigned long flags;
+
+ /*
+ * This routine either returns:
+ * 0 -- if the syscall is to be passed to Linux;
+ * >0 -- if the syscall should not be passed to Linux, and no
+ * tail work should be performed;
+ * <0 -- if the syscall should not be passed to Linux but the
+ * tail work has to be performed (for handling signals etc).
+ */
+
+ if (__ipipe_syscall_watched_p(current, regs->gpr[0]) &&
+ __ipipe_event_monitored_p(IPIPE_EVENT_SYSCALL) &&
+ __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL,regs) > 0) {
+ if (ipipe_current_domain == ipipe_root_domain && !in_atomic()) {
+ /*
+ * Sync pending VIRQs before _TIF_NEED_RESCHED
+ * is tested.
+ */
+ ipipe_lock_cpu(flags);
+ if ((ipipe_root_domain->cpudata[cpuid].irq_pending_hi & IPIPE_IRQMASK_VIRT) != 0)
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_VIRT);
+ ipipe_unlock_cpu(flags);
+ return -1;
+ }
+ return 1;
+ }
+
+ return 0;
+}
+
+void atomic_set_mask(unsigned long mask,
+ unsigned long *ptr);
+
+void atomic_clear_mask(unsigned long mask,
+ unsigned long *ptr);
+
+extern unsigned long context_map[];
+
+EXPORT_SYMBOL_GPL(__switch_to);
+EXPORT_SYMBOL_GPL(show_stack);
+EXPORT_SYMBOL_GPL(atomic_set_mask);
+EXPORT_SYMBOL_GPL(atomic_clear_mask);
+EXPORT_SYMBOL_GPL(context_map);
+EXPORT_SYMBOL_GPL(_switch);
+EXPORT_SYMBOL_GPL(last_task_used_math);
+#ifdef FEW_CONTEXTS
+EXPORT_SYMBOL_GPL(nr_free_contexts);
+EXPORT_SYMBOL_GPL(context_mm);
+EXPORT_SYMBOL_GPL(steal_context);
+#endif
+
+#ifdef CONFIG_IPIPE_TRACE_MCOUNT
+void notrace _mcount(void);
+EXPORT_SYMBOL(_mcount);
+#endif /* CONFIG_IPIPE_TRACE_MCOUNT */
diff -Naur linux-2.6.19.orig/arch/ppc/kernel/Makefile linux-2.6.19.ipipe/arch/ppc/kernel/Makefile
--- linux-2.6.19.orig/arch/ppc/kernel/Makefile 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/kernel/Makefile 2006-12-19 13:02:15.000000000 +0100
@@ -17,6 +17,8 @@
obj-$(CONFIG_RAPIDIO) += rio.o
obj-$(CONFIG_KGDB) += ppc-stub.o
obj-$(CONFIG_SMP) += smp.o smp-tbsync.o
+obj-$(CONFIG_IPIPE) += ipipe-core.o ipipe-root.o
+obj-$(CONFIG_IPIPE_TRACE_MCOUNT) += ipipe-mcount.o
obj-$(CONFIG_KEXEC) += machine_kexec.o relocate_kernel.o
ifndef CONFIG_MATH_EMULATION
diff -Naur linux-2.6.19.orig/arch/ppc/kernel/setup.c linux-2.6.19.ipipe/arch/ppc/kernel/setup.c
--- linux-2.6.19.orig/arch/ppc/kernel/setup.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/kernel/setup.c 2006-12-19 13:02:15.000000000 +0100
@@ -295,7 +295,7 @@
* to access static data (including strings). -- paulus
*/
__init
-unsigned long
+notrace unsigned long
early_init(int r3, int r4, int r5)
{
unsigned long phys;
diff -Naur linux-2.6.19.orig/arch/ppc/kernel/time.c linux-2.6.19.ipipe/arch/ppc/kernel/time.c
--- linux-2.6.19.orig/arch/ppc/kernel/time.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/kernel/time.c 2006-12-19 13:02:15.000000000 +0100
@@ -146,7 +146,11 @@
jiffy_stamp += tb_ticks_per_jiffy;
profile_tick(CPU_PROFILING);
+#ifdef CONFIG_IPIPE
+ update_process_times(user_mode(__ipipe_tick_regs + smp_processor_id()));
+#else
update_process_times(user_mode(regs));
+#endif
if (smp_processor_id())
continue;
diff -Naur linux-2.6.19.orig/arch/ppc/kernel/traps.c linux-2.6.19.ipipe/arch/ppc/kernel/traps.c
--- linux-2.6.19.orig/arch/ppc/kernel/traps.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/kernel/traps.c 2006-12-19 13:02:15.000000000 +0100
@@ -233,6 +233,9 @@
{
unsigned long reason = get_mc_reason(regs);
+ if (ipipe_trap_notify(IPIPE_TRAP_MCE,regs))
+ return;
+
if (user_mode(regs)) {
regs->msr |= MSR_RI;
_exception(SIGBUS, regs, BUS_ADRERR, regs->nip);
@@ -392,6 +395,8 @@
void unknown_exception(struct pt_regs *regs)
{
+ if (ipipe_trap_notify(IPIPE_TRAP_UNKNOWN,regs))
+ return;
printk("Bad trap at PC: %lx, MSR: %lx, vector=%lx %s\n",
regs->nip, regs->msr, regs->trap, print_tainted());
_exception(SIGTRAP, regs, 0, 0);
@@ -399,6 +404,8 @@
void instruction_breakpoint_exception(struct pt_regs *regs)
{
+ if (ipipe_trap_notify(IPIPE_TRAP_IABR,regs))
+ return;
if (debugger_iabr_match(regs))
return;
_exception(SIGTRAP, regs, TRAP_BRKPT, 0);
@@ -406,6 +413,8 @@
void RunModeException(struct pt_regs *regs)
{
+ if (ipipe_trap_notify(IPIPE_TRAP_RM,regs))
+ return;
_exception(SIGTRAP, regs, 0, 0);
}
@@ -555,6 +564,8 @@
{
if (single_stepping(regs)) {
clear_single_step(regs);
+ if (ipipe_trap_notify(IPIPE_TRAP_SSTEP,regs))
+ return;
_exception(SIGTRAP, regs, TRAP_TRACE, 0);
}
}
@@ -624,6 +635,9 @@
unsigned int reason = get_reason(regs);
extern int do_mathemu(struct pt_regs *regs);
+ if (ipipe_trap_notify(IPIPE_TRAP_PCE,regs))
+ return;
+
#ifdef CONFIG_MATH_EMULATION
/* (reason & REASON_ILLEGAL) would be the obvious thing here,
* but there seems to be a hardware bug on the 405GP (RevD)
@@ -701,6 +715,8 @@
void single_step_exception(struct pt_regs *regs)
{
regs->msr &= ~(MSR_SE | MSR_BE); /* Turn off 'trace' bits */
+ if (ipipe_trap_notify(IPIPE_TRAP_SSTEP,regs))
+ return;
if (debugger_sstep(regs))
return;
_exception(SIGTRAP, regs, TRAP_TRACE, 0);
@@ -716,6 +732,8 @@
emulate_single_step(regs);
return;
}
+ if (ipipe_trap_notify(IPIPE_TRAP_ALIGNMENT,regs))
+ return;
if (fixed == -EFAULT) {
sig = SIGSEGV;
code = SEGV_ACCERR;
@@ -740,6 +758,8 @@
void nonrecoverable_exception(struct pt_regs *regs)
{
+ if (ipipe_trap_notify(IPIPE_TRAP_NREC,regs))
+ return;
printk(KERN_ERR "Non-recoverable exception at PC=%lx MSR=%lx\n",
regs->nip, regs->msr);
debugger(regs);
@@ -760,6 +780,9 @@
extern int Soft_emulate_8xx(struct pt_regs *);
int errcode;
+ if (ipipe_trap_notify(IPIPE_TRAP_SOFTEMU,regs))
+ return;
+
CHECK_FULL_REGS(regs);
if (!user_mode(regs)) {
@@ -788,6 +811,8 @@
void DebugException(struct pt_regs *regs, unsigned long debug_status)
{
+ if (ipipe_trap_notify(IPIPE_TRAP_DEBUG,regs))
+ return;
if (debug_status & DBSR_IC) { /* instruction completion */
regs->msr &= ~MSR_DE;
if (user_mode(regs)) {
@@ -828,6 +853,9 @@
{
static int kernel_altivec_count;
+ if (ipipe_trap_notify(IPIPE_TRAP_ALTUNAVAIL,regs))
+ return;
+
#ifndef CONFIG_ALTIVEC
if (user_mode(regs)) {
/* A user program has executed an altivec instruction,
@@ -849,6 +877,8 @@
{
int err;
+ if (ipipe_trap_notify(IPIPE_TRAP_ALTASSIST,regs))
+ return;
preempt_disable();
if (regs->msr & MSR_VEC)
giveup_altivec(current);
@@ -909,6 +939,9 @@
int fpexc_mode;
int code = 0;
+ if (ipipe_trap_notify(IPIPE_TRAP_SPE,regs))
+ return;
+
spefscr = current->thread.spefscr;
fpexc_mode = current->thread.fpexc_mode;
diff -Naur linux-2.6.19.orig/arch/ppc/mm/fault.c linux-2.6.19.ipipe/arch/ppc/mm/fault.c
--- linux-2.6.19.orig/arch/ppc/mm/fault.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/mm/fault.c 2006-12-19 13:02:15.000000000 +0100
@@ -113,6 +113,9 @@
is_write = error_code & 0x02000000;
#endif /* CONFIG_4xx || CONFIG_BOOKE */
+ if (ipipe_trap_notify(IPIPE_TRAP_ACCESS,regs))
+ return 0;
+
#if defined(CONFIG_XMON) || defined(CONFIG_KGDB)
if (debugger_fault_handler && TRAP(regs) == 0x300) {
debugger_fault_handler(regs);
diff -Naur linux-2.6.19.orig/arch/ppc/platforms/85xx/sbc85xx.h linux-2.6.19.ipipe/arch/ppc/platforms/85xx/sbc85xx.h
--- linux-2.6.19.orig/arch/ppc/platforms/85xx/sbc85xx.h 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/platforms/85xx/sbc85xx.h 2006-12-19 13:02:15.000000000 +0100
@@ -15,7 +15,6 @@
#define __PLATFORMS_85XX_SBC85XX_H__
#include <linux/init.h>
-#include <linux/seq_file.h>
#include <asm/ppcboot.h>
#define BOARD_CCSRBAR ((uint)0xff700000)
diff -Naur linux-2.6.19.orig/arch/ppc/syslib/open_pic.c linux-2.6.19.ipipe/arch/ppc/syslib/open_pic.c
--- linux-2.6.19.orig/arch/ppc/syslib/open_pic.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/arch/ppc/syslib/open_pic.c 2006-12-19 13:02:15.000000000 +0100
@@ -821,7 +821,7 @@
*/
static void openpic_ack_irq(unsigned int irq_nr)
{
-#ifdef __SLOW_VERSION__
+#if defined(__SLOW_VERSION__) || defined(CONFIG_IPIPE)
openpic_disable_irq(irq_nr);
openpic_eoi();
#else
@@ -832,7 +832,7 @@
static void openpic_end_irq(unsigned int irq_nr)
{
-#ifdef __SLOW_VERSION__
+#if defined(__SLOW_VERSION__) || defined(CONFIG_IPIPE)
if (!(irq_desc[irq_nr].status & (IRQ_DISABLED|IRQ_INPROGRESS))
&& irq_desc[irq_nr].action)
openpic_enable_irq(irq_nr);
diff -Naur linux-2.6.19.orig/include/asm-powerpc/hw_irq.h linux-2.6.19.ipipe/include/asm-powerpc/hw_irq.h
--- linux-2.6.19.orig/include/asm-powerpc/hw_irq.h 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/include/asm-powerpc/hw_irq.h 2006-12-19 13:02:32.000000000 +0100
@@ -12,6 +12,166 @@
extern void timer_interrupt(struct pt_regs *);
+#ifdef CONFIG_IPIPE
+
+#ifdef CONFIG_PPC_ISERIES
+#error "iSeries machines not supported by adeos i-pipe."
+#endif
+
+void __ipipe_stall_root(void);
+void __ipipe_unstall_root(void);
+unsigned long __ipipe_test_root(void);
+unsigned long __ipipe_test_and_stall_root(void);
+void __ipipe_restore_root(unsigned long flags);
+
+#define irqs_disabled() __ipipe_test_root()
+
+static inline void local_irq_disable(void)
+{
+ __ipipe_stall_root();
+}
+
+static inline void local_irq_enable(void)
+{
+ __ipipe_unstall_root();
+}
+
+static inline void local_irq_save_ptr(unsigned long *flags)
+{
+ *flags = (!__ipipe_test_and_stall_root()) << 15;
+}
+
+static inline void local_irq_restore(unsigned long flags)
+{
+ __ipipe_restore_root(!(flags & MSR_EE));
+}
+
+#define local_save_flags(flags) ((flags) = (!__ipipe_test_root()) << 15)
+#define local_irq_save(flags) local_irq_save_ptr(&flags)
+
+#if defined(CONFIG_BOOKE)
+#define SET_MSR_EE(x) mtmsr(x)
+#elif defined(__powerpc64__)
+#define SET_MSR_EE(x) __mtmsrd(x, 1)
+#else
+#define SET_MSR_EE(x) mtmsr(x)
+#endif
+
+static inline void local_irq_save_ptr_hw(unsigned long *flags)
+{
+ unsigned long msr;
+ msr = mfmsr();
+ *flags = msr;
+#ifdef CONFIG_BOOKE
+ __asm__ __volatile__("wrteei 0": : :"memory");
+#else
+ SET_MSR_EE(msr & ~MSR_EE);
+#endif
+ __asm__ __volatile__("": : :"memory");
+}
+
+#define local_save_flags_hw(flags) ((flags) = mfmsr())
+#define local_test_iflag_hw(x) ((x) & MSR_EE)
+#define irqs_disabled_hw() ((mfmsr() & MSR_EE) == 0)
+
+static inline void local_irq_disable_hw_notrace(void)
+{
+#ifdef CONFIG_BOOKE
+ __asm__ __volatile__("wrteei 0": : :"memory");
+#else
+ unsigned long msr;
+ __asm__ __volatile__("": : :"memory");
+ msr = mfmsr();
+ SET_MSR_EE(msr & ~MSR_EE);
+#endif
+}
+
+static inline void local_irq_enable_hw_notrace(void)
+{
+#ifdef CONFIG_BOOKE
+ __asm__ __volatile__("wrteei 1": : :"memory");
+#else
+ unsigned long msr;
+ __asm__ __volatile__("": : :"memory");
+ msr = mfmsr();
+ SET_MSR_EE(msr | MSR_EE);
+#endif
+}
+
+#define local_irq_save_hw_notrace(flags) local_irq_save_ptr_hw(&flags)
+#if defined(CONFIG_BOOKE)
+#define local_irq_restore_hw_notrace(flags) __asm__ __volatile__("wrtee %0" : : "r" (flags) : "memory")
+#elif defined(__powerpc64__)
+#define local_irq_restore_hw_notrace(flags) do { \
+ __asm__ __volatile__("": : :"memory"); \
+ __mtmsrd((flags), 1); \
+} while(0)
+#else
+#define local_irq_restore_hw_notrace(flags) mtmsr(flags)
+#endif
+
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
+
+#include <linux/ipipe_trace.h>
+
+static inline void local_irq_disable_hw(void)
+{
+ if (!irqs_disabled_hw()) {
+#ifdef CONFIG_BOOKE
+ __asm__ __volatile__("wrteei 0": : :"memory");
+#else
+ unsigned long msr;
+ __asm__ __volatile__("": : :"memory");
+ msr = mfmsr();
+ SET_MSR_EE(msr & ~MSR_EE);
+#endif
+ ipipe_trace_begin(0x80000000);
+ }
+}
+
+static inline void local_irq_enable_hw(void)
+{
+ if (irqs_disabled_hw()) {
+#ifdef CONFIG_BOOKE
+ ipipe_trace_end(0x80000000);
+ __asm__ __volatile__("wrteei 1": : :"memory");
+#else
+ unsigned long msr;
+ ipipe_trace_end(0x80000000);
+ __asm__ __volatile__("": : :"memory");
+ msr = mfmsr();
+ SET_MSR_EE(msr | MSR_EE);
+#endif
+ }
+}
+
+#define local_irq_save_hw(flags) \
+do { \
+ local_irq_save_ptr_hw(&(flags)); \
+ if (local_test_iflag_hw(flags)) { \
+ ipipe_trace_begin(0x80000001); \
+ } \
+} while(0)
+
+static inline void local_irq_restore_hw(unsigned long flags)
+{
+ if (local_test_iflag_hw(flags)) {
+ ipipe_trace_end(0x80000001);
+ }
+ local_irq_restore_hw_notrace(flags);
+}
+
+#else /* !CONFIG_IPIPE_TRACE_IRQSOFF */
+
+#define local_irq_disable_hw local_irq_disable_hw_notrace
+#define local_irq_enable_hw local_irq_enable_hw_notrace
+#define local_irq_save_hw local_irq_save_hw_notrace
+#define local_irq_restore_hw local_irq_restore_hw_notrace
+
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
+
+#else /* !CONFIG_IPIPE */
+
#ifdef CONFIG_PPC_ISERIES
extern unsigned long local_get_flags(void);
@@ -83,6 +243,8 @@
#endif /* CONFIG_PPC_ISERIES */
+#endif /* CONFIG_IPIPE */
+
#define mask_irq(irq) \
({ \
irq_desc_t *desc = get_irq_desc(irq); \
diff -Naur linux-2.6.19.orig/include/asm-powerpc/ipipe.h linux-2.6.19.ipipe/include/asm-powerpc/ipipe.h
--- linux-2.6.19.orig/include/asm-powerpc/ipipe.h 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.19.ipipe/include/asm-powerpc/ipipe.h 2006-12-19 13:02:32.000000000 +0100
@@ -0,0 +1,216 @@
+/* -*- linux-c -*-
+ * include/asm-powerpc/ipipe.h
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __POWERPC_IPIPE_H
+#define __POWERPC_IPIPE_H
+
+#ifdef CONFIG_IPIPE
+
+#ifdef FIXME
+#include <asm/ptrace.h>
+#include <asm/irq.h>
+#include <asm/bitops.h>
+#include <asm/time.h>
+#include <linux/list.h>
+#include <linux/cpumask.h>
+#include <linux/threads.h>
+#else
+#include <asm/irq.h>
+#include <linux/cpumask.h>
+#endif
+
+#define IPIPE_ARCH_STRING "1.5-01"
+#define IPIPE_MAJOR_NUMBER 1
+#define IPIPE_MINOR_NUMBER 5
+#define IPIPE_PATCH_NUMBER 0
+
+#define IPIPE_NR_XIRQS NR_IRQS
+#define IPIPE_IRQ_ISHIFT 5 /* 25 for 32bits arch. */
+
+/*
+ * The first virtual interrupt is reserved for the timer (see
+ * __ipipe_init_platform).
+ */
+#define IPIPE_TIMER_VIRQ IPIPE_VIRQ_BASE
+
+#ifdef CONFIG_SMP
+#error "I-pipe/ppc: SMP not yet implemented"
+#define ipipe_processor_id() (current_thread_info()->cpu)
+#else /* !CONFIG_SMP */
+#define ipipe_processor_id() 0
+#endif /* CONFIG_SMP */
+
+#define prepare_arch_switch(next) \
+do { \
+ ipipe_schedule_notify(current, next); \
+ local_irq_disable_hw(); \
+} while(0)
+
+#define task_hijacked(p) \
+ ( { \
+ int x = ipipe_current_domain != ipipe_root_domain; \
+ __clear_bit(IPIPE_SYNC_FLAG, \
+ &ipipe_root_domain->cpudata[task_cpu(p)].status); \
+ local_irq_enable_hw(); x; \
+ } )
+
+ /* PPC traps */
+#define IPIPE_TRAP_ACCESS 0 /* Data or instruction access exception */
+#define IPIPE_TRAP_ALIGNMENT 1 /* Alignment exception */
+#define IPIPE_TRAP_ALTUNAVAIL 2 /* Altivec unavailable */
+#define IPIPE_TRAP_PCE 3 /* Program check exception */
+#define IPIPE_TRAP_MCE 4 /* Machine check exception */
+#define IPIPE_TRAP_UNKNOWN 5 /* Unknown exception */
+#define IPIPE_TRAP_IABR 6 /* Instruction breakpoint */
+#define IPIPE_TRAP_RM 7 /* Run mode exception */
+#define IPIPE_TRAP_SSTEP 8 /* Single-step exception */
+#define IPIPE_TRAP_NREC 9 /* Non-recoverable exception */
+#define IPIPE_TRAP_SOFTEMU 10 /* Software emulation */
+#define IPIPE_TRAP_DEBUG 11 /* Debug exception */
+#define IPIPE_TRAP_SPE 12 /* SPE exception */
+#define IPIPE_TRAP_ALTASSIST 13 /* Altivec assist exception */
+#define IPIPE_NR_FAULTS 14
+/* Pseudo-vectors used for kernel events */
+#define IPIPE_FIRST_EVENT IPIPE_NR_FAULTS
+#define IPIPE_EVENT_SYSCALL (IPIPE_FIRST_EVENT)
+#define IPIPE_EVENT_SCHEDULE (IPIPE_FIRST_EVENT + 1)
+#define IPIPE_EVENT_SIGWAKE (IPIPE_FIRST_EVENT + 2)
+#define IPIPE_EVENT_SETSCHED (IPIPE_FIRST_EVENT + 3)
+#define IPIPE_EVENT_INIT (IPIPE_FIRST_EVENT + 4)
+#define IPIPE_EVENT_EXIT (IPIPE_FIRST_EVENT + 5)
+#define IPIPE_EVENT_CLEANUP (IPIPE_FIRST_EVENT + 6)
+#define IPIPE_LAST_EVENT IPIPE_EVENT_CLEANUP
+#define IPIPE_NR_EVENTS (IPIPE_LAST_EVENT + 1)
+
+struct ipipe_domain;
+
+struct ipipe_sysinfo {
+
+ int ncpus; /* Number of CPUs on board */
+ u64 cpufreq; /* CPU frequency (in Hz) */
+
+ /* Arch-dependent block */
+
+ struct {
+ unsigned tmirq; /* Timer tick IRQ */
+ u64 tmfreq; /* Timer frequency */
+ } archdep;
+};
+
+#define ipipe_read_tsc(t) \
+ ({ \
+ unsigned long __tbu; \
+ __asm__ __volatile__ ("1: mftbu %0\n" \
+ "mftb %1\n" \
+ "mftbu %2\n" \
+ "cmpw %2,%0\n" \
+ "bne- 1b\n" \
+ :"=r" (((unsigned long *)&t)[0]), \
+ "=r" (((unsigned long *)&t)[1]), \
+ "=r" (__tbu)); \
+ t; \
+ })
+
+#define __ipipe_read_timebase() \
+ ({ \
+ unsigned long long t; \
+ ipipe_read_tsc(t); \
+ t; \
+ })
+
+#define ipipe_cpu_freq() (HZ * tb_ticks_per_jiffy)
+#define ipipe_tsc2ns(t) ((((unsigned long)(t)) * 1000) / (ipipe_cpu_freq() / 1000000))
+
+#define ipipe_tsc2us(t) \
+({ \
+ unsigned long long delta = (t); \
+ do_div(delta, ipipe_cpu_freq()/1000000+1); \
+ (unsigned long)delta; \
+})
+
+/* Private interface -- Internal use only */
+
+#define __ipipe_check_platform() do { } while(0)
+
+#define __ipipe_enable_irq(irq) enable_irq(irq)
+
+#define __ipipe_disable_irq(irq) disable_irq(irq)
+
+void __ipipe_enable_irqdesc(unsigned irq);
+
+void __ipipe_init_platform(void);
+
+void __ipipe_enable_pipeline(void);
+
+int __ipipe_ack_irq(unsigned irq);
+
+void __ipipe_do_IRQ(int irq,
+ struct pt_regs *regs);
+
+void __ipipe_do_timer(int irq,
+ struct pt_regs *regs);
+
+void __ipipe_do_critical_sync(unsigned irq);
+
+extern unsigned long __ipipe_decr_ticks;
+
+extern unsigned long long __ipipe_decr_next[];
+
+extern struct pt_regs __ipipe_tick_regs[];
+
+void __ipipe_handle_irq(int irq,
+ struct pt_regs *regs);
+
+#define __ipipe_tick_irq IPIPE_TIMER_VIRQ
+
+static inline unsigned long __ipipe_ffnz(unsigned long ul)
+{
+ __asm__ __volatile__("cntlzw %0, %1":"=r"(ul):"r"(ul & (-ul)));
+ return 31 - ul;
+}
+
+/* When running handlers, enable hw interrupts for all domains but the
+ * one heading the pipeline, so that IRQs can never be significantly
+ * deferred for the latter. */
+#define __ipipe_run_isr(ipd, irq, cpuid) \
+do { \
+ local_irq_enable_nohead(ipd); \
+ if (ipd == ipipe_root_domain) { \
+ ((void (*)(unsigned, struct pt_regs *)) \
+ ipd->irqs[irq].handler) (irq, __ipipe_tick_regs + cpuid); \
+ } else { \
+ __clear_bit(IPIPE_SYNC_FLAG, &cpudata->status); \
+ ipd->irqs[irq].handler(irq,ipd->irqs[irq].cookie); \
+ __set_bit(IPIPE_SYNC_FLAG, &cpudata->status); \
+ } \
+ local_irq_disable_nohead(ipd); \
+} while(0)
+
+#define __ipipe_syscall_watched_p(p, sc) \
+ (((p)->flags & PF_EVNOTIFY) || (unsigned long)sc >= NR_syscalls)
+
+#else /* !CONFIG_IPIPE */
+
+#define task_hijacked(p) 0
+
+#endif /* CONFIG_IPIPE */
+
+#endif /* !__POWERPC_IPIPE_H */
diff -Naur linux-2.6.19.orig/include/asm-powerpc/irq.h linux-2.6.19.ipipe/include/asm-powerpc/irq.h
--- linux-2.6.19.orig/include/asm-powerpc/irq.h 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/include/asm-powerpc/irq.h 2006-12-19 13:02:32.000000000 +0100
@@ -11,7 +11,9 @@
#include <linux/threads.h>
#include <linux/list.h>
+#ifdef CONFIG_PPC_MERGE
#include <linux/radix-tree.h>
+#endif
#include <asm/types.h>
#include <asm/atomic.h>
diff -Naur linux-2.6.19.orig/include/asm-ppc/ipipe.h linux-2.6.19.ipipe/include/asm-ppc/ipipe.h
--- linux-2.6.19.orig/include/asm-ppc/ipipe.h 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.19.ipipe/include/asm-ppc/ipipe.h 2006-12-19 13:02:32.000000000 +0100
@@ -0,0 +1,220 @@
+/* -*- linux-c -*-
+ * include/asm-ppc/ipipe.h
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __PPC_IPIPE_H
+#define __PPC_IPIPE_H
+
+#ifdef CONFIG_IPIPE
+
+#ifdef FIXME
+#include <asm/ptrace.h>
+#include <asm/irq.h>
+#include <asm/bitops.h>
+#include <asm/time.h>
+#include <linux/list.h>
+#include <linux/cpumask.h>
+#include <linux/threads.h>
+#else
+#include <asm/irq.h>
+#include <linux/cpumask.h>
+#endif
+
+#define IPIPE_ARCH_STRING "1.5-00"
+#define IPIPE_MAJOR_NUMBER 1
+#define IPIPE_MINOR_NUMBER 5
+#define IPIPE_PATCH_NUMBER 0
+
+#define IPIPE_NR_XIRQS NR_IRQS
+#define IPIPE_IRQ_ISHIFT 5 /* 25 for 32bits arch. */
+
+/*
+ * The first virtual interrupt is reserved for the timer (see
+ * __ipipe_init_platform).
+ */
+#define IPIPE_TIMER_VIRQ IPIPE_VIRQ_BASE
+
+#ifdef CONFIG_PPC_MERGE
+#error "I-pipe/ppc: merged PowerPC tree not yet supported"
+#endif
+
+#ifdef CONFIG_SMP
+#error "I-pipe/ppc: SMP not yet implemented"
+#define ipipe_processor_id() (current_thread_info()->cpu)
+#else /* !CONFIG_SMP */
+#define ipipe_processor_id() 0
+#endif /* CONFIG_SMP */
+
+#define prepare_arch_switch(next) \
+do { \
+ ipipe_schedule_notify(current, next); \
+ local_irq_disable_hw(); \
+} while(0)
+
+#define task_hijacked(p) \
+ ( { \
+ int x = ipipe_current_domain != ipipe_root_domain; \
+ __clear_bit(IPIPE_SYNC_FLAG, \
+ &ipipe_root_domain->cpudata[task_cpu(p)].status); \
+ local_irq_enable_hw(); x; \
+ } )
+
+ /* PPC traps */
+#define IPIPE_TRAP_ACCESS 0 /* Data or instruction access exception */
+#define IPIPE_TRAP_ALIGNMENT 1 /* Alignment exception */
+#define IPIPE_TRAP_ALTUNAVAIL 2 /* Altivec unavailable */
+#define IPIPE_TRAP_PCE 3 /* Program check exception */
+#define IPIPE_TRAP_MCE 4 /* Machine check exception */
+#define IPIPE_TRAP_UNKNOWN 5 /* Unknown exception */
+#define IPIPE_TRAP_IABR 6 /* Instruction breakpoint */
+#define IPIPE_TRAP_RM 7 /* Run mode exception */
+#define IPIPE_TRAP_SSTEP 8 /* Single-step exception */
+#define IPIPE_TRAP_NREC 9 /* Non-recoverable exception */
+#define IPIPE_TRAP_SOFTEMU 10 /* Software emulation */
+#define IPIPE_TRAP_DEBUG 11 /* Debug exception */
+#define IPIPE_TRAP_SPE 12 /* SPE exception */
+#define IPIPE_TRAP_ALTASSIST 13 /* Altivec assist exception */
+#define IPIPE_NR_FAULTS 14
+/* Pseudo-vectors used for kernel events */
+#define IPIPE_FIRST_EVENT IPIPE_NR_FAULTS
+#define IPIPE_EVENT_SYSCALL (IPIPE_FIRST_EVENT)
+#define IPIPE_EVENT_SCHEDULE (IPIPE_FIRST_EVENT + 1)
+#define IPIPE_EVENT_SIGWAKE (IPIPE_FIRST_EVENT + 2)
+#define IPIPE_EVENT_SETSCHED (IPIPE_FIRST_EVENT + 3)
+#define IPIPE_EVENT_INIT (IPIPE_FIRST_EVENT + 4)
+#define IPIPE_EVENT_EXIT (IPIPE_FIRST_EVENT + 5)
+#define IPIPE_EVENT_CLEANUP (IPIPE_FIRST_EVENT + 6)
+#define IPIPE_LAST_EVENT IPIPE_EVENT_CLEANUP
+#define IPIPE_NR_EVENTS (IPIPE_LAST_EVENT + 1)
+
+struct ipipe_domain;
+
+struct ipipe_sysinfo {
+
+ int ncpus; /* Number of CPUs on board */
+ u64 cpufreq; /* CPU frequency (in Hz) */
+
+ /* Arch-dependent block */
+
+ struct {
+ unsigned tmirq; /* Timer tick IRQ */
+ u64 tmfreq; /* Timer frequency */
+ } archdep;
+};
+
+#define ipipe_read_tsc(t) \
+ ({ \
+ unsigned long __tbu; \
+ __asm__ __volatile__ ("1: mftbu %0\n" \
+ "mftb %1\n" \
+ "mftbu %2\n" \
+ "cmpw %2,%0\n" \
+ "bne- 1b\n" \
+ :"=r" (((unsigned long *)&t)[0]), \
+ "=r" (((unsigned long *)&t)[1]), \
+ "=r" (__tbu)); \
+ t; \
+ })
+
+#define __ipipe_read_timebase() \
+ ({ \
+ unsigned long long t; \
+ ipipe_read_tsc(t); \
+ t; \
+ })
+
+#define ipipe_cpu_freq() (HZ * tb_ticks_per_jiffy)
+#define ipipe_tsc2ns(t) ((((unsigned long)(t)) * 1000) / (ipipe_cpu_freq() / 1000000))
+
+#define ipipe_tsc2us(t) \
+({ \
+ unsigned long long delta = (t); \
+ do_div(delta, ipipe_cpu_freq()/1000000+1); \
+ (unsigned long)delta; \
+})
+
+/* Private interface -- Internal use only */
+
+#define __ipipe_check_platform() do { } while(0)
+
+#define __ipipe_enable_irq(irq) enable_irq(irq)
+
+#define __ipipe_disable_irq(irq) disable_irq(irq)
+
+void __ipipe_enable_irqdesc(unsigned irq);
+
+void __ipipe_init_platform(void);
+
+void __ipipe_enable_pipeline(void);
+
+int __ipipe_ack_irq(unsigned irq);
+
+void __ipipe_do_IRQ(int irq,
+ struct pt_regs *regs);
+
+void __ipipe_do_timer(int irq,
+ struct pt_regs *regs);
+
+void __ipipe_do_critical_sync(unsigned irq);
+
+extern unsigned long __ipipe_decr_ticks;
+
+extern unsigned long long __ipipe_decr_next[];
+
+extern struct pt_regs __ipipe_tick_regs[];
+
+void __ipipe_handle_irq(int irq,
+ struct pt_regs *regs);
+
+#define __ipipe_tick_irq IPIPE_TIMER_VIRQ
+
+static inline unsigned long __ipipe_ffnz(unsigned long ul)
+{
+ __asm__ __volatile__("cntlzw %0, %1":"=r"(ul):"r"(ul & (-ul)));
+ return 31 - ul;
+}
+
+/* When running handlers, enable hw interrupts for all domains but the
+ * one heading the pipeline, so that IRQs can never be significantly
+ * deferred for the latter. */
+#define __ipipe_run_isr(ipd, irq, cpuid) \
+do { \
+ local_irq_enable_nohead(ipd); \
+ if (ipd == ipipe_root_domain) { \
+ ((void (*)(unsigned, struct pt_regs *)) \
+ ipd->irqs[irq].handler) (irq, __ipipe_tick_regs + cpuid); \
+ } else { \
+ __clear_bit(IPIPE_SYNC_FLAG, &cpudata->status); \
+ ipd->irqs[irq].handler(irq,ipd->irqs[irq].cookie); \
+ __set_bit(IPIPE_SYNC_FLAG, &cpudata->status); \
+ } \
+ local_irq_disable_nohead(ipd); \
+} while(0)
+
+#define __ipipe_syscall_watched_p(p, sc) \
+ (((p)->flags & PF_EVNOTIFY) || (unsigned long)sc >= NR_syscalls)
+
+#else /* !CONFIG_IPIPE */
+
+#define task_hijacked(p) 0
+
+#endif /* CONFIG_IPIPE */
+
+#endif /* !__PPC_IPIPE_H */
diff -Naur linux-2.6.19.orig/include/asm-ppc/mmu_context.h linux-2.6.19.ipipe/include/asm-ppc/mmu_context.h
--- linux-2.6.19.orig/include/asm-ppc/mmu_context.h 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/include/asm-ppc/mmu_context.h 2006-12-19 13:02:32.000000000 +0100
@@ -153,7 +153,10 @@
*/
static inline void destroy_context(struct mm_struct *mm)
{
+ unsigned long flags;
+
preempt_disable();
+ local_irq_save_hw_cond(flags);
if (mm->context.id != NO_CONTEXT) {
clear_bit(mm->context.id, context_map);
mm->context.id = NO_CONTEXT;
@@ -161,6 +164,7 @@
atomic_inc(&nr_free_contexts);
#endif
}
+ local_irq_restore_hw_cond(flags);
preempt_enable();
}
@@ -193,7 +197,13 @@
* After we have set current->mm to a new value, this activates
* the context for the new mm so we see the new mappings.
*/
-#define activate_mm(active_mm, mm) switch_mm(active_mm, mm, current)
+#define activate_mm(active_mm, mm) \
+do { \
+ unsigned long flags; \
+ local_irq_save_hw_cond(flags); \
+ switch_mm(active_mm, mm, current); \
+ local_irq_restore_hw_cond(flags); \
+} while(0)
extern void mmu_context_init(void);
diff -Naur linux-2.6.19.orig/include/asm-ppc/mv64x60.h linux-2.6.19.ipipe/include/asm-ppc/mv64x60.h
--- linux-2.6.19.orig/include/asm-ppc/mv64x60.h 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/include/asm-ppc/mv64x60.h 2006-12-19 13:02:32.000000000 +0100
@@ -255,9 +255,9 @@
mv64x60_write(struct mv64x60_handle *bh, u32 offset, u32 val) {
ulong flags;
- spin_lock_irqsave(&mv64x60_lock, flags);
+ spin_lock_irqsave_hw(&mv64x60_lock, flags);
out_le32(bh->v_base + offset, val);
- spin_unlock_irqrestore(&mv64x60_lock, flags);
+ spin_unlock_irqrestore_hw(&mv64x60_lock, flags);
}
extern inline u32
@@ -265,9 +265,9 @@
ulong flags;
u32 reg;
- spin_lock_irqsave(&mv64x60_lock, flags);
+ spin_lock_irqsave_hw(&mv64x60_lock, flags);
reg = in_le32(bh->v_base + offset);
- spin_unlock_irqrestore(&mv64x60_lock, flags);
+ spin_unlock_irqrestore_hw(&mv64x60_lock, flags);
return reg;
}
@@ -277,11 +277,11 @@
u32 reg;
ulong flags;
- spin_lock_irqsave(&mv64x60_lock, flags);
+ spin_lock_irqsave_hw(&mv64x60_lock, flags);
reg = in_le32(bh->v_base + offs) & (~mask);
reg |= data & mask;
out_le32(bh->v_base + offs, reg);
- spin_unlock_irqrestore(&mv64x60_lock, flags);
+ spin_unlock_irqrestore_hw(&mv64x60_lock, flags);
}
#define mv64x60_set_bits(bh, offs, bits) mv64x60_modify(bh, offs, ~0, bits)
diff -Naur linux-2.6.19.orig/include/asm-ppc/pgalloc.h linux-2.6.19.ipipe/include/asm-ppc/pgalloc.h
--- linux-2.6.19.orig/include/asm-ppc/pgalloc.h 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/include/asm-ppc/pgalloc.h 2006-12-19 13:02:32.000000000 +0100
@@ -39,5 +39,10 @@
#define check_pgt_cache() do { } while (0)
+static inline void set_pgdir(unsigned long address, pgd_t entry)
+{
+ /* nop */
+}
+
#endif /* _PPC_PGALLOC_H */
#endif /* __KERNEL__ */
diff -Naur linux-2.6.19.orig/include/linux/hardirq.h linux-2.6.19.ipipe/include/linux/hardirq.h
--- linux-2.6.19.orig/include/linux/hardirq.h 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/include/linux/hardirq.h 2006-12-19 13:02:33.000000000 +0100
@@ -128,7 +128,20 @@
*/
extern void irq_exit(void);
+#ifdef CONFIG_IPIPE
+#define nmi_enter() \
+do { \
+ if (ipipe_current_domain == ipipe_root_domain) \
+ { lockdep_off(); irq_enter(); } \
+} while(0)
+#define nmi_exit() \
+do { \
+ if (ipipe_current_domain == ipipe_root_domain) \
+ { __irq_exit(); lockdep_on(); } \
+} while(0)
+#else /* !CONFIG_IPIPE */
#define nmi_enter() do { lockdep_off(); irq_enter(); } while (0)
#define nmi_exit() do { __irq_exit(); lockdep_on(); } while (0)
+#endif /* CONFIG_IPIPE */
#endif /* LINUX_HARDIRQ_H */
diff -Naur linux-2.6.19.orig/include/linux/ipipe.h linux-2.6.19.ipipe/include/linux/ipipe.h
--- linux-2.6.19.orig/include/linux/ipipe.h 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.19.ipipe/include/linux/ipipe.h 2006-12-19 13:02:34.000000000 +0100
@@ -0,0 +1,769 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe.h
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __LINUX_IPIPE_H
+#define __LINUX_IPIPE_H
+
+#include <linux/spinlock.h>
+#include <linux/cache.h>
+#include <linux/percpu.h>
+#include <asm/ipipe.h>
+
+#ifdef CONFIG_IPIPE
+
+#define IPIPE_VERSION_STRING IPIPE_ARCH_STRING
+#define IPIPE_RELEASE_NUMBER ((IPIPE_MAJOR_NUMBER << 16) | \
+ (IPIPE_MINOR_NUMBER << 8) | \
+ (IPIPE_PATCH_NUMBER))
+
+#ifndef BROKEN_BUILTIN_RETURN_ADDRESS
+#define __BUILTIN_RETURN_ADDRESS0 ((unsigned long)__builtin_return_address(0))
+#define __BUILTIN_RETURN_ADDRESS1 ((unsigned long)__builtin_return_address(1))
+#endif /* !BUILTIN_RETURN_ADDRESS */
+
+#define IPIPE_ROOT_PRIO 100
+#define IPIPE_ROOT_ID 0
+#define IPIPE_ROOT_NPTDKEYS 4 /* Must be <= BITS_PER_LONG */
+
+#define IPIPE_RESET_TIMER 0x1
+#define IPIPE_GRAB_TIMER 0x2
+
+/* Global domain flags */
+#define IPIPE_SPRINTK_FLAG 0 /* Synchronous printk() allowed */
+#define IPIPE_AHEAD_FLAG 1 /* Domain always heads the pipeline */
+
+/* Per-cpu pipeline status */
+#define IPIPE_STALL_FLAG 0 /* Stalls a pipeline stage -- guaranteed at bit #0 */
+#define IPIPE_SYNC_FLAG 1 /* The interrupt syncer is running for the domain */
+#define IPIPE_NOSTACK_FLAG 2 /* Domain currently runs on a foreign stack */
+
+#define IPIPE_SYNC_MASK (1 << IPIPE_SYNC_FLAG)
+
+/* Interrupt control bits */
+#define IPIPE_HANDLE_FLAG 0
+#define IPIPE_PASS_FLAG 1
+#define IPIPE_ENABLE_FLAG 2
+#define IPIPE_DYNAMIC_FLAG IPIPE_HANDLE_FLAG
+#define IPIPE_STICKY_FLAG 3
+#define IPIPE_SYSTEM_FLAG 4
+#define IPIPE_LOCK_FLAG 5
+#define IPIPE_SHARED_FLAG 6
+#define IPIPE_WIRED_FLAG 7
+#define IPIPE_EXCLUSIVE_FLAG 8
+
+#define IPIPE_HANDLE_MASK (1 << IPIPE_HANDLE_FLAG)
+#define IPIPE_PASS_MASK (1 << IPIPE_PASS_FLAG)
+#define IPIPE_ENABLE_MASK (1 << IPIPE_ENABLE_FLAG)
+#define IPIPE_DYNAMIC_MASK IPIPE_HANDLE_MASK
+#define IPIPE_STICKY_MASK (1 << IPIPE_STICKY_FLAG)
+#define IPIPE_SYSTEM_MASK (1 << IPIPE_SYSTEM_FLAG)
+#define IPIPE_LOCK_MASK (1 << IPIPE_LOCK_FLAG)
+#define IPIPE_SHARED_MASK (1 << IPIPE_SHARED_FLAG)
+#define IPIPE_WIRED_MASK (1 << IPIPE_WIRED_FLAG)
+#define IPIPE_EXCLUSIVE_MASK (1 << IPIPE_EXCLUSIVE_FLAG)
+
+#define IPIPE_DEFAULT_MASK (IPIPE_HANDLE_MASK|IPIPE_PASS_MASK)
+#define IPIPE_STDROOT_MASK (IPIPE_HANDLE_MASK|IPIPE_PASS_MASK|IPIPE_SYSTEM_MASK)
+
+#define IPIPE_EVENT_SELF 0x80000000
+
+/* Number of virtual IRQs */
+#define IPIPE_NR_VIRQS BITS_PER_LONG
+/* First virtual IRQ # */
+#define IPIPE_VIRQ_BASE (((IPIPE_NR_XIRQS + BITS_PER_LONG - 1) / BITS_PER_LONG) * BITS_PER_LONG)
+/* Total number of IRQ slots */
+#define IPIPE_NR_IRQS (IPIPE_VIRQ_BASE + IPIPE_NR_VIRQS)
+/* Number of indirect words needed to map the whole IRQ space. */
+#define IPIPE_IRQ_IWORDS ((IPIPE_NR_IRQS + BITS_PER_LONG - 1) / BITS_PER_LONG)
+#define IPIPE_IRQ_IMASK (BITS_PER_LONG - 1)
+#define IPIPE_IRQMASK_ANY (~0L)
+#define IPIPE_IRQMASK_VIRT (IPIPE_IRQMASK_ANY << (IPIPE_VIRQ_BASE / BITS_PER_LONG))
+
+#ifdef CONFIG_SMP
+
+#define IPIPE_NR_CPUS NR_CPUS
+#define ipipe_declare_cpuid int cpuid
+#define ipipe_load_cpuid() do { \
+ (cpuid) = ipipe_processor_id(); \
+ } while(0)
+#define ipipe_lock_cpu(flags) do { \
+ local_irq_save_hw(flags); \
+ (cpuid) = ipipe_processor_id(); \
+ } while(0)
+#define ipipe_unlock_cpu(flags) local_irq_restore_hw(flags)
+#define ipipe_get_cpu(flags) ipipe_lock_cpu(flags)
+#define ipipe_put_cpu(flags) ipipe_unlock_cpu(flags)
+#define ipipe_current_domain per_cpu(ipipe_percpu_domain, ipipe_processor_id())
+
+#else /* !CONFIG_SMP */
+
+#define IPIPE_NR_CPUS 1
+#define ipipe_declare_cpuid const int cpuid = 0
+#define ipipe_load_cpuid() do { } while(0)
+#define ipipe_lock_cpu(flags) local_irq_save_hw(flags)
+#define ipipe_unlock_cpu(flags) local_irq_restore_hw(flags)
+#define ipipe_get_cpu(flags) do { (void)(flags); } while(0)
+#define ipipe_put_cpu(flags) do { } while(0)
+#define ipipe_current_domain per_cpu(ipipe_percpu_domain, 0)
+
+#endif /* CONFIG_SMP */
+
+#define ipipe_virtual_irq_p(irq) ((irq) >= IPIPE_VIRQ_BASE && \
+ (irq) < IPIPE_NR_IRQS)
+
+typedef void (*ipipe_irq_handler_t)(unsigned irq,
+ void *cookie);
+
+#define IPIPE_SAME_HANDLER ((ipipe_irq_handler_t)(-1))
+
+typedef int (*ipipe_irq_ackfn_t)(unsigned irq);
+
+typedef int (*ipipe_event_handler_t)(unsigned event,
+ struct ipipe_domain *from,
+ void *data);
+struct ipipe_domain {
+
+ struct list_head p_link; /* Link in pipeline */
+
+ struct ipcpudata {
+ unsigned long status;
+ unsigned long irq_pending_hi;
+ unsigned long irq_pending_lo[IPIPE_IRQ_IWORDS];
+ struct ipirqcnt {
+ unsigned long pending_hits;
+ unsigned long total_hits;
+ } irq_counters[IPIPE_NR_IRQS];
+ unsigned long long evsync;
+ } ____cacheline_aligned_in_smp cpudata[IPIPE_NR_CPUS];
+
+ struct {
+ unsigned long control;
+ ipipe_irq_ackfn_t acknowledge;
+ ipipe_irq_handler_t handler;
+ void *cookie;
+ } ____cacheline_aligned irqs[IPIPE_NR_IRQS];
+
+ ipipe_event_handler_t evhand[IPIPE_NR_EVENTS]; /* Event handlers. */
+ unsigned long long evself; /* Self-monitored event bits. */
+ unsigned long flags;
+ unsigned domid;
+ const char *name;
+ int priority;
+ void *pdd;
+};
+
+#define IPIPE_HEAD_PRIORITY (-1) /* For domains always heading the pipeline */
+
+struct ipipe_domain_attr {
+
+ unsigned domid; /* Domain identifier -- Magic value set by caller */
+ const char *name; /* Domain name -- Warning: won't be dup'ed! */
+ int priority; /* Priority in interrupt pipeline */
+ void (*entry) (void); /* Domain entry point */
+ void *pdd; /* Per-domain (opaque) data pointer */
+};
+
+/* The following macros must be used hw interrupts off. */
+
+#define __ipipe_irq_cookie(ipd,irq) (ipd)->irqs[irq].cookie
+#define __ipipe_irq_handler(ipd,irq) (ipd)->irqs[irq].handler
+
+#define __ipipe_cpudata_irq_hits(ipd,cpuid,irq) ((ipd)->cpudata[cpuid].irq_counters[irq].total_hits)
+
+#define __ipipe_set_irq_bit(ipd,cpuid,irq) \
+do { \
+ if (!test_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control)) { \
+ __set_bit(irq & IPIPE_IRQ_IMASK,&(ipd)->cpudata[cpuid].irq_pending_lo[irq >> IPIPE_IRQ_ISHIFT]); \
+ __set_bit(irq >> IPIPE_IRQ_ISHIFT,&(ipd)->cpudata[cpuid].irq_pending_hi); \
+ } \
+} while(0)
+
+#define __ipipe_clear_pend(ipd,cpuid,irq) \
+do { \
+ __clear_bit(irq & IPIPE_IRQ_IMASK,&(ipd)->cpudata[cpuid].irq_pending_lo[irq >> IPIPE_IRQ_ISHIFT]); \
+ if ((ipd)->cpudata[cpuid].irq_pending_lo[irq >> IPIPE_IRQ_ISHIFT] == 0) \
+ __clear_bit(irq >> IPIPE_IRQ_ISHIFT,&(ipd)->cpudata[cpuid].irq_pending_hi); \
+} while(0)
+
+#define __ipipe_lock_irq(ipd,cpuid,irq) \
+do { \
+ if (!test_and_set_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control)) \
+ __ipipe_clear_pend(ipd,cpuid,irq); \
+} while(0)
+
+#define __ipipe_unlock_irq(ipd,irq) \
+do { \
+ int __cpuid, __nr_cpus = num_online_cpus(); \
+ if (test_and_clear_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control)) \
+ for (__cpuid = 0; __cpuid < __nr_cpus; __cpuid++) \
+ if ((ipd)->cpudata[__cpuid].irq_counters[irq].pending_hits > 0) { /* We need atomic ops next. */ \
+ set_bit(irq & IPIPE_IRQ_IMASK,&(ipd)->cpudata[__cpuid].irq_pending_lo[irq >> IPIPE_IRQ_ISHIFT]); \
+ set_bit(irq >> IPIPE_IRQ_ISHIFT,&(ipd)->cpudata[__cpuid].irq_pending_hi); \
+ } \
+} while(0)
+
+#define __ipipe_clear_irq(ipd,irq) \
+do { \
+ int __cpuid, __nr_cpus = num_online_cpus(); \
+ clear_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control); \
+ for (__cpuid = 0; __cpuid < __nr_cpus; __cpuid++) { \
+ (ipd)->cpudata[__cpuid].irq_counters[irq].pending_hits = 0; \
+ __ipipe_clear_pend(ipd,__cpuid,irq); \
+ } \
+} while(0)
+
+#ifdef __RAW_SPIN_LOCK_UNLOCKED
+#define spin_lock_hw(x) __raw_spin_lock(&(x)->raw_lock)
+#define spin_trylock_hw(x) __raw_spin_trylock(&(x)->raw_lock)
+#define spin_unlock_hw(x) __raw_spin_unlock(&(x)->raw_lock)
+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
+#define write_lock_hw(x) __raw_write_lock(&(x)->raw_lock)
+#define write_trylock_hw(x) __raw_write_trylock(&(x)->raw_lock)
+#define write_unlock_hw(x) __raw_write_unlock(&(x)->raw_lock)
+#define read_lock_hw(x) __raw_read_lock(&(x)->raw_lock)
+#define read_trylock_hw(x) __raw_read_trylock(&(x)->raw_lock)
+#define read_unlock_hw(x) __raw_read_unlock(&(x)->raw_lock)
+#else /* UP non-debug */
+#define write_lock_hw(lock) do { (void)(lock); } while (0)
+#define write_trylock_hw(lock) ({ (void)(lock); 1; })
+#define write_unlock_hw(lock) do { (void)(lock); } while (0)
+#define read_lock_hw(lock) do { (void)(lock); } while (0)
+#define read_trylock_hw(lock) ({ (void)(lock); 1; })
+#define read_unlock_hw(lock) do { (void)(lock); } while (0)
+#endif /* CONFIG_SMP || CONFIG_DEBUG_SPINLOCK */
+#else /* !__RAW_SPIN_LOCK_UNLOCKED */
+#define spin_lock_hw(x) _raw_spin_lock(x)
+#define spin_unlock_hw(x) _raw_spin_unlock(x)
+#define spin_trylock_hw(x) _raw_spin_trylock(x)
+#define write_lock_hw(x) _raw_write_lock(x)
+#define write_unlock_hw(x) _raw_write_unlock(x)
+#define write_trylock_hw(x) _raw_write_trylock(x)
+#define read_lock_hw(x) _raw_read_lock(x)
+#define read_unlock_hw(x) _raw_read_unlock(x)
+#endif /* __RAW_SPIN_LOCK_UNLOCKED */
+
+typedef spinlock_t ipipe_spinlock_t;
+typedef rwlock_t ipipe_rwlock_t;
+#define IPIPE_SPIN_LOCK_UNLOCKED SPIN_LOCK_UNLOCKED
+#define IPIPE_RW_LOCK_UNLOCKED RW_LOCK_UNLOCKED
+
+#define spin_lock_irqsave_hw(x,flags) \
+do { \
+ local_irq_save_hw(flags); \
+ spin_lock_hw(x); \
+} while (0)
+
+#define spin_unlock_irqrestore_hw(x,flags) \
+do { \
+ spin_unlock_hw(x); \
+ local_irq_restore_hw(flags); \
+} while (0)
+
+#define spin_lock_irq_hw(x) \
+do { \
+ local_irq_disable_hw(); \
+ spin_lock_hw(x); \
+} while (0)
+
+#define spin_unlock_irq_hw(x) \
+do { \
+ spin_unlock_hw(x); \
+ local_irq_enable_hw(); \
+} while (0)
+
+#define read_lock_irqsave_hw(lock, flags) \
+do { \
+ local_irq_save_hw(flags); \
+ read_lock_hw(lock); \
+} while (0)
+
+#define read_unlock_irqrestore_hw(lock, flags) \
+do { \
+ read_unlock_hw(lock); \
+ local_irq_restore_hw(flags); \
+} while (0)
+
+#define write_lock_irqsave_hw(lock, flags) \
+do { \
+ local_irq_save_hw(flags); \
+ write_lock_hw(lock); \
+} while (0)
+
+#define write_unlock_irqrestore_hw(lock, flags) \
+do { \
+ write_unlock_hw(lock); \
+ local_irq_restore_hw(flags); \
+} while (0)
+
+DECLARE_PER_CPU(struct ipipe_domain *, ipipe_percpu_domain);
+
+extern struct ipipe_domain ipipe_root;
+
+#define ipipe_root_domain (&ipipe_root)
+
+extern unsigned __ipipe_printk_virq;
+
+extern unsigned long __ipipe_virtual_irq_map;
+
+extern struct list_head __ipipe_pipeline;
+
+extern ipipe_spinlock_t __ipipe_pipelock;
+
+extern int __ipipe_event_monitors[];
+
+/* Private interface */
+
+void ipipe_init(void);
+
+#ifdef CONFIG_PROC_FS
+void ipipe_init_proc(void);
+
+#ifdef CONFIG_IPIPE_TRACE
+void __ipipe_init_tracer(void);
+#else /* !CONFIG_IPIPE_TRACE */
+#define __ipipe_init_tracer() do { } while(0)
+#endif /* CONFIG_IPIPE_TRACE */
+
+#else /* !CONFIG_PROC_FS */
+#define ipipe_init_proc() do { } while(0)
+#endif /* CONFIG_PROC_FS */
+
+void __ipipe_init_stage(struct ipipe_domain *ipd);
+
+void __ipipe_cleanup_domain(struct ipipe_domain *ipd);
+
+void __ipipe_add_domain_proc(struct ipipe_domain *ipd);
+
+void __ipipe_remove_domain_proc(struct ipipe_domain *ipd);
+
+void __ipipe_flush_printk(unsigned irq, void *cookie);
+
+void __ipipe_stall_root(void);
+
+void __ipipe_unstall_root(void);
+
+unsigned long __ipipe_test_root(void);
+
+unsigned long __ipipe_test_and_stall_root(void);
+
+void fastcall __ipipe_walk_pipeline(struct list_head *pos, int cpuid);
+
+void fastcall __ipipe_restore_root(unsigned long x);
+
+int fastcall __ipipe_schedule_irq(unsigned irq, struct list_head *head);
+
+int fastcall __ipipe_dispatch_event(unsigned event, void *data);
+
+int fastcall __ipipe_dispatch_wired(struct ipipe_domain *head, unsigned irq);
+
+void fastcall __ipipe_sync_stage(unsigned long syncmask);
+
+#ifndef __ipipe_sync_pipeline
+#define __ipipe_sync_pipeline(syncmask) __ipipe_sync_stage(syncmask)
+#endif
+
+#ifndef __ipipe_run_irqtail
+#define __ipipe_run_irqtail() do { } while(0)
+#endif
+
+#define __ipipe_pipeline_head_p(ipd) (&(ipd)->p_link == __ipipe_pipeline.next)
+
+/*
+ * Keep the following as a macro, so that client code could check for
+ * the support of the invariant pipeline head optimization.
+ */
+#define __ipipe_pipeline_head() list_entry(__ipipe_pipeline.next,struct ipipe_domain,p_link)
+
+#define __ipipe_event_monitored_p(ev) \
+ (__ipipe_event_monitors[ev] > 0 || (ipipe_current_domain->evself & (1LL << ev)))
+
+#ifdef CONFIG_SMP
+
+cpumask_t __ipipe_set_irq_affinity(unsigned irq,
+ cpumask_t cpumask);
+
+int fastcall __ipipe_send_ipi(unsigned ipi,
+ cpumask_t cpumask);
+
+#endif /* CONFIG_SMP */
+
+/* Called with hw interrupts off. */
+static inline void __ipipe_switch_to(struct ipipe_domain *out,
+ struct ipipe_domain *in, int cpuid)
+{
+ void ipipe_suspend_domain(void);
+
+ /*
+ * "in" is guaranteed to be closer than "out" from the head of the
+ * pipeline (and obviously different).
+ */
+
+ out->cpudata[cpuid].evsync = 0;
+ per_cpu(ipipe_percpu_domain, cpuid) = in;
+
+ ipipe_suspend_domain(); /* Sync stage and propagate interrupts. */
+ ipipe_load_cpuid(); /* Processor might have changed. */
+
+ if (per_cpu(ipipe_percpu_domain, cpuid) == in)
+ /*
+ * Otherwise, something has changed the current domain under
+ * our feet recycling the register set; do not override.
+ */
+ per_cpu(ipipe_percpu_domain, cpuid) = out;
+}
+
+#define ipipe_sigwake_notify(p) \
+do { \
+ if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_SIGWAKE)) \
+ __ipipe_dispatch_event(IPIPE_EVENT_SIGWAKE,p); \
+} while(0)
+
+#define ipipe_exit_notify(p) \
+do { \
+ if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_EXIT)) \
+ __ipipe_dispatch_event(IPIPE_EVENT_EXIT,p); \
+} while(0)
+
+#define ipipe_setsched_notify(p) \
+do { \
+ if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_SETSCHED)) \
+ __ipipe_dispatch_event(IPIPE_EVENT_SETSCHED,p); \
+} while(0)
+
+#define ipipe_schedule_notify(prev, next) \
+do { \
+ if ((((prev)->flags|(next)->flags) & PF_EVNOTIFY) && \
+ __ipipe_event_monitored_p(IPIPE_EVENT_SCHEDULE)) \
+ __ipipe_dispatch_event(IPIPE_EVENT_SCHEDULE,next); \
+} while(0)
+
+#define ipipe_trap_notify(ex, regs) \
+({ \
+ ipipe_declare_cpuid; \
+ int ret = 0; \
+ ipipe_load_cpuid(); \
+ if ((test_bit(IPIPE_NOSTACK_FLAG, &ipipe_current_domain->cpudata[cpuid].status) || \
+ ((current)->flags & PF_EVNOTIFY)) && \
+ __ipipe_event_monitored_p(ex)) \
+ ret = __ipipe_dispatch_event(ex, regs); \
+ ret; \
+})
+
+static inline void ipipe_init_notify(struct task_struct *p)
+{
+ if (__ipipe_event_monitored_p(IPIPE_EVENT_INIT))
+ __ipipe_dispatch_event(IPIPE_EVENT_INIT,p);
+}
+
+struct mm_struct;
+
+static inline void ipipe_cleanup_notify(struct mm_struct *mm)
+{
+ if (__ipipe_event_monitored_p(IPIPE_EVENT_CLEANUP))
+ __ipipe_dispatch_event(IPIPE_EVENT_CLEANUP,mm);
+}
+
+/* Public interface */
+
+int ipipe_register_domain(struct ipipe_domain *ipd,
+ struct ipipe_domain_attr *attr);
+
+int ipipe_unregister_domain(struct ipipe_domain *ipd);
+
+void ipipe_suspend_domain(void);
+
+int ipipe_virtualize_irq(struct ipipe_domain *ipd,
+ unsigned irq,
+ ipipe_irq_handler_t handler,
+ void *cookie,
+ ipipe_irq_ackfn_t acknowledge,
+ unsigned modemask);
+
+static inline int ipipe_share_irq(unsigned irq,
+ ipipe_irq_ackfn_t acknowledge)
+{
+ return ipipe_virtualize_irq(ipipe_current_domain,
+ irq,
+ IPIPE_SAME_HANDLER,
+ NULL,
+ acknowledge,
+ IPIPE_SHARED_MASK | IPIPE_HANDLE_MASK |
+ IPIPE_PASS_MASK);
+}
+
+int ipipe_control_irq(unsigned irq,
+ unsigned clrmask,
+ unsigned setmask);
+
+unsigned ipipe_alloc_virq(void);
+
+int ipipe_free_virq(unsigned virq);
+
+int fastcall ipipe_trigger_irq(unsigned irq);
+
+static inline int ipipe_propagate_irq(unsigned irq)
+{
+ return __ipipe_schedule_irq(irq, ipipe_current_domain->p_link.next);
+}
+
+static inline int ipipe_schedule_irq(unsigned irq)
+{
+ return __ipipe_schedule_irq(irq, &ipipe_current_domain->p_link);
+}
+
+void fastcall ipipe_stall_pipeline_from(struct ipipe_domain *ipd);
+
+unsigned long fastcall ipipe_test_and_stall_pipeline_from(struct ipipe_domain *ipd);
+
+void fastcall ipipe_unstall_pipeline_from(struct ipipe_domain *ipd);
+
+unsigned long fastcall ipipe_test_and_unstall_pipeline_from(struct ipipe_domain *ipd);
+
+void fastcall ipipe_restore_pipeline_from(struct ipipe_domain *ipd,
+ unsigned long x);
+
+static inline unsigned long ipipe_test_pipeline_from(struct ipipe_domain *ipd)
+{
+ unsigned long flags, x;
+ ipipe_declare_cpuid;
+
+ ipipe_get_cpu(flags);
+ x = test_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+ ipipe_put_cpu(flags);
+
+ return x;
+}
+
+static inline void ipipe_restore_pipeline_nosync(struct ipipe_domain *ipd,
+ unsigned long x, int cpuid)
+{
+ /*
+ * If cpuid is current, then it must be held on entry
+ * (ipipe_get_cpu/local_irq_save_hw/local_irq_disable_hw).
+ */
+
+ if (x)
+ __set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+ else
+ __clear_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+}
+
+static inline void ipipe_stall_pipeline_head(void)
+{
+ ipipe_declare_cpuid;
+ unsigned long flags;
+
+ ipipe_lock_cpu(flags);
+ __set_bit(IPIPE_STALL_FLAG, &__ipipe_pipeline_head()->cpudata[cpuid].status);
+}
+
+static inline unsigned long ipipe_test_and_stall_pipeline_head(void)
+{
+ unsigned long flags;
+ ipipe_declare_cpuid;
+
+ ipipe_lock_cpu(flags);
+ return __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_pipeline_head()->cpudata[cpuid].status);
+}
+
+void ipipe_unstall_pipeline_head(void);
+
+void fastcall __ipipe_restore_pipeline_head(struct ipipe_domain *head,
+ unsigned long x);
+
+static inline void ipipe_restore_pipeline_head(unsigned long x)
+{
+ struct ipipe_domain *head = __ipipe_pipeline_head();
+ /* On some archs, __test_and_set_bit() might return different
+ * truth value than test_bit(), so we test the exclusive OR of
+ * both statuses, assuming that the lowest bit is always set in
+ * the truth value (if this is wrong, the failed optimization will
+ * be caught in __ipipe_restore_pipeline_head() if
+ * CONFIG_DEBUG_KERNEL is set). */
+ if ((x ^ test_bit(IPIPE_STALL_FLAG, &head->cpudata[ipipe_processor_id()].status)) & 1)
+ __ipipe_restore_pipeline_head(head,x);
+}
+
+#define ipipe_unstall_pipeline() \
+ ipipe_unstall_pipeline_from(ipipe_current_domain)
+
+#define ipipe_test_and_unstall_pipeline() \
+ ipipe_test_and_unstall_pipeline_from(ipipe_current_domain)
+
+#define ipipe_test_pipeline() \
+ ipipe_test_pipeline_from(ipipe_current_domain)
+
+#define ipipe_test_and_stall_pipeline() \
+ ipipe_test_and_stall_pipeline_from(ipipe_current_domain)
+
+#define ipipe_stall_pipeline() \
+ ipipe_stall_pipeline_from(ipipe_current_domain)
+
+#define ipipe_restore_pipeline(x) \
+ ipipe_restore_pipeline_from(ipipe_current_domain, (x))
+
+void ipipe_init_attr(struct ipipe_domain_attr *attr);
+
+int ipipe_get_sysinfo(struct ipipe_sysinfo *sysinfo);
+
+int ipipe_tune_timer(unsigned long ns,
+ int flags);
+
+unsigned long ipipe_critical_enter(void (*syncfn) (void));
+
+void ipipe_critical_exit(unsigned long flags);
+
+static inline void ipipe_set_printk_sync(struct ipipe_domain *ipd)
+{
+ set_bit(IPIPE_SPRINTK_FLAG, &ipd->flags);
+}
+
+static inline void ipipe_set_printk_async(struct ipipe_domain *ipd)
+{
+ clear_bit(IPIPE_SPRINTK_FLAG, &ipd->flags);
+}
+
+static inline void ipipe_set_foreign_stack(struct ipipe_domain *ipd)
+{
+ /* Must be called hw interrupts off. */
+ ipipe_declare_cpuid;
+ ipipe_load_cpuid();
+ __set_bit(IPIPE_NOSTACK_FLAG, &ipd->cpudata[cpuid].status);
+}
+
+static inline void ipipe_clear_foreign_stack(struct ipipe_domain *ipd)
+{
+ /* Must be called hw interrupts off. */
+ ipipe_declare_cpuid;
+ ipipe_load_cpuid();
+ __clear_bit(IPIPE_NOSTACK_FLAG, &ipd->cpudata[cpuid].status);
+}
+
+#define ipipe_safe_current() \
+({ \
+ ipipe_declare_cpuid; \
+ struct task_struct *p; \
+ ipipe_load_cpuid(); \
+ p = test_bit(IPIPE_NOSTACK_FLAG, \
+ &per_cpu(ipipe_percpu_domain, cpuid)->cpudata[cpuid].status) ? &init_task : current; \
+ p; \
+})
+
+ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd,
+ unsigned event,
+ ipipe_event_handler_t handler);
+
+cpumask_t ipipe_set_irq_affinity(unsigned irq,
+ cpumask_t cpumask);
+
+int fastcall ipipe_send_ipi(unsigned ipi,
+ cpumask_t cpumask);
+
+int ipipe_setscheduler_root(struct task_struct *p,
+ int policy,
+ int prio);
+
+int ipipe_reenter_root(struct task_struct *prev,
+ int policy,
+ int prio);
+
+int ipipe_alloc_ptdkey(void);
+
+int ipipe_free_ptdkey(int key);
+
+int fastcall ipipe_set_ptd(int key,
+ void *value);
+
+void fastcall *ipipe_get_ptd(int key);
+
+#define local_irq_enable_hw_cond() local_irq_enable_hw()
+#define local_irq_disable_hw_cond() local_irq_disable_hw()
+#define local_irq_save_hw_cond(flags) local_irq_save_hw(flags)
+#define local_irq_restore_hw_cond(flags) local_irq_restore_hw(flags)
+
+#define local_irq_enable_nohead(ipd) \
+ do { \
+ if (!__ipipe_pipeline_head_p(ipd)) \
+ local_irq_enable_hw(); \
+ } while(0)
+
+#define local_irq_disable_nohead(ipd) \
+ do { \
+ if (!__ipipe_pipeline_head_p(ipd)) \
+ local_irq_disable_hw(); \
+ } while(0)
+
+#define spin_lock_irqsave_hw_cond(lock,flags) spin_lock_irqsave_hw(lock,flags)
+#define spin_unlock_irqrestore_hw_cond(lock,flags) spin_unlock_irqrestore_hw(lock,flags)
+#define smp_processor_id_hw() ipipe_processor_id()
+
+#define ipipe_irq_lock(irq) \
+ do { \
+ ipipe_declare_cpuid; \
+ ipipe_load_cpuid(); \
+ __ipipe_lock_irq(per_cpu(ipipe_percpu_domain, cpuid), cpuid, irq);\
+ } while(0)
+
+#define ipipe_irq_unlock(irq) \
+ do { \
+ ipipe_declare_cpuid; \
+ ipipe_load_cpuid(); \
+ __ipipe_unlock_irq(per_cpu(ipipe_percpu_domain, cpuid), irq); \
+ } while(0)
+
+#define ipipe_root_domain_p (ipipe_current_domain == ipipe_root_domain)
+
+#else /* !CONFIG_IPIPE */
+
+#define ipipe_init() do { } while(0)
+#define ipipe_suspend_domain() do { } while(0)
+#define ipipe_sigwake_notify(p) do { } while(0)
+#define ipipe_setsched_notify(p) do { } while(0)
+#define ipipe_init_notify(p) do { } while(0)
+#define ipipe_exit_notify(p) do { } while(0)
+#define ipipe_cleanup_notify(mm) do { } while(0)
+#define ipipe_trap_notify(t,r) 0
+#define ipipe_init_proc() do { } while(0)
+
+#define spin_lock_hw(lock) spin_lock(lock)
+#define spin_unlock_hw(lock) spin_unlock(lock)
+#define spin_lock_irq_hw(lock) spin_lock_irq(lock)
+#define spin_unlock_irq_hw(lock) spin_unlock_irq(lock)
+#define spin_lock_irqsave_hw(lock,flags) spin_lock_irqsave(lock, flags)
+#define spin_unlock_irqrestore_hw(lock,flags) spin_unlock_irqrestore(lock, flags)
+
+#define local_irq_enable_hw_cond() do { } while(0)
+#define local_irq_disable_hw_cond() do { } while(0)
+#define local_irq_save_hw_cond(flags) do { (void)(flags); } while(0)
+#define local_irq_restore_hw_cond(flags) do { } while(0)
+#define spin_lock_irqsave_hw_cond(lock,flags) do { (void)(flags); spin_lock(lock); } while(0)
+#define spin_unlock_irqrestore_hw_cond(lock,flags) spin_unlock(lock)
+#define smp_processor_id_hw() smp_processor_id()
+
+#define ipipe_irq_lock(irq) do { } while(0)
+#define ipipe_irq_unlock(irq) do { } while(0)
+
+#define ipipe_root_domain_p 1
+#define ipipe_safe_current current
+
+#endif /* CONFIG_IPIPE */
+
+#endif /* !__LINUX_IPIPE_H */
diff -Naur linux-2.6.19.orig/include/linux/ipipe_trace.h linux-2.6.19.ipipe/include/linux/ipipe_trace.h
--- linux-2.6.19.orig/include/linux/ipipe_trace.h 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.19.ipipe/include/linux/ipipe_trace.h 2006-12-19 13:02:34.000000000 +0100
@@ -0,0 +1,40 @@
+/* -*- linux-c -*-
+ * include/linux/ipipe_trace.h
+ *
+ * Copyright (C) 2005 Luotao Fu.
+ * 2005, 2006 Jan Kiszka.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#ifndef _LINUX_IPIPE_TRACE_H
+#define _LINUX_IPIPE_TRACE_H
+
+#include <linux/types.h>
+
+void ipipe_trace_begin(unsigned long v);
+void ipipe_trace_end(unsigned long v);
+void ipipe_trace_freeze(unsigned long v);
+void ipipe_trace_special(unsigned char special_id, unsigned long v);
+void ipipe_trace_pid(pid_t pid, short prio);
+
+int ipipe_trace_max_reset(void);
+int ipipe_trace_frozen_reset(void);
+
+void ipipe_trace_panic_freeze(void);
+void ipipe_trace_panic_dump(void);
+
+#endif /* !__LINUX_IPIPE_H */
diff -Naur linux-2.6.19.orig/include/linux/linkage.h linux-2.6.19.ipipe/include/linux/linkage.h
--- linux-2.6.19.orig/include/linux/linkage.h 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/include/linux/linkage.h 2006-12-19 13:02:33.000000000 +0100
@@ -64,4 +64,8 @@
#define fastcall
#endif
+#ifndef notrace
+#define notrace __attribute__((no_instrument_function))
+#endif
+
#endif
diff -Naur linux-2.6.19.orig/include/linux/preempt.h linux-2.6.19.ipipe/include/linux/preempt.h
--- linux-2.6.19.orig/include/linux/preempt.h 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/include/linux/preempt.h 2006-12-19 13:02:33.000000000 +0100
@@ -26,22 +26,37 @@
asmlinkage void preempt_schedule(void);
+#ifdef CONFIG_IPIPE
+#include <asm/ipipe.h>
+DECLARE_PER_CPU(struct ipipe_domain *, ipipe_percpu_domain);
+extern struct ipipe_domain ipipe_root;
+#define ipipe_preempt_guard() (per_cpu(ipipe_percpu_domain, ipipe_processor_id()) == &ipipe_root)
+#else /* !CONFIG_IPIPE */
+#define ipipe_preempt_guard() 1
+#endif /* CONFIG_IPIPE */
+
#define preempt_disable() \
do { \
- inc_preempt_count(); \
- barrier(); \
+ if (ipipe_preempt_guard()) { \
+ inc_preempt_count(); \
+ barrier(); \
+ } \
} while (0)
#define preempt_enable_no_resched() \
do { \
- barrier(); \
- dec_preempt_count(); \
+ if (ipipe_preempt_guard()) { \
+ barrier(); \
+ dec_preempt_count(); \
+ } \
} while (0)
#define preempt_check_resched() \
do { \
- if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \
- preempt_schedule(); \
+ if (ipipe_preempt_guard()) { \
+ if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \
+ preempt_schedule(); \
+ } \
} while (0)
#define preempt_enable() \
diff -Naur linux-2.6.19.orig/include/linux/radix-tree.h linux-2.6.19.ipipe/include/linux/radix-tree.h
--- linux-2.6.19.orig/include/linux/radix-tree.h 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/include/linux/radix-tree.h 2006-12-19 13:02:33.000000000 +0100
@@ -19,7 +19,9 @@
#ifndef _LINUX_RADIX_TREE_H
#define _LINUX_RADIX_TREE_H
+#ifndef CONFIG_PPC_MERGE
#include <linux/sched.h>
+#endif /* CONFIG_PPC_MERGE */
#include <linux/preempt.h>
#include <linux/types.h>
diff -Naur linux-2.6.19.orig/include/linux/sched.h linux-2.6.19.ipipe/include/linux/sched.h
--- linux-2.6.19.orig/include/linux/sched.h 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/include/linux/sched.h 2006-12-19 13:02:33.000000000 +0100
@@ -43,6 +43,7 @@
#include <asm/param.h> /* for HZ */
+#include <linux/ipipe.h>
#include <linux/capability.h>
#include <linux/threads.h>
#include <linux/kernel.h>
@@ -150,6 +151,14 @@
/* in tsk->state again */
#define TASK_NONINTERACTIVE 64
#define TASK_DEAD 128
+#ifdef CONFIG_IPIPE
+#define TASK_ATOMICSWITCH 512
+#define TASK_NOWAKEUP 1024
+#else /* !CONFIG_IPIPE */
+#define TASK_ATOMICSWITCH 0
+#define TASK_NOWAKEUP 0
+#endif /* CONFIG_IPIPE */
+
#define __set_task_state(tsk, state_value) \
do { (tsk)->state = (state_value); } while (0)
@@ -1023,6 +1032,9 @@
#ifdef CONFIG_TASK_DELAY_ACCT
struct task_delay_info *delays;
#endif
+#ifdef CONFIG_IPIPE
+ void *ptd[IPIPE_ROOT_NPTDKEYS];
+#endif
};
static inline pid_t process_group(struct task_struct *tsk)
@@ -1115,6 +1127,11 @@
#define PF_SPREAD_SLAB 0x02000000 /* Spread some slab caches over cpuset */
#define PF_MEMPOLICY 0x10000000 /* Non-default NUMA mempolicy */
#define PF_MUTEX_TESTER 0x20000000 /* Thread belongs to the rt mutex tester */
+#ifdef CONFIG_IPIPE
+#define PF_EVNOTIFY 0x40000000 /* Notify other domains about internal events */
+#else
+#define PF_EVNOTIFY 0
+#endif /* CONFIG_IPIPE */
/*
* Only the _current_ task can read/write to tsk->flags, but other
diff -Naur linux-2.6.19.orig/init/Kconfig linux-2.6.19.ipipe/init/Kconfig
--- linux-2.6.19.orig/init/Kconfig 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/init/Kconfig 2006-12-19 13:02:34.000000000 +0100
@@ -67,6 +67,7 @@
config LOCALVERSION
string "Local version - append to kernel release"
+ default "-ipipe"
help
Append an extra string to the end of your kernel version.
This will show up when you type uname, for example.
@@ -543,3 +544,4 @@
menu "Block layer"
source "block/Kconfig"
endmenu
+
diff -Naur linux-2.6.19.orig/init/main.c linux-2.6.19.ipipe/init/main.c
--- linux-2.6.19.orig/init/main.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/init/main.c 2006-12-19 13:02:34.000000000 +0100
@@ -535,6 +535,11 @@
softirq_init();
timekeeping_init();
time_init();
+ /*
+ * We need to wait for the interrupt and time subsystems to be
+ * initialized before enabling the pipeline.
+ */
+ ipipe_init();
profile_init();
if (!irqs_disabled())
printk("start_kernel(): bug: interrupts were enabled early\n");
@@ -683,6 +688,7 @@
#ifdef CONFIG_SYSCTL
sysctl_init();
#endif
+ ipipe_init_proc();
do_initcalls();
}
diff -Naur linux-2.6.19.orig/kernel/exit.c linux-2.6.19.ipipe/kernel/exit.c
--- linux-2.6.19.orig/kernel/exit.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/kernel/exit.c 2006-12-19 13:02:35.000000000 +0100
@@ -914,6 +914,7 @@
taskstats_exit_send(tsk, tidstats, group_dead, mycpu);
taskstats_exit_free(tidstats);
+ ipipe_exit_notify(tsk);
exit_mm(tsk);
if (group_dead)
diff -Naur linux-2.6.19.orig/kernel/fork.c linux-2.6.19.ipipe/kernel/fork.c
--- linux-2.6.19.orig/kernel/fork.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/kernel/fork.c 2006-12-19 13:02:35.000000000 +0100
@@ -384,6 +384,7 @@
might_sleep();
if (atomic_dec_and_test(&mm->mm_users)) {
+ ipipe_cleanup_notify(mm);
exit_aio(mm);
exit_mmap(mm);
if (!list_empty(&mm->mmlist)) {
@@ -914,7 +915,7 @@
{
unsigned long new_flags = p->flags;
- new_flags &= ~(PF_SUPERPRIV | PF_NOFREEZE);
+ new_flags &= ~(PF_SUPERPRIV | PF_NOFREEZE | PF_EVNOTIFY);
new_flags |= PF_FORKNOEXEC;
if (!(clone_flags & CLONE_PTRACE))
p->ptrace = 0;
@@ -1258,6 +1259,15 @@
spin_unlock(¤t->sighand->siglock);
write_unlock_irq(&tasklist_lock);
proc_fork_connector(p);
+#ifdef CONFIG_IPIPE
+ {
+ int k;
+
+ for (k = 0; k < IPIPE_ROOT_NPTDKEYS; k++)
+ p->ptd[k] = NULL;
+ }
+#endif /* CONFIG_IPIPE */
+
return p;
bad_fork_cleanup_namespaces:
diff -Naur linux-2.6.19.orig/kernel/ipipe/core.c linux-2.6.19.ipipe/kernel/ipipe/core.c
--- linux-2.6.19.orig/kernel/ipipe/core.c 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.19.ipipe/kernel/ipipe/core.c 2006-12-19 13:02:36.000000000 +0100
@@ -0,0 +1,1049 @@
+/* -*- linux-c -*-
+ * linux/kernel/ipipe/core.c
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-independent I-PIPE core support.
+ */
+
+#include <linux/sched.h>
+#include <linux/module.h>
+#include <linux/kallsyms.h>
+#include <linux/interrupt.h>
+#ifdef CONFIG_PROC_FS
+#include <linux/proc_fs.h>
+#endif /* CONFIG_PROC_FS */
+
+struct ipipe_domain ipipe_root =
+ { .cpudata = {[0 ... IPIPE_NR_CPUS-1] =
+ { .status = (1<<IPIPE_STALL_FLAG) } } };
+
+DEFINE_PER_CPU(struct ipipe_domain *, ipipe_percpu_domain) =
+ { &ipipe_root };
+
+ipipe_spinlock_t __ipipe_pipelock = IPIPE_SPIN_LOCK_UNLOCKED;
+
+LIST_HEAD(__ipipe_pipeline);
+
+unsigned long __ipipe_virtual_irq_map;
+
+#ifdef CONFIG_PRINTK
+unsigned __ipipe_printk_virq;
+#endif /* CONFIG_PRINTK */
+
+int __ipipe_event_monitors[IPIPE_NR_EVENTS];
+
+/*
+ * ipipe_init() -- Initialization routine of the IPIPE layer. Called
+ * by the host kernel early during the boot procedure.
+ */
+void ipipe_init(void)
+{
+ struct ipipe_domain *ipd = &ipipe_root;
+
+ __ipipe_check_platform(); /* Do platform dependent checks first. */
+
+ /*
+ * A lightweight registration code for the root domain. We are
+ * running on the boot CPU, hw interrupts are off, and
+ * secondary CPUs are still lost in space.
+ */
+
+ ipd->name = "Linux";
+ ipd->domid = IPIPE_ROOT_ID;
+ ipd->priority = IPIPE_ROOT_PRIO;
+
+ __ipipe_init_stage(ipd);
+
+ INIT_LIST_HEAD(&ipd->p_link);
+ list_add_tail(&ipd->p_link, &__ipipe_pipeline);
+
+ __ipipe_init_platform();
+
+#ifdef CONFIG_PRINTK
+ __ipipe_printk_virq = ipipe_alloc_virq(); /* Cannot fail here. */
+ ipd->irqs[__ipipe_printk_virq].handler = &__ipipe_flush_printk;
+ ipd->irqs[__ipipe_printk_virq].cookie = NULL;
+ ipd->irqs[__ipipe_printk_virq].acknowledge = NULL;
+ ipd->irqs[__ipipe_printk_virq].control = IPIPE_HANDLE_MASK;
+#endif /* CONFIG_PRINTK */
+
+ __ipipe_enable_pipeline();
+
+ printk(KERN_INFO "I-pipe %s: pipeline enabled.\n",
+ IPIPE_VERSION_STRING);
+}
+
+void __ipipe_init_stage(struct ipipe_domain *ipd)
+{
+ int cpuid, n;
+
+ for (cpuid = 0; cpuid < IPIPE_NR_CPUS; cpuid++) {
+ ipd->cpudata[cpuid].irq_pending_hi = 0;
+
+ for (n = 0; n < IPIPE_IRQ_IWORDS; n++)
+ ipd->cpudata[cpuid].irq_pending_lo[n] = 0;
+
+ for (n = 0; n < IPIPE_NR_IRQS; n++) {
+ ipd->cpudata[cpuid].irq_counters[n].total_hits = 0;
+ ipd->cpudata[cpuid].irq_counters[n].pending_hits = 0;
+ }
+
+ ipd->cpudata[cpuid].evsync = 0;
+ }
+
+ for (n = 0; n < IPIPE_NR_IRQS; n++) {
+ ipd->irqs[n].acknowledge = NULL;
+ ipd->irqs[n].handler = NULL;
+ ipd->irqs[n].control = IPIPE_PASS_MASK; /* Pass but don't handle */
+ }
+
+ for (n = 0; n < IPIPE_NR_EVENTS; n++)
+ ipd->evhand[n] = NULL;
+
+ ipd->evself = 0LL;
+
+#ifdef CONFIG_SMP
+ ipd->irqs[IPIPE_CRITICAL_IPI].acknowledge = &__ipipe_ack_system_irq;
+ ipd->irqs[IPIPE_CRITICAL_IPI].handler = &__ipipe_do_critical_sync;
+ ipd->irqs[IPIPE_CRITICAL_IPI].cookie = NULL;
+ /* Immediately handle in the current domain but *never* pass */
+ ipd->irqs[IPIPE_CRITICAL_IPI].control =
+ IPIPE_HANDLE_MASK|IPIPE_STICKY_MASK|IPIPE_SYSTEM_MASK;
+#endif /* CONFIG_SMP */
+}
+
+void __ipipe_stall_root(void)
+{
+ ipipe_declare_cpuid;
+ unsigned long flags;
+
+ ipipe_get_cpu(flags); /* Care for migration. */
+ set_bit(IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].status);
+ ipipe_put_cpu(flags);
+}
+
+void __ipipe_cleanup_domain(struct ipipe_domain *ipd)
+{
+ ipipe_unstall_pipeline_from(ipd);
+
+#ifdef CONFIG_SMP
+ {
+ int cpu;
+
+ for_each_online_cpu(cpu) {
+ while (ipd->cpudata[cpu].irq_pending_hi != 0)
+ cpu_relax();
+ }
+ }
+#endif /* CONFIG_SMP */
+}
+
+void __ipipe_unstall_root(void)
+{
+ ipipe_declare_cpuid;
+
+ local_irq_disable_hw();
+
+ ipipe_load_cpuid();
+
+ __clear_bit(IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].status);
+
+ if (unlikely(ipipe_root_domain->cpudata[cpuid].irq_pending_hi != 0))
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+
+ local_irq_enable_hw();
+}
+
+unsigned long __ipipe_test_root(void)
+{
+ unsigned long flags, x;
+ ipipe_declare_cpuid;
+
+ ipipe_get_cpu(flags); /* Care for migration. */
+ x = test_bit(IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].status);
+ ipipe_put_cpu(flags);
+
+ return x;
+}
+
+unsigned long __ipipe_test_and_stall_root(void)
+{
+ unsigned long flags, x;
+ ipipe_declare_cpuid;
+
+ ipipe_get_cpu(flags); /* Care for migration. */
+ x = test_and_set_bit(IPIPE_STALL_FLAG,
+ &ipipe_root_domain->cpudata[cpuid].status);
+ ipipe_put_cpu(flags);
+
+ return x;
+}
+
+void fastcall __ipipe_restore_root(unsigned long x)
+{
+ if (x)
+ __ipipe_stall_root();
+ else
+ __ipipe_unstall_root();
+}
+
+void fastcall ipipe_stall_pipeline_from(struct ipipe_domain *ipd)
+{
+ ipipe_declare_cpuid;
+#ifdef CONFIG_SMP
+ unsigned long flags;
+
+ ipipe_lock_cpu(flags); /* Care for migration. */
+
+ __set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+
+ if (!__ipipe_pipeline_head_p(ipd))
+ ipipe_unlock_cpu(flags);
+#else /* CONFIG_SMP */
+ set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+
+ if (__ipipe_pipeline_head_p(ipd))
+ local_irq_disable_hw();
+#endif /* CONFIG_SMP */
+}
+
+unsigned long fastcall ipipe_test_and_stall_pipeline_from(struct ipipe_domain *ipd)
+{
+ ipipe_declare_cpuid;
+ unsigned long s;
+#ifdef CONFIG_SMP
+ unsigned long flags;
+
+ ipipe_lock_cpu(flags); /* Care for migration. */
+
+ s = __test_and_set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+
+ if (!__ipipe_pipeline_head_p(ipd))
+ ipipe_unlock_cpu(flags);
+#else /* CONFIG_SMP */
+ s = test_and_set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+
+ if (__ipipe_pipeline_head_p(ipd))
+ local_irq_disable_hw();
+#endif /* CONFIG_SMP */
+
+ return s;
+}
+
+/*
+ * ipipe_unstall_pipeline_from() -- Unstall the pipeline and
+ * synchronize pending interrupts for a given domain. See
+ * __ipipe_walk_pipeline() for more information.
+ */
+void fastcall ipipe_unstall_pipeline_from(struct ipipe_domain *ipd)
+{
+ struct list_head *pos;
+ unsigned long flags;
+ ipipe_declare_cpuid;
+
+ ipipe_lock_cpu(flags);
+
+ __clear_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+
+ if (ipd == per_cpu(ipipe_percpu_domain, cpuid))
+ pos = &ipd->p_link;
+ else
+ pos = __ipipe_pipeline.next;
+
+ __ipipe_walk_pipeline(pos, cpuid);
+
+ if (__ipipe_pipeline_head_p(ipd))
+ local_irq_enable_hw();
+ else
+ ipipe_unlock_cpu(flags);
+}
+
+unsigned long fastcall ipipe_test_and_unstall_pipeline_from(struct ipipe_domain *ipd)
+{
+ unsigned long flags, x;
+ ipipe_declare_cpuid;
+
+ ipipe_get_cpu(flags);
+ x = test_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status);
+ ipipe_unstall_pipeline_from(ipd);
+ ipipe_put_cpu(flags);
+
+ return x;
+}
+
+void fastcall ipipe_restore_pipeline_from(struct ipipe_domain *ipd,
+ unsigned long x)
+{
+ if (x)
+ ipipe_stall_pipeline_from(ipd);
+ else
+ ipipe_unstall_pipeline_from(ipd);
+}
+
+void ipipe_unstall_pipeline_head(void)
+{
+ struct ipipe_domain *head;
+ unsigned long flags;
+ ipipe_declare_cpuid;
+
+ ipipe_lock_cpu(flags);
+ head = __ipipe_pipeline_head();
+ __clear_bit(IPIPE_STALL_FLAG, &head->cpudata[cpuid].status);
+
+ if (unlikely(head->cpudata[cpuid].irq_pending_hi != 0)) {
+ if (likely(head == per_cpu(ipipe_percpu_domain, cpuid)))
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+ else
+ __ipipe_walk_pipeline(&head->p_link, cpuid);
+ }
+
+ local_irq_enable_hw();
+}
+
+void fastcall __ipipe_restore_pipeline_head(struct ipipe_domain *head, unsigned long x)
+{
+ ipipe_declare_cpuid;
+ unsigned long flags;
+
+ ipipe_lock_cpu(flags);
+
+ if (x) {
+#ifdef CONFIG_DEBUG_KERNEL
+ static int warned;
+ if (!warned && test_and_set_bit(IPIPE_STALL_FLAG, &head->cpudata[cpuid].status)) {
+ /*
+ * Already stalled albeit ipipe_restore_pipeline_head()
+ * should have detected it? Send a warning once.\n");
+ */
+ warned = 1;
+ printk(KERN_WARNING
+ "I-pipe: ipipe_restore_pipeline_head() optimization failed.\n");
+ dump_stack();
+ }
+#else /* !CONFIG_DEBUG_KERNEL */
+ set_bit(IPIPE_STALL_FLAG, &head->cpudata[cpuid].status);
+#endif /* CONFIG_DEBUG_KERNEL */
+ }
+ else {
+ __clear_bit(IPIPE_STALL_FLAG, &head->cpudata[cpuid].status);
+ if (unlikely(head->cpudata[cpuid].irq_pending_hi != 0)) {
+ if (likely(head == per_cpu(ipipe_percpu_domain, cpuid)))
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+ else
+ __ipipe_walk_pipeline(&head->p_link, cpuid);
+ }
+ local_irq_enable_hw();
+ }
+}
+
+/* __ipipe_walk_pipeline(): Plays interrupts pending in the log. Must
+ be called with local hw interrupts disabled. */
+
+void fastcall __ipipe_walk_pipeline(struct list_head *pos, int cpuid)
+{
+ struct ipipe_domain *this_domain = per_cpu(ipipe_percpu_domain, cpuid);
+
+ while (pos != &__ipipe_pipeline) {
+ struct ipipe_domain *next_domain =
+ list_entry(pos, struct ipipe_domain, p_link);
+
+ if (test_bit
+ (IPIPE_STALL_FLAG, &next_domain->cpudata[cpuid].status))
+ break; /* Stalled stage -- do not go further. */
+
+ if (next_domain->cpudata[cpuid].irq_pending_hi != 0) {
+
+ if (next_domain == this_domain)
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+ else {
+ __ipipe_switch_to(this_domain, next_domain,
+ cpuid);
+
+ ipipe_load_cpuid(); /* Processor might have changed. */
+
+ if (this_domain->cpudata[cpuid].
+ irq_pending_hi != 0
+ && !test_bit(IPIPE_STALL_FLAG,
+ &this_domain->cpudata[cpuid].status))
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+ }
+
+ break;
+ } else if (next_domain == this_domain)
+ break;
+
+ pos = next_domain->p_link.next;
+ }
+}
+
+/*
+ * ipipe_suspend_domain() -- Suspend the current domain, switching to
+ * the next one which has pending work down the pipeline.
+ */
+void ipipe_suspend_domain(void)
+{
+ struct ipipe_domain *this_domain, *next_domain;
+ struct list_head *ln;
+ unsigned long flags;
+ ipipe_declare_cpuid;
+
+ ipipe_lock_cpu(flags);
+
+ this_domain = next_domain = per_cpu(ipipe_percpu_domain, cpuid);
+
+ __clear_bit(IPIPE_STALL_FLAG, &this_domain->cpudata[cpuid].status);
+
+ if (this_domain->cpudata[cpuid].irq_pending_hi != 0)
+ goto sync_stage;
+
+ for (;;) {
+ ln = next_domain->p_link.next;
+
+ if (ln == &__ipipe_pipeline)
+ break;
+
+ next_domain = list_entry(ln, struct ipipe_domain, p_link);
+
+ if (test_bit(IPIPE_STALL_FLAG,
+ &next_domain->cpudata[cpuid].status))
+ break;
+
+ if (next_domain->cpudata[cpuid].irq_pending_hi == 0)
+ continue;
+
+ per_cpu(ipipe_percpu_domain, cpuid) = next_domain;
+
+sync_stage:
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+
+ ipipe_load_cpuid(); /* Processor might have changed. */
+
+ if (per_cpu(ipipe_percpu_domain, cpuid) != next_domain)
+ /*
+ * Something has changed the current domain under our
+ * feet, recycling the register set; take note.
+ */
+ this_domain = per_cpu(ipipe_percpu_domain, cpuid);
+ }
+
+ per_cpu(ipipe_percpu_domain, cpuid) = this_domain;
+
+ ipipe_unlock_cpu(flags);
+}
+
+/* ipipe_alloc_virq() -- Allocate a pipelined virtual/soft interrupt.
+ * Virtual interrupts are handled in exactly the same way than their
+ * hw-generated counterparts wrt pipelining.
+ */
+unsigned ipipe_alloc_virq(void)
+{
+ unsigned long flags, irq = 0;
+ int ipos;
+
+ spin_lock_irqsave_hw(&__ipipe_pipelock, flags);
+
+ if (__ipipe_virtual_irq_map != ~0) {
+ ipos = ffz(__ipipe_virtual_irq_map);
+ set_bit(ipos, &__ipipe_virtual_irq_map);
+ irq = ipos + IPIPE_VIRQ_BASE;
+ }
+
+ spin_unlock_irqrestore_hw(&__ipipe_pipelock, flags);
+
+ return irq;
+}
+
+/* ipipe_virtualize_irq() -- Attach a handler (and optionally a hw
+ acknowledge routine) to an interrupt for a given domain. */
+
+int ipipe_virtualize_irq(struct ipipe_domain *ipd,
+ unsigned irq,
+ ipipe_irq_handler_t handler,
+ void *cookie,
+ ipipe_irq_ackfn_t acknowledge,
+ unsigned modemask)
+{
+ unsigned long flags;
+ int err;
+
+ if (irq >= IPIPE_NR_IRQS)
+ return -EINVAL;
+
+ if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK)
+ return -EPERM;
+
+ if (!test_bit(IPIPE_AHEAD_FLAG, &ipd->flags))
+ /* Silently unwire interrupts for non-heading domains. */
+ modemask &= ~IPIPE_WIRED_MASK;
+
+ spin_lock_irqsave_hw(&__ipipe_pipelock, flags);
+
+ if (handler != NULL) {
+
+ if (handler == IPIPE_SAME_HANDLER) {
+ handler = ipd->irqs[irq].handler;
+ cookie = ipd->irqs[irq].cookie;
+
+ if (handler == NULL) {
+ err = -EINVAL;
+ goto unlock_and_exit;
+ }
+ } else if ((modemask & IPIPE_EXCLUSIVE_MASK) != 0 &&
+ ipd->irqs[irq].handler != NULL) {
+ err = -EBUSY;
+ goto unlock_and_exit;
+ }
+
+ if ((modemask & (IPIPE_SHARED_MASK | IPIPE_PASS_MASK)) ==
+ IPIPE_SHARED_MASK) {
+ err = -EINVAL;
+ goto unlock_and_exit;
+ }
+
+ /* Wired interrupts can only be delivered to domains
+ * always heading the pipeline. */
+
+ if ((modemask & IPIPE_WIRED_MASK) != 0) {
+ if ((modemask & (IPIPE_SHARED_MASK | IPIPE_PASS_MASK | IPIPE_STICKY_MASK)) != 0) {
+ err = -EINVAL;
+ goto unlock_and_exit;
+ }
+ modemask |= (IPIPE_HANDLE_MASK);
+ }
+
+ if ((modemask & IPIPE_STICKY_MASK) != 0)
+ modemask |= IPIPE_HANDLE_MASK;
+ } else
+ modemask &=
+ ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK |
+ IPIPE_SHARED_MASK | IPIPE_EXCLUSIVE_MASK | IPIPE_WIRED_MASK);
+
+ if (acknowledge == NULL) {
+ if ((modemask & IPIPE_SHARED_MASK) == 0) {
+ if (!ipipe_virtual_irq_p(irq)) {
+ /* Acknowledge handler unspecified for a hw
+ interrupt -- this is ok in non-shared
+ management mode, but we will force the use
+ of the Linux-defined handler instead. */
+ acknowledge = ipipe_root_domain->irqs[irq].acknowledge;
+ }
+ }
+ else {
+ /* A valid acknowledge handler to be called in shared mode
+ is required when declaring a shared IRQ. */
+ err = -EINVAL;
+ goto unlock_and_exit;
+ }
+ }
+
+ ipd->irqs[irq].handler = handler;
+ ipd->irqs[irq].cookie = cookie;
+ ipd->irqs[irq].acknowledge = acknowledge;
+ ipd->irqs[irq].control = modemask;
+
+ if (irq < NR_IRQS && handler != NULL && !ipipe_virtual_irq_p(irq)) {
+ __ipipe_enable_irqdesc(irq);
+
+ if ((modemask & IPIPE_ENABLE_MASK) != 0) {
+ if (ipd != ipipe_current_domain) {
+ /* IRQ enable/disable state is domain-sensitive, so we may
+ not change it for another domain. What is allowed
+ however is forcing some domain to handle an interrupt
+ source, by passing the proper 'ipd' descriptor which
+ thus may be different from ipipe_current_domain. */
+ err = -EPERM;
+ goto unlock_and_exit;
+ }
+
+ __ipipe_enable_irq(irq);
+ }
+ }
+
+ err = 0;
+
+ unlock_and_exit:
+
+ spin_unlock_irqrestore_hw(&__ipipe_pipelock, flags);
+
+ return err;
+}
+
+/* ipipe_control_irq() -- Change modes of a pipelined interrupt for
+ * the current domain. */
+
+int ipipe_control_irq(unsigned irq, unsigned clrmask, unsigned setmask)
+{
+ struct ipipe_domain *ipd;
+ unsigned long flags;
+
+ if (irq >= IPIPE_NR_IRQS)
+ return -EINVAL;
+
+ ipd = ipipe_current_domain;
+
+ if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK)
+ return -EPERM;
+
+ if (((setmask | clrmask) & IPIPE_SHARED_MASK) != 0)
+ return -EINVAL;
+
+ if (ipd->irqs[irq].handler == NULL)
+ setmask &= ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK);
+
+ if ((setmask & IPIPE_STICKY_MASK) != 0)
+ setmask |= IPIPE_HANDLE_MASK;
+
+ if ((clrmask & (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK)) != 0) /* If one goes, both go. */
+ clrmask |= (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK);
+
+ spin_lock_irqsave_hw(&__ipipe_pipelock, flags);
+
+ ipd->irqs[irq].control &= ~clrmask;
+ ipd->irqs[irq].control |= setmask;
+
+ if ((setmask & IPIPE_ENABLE_MASK) != 0)
+ __ipipe_enable_irq(irq);
+ else if ((clrmask & IPIPE_ENABLE_MASK) != 0)
+ __ipipe_disable_irq(irq);
+
+ spin_unlock_irqrestore_hw(&__ipipe_pipelock, flags);
+
+ return 0;
+}
+
+/* __ipipe_dispatch_event() -- Low-level event dispatcher. */
+
+int fastcall __ipipe_dispatch_event (unsigned event, void *data)
+{
+ struct ipipe_domain *start_domain, *this_domain, *next_domain;
+ ipipe_event_handler_t evhand;
+ struct list_head *pos, *npos;
+ unsigned long flags;
+ ipipe_declare_cpuid;
+ int propagate = 1;
+
+ ipipe_lock_cpu(flags);
+
+ start_domain = this_domain = per_cpu(ipipe_percpu_domain, cpuid);
+
+ list_for_each_safe(pos,npos,&__ipipe_pipeline) {
+
+ /*
+ * Note: Domain migration may occur while running
+ * event or interrupt handlers, in which case the
+ * current register set is going to be recycled for a
+ * different domain than the initiating one. We do
+ * care for that, always tracking the current domain
+ * descriptor upon return from those handlers.
+ */
+ next_domain = list_entry(pos,struct ipipe_domain,p_link);
+
+ /*
+ * Keep a cached copy of the handler's address since
+ * ipipe_catch_event() may clear it under our feet.
+ */
+
+ evhand = next_domain->evhand[event];
+
+ if (evhand != NULL) {
+ per_cpu(ipipe_percpu_domain, cpuid) = next_domain;
+ next_domain->cpudata[cpuid].evsync |= (1LL << event);
+ ipipe_unlock_cpu(flags);
+ propagate = !evhand(event,start_domain,data);
+ ipipe_lock_cpu(flags);
+ next_domain->cpudata[cpuid].evsync &= ~(1LL << event);
+ if (per_cpu(ipipe_percpu_domain, cpuid) != next_domain)
+ this_domain = per_cpu(ipipe_percpu_domain, cpuid);
+ }
+
+ if (next_domain != ipipe_root_domain && /* NEVER sync the root stage here. */
+ next_domain->cpudata[cpuid].irq_pending_hi != 0 &&
+ !test_bit(IPIPE_STALL_FLAG,&next_domain->cpudata[cpuid].status)) {
+ per_cpu(ipipe_percpu_domain, cpuid) = next_domain;
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+ ipipe_load_cpuid();
+ if (per_cpu(ipipe_percpu_domain, cpuid) != next_domain)
+ this_domain = per_cpu(ipipe_percpu_domain, cpuid);
+ }
+
+ per_cpu(ipipe_percpu_domain, cpuid) = this_domain;
+
+ if (next_domain == this_domain || !propagate)
+ break;
+ }
+
+ ipipe_unlock_cpu(flags);
+
+ return !propagate;
+}
+
+/*
+ * __ipipe_dispatch_wired -- Wired interrupt dispatcher. Wired
+ * interrupts are immediately and unconditionally delivered to the
+ * domain heading the pipeline upon receipt, and such domain must have
+ * been registered as an invariant head for the system (priority ==
+ * IPIPE_HEAD_PRIORITY). The motivation for using wired interrupts is
+ * to get an extra-fast dispatching path for those IRQs, by relying on
+ * a straightforward logic based on assumptions that must always be
+ * true for invariant head domains. The following assumptions are
+ * made when dealing with such interrupts:
+ *
+ * 1- Wired interrupts are purely dynamic, i.e. the decision to
+ * propagate them down the pipeline must be done from the head domain
+ * ISR.
+ * 2- Wired interrupts cannot be shared or sticky.
+ * 3- The root domain cannot be an invariant pipeline head, in
+ * consequence of what the root domain cannot handle wired
+ * interrupts.
+ * 4- Wired interrupts must have a valid acknowledge handler for the
+ * head domain (if needed), and in any case, must not rely on handlers
+ * provided by lower priority domains during the acknowledge cycle
+ * (see __ipipe_handle_irq).
+ *
+ * Called with hw interrupts off.
+ */
+
+int fastcall __ipipe_dispatch_wired(struct ipipe_domain *head, unsigned irq)
+{
+ struct ipcpudata *cpudata;
+ struct ipipe_domain *old;
+ ipipe_declare_cpuid;
+
+ ipipe_load_cpuid();
+ cpudata = &head->cpudata[cpuid];
+ cpudata->irq_counters[irq].total_hits++;
+
+ if (test_bit(IPIPE_LOCK_FLAG, &head->irqs[irq].control)) {
+ /* If we can't process this IRQ right now, we must
+ * mark it as pending, so that it will get played
+ * during normal log sync when the corresponding
+ * interrupt source is eventually unlocked. */
+ cpudata->irq_counters[irq].pending_hits++;
+ return 0;
+ }
+
+ if (test_bit(IPIPE_STALL_FLAG, &cpudata->status)) {
+ cpudata->irq_counters[irq].pending_hits++;
+ __ipipe_set_irq_bit(head, cpuid, irq);
+ return 0;
+ }
+
+ old = per_cpu(ipipe_percpu_domain, cpuid);
+ per_cpu(ipipe_percpu_domain, cpuid) = head; /* Switch to the head domain. */
+
+ __set_bit(IPIPE_STALL_FLAG, &cpudata->status);
+ head->irqs[irq].handler(irq,head->irqs[irq].cookie); /* Call the ISR. */
+ __ipipe_run_irqtail();
+ __clear_bit(IPIPE_STALL_FLAG, &cpudata->status);
+
+ /* We expect the caller to start a complete pipeline walk upon
+ * return, so that propagated interrupts will get played. */
+
+ if (per_cpu(ipipe_percpu_domain, cpuid) == head)
+ per_cpu(ipipe_percpu_domain, cpuid) = old; /* Back to the preempted domain. */
+
+ return 1;
+}
+
+/*
+ * __ipipe_sync_stage() -- Flush the pending IRQs for the current
+ * domain (and processor). This routine flushes the interrupt log
+ * (see "Optimistic interrupt protection" from D. Stodolsky et al. for
+ * more on the deferred interrupt scheme). Every interrupt that
+ * occurred while the pipeline was stalled gets played. WARNING:
+ * callers on SMP boxen should always check for CPU migration on
+ * return of this routine. One can control the kind of interrupts
+ * which are going to be sync'ed using the syncmask
+ * parameter. IPIPE_IRQMASK_ANY plays them all, IPIPE_IRQMASK_VIRT
+ * plays virtual interrupts only.
+ *
+ * This routine must be called with hw interrupts off.
+ */
+void fastcall __ipipe_sync_stage(unsigned long syncmask)
+{
+ unsigned long mask, submask;
+ struct ipcpudata *cpudata;
+ struct ipipe_domain *ipd;
+ ipipe_declare_cpuid;
+ int level, rank;
+ unsigned irq;
+
+ ipipe_load_cpuid();
+ ipd = per_cpu(ipipe_percpu_domain, cpuid);
+ cpudata = &ipd->cpudata[cpuid];
+
+ if (__test_and_set_bit(IPIPE_SYNC_FLAG, &cpudata->status))
+ return;
+
+ /*
+ * The policy here is to keep the dispatching code interrupt-free
+ * by stalling the current stage. If the upper domain handler
+ * (which we call) wants to re-enable interrupts while in a safe
+ * portion of the code (e.g. SA_INTERRUPT flag unset for Linux's
+ * sigaction()), it will have to unstall (then stall again before
+ * returning to us!) the stage when it sees fit.
+ */
+ while ((mask = (cpudata->irq_pending_hi & syncmask)) != 0) {
+ level = __ipipe_ffnz(mask);
+ __clear_bit(level, &cpudata->irq_pending_hi);
+
+ while ((submask = cpudata->irq_pending_lo[level]) != 0) {
+ rank = __ipipe_ffnz(submask);
+ irq = (level << IPIPE_IRQ_ISHIFT) + rank;
+
+ if (test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control)) {
+ __clear_bit(rank, &cpudata->irq_pending_lo[level]);
+ continue;
+ }
+
+ if (--cpudata->irq_counters[irq].pending_hits == 0)
+ __clear_bit(rank, &cpudata->irq_pending_lo[level]);
+
+ __set_bit(IPIPE_STALL_FLAG, &cpudata->status);
+ __ipipe_run_isr(ipd, irq, cpuid);
+#ifdef CONFIG_SMP
+ {
+ int _cpuid = ipipe_processor_id();
+
+ if (_cpuid != cpuid) { /* Handle CPU migration. */
+ /*
+ * We expect any domain to clear the SYNC bit each
+ * time it switches in a new task, so that preemptions
+ * and/or CPU migrations (in the SMP case) over the
+ * ISR do not lock out the log syncer for some
+ * indefinite amount of time. In the Linux case,
+ * schedule() handles this (see kernel/sched.c). For
+ * this reason, we don't bother clearing it here for
+ * the source CPU in the migration handling case,
+ * since it must have scheduled another task in by
+ * now.
+ */
+ cpuid = _cpuid;
+ cpudata = &ipd->cpudata[cpuid];
+ __set_bit(IPIPE_SYNC_FLAG, &cpudata->status);
+ }
+ }
+#endif /* CONFIG_SMP */
+
+ __clear_bit(IPIPE_STALL_FLAG, &cpudata->status);
+ }
+ }
+
+ __clear_bit(IPIPE_SYNC_FLAG, &cpudata->status);
+}
+
+#ifdef CONFIG_PROC_FS
+
+#include <linux/proc_fs.h>
+
+struct proc_dir_entry *ipipe_proc_root;
+
+static int __ipipe_version_info_proc(char *page,
+ char **start,
+ off_t off, int count, int *eof, void *data)
+{
+ int len = sprintf(page, "%s\n", IPIPE_VERSION_STRING);
+
+ len -= off;
+
+ if (len <= off + count)
+ *eof = 1;
+
+ *start = page + off;
+
+ if(len > count)
+ len = count;
+
+ if(len < 0)
+ len = 0;
+
+ return len;
+}
+
+static int __ipipe_common_info_proc(char *page,
+ char **start,
+ off_t off, int count, int *eof, void *data)
+{
+ struct ipipe_domain *ipd = (struct ipipe_domain *)data;
+ unsigned long ctlbits;
+ unsigned irq, _irq;
+ char *p = page;
+ int len;
+
+ spin_lock(&__ipipe_pipelock);
+
+ if (test_bit(IPIPE_AHEAD_FLAG,&ipd->flags))
+ p += sprintf(p, "Invariant head");
+ else
+ p += sprintf(p, "Priority=%d", ipd->priority);
+
+ p += sprintf(p, ", Id=0x%.8x\n", ipd->domid);
+
+ irq = 0;
+
+ while (irq < IPIPE_NR_IRQS) {
+ ctlbits =
+ (ipd->irqs[irq].
+ control & (IPIPE_HANDLE_MASK | IPIPE_PASS_MASK |
+ IPIPE_STICKY_MASK | IPIPE_WIRED_MASK));
+ if (irq >= IPIPE_NR_XIRQS && !ipipe_virtual_irq_p(irq)) {
+ /*
+ * There might be a hole between the last external
+ * IRQ and the first virtual one; skip it.
+ */
+ irq++;
+ continue;
+ }
+
+ if (ipipe_virtual_irq_p(irq)
+ && !test_bit(irq - IPIPE_VIRQ_BASE,
+ &__ipipe_virtual_irq_map)) {
+ /* Non-allocated virtual IRQ; skip it. */
+ irq++;
+ continue;
+ }
+
+ /*
+ * Attempt to group consecutive IRQ numbers having the
+ * same virtualization settings in a single line.
+ */
+
+ _irq = irq;
+
+ while (++_irq < IPIPE_NR_IRQS) {
+ if (ipipe_virtual_irq_p(_irq) !=
+ ipipe_virtual_irq_p(irq)
+ || (ipipe_virtual_irq_p(_irq)
+ && !test_bit(_irq - IPIPE_VIRQ_BASE,
+ &__ipipe_virtual_irq_map))
+ || ctlbits != (ipd->irqs[_irq].
+ control & (IPIPE_HANDLE_MASK |
+ IPIPE_PASS_MASK |
+ IPIPE_STICKY_MASK)))
+ break;
+ }
+
+ if (_irq == irq + 1)
+ p += sprintf(p, "irq%u: ", irq);
+ else
+ p += sprintf(p, "irq%u-%u: ", irq, _irq - 1);
+
+ /*
+ * Statuses are as follows:
+ * o "accepted" means handled _and_ passed down the pipeline.
+ * o "grabbed" means handled, but the interrupt might be
+ * terminated _or_ passed down the pipeline depending on
+ * what the domain handler asks for to the I-pipe.
+ * o "wired" is basically the same as "grabbed", except that
+ * the interrupt is unconditionally delivered to an invariant
+ * pipeline head domain.
+ * o "passed" means unhandled by the domain but passed
+ * down the pipeline.
+ * o "discarded" means unhandled and _not_ passed down the
+ * pipeline. The interrupt merely disappears from the
+ * current domain down to the end of the pipeline.
+ */
+ if (ctlbits & IPIPE_HANDLE_MASK) {
+ if (ctlbits & IPIPE_PASS_MASK)
+ p += sprintf(p, "accepted");
+ else if (ctlbits & IPIPE_WIRED_MASK)
+ p += sprintf(p, "wired");
+ else
+ p += sprintf(p, "grabbed");
+ } else if (ctlbits & IPIPE_PASS_MASK)
+ p += sprintf(p, "passed");
+ else
+ p += sprintf(p, "discarded");
+
+ if (ctlbits & IPIPE_STICKY_MASK)
+ p += sprintf(p, ", sticky");
+
+ if (ipipe_virtual_irq_p(irq))
+ p += sprintf(p, ", virtual");
+
+ p += sprintf(p, "\n");
+
+ irq = _irq;
+ }
+
+ spin_unlock(&__ipipe_pipelock);
+
+ len = p - page;
+
+ if (len <= off + count)
+ *eof = 1;
+
+ *start = page + off;
+
+ len -= off;
+
+ if (len > count)
+ len = count;
+
+ if (len < 0)
+ len = 0;
+
+ return len;
+}
+
+void __ipipe_add_domain_proc(struct ipipe_domain *ipd)
+{
+ create_proc_read_entry(ipd->name,0444,ipipe_proc_root,&__ipipe_common_info_proc,ipd);
+}
+
+void __ipipe_remove_domain_proc(struct ipipe_domain *ipd)
+{
+ remove_proc_entry(ipd->name,ipipe_proc_root);
+}
+
+void ipipe_init_proc(void)
+{
+ ipipe_proc_root = create_proc_entry("ipipe",S_IFDIR, 0);
+ create_proc_read_entry("version",0444,ipipe_proc_root,&__ipipe_version_info_proc,NULL);
+ __ipipe_add_domain_proc(ipipe_root_domain);
+
+ __ipipe_init_tracer();
+}
+
+#endif /* CONFIG_PROC_FS */
+
+EXPORT_SYMBOL(ipipe_virtualize_irq);
+EXPORT_SYMBOL(ipipe_control_irq);
+EXPORT_SYMBOL(ipipe_suspend_domain);
+EXPORT_SYMBOL(ipipe_alloc_virq);
+EXPORT_PER_CPU_SYMBOL(ipipe_percpu_domain);
+EXPORT_SYMBOL(ipipe_root);
+EXPORT_SYMBOL(ipipe_stall_pipeline_from);
+EXPORT_SYMBOL(ipipe_test_and_stall_pipeline_from);
+EXPORT_SYMBOL(ipipe_unstall_pipeline_from);
+EXPORT_SYMBOL(ipipe_restore_pipeline_from);
+EXPORT_SYMBOL(ipipe_test_and_unstall_pipeline_from);
+EXPORT_SYMBOL(ipipe_unstall_pipeline_head);
+EXPORT_SYMBOL(__ipipe_restore_pipeline_head);
+EXPORT_SYMBOL(__ipipe_unstall_root);
+EXPORT_SYMBOL(__ipipe_stall_root);
+EXPORT_SYMBOL(__ipipe_restore_root);
+EXPORT_SYMBOL(__ipipe_test_and_stall_root);
+EXPORT_SYMBOL(__ipipe_test_root);
+EXPORT_SYMBOL(__ipipe_dispatch_event);
+EXPORT_SYMBOL(__ipipe_dispatch_wired);
+EXPORT_SYMBOL(__ipipe_sync_stage);
+EXPORT_SYMBOL(__ipipe_pipeline);
+EXPORT_SYMBOL(__ipipe_pipelock);
+EXPORT_SYMBOL(__ipipe_virtual_irq_map);
diff -Naur linux-2.6.19.orig/kernel/ipipe/generic.c linux-2.6.19.ipipe/kernel/ipipe/generic.c
--- linux-2.6.19.orig/kernel/ipipe/generic.c 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.19.ipipe/kernel/ipipe/generic.c 2006-12-19 13:02:36.000000000 +0100
@@ -0,0 +1,424 @@
+/* -*- linux-c -*-
+ * linux/kernel/ipipe/generic.c
+ *
+ * Copyright (C) 2002-2005 Philippe Gerum.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Architecture-independent I-PIPE services.
+ */
+
+#include <linux/version.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#ifdef CONFIG_PROC_FS
+#include <linux/proc_fs.h>
+#endif /* CONFIG_PROC_FS */
+
+MODULE_DESCRIPTION("I-pipe");
+MODULE_LICENSE("GPL");
+
+static int __ipipe_ptd_key_count;
+
+static unsigned long __ipipe_ptd_key_map;
+
+/* ipipe_register_domain() -- Link a new domain to the pipeline. */
+
+int ipipe_register_domain(struct ipipe_domain *ipd,
+ struct ipipe_domain_attr *attr)
+{
+ struct list_head *pos;
+ unsigned long flags;
+
+ if (ipipe_current_domain != ipipe_root_domain) {
+ printk(KERN_WARNING
+ "I-pipe: Only the root domain may register a new domain.\n");
+ return -EPERM;
+ }
+
+ if (attr->priority == IPIPE_HEAD_PRIORITY &&
+ test_bit(IPIPE_AHEAD_FLAG,&__ipipe_pipeline_head()->flags))
+ return -EAGAIN; /* Cannot override current head. */
+
+ flags = ipipe_critical_enter(NULL);
+
+ list_for_each(pos, &__ipipe_pipeline) {
+ struct ipipe_domain *_ipd =
+ list_entry(pos, struct ipipe_domain, p_link);
+ if (_ipd->domid == attr->domid)
+ break;
+ }
+
+ ipipe_critical_exit(flags);
+
+ if (pos != &__ipipe_pipeline)
+ /* A domain with the given id already exists -- fail. */
+ return -EBUSY;
+
+ ipd->name = attr->name;
+ ipd->domid = attr->domid;
+ ipd->pdd = attr->pdd;
+ ipd->flags = 0;
+
+ if (attr->priority == IPIPE_HEAD_PRIORITY) {
+ ipd->priority = INT_MAX;
+ __set_bit(IPIPE_AHEAD_FLAG,&ipd->flags);
+ }
+ else
+ ipd->priority = attr->priority;
+
+ __ipipe_init_stage(ipd);
+
+ INIT_LIST_HEAD(&ipd->p_link);
+
+#ifdef CONFIG_PROC_FS
+ __ipipe_add_domain_proc(ipd);
+#endif /* CONFIG_PROC_FS */
+
+ flags = ipipe_critical_enter(NULL);
+
+ list_for_each(pos, &__ipipe_pipeline) {
+ struct ipipe_domain *_ipd =
+ list_entry(pos, struct ipipe_domain, p_link);
+ if (ipd->priority > _ipd->priority)
+ break;
+ }
+
+ list_add_tail(&ipd->p_link, pos);
+
+ ipipe_critical_exit(flags);
+
+ printk(KERN_WARNING "I-pipe: Domain %s registered.\n", ipd->name);
+
+ /*
+ * Finally, allow the new domain to perform its initialization
+ * chores.
+ */
+
+ if (attr->entry != NULL) {
+ ipipe_declare_cpuid;
+
+ ipipe_lock_cpu(flags);
+
+ per_cpu(ipipe_percpu_domain, cpuid) = ipd;
+ attr->entry();
+ per_cpu(ipipe_percpu_domain, cpuid) = ipipe_root_domain;
+
+ ipipe_load_cpuid(); /* Processor might have changed. */
+
+ if (ipipe_root_domain->cpudata[cpuid].irq_pending_hi != 0 &&
+ !test_bit(IPIPE_STALL_FLAG,
+ &ipipe_root_domain->cpudata[cpuid].status))
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
+
+ ipipe_unlock_cpu(flags);
+ }
+
+ return 0;
+}
+
+/* ipipe_unregister_domain() -- Remove a domain from the pipeline. */
+
+int ipipe_unregister_domain(struct ipipe_domain *ipd)
+{
+ unsigned long flags;
+
+ if (ipipe_current_domain != ipipe_root_domain) {
+ printk(KERN_WARNING
+ "I-pipe: Only the root domain may unregister a domain.\n");
+ return -EPERM;
+ }
+
+ if (ipd == ipipe_root_domain) {
+ printk(KERN_WARNING
+ "I-pipe: Cannot unregister the root domain.\n");
+ return -EPERM;
+ }
+#ifdef CONFIG_SMP
+ {
+ int nr_cpus = num_online_cpus(), _cpuid;
+ unsigned irq;
+
+ /*
+ * In the SMP case, wait for the logged events to drain on
+ * other processors before eventually removing the domain
+ * from the pipeline.
+ */
+
+ ipipe_unstall_pipeline_from(ipd);
+
+ flags = ipipe_critical_enter(NULL);
+
+ for (irq = 0; irq < IPIPE_NR_IRQS; irq++) {
+ clear_bit(IPIPE_HANDLE_FLAG, &ipd->irqs[irq].control);
+ clear_bit(IPIPE_STICKY_FLAG, &ipd->irqs[irq].control);
+ set_bit(IPIPE_PASS_FLAG, &ipd->irqs[irq].control);
+ }
+
+ ipipe_critical_exit(flags);
+
+ for (_cpuid = 0; _cpuid < nr_cpus; _cpuid++)
+ for (irq = 0; irq < IPIPE_NR_IRQS; irq++)
+ while (ipd->cpudata[_cpuid].irq_counters[irq].pending_hits > 0)
+ cpu_relax();
+ }
+#endif /* CONFIG_SMP */
+
+#ifdef CONFIG_PROC_FS
+ __ipipe_remove_domain_proc(ipd);
+#endif /* CONFIG_PROC_FS */
+
+ /*
+ * Simply remove the domain from the pipeline and we are almost done.
+ */
+
+ flags = ipipe_critical_enter(NULL);
+ list_del_init(&ipd->p_link);
+ ipipe_critical_exit(flags);
+
+ __ipipe_cleanup_domain(ipd);
+
+ printk(KERN_WARNING "I-pipe: Domain %s unregistered.\n", ipd->name);
+
+ return 0;
+}
+
+/*
+ * ipipe_propagate_irq() -- Force a given IRQ propagation on behalf of
+ * a running interrupt handler to the next domain down the pipeline.
+ * ipipe_schedule_irq() -- Does almost the same as above, but attempts
+ * to pend the interrupt for the current domain first.
+ */
+int fastcall __ipipe_schedule_irq(unsigned irq, struct list_head *head)
+{
+ struct list_head *ln;
+ unsigned long flags;
+ ipipe_declare_cpuid;
+
+ if (irq >= IPIPE_NR_IRQS ||
+ (ipipe_virtual_irq_p(irq)
+ && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)))
+ return -EINVAL;
+
+ ipipe_lock_cpu(flags);
+
+ ln = head;
+
+ while (ln != &__ipipe_pipeline) {
+ struct ipipe_domain *ipd =
+ list_entry(ln, struct ipipe_domain, p_link);
+
+ if (test_bit(IPIPE_HANDLE_FLAG, &ipd->irqs[irq].control)) {
+ ipd->cpudata[cpuid].irq_counters[irq].total_hits++;
+ ipd->cpudata[cpuid].irq_counters[irq].pending_hits++;
+ __ipipe_set_irq_bit(ipd, cpuid, irq);
+ ipipe_unlock_cpu(flags);
+ return 1;
+ }
+
+ ln = ipd->p_link.next;
+ }
+
+ ipipe_unlock_cpu(flags);
+
+ return 0;
+}
+
+/* ipipe_free_virq() -- Release a virtual/soft interrupt. */
+
+int ipipe_free_virq(unsigned virq)
+{
+ if (!ipipe_virtual_irq_p(virq))
+ return -EINVAL;
+
+ clear_bit(virq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map);
+
+ return 0;
+}
+
+void ipipe_init_attr(struct ipipe_domain_attr *attr)
+{
+ attr->name = "anon";
+ attr->domid = 1;
+ attr->entry = NULL;
+ attr->priority = IPIPE_ROOT_PRIO;
+ attr->pdd = NULL;
+}
+
+/*
+ * ipipe_catch_event() -- Interpose or remove an event handler for a
+ * given domain.
+ */
+ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd,
+ unsigned event,
+ ipipe_event_handler_t handler)
+{
+ ipipe_event_handler_t old_handler;
+ unsigned long flags;
+ int self = 0, cpuid;
+
+ if (event & IPIPE_EVENT_SELF) {
+ event &= ~IPIPE_EVENT_SELF;
+ self = 1;
+ }
+
+ if (event >= IPIPE_NR_EVENTS)
+ return NULL;
+
+ flags = ipipe_critical_enter(NULL);
+
+ if (!(old_handler = xchg(&ipd->evhand[event],handler))) {
+ if (handler) {
+ if (self)
+ ipd->evself |= (1LL << event);
+ else
+ __ipipe_event_monitors[event]++;
+ }
+ }
+ else if (!handler) {
+ if (ipd->evself & (1LL << event))
+ ipd->evself &= ~(1LL << event);
+ else
+ __ipipe_event_monitors[event]--;
+ } else if ((ipd->evself & (1LL << event)) && !self) {
+ __ipipe_event_monitors[event]++;
+ ipd->evself &= ~(1LL << event);
+ } else if (!(ipd->evself & (1LL << event)) && self) {
+ __ipipe_event_monitors[event]--;
+ ipd->evself |= (1LL << event);
+ }
+
+ ipipe_critical_exit(flags);
+
+ if (!handler && ipipe_root_domain_p) {
+ /*
+ * If we cleared a handler on behalf of the root
+ * domain, we have to wait for any current invocation
+ * to drain, since our caller might subsequently unmap
+ * the target domain. To this aim, this code
+ * synchronizes with __ipipe_dispatch_event(),
+ * guaranteeing that either the dispatcher sees a null
+ * handler in which case it discards the invocation
+ * (which also prevents from entering a livelock), or
+ * finds a valid handler and calls it. Symmetrically,
+ * ipipe_catch_event() ensures that the called code
+ * won't be unmapped under our feet until the event
+ * synchronization flag is cleared for the given event
+ * on all CPUs.
+ */
+
+ for_each_online_cpu(cpuid) {
+ while (ipd->cpudata[cpuid].evsync & (1LL << event))
+ schedule_timeout_interruptible(HZ / 50);
+ }
+ }
+
+ return old_handler;
+}
+
+cpumask_t ipipe_set_irq_affinity (unsigned irq, cpumask_t cpumask)
+{
+#ifdef CONFIG_SMP
+ if (irq >= IPIPE_NR_XIRQS)
+ /* Allow changing affinity of external IRQs only. */
+ return CPU_MASK_NONE;
+
+ if (num_online_cpus() > 1)
+ return __ipipe_set_irq_affinity(irq,cpumask);
+#endif /* CONFIG_SMP */
+
+ return CPU_MASK_NONE;
+}
+
+int fastcall ipipe_send_ipi (unsigned ipi, cpumask_t cpumask)
+
+{
+#ifdef CONFIG_SMP
+ return __ipipe_send_ipi(ipi,cpumask);
+#else /* !CONFIG_SMP */
+ return -EINVAL;
+#endif /* CONFIG_SMP */
+}
+
+int ipipe_alloc_ptdkey (void)
+{
+ unsigned long flags;
+ int key = -1;
+
+ spin_lock_irqsave_hw(&__ipipe_pipelock,flags);
+
+ if (__ipipe_ptd_key_count < IPIPE_ROOT_NPTDKEYS) {
+ key = ffz(__ipipe_ptd_key_map);
+ set_bit(key,&__ipipe_ptd_key_map);
+ __ipipe_ptd_key_count++;
+ }
+
+ spin_unlock_irqrestore_hw(&__ipipe_pipelock,flags);
+
+ return key;
+}
+
+int ipipe_free_ptdkey (int key)
+{
+ unsigned long flags;
+
+ if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+ return -EINVAL;
+
+ spin_lock_irqsave_hw(&__ipipe_pipelock,flags);
+
+ if (test_and_clear_bit(key,&__ipipe_ptd_key_map))
+ __ipipe_ptd_key_count--;
+
+ spin_unlock_irqrestore_hw(&__ipipe_pipelock,flags);
+
+ return 0;
+}
+
+int fastcall ipipe_set_ptd (int key, void *value)
+
+{
+ if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+ return -EINVAL;
+
+ current->ptd[key] = value;
+
+ return 0;
+}
+
+void fastcall *ipipe_get_ptd (int key)
+
+{
+ if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
+ return NULL;
+
+ return current->ptd[key];
+}
+
+EXPORT_SYMBOL(ipipe_register_domain);
+EXPORT_SYMBOL(ipipe_unregister_domain);
+EXPORT_SYMBOL(ipipe_free_virq);
+EXPORT_SYMBOL(ipipe_init_attr);
+EXPORT_SYMBOL(ipipe_catch_event);
+EXPORT_SYMBOL(ipipe_alloc_ptdkey);
+EXPORT_SYMBOL(ipipe_free_ptdkey);
+EXPORT_SYMBOL(ipipe_set_ptd);
+EXPORT_SYMBOL(ipipe_get_ptd);
+EXPORT_SYMBOL(ipipe_set_irq_affinity);
+EXPORT_SYMBOL(ipipe_send_ipi);
+EXPORT_SYMBOL(__ipipe_schedule_irq);
diff -Naur linux-2.6.19.orig/kernel/ipipe/Kconfig linux-2.6.19.ipipe/kernel/ipipe/Kconfig
--- linux-2.6.19.orig/kernel/ipipe/Kconfig 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.19.ipipe/kernel/ipipe/Kconfig 2006-12-19 13:02:36.000000000 +0100
@@ -0,0 +1,6 @@
+config IPIPE
+ bool "Interrupt pipeline"
+ default y
+ ---help---
+ Activate this option if you want the interrupt pipeline to be
+ compiled in.
diff -Naur linux-2.6.19.orig/kernel/ipipe/Kconfig.debug linux-2.6.19.ipipe/kernel/ipipe/Kconfig.debug
--- linux-2.6.19.orig/kernel/ipipe/Kconfig.debug 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.19.ipipe/kernel/ipipe/Kconfig.debug 2006-12-19 13:02:36.000000000 +0100
@@ -0,0 +1,69 @@
+config IPIPE_DEBUG
+ bool "I-pipe debugging"
+ depends on IPIPE
+
+config IPIPE_TRACE
+ bool "Latency tracing"
+ depends on IPIPE_DEBUG
+ select FRAME_POINTER
+ select KALLSYMS
+ select PROC_FS
+ ---help---
+ Activate this option if you want to use per-function tracing of
+ the kernel. The tracer will collect data via instrumentation
+ features like the one below or with the help of explicite calls
+ of ipipe_trace_xxx(). See include/linux/ipipe_trace.h for the
+ in-kernel tracing API. The collected data and runtime control
+ is available via /proc/ipipe/trace/*.
+
+config IPIPE_TRACE_ENABLE
+ bool "Enable tracing on boot"
+ depends on IPIPE_TRACE
+ default y
+ ---help---
+ Disable this option if you want to arm the tracer after booting
+ manually ("echo 1 > /proc/ipipe/tracer/enable"). This can reduce
+ boot time on slow embedded devices due to the tracer overhead.
+
+config IPIPE_TRACE_MCOUNT
+ bool "Instrument function entries"
+ depends on IPIPE_TRACE
+ default y
+ ---help---
+ When enabled, records every kernel function entry in the tracer
+ log. While this slows down the system noticeably, it provides
+ the highest level of information about the flow of events.
+ However, it can be switch off in order to record only explicit
+ I-pipe trace points.
+
+config IPIPE_TRACE_IRQSOFF
+ bool "Trace IRQs-off times"
+ depends on IPIPE_TRACE
+ default y
+ ---help---
+ Activate this option if I-pipe shall trace the longest path
+ with hard-IRQs switched off.
+
+config IPIPE_TRACE_SHIFT
+ int "Depth of trace log (14 => 16Kpoints, 15 => 32Kpoints)"
+ range 10 18
+ default 14
+ depends on IPIPE_TRACE
+ ---help---
+ The number of trace points to hold tracing data for each
+ trace path, as a power of 2.
+
+config IPIPE_TRACE_VMALLOC
+ bool "Use vmalloc'ed trace buffer"
+ depends on IPIPE_TRACE
+ ---help---
+ Instead of reserving static kernel data, the required buffer
+ is allocated via vmalloc during boot-up when this option is
+ enabled. This can help to start systems that are low on memory,
+ but it slightly degrades overall performance. Try this option
+ when a traced kernel hangs unexpectedly at boot time.
+
+config IPIPE_TRACE_ENABLE_VALUE
+ int
+ default 0 if !IPIPE_TRACE_ENABLE
+ default 1 if IPIPE_TRACE_ENABLE
diff -Naur linux-2.6.19.orig/kernel/ipipe/Makefile linux-2.6.19.ipipe/kernel/ipipe/Makefile
--- linux-2.6.19.orig/kernel/ipipe/Makefile 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.19.ipipe/kernel/ipipe/Makefile 2006-12-19 13:02:36.000000000 +0100
@@ -0,0 +1,3 @@
+
+obj-$(CONFIG_IPIPE) += core.o generic.o
+obj-$(CONFIG_IPIPE_TRACE) += tracer.o
diff -Naur linux-2.6.19.orig/kernel/ipipe/tracer.c linux-2.6.19.ipipe/kernel/ipipe/tracer.c
--- linux-2.6.19.orig/kernel/ipipe/tracer.c 1970-01-01 01:00:00.000000000 +0100
+++ linux-2.6.19.ipipe/kernel/ipipe/tracer.c 2006-12-19 13:02:36.000000000 +0100
@@ -0,0 +1,1245 @@
+/* -*- linux-c -*-
+ * kernel/ipipe/tracer.c
+ *
+ * Copyright (C) 2005 Luotao Fu.
+ * 2005, 2006 Jan Kiszka.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
+ * USA; either version 2 of the License, or (at your option) any later
+ * version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/version.h>
+#include <linux/kallsyms.h>
+#include <linux/seq_file.h>
+#include <linux/proc_fs.h>
+#include <linux/ctype.h>
+#include <linux/vmalloc.h>
+#include <linux/vermagic.h>
+#include <linux/ipipe_trace.h>
+#include <asm/uaccess.h>
+#include <asm/time.h>
+
+#define IPIPE_TRACE_PATHS 4 /* <!> Do not lower below 3 */
+#define IPIPE_DEFAULT_ACTIVE 0
+#define IPIPE_DEFAULT_MAX 1
+#define IPIPE_DEFAULT_FROZEN 2
+
+#define IPIPE_TRACE_POINTS (1 << CONFIG_IPIPE_TRACE_SHIFT)
+#define WRAP_POINT_NO(point) ((point) & (IPIPE_TRACE_POINTS-1))
+
+#define IPIPE_DEFAULT_PRE_TRACE 10
+#define IPIPE_DEFAULT_POST_TRACE 10
+#define IPIPE_DEFAULT_BACK_TRACE 30
+
+#define IPIPE_DELAY_NOTE 1000 /* in nanoseconds */
+#define IPIPE_DELAY_WARN 10000 /* in nanoseconds */
+
+#define IPIPE_TFLG_NMI_LOCK 0x0001
+#define IPIPE_TFLG_NMI_HIT 0x0002
+#define IPIPE_TFLG_NMI_FREEZE_REQ 0x0004
+
+#define IPIPE_TFLG_HWIRQ_OFF 0x0100
+#define IPIPE_TFLG_FREEZING 0x0200
+#define IPIPE_TFLG_CURRDOM_SHIFT 10 /* bits 10..11: current domain */
+#define IPIPE_TFLG_CURRDOM_MASK 0x0C00
+#define IPIPE_TFLG_DOMSTATE_SHIFT 12 /* bits 12..15: domain stalled? */
+#define IPIPE_TFLG_DOMSTATE_BITS 3
+
+#define IPIPE_TFLG_DOMAIN_STALLED(point, n) \
+ (point->flags & (1 << (n + IPIPE_TFLG_DOMSTATE_SHIFT)))
+#define IPIPE_TFLG_CURRENT_DOMAIN(point) \
+ ((point->flags & IPIPE_TFLG_CURRDOM_MASK) >> IPIPE_TFLG_CURRDOM_SHIFT)
+
+
+struct ipipe_trace_point{
+ short type;
+ short flags;
+ unsigned long eip;
+ unsigned long parent_eip;
+ unsigned long v;
+ unsigned long long timestamp;
+};
+
+struct ipipe_trace_path{
+ volatile int flags;
+ int dump_lock; /* separated from flags due to cross-cpu access */
+ int trace_pos; /* next point to fill */
+ int begin, end; /* finalised path begin and end */
+ int post_trace; /* non-zero when in post-trace phase */
+ unsigned long long length; /* max path length in cycles */
+ unsigned long nmi_saved_eip; /* for deferred requests from NMIs */
+ unsigned long nmi_saved_parent_eip;
+ unsigned long nmi_saved_v;
+ struct ipipe_trace_point point[IPIPE_TRACE_POINTS];
+} ____cacheline_aligned_in_smp;
+
+enum ipipe_trace_type
+{
+ IPIPE_TRACE_FUNC = 0,
+ IPIPE_TRACE_BEGIN,
+ IPIPE_TRACE_END,
+ IPIPE_TRACE_FREEZE,
+ IPIPE_TRACE_SPECIAL,
+ IPIPE_TRACE_PID,
+};
+
+#define IPIPE_TYPE_MASK 0x0007
+#define IPIPE_TYPE_BITS 3
+
+
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
+#define IPIPE_DEFAULT_TRACE_STATE 0
+
+static struct ipipe_trace_path *trace_paths[NR_CPUS];
+
+#else /* !CONFIG_IPIPE_TRACE_VMALLOC */
+#define IPIPE_DEFAULT_TRACE_STATE CONFIG_IPIPE_TRACE_ENABLE_VALUE
+
+static struct ipipe_trace_path trace_paths[NR_CPUS][IPIPE_TRACE_PATHS] =
+ { [0 ... NR_CPUS-1] =
+ { [0 ... IPIPE_TRACE_PATHS-1] =
+ { .begin = -1, .end = -1 }
+ }
+ };
+#endif /* CONFIG_IPIPE_TRACE_VMALLOC */
+
+int ipipe_trace_enable = IPIPE_DEFAULT_TRACE_STATE;
+
+static int active_path[NR_CPUS] =
+ { [0 ... NR_CPUS-1] = IPIPE_DEFAULT_ACTIVE };
+static int max_path[NR_CPUS] =
+ { [0 ... NR_CPUS-1] = IPIPE_DEFAULT_MAX };
+static int frozen_path[NR_CPUS] =
+ { [0 ... NR_CPUS-1] = IPIPE_DEFAULT_FROZEN };
+static ipipe_spinlock_t global_path_lock = IPIPE_SPIN_LOCK_UNLOCKED;
+static int pre_trace = IPIPE_DEFAULT_PRE_TRACE;
+static int post_trace = IPIPE_DEFAULT_POST_TRACE;
+static int back_trace = IPIPE_DEFAULT_BACK_TRACE;
+static int verbose_trace = 0;
+
+static DECLARE_MUTEX(out_mutex);
+static struct ipipe_trace_path *print_path;
+static struct ipipe_trace_path *panic_path;
+static int print_pre_trace;
+static int print_post_trace;
+
+
+static long __ipipe_signed_tsc2us(long long tsc);
+static void
+__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point);
+static void __ipipe_print_symname(struct seq_file *m, unsigned long eip);
+
+
+static notrace void
+__ipipe_store_domain_states(struct ipipe_trace_point *point, int cpu_id)
+{
+ struct list_head *pos;
+ int i = 0;
+
+ list_for_each_prev(pos, &__ipipe_pipeline) {
+ struct ipipe_domain *ipd =
+ list_entry(pos, struct ipipe_domain, p_link);
+
+ if (test_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpu_id].status))
+ point->flags |= 1 << (i + IPIPE_TFLG_DOMSTATE_SHIFT);
+
+ if (ipd == per_cpu(ipipe_percpu_domain, cpu_id))
+ point->flags |= i << IPIPE_TFLG_CURRDOM_SHIFT;
+
+ if (++i > IPIPE_TFLG_DOMSTATE_BITS)
+ break;
+ }
+}
+
+static notrace int __ipipe_get_free_trace_path(int old, int cpu_id)
+{
+ int new_active = old;
+ struct ipipe_trace_path *tp;
+
+ do {
+ if (++new_active == IPIPE_TRACE_PATHS)
+ new_active = 0;
+ tp = &trace_paths[cpu_id][new_active];
+ } while ((new_active == max_path[cpu_id]) ||
+ (new_active == frozen_path[cpu_id]) ||
+ tp->dump_lock);
+
+ return new_active;
+}
+
+static notrace void
+__ipipe_migrate_pre_trace(struct ipipe_trace_path *new_tp,
+ struct ipipe_trace_path *old_tp, int old_pos)
+{
+ int i;
+
+ new_tp->trace_pos = pre_trace+1;
+
+ for (i = new_tp->trace_pos; i > 0; i--)
+ memcpy(&new_tp->point[WRAP_POINT_NO(new_tp->trace_pos-i)],
+ &old_tp->point[WRAP_POINT_NO(old_pos-i)],
+ sizeof(struct ipipe_trace_point));
+
+ /* mark the end (i.e. the point before point[0]) invalid */
+ new_tp->point[IPIPE_TRACE_POINTS-1].eip = 0;
+}
+
+static notrace struct ipipe_trace_path *
+__ipipe_trace_end(int cpu_id, struct ipipe_trace_path *tp, int pos)
+{
+ struct ipipe_trace_path *old_tp = tp;
+ long active = active_path[cpu_id];
+ unsigned long long length;
+
+ /* do we have a new worst case? */
+ length = tp->point[tp->end].timestamp -
+ tp->point[tp->begin].timestamp;
+ if (length > (trace_paths[cpu_id][max_path[cpu_id]]).length) {
+ /* we need protection here against other cpus trying
+ to start a proc dump */
+ spin_lock_hw(&global_path_lock);
+
+ /* active path holds new worst case */
+ tp->length = length;
+ max_path[cpu_id] = active;
+
+ /* find next unused trace path */
+ active = __ipipe_get_free_trace_path(active, cpu_id);
+
+ spin_unlock_hw(&global_path_lock);
+
+ tp = &trace_paths[cpu_id][active];
+
+ /* migrate last entries for pre-tracing */
+ __ipipe_migrate_pre_trace(tp, old_tp, pos);
+ }
+
+ return tp;
+}
+
+static notrace struct ipipe_trace_path *
+__ipipe_trace_freeze(int cpu_id, struct ipipe_trace_path *tp, int pos)
+{
+ struct ipipe_trace_path *old_tp = tp;
+ long active = active_path[cpu_id];
+ int i;
+
+ /* frozen paths have no core (begin=end) */
+ tp->begin = tp->end;
+
+ /* we need protection here against other cpus trying
+ * to set their frozen path or to start a proc dump */
+ spin_lock_hw(&global_path_lock);
+
+ frozen_path[cpu_id] = active;
+
+ /* find next unused trace path */
+ active = __ipipe_get_free_trace_path(active, cpu_id);
+
+ /* check if this is the first frozen path */
+ for_each_online_cpu(i) {
+ if ((i != cpu_id) &&
+ (trace_paths[i][frozen_path[i]].end >= 0))
+ tp->end = -1;
+ }
+
+ spin_unlock_hw(&global_path_lock);
+
+ tp = &trace_paths[cpu_id][active];
+
+ /* migrate last entries for pre-tracing */
+ __ipipe_migrate_pre_trace(tp, old_tp, pos);
+
+ return tp;
+}
+
+void notrace
+__ipipe_trace(enum ipipe_trace_type type, unsigned long eip,
+ unsigned long parent_eip, unsigned long v)
+{
+ struct ipipe_trace_path *tp, *old_tp;
+ int pos, next_pos, begin;
+ struct ipipe_trace_point *point;
+ unsigned long flags;
+ int cpu_id;
+
+ local_irq_save_hw_notrace(flags);
+
+ cpu_id = ipipe_processor_id();
+restart:
+ tp = old_tp = &trace_paths[cpu_id][active_path[cpu_id]];
+
+ /* here starts a race window with NMIs - catched below */
+
+ /* check for NMI recursion */
+ if (unlikely(tp->flags & IPIPE_TFLG_NMI_LOCK)) {
+ tp->flags |= IPIPE_TFLG_NMI_HIT;
+
+ /* first freeze request from NMI context? */
+ if ((type == IPIPE_TRACE_FREEZE) &&
+ !(tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)) {
+ /* save arguments and mark deferred freezing */
+ tp->flags |= IPIPE_TFLG_NMI_FREEZE_REQ;
+ tp->nmi_saved_eip = eip;
+ tp->nmi_saved_parent_eip = parent_eip;
+ tp->nmi_saved_v = v;
+ }
+ return; /* no need for restoring flags inside IRQ */
+ }
+
+ /* clear NMI events and set lock (atomically per cpu) */
+ tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT |
+ IPIPE_TFLG_NMI_FREEZE_REQ))
+ | IPIPE_TFLG_NMI_LOCK;
+
+ /* check active_path again - some nasty NMI may have switched
+ * it meanwhile */
+ if (unlikely(tp != &trace_paths[cpu_id][active_path[cpu_id]])) {
+ /* release lock on wrong path and restart */
+ tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+ /* there is no chance that the NMI got deferred
+ * => no need to check for pending freeze requests */
+ goto restart;
+ }
+
+ /* get the point buffer */
+ pos = tp->trace_pos;
+ point = &tp->point[pos];
+
+ /* store all trace point data */
+ point->type = type;
+ point->flags = local_test_iflag_hw(flags) ? 0 : IPIPE_TFLG_HWIRQ_OFF;
+ point->eip = eip;
+ point->parent_eip = parent_eip;
+ point->v = v;
+ ipipe_read_tsc(point->timestamp);
+
+ __ipipe_store_domain_states(point, cpu_id);
+
+ /* forward to next point buffer */
+ next_pos = WRAP_POINT_NO(pos+1);
+ tp->trace_pos = next_pos;
+
+ /* only mark beginning if we haven't started yet */
+ begin = tp->begin;
+ if (unlikely(type == IPIPE_TRACE_BEGIN) && (begin < 0))
+ tp->begin = pos;
+
+ /* end of critical path, start post-trace if not already started */
+ if (unlikely(type == IPIPE_TRACE_END) &&
+ (begin >= 0) && !tp->post_trace)
+ tp->post_trace = post_trace + 1;
+
+ /* freeze only if the slot is free and we are not already freezing */
+ if (unlikely(type == IPIPE_TRACE_FREEZE) &&
+ (trace_paths[cpu_id][frozen_path[cpu_id]].begin < 0) &&
+ !(tp->flags & IPIPE_TFLG_FREEZING)) {
+ tp->post_trace = post_trace + 1;
+ tp->flags |= IPIPE_TFLG_FREEZING;
+ }
+
+ /* enforce end of trace in case of overflow */
+ if (unlikely(WRAP_POINT_NO(next_pos + 1) == begin)) {
+ tp->end = pos;
+ goto enforce_end;
+ }
+
+ /* stop tracing this path if we are in post-trace and
+ * a) that phase is over now or
+ * b) a new TRACE_BEGIN came in but we are not freezing this path */
+ if (unlikely((tp->post_trace > 0) && ((--tp->post_trace == 0) ||
+ ((type == IPIPE_TRACE_BEGIN) &&
+ !(tp->flags & IPIPE_TFLG_FREEZING))))) {
+ /* store the path's end (i.e. excluding post-trace) */
+ tp->end = WRAP_POINT_NO(pos - post_trace + tp->post_trace);
+
+enforce_end:
+ if (tp->flags & IPIPE_TFLG_FREEZING)
+ tp = __ipipe_trace_freeze(cpu_id, tp, pos);
+ else
+ tp = __ipipe_trace_end(cpu_id, tp, pos);
+
+ /* reset the active path, maybe already start a new one */
+ tp->begin = (type == IPIPE_TRACE_BEGIN) ?
+ WRAP_POINT_NO(tp->trace_pos - 1) : -1;
+ tp->end = -1;
+ tp->post_trace = 0;
+ tp->flags = 0;
+
+ /* update active_path not earlier to avoid races with NMIs */
+ active_path[cpu_id] = tp - trace_paths[cpu_id];
+ }
+
+ /* we still have old_tp and point,
+ * let's reset NMI lock and check for catches */
+ old_tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+ if (unlikely(old_tp->flags & IPIPE_TFLG_NMI_HIT)) {
+ /* well, this late tagging may not immediately be visible for
+ * other cpus already dumping this path - a minor issue */
+ point->flags |= IPIPE_TFLG_NMI_HIT;
+
+ /* handle deferred freezing from NMI context */
+ if (old_tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)
+ __ipipe_trace(IPIPE_TRACE_FREEZE, old_tp->nmi_saved_eip,
+ old_tp->nmi_saved_parent_eip,
+ old_tp->nmi_saved_v);
+ }
+
+ local_irq_restore_hw_notrace(flags);
+}
+
+static unsigned long __ipipe_global_path_lock(void)
+{
+ unsigned long flags;
+ int cpu_id;
+ struct ipipe_trace_path *tp;
+
+ spin_lock_irqsave_hw(&global_path_lock, flags);
+
+ cpu_id = ipipe_processor_id();
+restart:
+ tp = &trace_paths[cpu_id][active_path[cpu_id]];
+
+ /* here is small race window with NMIs - catched below */
+
+ /* clear NMI events and set lock (atomically per cpu) */
+ tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT |
+ IPIPE_TFLG_NMI_FREEZE_REQ))
+ | IPIPE_TFLG_NMI_LOCK;
+
+ /* check active_path again - some nasty NMI may have switched
+ * it meanwhile */
+ if (tp != &trace_paths[cpu_id][active_path[cpu_id]]) {
+ /* release lock on wrong path and restart */
+ tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+ /* there is no chance that the NMI got deferred
+ * => no need to check for pending freeze requests */
+ goto restart;
+ }
+
+ return flags;
+}
+
+static void __ipipe_global_path_unlock(unsigned long flags)
+{
+ int cpu_id;
+ struct ipipe_trace_path *tp;
+
+ /* release spinlock first - it's not involved in the NMI issue */
+ spin_unlock_hw(&global_path_lock);
+
+ cpu_id = ipipe_processor_id();
+ tp = &trace_paths[cpu_id][active_path[cpu_id]];
+
+ tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
+
+ /* handle deferred freezing from NMI context */
+ if (tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)
+ __ipipe_trace(IPIPE_TRACE_FREEZE, tp->nmi_saved_eip,
+ tp->nmi_saved_parent_eip, tp->nmi_saved_v);
+
+ local_irq_restore_hw(flags);
+}
+
+void notrace ipipe_trace_begin(unsigned long v)
+{
+ if (!ipipe_trace_enable)
+ return;
+ __ipipe_trace(IPIPE_TRACE_BEGIN, __BUILTIN_RETURN_ADDRESS0,
+ __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_begin);
+
+void notrace ipipe_trace_end(unsigned long v)
+{
+ if (!ipipe_trace_enable)
+ return;
+ __ipipe_trace(IPIPE_TRACE_END, __BUILTIN_RETURN_ADDRESS0,
+ __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_end);
+
+void notrace ipipe_trace_freeze(unsigned long v)
+{
+ if (!ipipe_trace_enable)
+ return;
+ __ipipe_trace(IPIPE_TRACE_FREEZE, __BUILTIN_RETURN_ADDRESS0,
+ __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_freeze);
+
+void notrace ipipe_trace_special(unsigned char id, unsigned long v)
+{
+ if (!ipipe_trace_enable)
+ return;
+ __ipipe_trace(IPIPE_TRACE_SPECIAL | (id << IPIPE_TYPE_BITS),
+ __BUILTIN_RETURN_ADDRESS0,
+ __BUILTIN_RETURN_ADDRESS1, v);
+}
+EXPORT_SYMBOL(ipipe_trace_special);
+
+void notrace ipipe_trace_pid(pid_t pid, short prio)
+{
+ if (!ipipe_trace_enable)
+ return;
+ __ipipe_trace(IPIPE_TRACE_PID | (prio << IPIPE_TYPE_BITS),
+ __BUILTIN_RETURN_ADDRESS0,
+ __BUILTIN_RETURN_ADDRESS1, pid);
+}
+EXPORT_SYMBOL(ipipe_trace_pid);
+
+int ipipe_trace_max_reset(void)
+{
+ int cpu_id;
+ unsigned long flags;
+ struct ipipe_trace_path *path;
+ int ret = 0;
+
+ flags = __ipipe_global_path_lock();
+
+ for_each_online_cpu(cpu_id) {
+ path = &trace_paths[cpu_id][max_path[cpu_id]];
+
+ if (path->dump_lock) {
+ ret = -EBUSY;
+ break;
+ }
+
+ path->begin = -1;
+ path->end = -1;
+ path->trace_pos = 0;
+ path->length = 0;
+ }
+
+ __ipipe_global_path_unlock(flags);
+
+ return ret;
+}
+EXPORT_SYMBOL(ipipe_trace_max_reset);
+
+int ipipe_trace_frozen_reset(void)
+{
+ int cpu_id;
+ unsigned long flags;
+ struct ipipe_trace_path *path;
+ int ret = 0;
+
+ flags = __ipipe_global_path_lock();
+
+ for_each_online_cpu(cpu_id) {
+ path = &trace_paths[cpu_id][frozen_path[cpu_id]];
+
+ if (path->dump_lock) {
+ ret = -EBUSY;
+ break;
+ }
+
+ path->begin = -1;
+ path->end = -1;
+ path->trace_pos = 0;
+ path->length = 0;
+ }
+
+ __ipipe_global_path_unlock(flags);
+
+ return ret;
+}
+EXPORT_SYMBOL(ipipe_trace_frozen_reset);
+
+void ipipe_trace_panic_freeze(void)
+{
+ unsigned long flags;
+ int cpu_id;
+
+ ipipe_trace_enable = 0;
+ local_irq_save_hw_notrace(flags);
+
+ cpu_id = ipipe_processor_id();
+
+ panic_path = &trace_paths[cpu_id][active_path[cpu_id]];
+
+ local_irq_restore_hw(flags);
+}
+EXPORT_SYMBOL(ipipe_trace_panic_freeze);
+
+static void
+__ipipe_get_task_info(char *task_info, struct ipipe_trace_point *point,
+ int trylock)
+{
+ struct task_struct *task = NULL;
+ char buf[8];
+ int i;
+ int locked = 1;
+
+ if (trylock && !read_trylock(&tasklist_lock))
+ locked = 0;
+ else
+ read_lock(&tasklist_lock);
+
+ if (locked)
+ task = find_task_by_pid((pid_t)point->v);
+
+ if (task)
+ strncpy(task_info, task->comm, 11);
+ else
+ strcpy(task_info, "-<?>-");
+
+ if (locked)
+ read_unlock(&tasklist_lock);
+
+ for (i = strlen(task_info); i < 11; i++)
+ task_info[i] = ' ';
+
+ sprintf(buf, " %d ", point->type >> IPIPE_TYPE_BITS);
+ strcpy(task_info + (11 - strlen(buf)), buf);
+}
+
+void ipipe_trace_panic_dump(void)
+{
+ int cnt = back_trace;
+ int start, pos;
+ char task_info[12];
+
+ printk("I-pipe tracer log (%d points):\n", cnt);
+
+ start = pos = WRAP_POINT_NO(panic_path->trace_pos-1);
+
+ while (cnt-- > 0) {
+ struct ipipe_trace_point *point = &panic_path->point[pos];
+ long time;
+ char buf[16];
+
+ if (!point->eip)
+ printk("-<invalid>-\n");
+ else {
+ __ipipe_trace_point_type(buf, point);
+ printk(buf);
+
+ switch (point->type & IPIPE_TYPE_MASK) {
+ case IPIPE_TRACE_FUNC:
+ printk(" ");
+ break;
+
+ case IPIPE_TRACE_PID:
+ __ipipe_get_task_info(task_info,
+ point, 1);
+ printk(task_info);
+ break;
+
+ default:
+ printk("0x%08lx ", point->v);
+ }
+
+ time = __ipipe_signed_tsc2us(point->timestamp -
+ panic_path->point[start].timestamp);
+ printk(" %5ld ", time);
+
+ __ipipe_print_symname(NULL, point->eip);
+ printk(" (");
+ __ipipe_print_symname(NULL, point->parent_eip);
+ printk(")\n");
+ }
+ pos = WRAP_POINT_NO(pos - 1);
+ }
+}
+EXPORT_SYMBOL(ipipe_trace_panic_dump);
+
+
+/* --- /proc output --- */
+
+static notrace int __ipipe_in_critical_trpath(long point_no)
+{
+ return ((WRAP_POINT_NO(point_no-print_path->begin) <
+ WRAP_POINT_NO(print_path->end-print_path->begin)) ||
+ ((print_path->end == print_path->begin) &&
+ (WRAP_POINT_NO(point_no-print_path->end) >
+ print_post_trace)));
+}
+
+static long __ipipe_signed_tsc2us(long long tsc)
+{
+ unsigned long long abs_tsc;
+ long us;
+
+ /* ipipe_tsc2us works on unsigned => handle sign separately */
+ abs_tsc = (tsc >= 0) ? tsc : -tsc;
+ us = ipipe_tsc2us(abs_tsc);
+ if (tsc < 0)
+ return -us;
+ else
+ return us;
+}
+
+static void
+__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point)
+{
+ switch (point->type & IPIPE_TYPE_MASK) {
+ case IPIPE_TRACE_FUNC:
+ strcpy(buf, "func ");
+ break;
+
+ case IPIPE_TRACE_BEGIN:
+ strcpy(buf, "begin ");
+ break;
+
+ case IPIPE_TRACE_END:
+ strcpy(buf, "end ");
+ break;
+
+ case IPIPE_TRACE_FREEZE:
+ strcpy(buf, "freeze ");
+ break;
+
+ case IPIPE_TRACE_SPECIAL:
+ sprintf(buf, "(0x%02x) ",
+ point->type >> IPIPE_TYPE_BITS);
+ break;
+
+ case IPIPE_TRACE_PID:
+ sprintf(buf, "[%5d] ", (pid_t)point->v);
+ break;
+ }
+}
+
+static void
+__ipipe_print_pathmark(struct seq_file *m, struct ipipe_trace_point *point)
+{
+ char mark = ' ';
+ int point_no = point - print_path->point;
+ int i;
+
+ if (print_path->end == point_no)
+ mark = '<';
+ else if (print_path->begin == point_no)
+ mark = '>';
+ else if (__ipipe_in_critical_trpath(point_no))
+ mark = ':';
+ seq_printf(m, "%c%c", mark,
+ (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' ');
+
+ if (!verbose_trace)
+ return;
+
+ for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--)
+ seq_printf(m, "%c",
+ (IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ?
+ (IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
+ '#' : '+') :
+ (IPIPE_TFLG_DOMAIN_STALLED(point, i) ? '*' : ' '));
+}
+
+static void
+__ipipe_print_delay(struct seq_file *m, struct ipipe_trace_point *point)
+{
+ unsigned long delay = 0;
+ int next;
+ char *mark = " ";
+
+ next = WRAP_POINT_NO(point+1 - print_path->point);
+
+ if (next != print_path->trace_pos)
+ delay = ipipe_tsc2ns(print_path->point[next].timestamp -
+ point->timestamp);
+
+ if (__ipipe_in_critical_trpath(point - print_path->point)) {
+ if (delay > IPIPE_DELAY_WARN)
+ mark = "! ";
+ else if (delay > IPIPE_DELAY_NOTE)
+ mark = "+ ";
+ }
+ seq_puts(m, mark);
+
+ if (verbose_trace)
+ seq_printf(m, "%3lu.%03lu%c ", delay/1000, delay%1000,
+ (point->flags & IPIPE_TFLG_NMI_HIT) ? 'N' : ' ');
+ else
+ seq_puts(m, " ");
+}
+
+static void __ipipe_print_symname(struct seq_file *m, unsigned long eip)
+{
+ char namebuf[KSYM_NAME_LEN+1];
+ unsigned long size, offset;
+ const char *sym_name;
+ char *modname;
+
+ sym_name = kallsyms_lookup(eip, &size, &offset, &modname, namebuf);
+
+ /* printing to /proc? */
+ if (m) {
+ if (sym_name) {
+ if (verbose_trace) {
+ seq_printf(m, "%s+0x%lx", sym_name, offset);
+ if (modname)
+ seq_printf(m, " [%s]", modname);
+ } else
+ seq_puts(m, sym_name);
+ } else
+ seq_printf(m, "<%08lx>", eip);
+ } else {
+ /* panic dump */
+ if (sym_name) {
+ printk("%s+0x%lx", sym_name, offset);
+ if (modname)
+ printk(" [%s]", modname);
+ }
+ }
+}
+
+#if defined(CONFIG_XENO_OPT_DEBUG) || defined(CONFIG_DEBUG_PREEMPT)
+static void __ipipe_print_dbgwarning(struct seq_file *m)
+{
+ seq_puts(m, "\n******** WARNING ********\n"
+ "The following debugging options will increase the observed "
+ "latencies:\n"
+#ifdef CONFIG_XENO_OPT_DEBUG
+ " o CONFIG_XENO_OPT_DEBUG\n"
+#endif /* CONFIG_XENO_OPT_DEBUG */
+#ifdef CONFIG_XENO_OPT_DEBUG_QUEUES
+ " o CONFIG_XENO_OPT_DEBUG_QUEUES (very costly)\n"
+#endif /* CONFIG_XENO_OPT_DEBUG */
+#ifdef CONFIG_DEBUG_PREEMPT
+ " o CONFIG_DEBUG_PREEMPT\n"
+#endif /* CONFIG_DEBUG_PREEMPT */
+ "\n");
+}
+#else /* !WARN_ON_DEBUGGING_LATENCIES */
+# define __ipipe_print_dbgwarning(m)
+#endif /* WARN_ON_DEBUGGING_LATENCIES */
+
+static void __ipipe_print_headline(struct seq_file *m)
+{
+ if (verbose_trace) {
+ const char *name[4] = { [0 ... 3] = "<unused>" };
+ struct list_head *pos;
+ int i = 0;
+
+ list_for_each_prev(pos, &__ipipe_pipeline) {
+ struct ipipe_domain *ipd =
+ list_entry(pos, struct ipipe_domain, p_link);
+
+ name[i] = ipd->name;
+ if (++i > 3)
+ break;
+ }
+
+ seq_printf(m,
+ " +----- Hard IRQs ('|': locked)\n"
+ " |+---- %s\n"
+ " ||+--- %s\n"
+ " |||+-- %s\n"
+ " ||||+- %s%s\n"
+ " ||||| +---------- "
+ "Delay flag ('+': > %d us, '!': > %d us)\n"
+ " ||||| | +- "
+ "NMI noise ('N')\n"
+ " ||||| | |\n"
+ " Type User Val. Time Delay Function "
+ "(Parent)\n",
+ name[3], name[2], name[1], name[0],
+ name[0] ? " ('*': domain stalled, '+': current, "
+ "'#': current+stalled)" : "",
+ IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000);
+ } else
+ seq_printf(m,
+ " +--------------- Hard IRQs ('|': locked)\n"
+ " | +- Delay flag "
+ "('+': > %d us, '!': > %d us)\n"
+ " | |\n"
+ " Type Time Function (Parent)\n",
+ IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000);
+}
+
+static void *__ipipe_max_prtrace_start(struct seq_file *m, loff_t *pos)
+{
+ loff_t n = *pos;
+
+ down(&out_mutex);
+
+ if (!n) {
+ struct ipipe_trace_path *path;
+ unsigned long length_usecs;
+ int points, i;
+ unsigned long flags;
+
+ /* protect against max_path/frozen_path updates while we
+ * haven't locked our target path, also avoid recursively
+ * taking global_path_lock from NMI context */
+ flags = __ipipe_global_path_lock();
+
+ /* find the longest of all per-cpu paths */
+ print_path = NULL;
+ for_each_online_cpu(i) {
+ path = &trace_paths[i][max_path[i]];
+ if ((print_path == NULL) ||
+ (path->length > print_path->length))
+ print_path = path;
+ }
+ print_path->dump_lock = 1;
+
+ __ipipe_global_path_unlock(flags);
+
+ /* does this path actually contain data? */
+ if (print_path->end == print_path->begin)
+ return NULL;
+
+ /* number of points inside the critical path */
+ points = WRAP_POINT_NO(print_path->end-print_path->begin+1);
+
+ /* pre- and post-tracing length, post-trace length was frozen
+ in __ipipe_trace, pre-trace may have to be reduced due to
+ buffer overrun */
+ print_pre_trace = pre_trace;
+ print_post_trace = WRAP_POINT_NO(print_path->trace_pos -
+ print_path->end - 1);
+ if (points+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1)
+ print_pre_trace = IPIPE_TRACE_POINTS - 1 - points -
+ print_post_trace;
+
+ length_usecs = ipipe_tsc2us(print_path->length);
+ seq_printf(m, "I-pipe worst-case tracing service on %s/ipipe-%s\n"
+ "------------------------------------------------------------\n",
+ UTS_RELEASE, IPIPE_ARCH_STRING);
+ __ipipe_print_dbgwarning(m);
+ seq_printf(m, "Begin: %lld cycles, Trace Points: %d (-%d/+%d), "
+ "Length: %lu us\n\n",
+ print_path->point[print_path->begin].timestamp,
+ points, print_pre_trace, print_post_trace, length_usecs);
+ __ipipe_print_headline(m);
+ }
+
+ /* check if we are inside the trace range */
+ if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 +
+ print_pre_trace + print_post_trace))
+ return NULL;
+
+ /* return the next point to be shown */
+ return &print_path->point[WRAP_POINT_NO(print_path->begin -
+ print_pre_trace + n)];
+}
+
+static void *__ipipe_prtrace_next(struct seq_file *m, void *p, loff_t *pos)
+{
+ loff_t n = ++*pos;
+
+ /* check if we are inside the trace range with the next entry */
+ if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 +
+ print_pre_trace + print_post_trace))
+ return NULL;
+
+ /* return the next point to be shown */
+ return &print_path->point[WRAP_POINT_NO(print_path->begin -
+ print_pre_trace + *pos)];
+}
+
+static void __ipipe_prtrace_stop(struct seq_file *m, void *p)
+{
+ if (print_path)
+ print_path->dump_lock = 0;
+ up(&out_mutex);
+}
+
+static int __ipipe_prtrace_show(struct seq_file *m, void *p)
+{
+ long time;
+ struct ipipe_trace_point *point = p;
+ char buf[16];
+
+ if (!point->eip) {
+ seq_puts(m, "-<invalid>-\n");
+ return 0;
+ }
+
+ __ipipe_print_pathmark(m, point);
+ __ipipe_trace_point_type(buf, point);
+ seq_puts(m, buf);
+ if (verbose_trace)
+ switch (point->type & IPIPE_TYPE_MASK) {
+ case IPIPE_TRACE_FUNC:
+ seq_puts(m, " ");
+ break;
+
+ case IPIPE_TRACE_PID:
+ __ipipe_get_task_info(buf, point, 0);
+ seq_puts(m, buf);
+ break;
+
+ default:
+ seq_printf(m, "0x%08lx ", point->v);
+ }
+
+ time = __ipipe_signed_tsc2us(point->timestamp -
+ print_path->point[print_path->begin].timestamp);
+ seq_printf(m, "%5ld", time);
+
+ __ipipe_print_delay(m, point);
+ __ipipe_print_symname(m, point->eip);
+ seq_puts(m, " (");
+ __ipipe_print_symname(m, point->parent_eip);
+ seq_puts(m, ")\n");
+
+ return 0;
+}
+
+static struct seq_operations __ipipe_max_ptrace_ops = {
+ .start = __ipipe_max_prtrace_start,
+ .next = __ipipe_prtrace_next,
+ .stop = __ipipe_prtrace_stop,
+ .show = __ipipe_prtrace_show
+};
+
+static int __ipipe_max_prtrace_open(struct inode *inode, struct file *file)
+{
+ return seq_open(file, &__ipipe_max_ptrace_ops);
+}
+
+static ssize_t
+__ipipe_max_reset(struct file *file, const char __user *pbuffer,
+ size_t count, loff_t *data)
+{
+ down(&out_mutex);
+ ipipe_trace_max_reset();
+ up(&out_mutex);
+
+ return count;
+}
+
+struct file_operations __ipipe_max_prtrace_fops = {
+ .open = __ipipe_max_prtrace_open,
+ .read = seq_read,
+ .write = __ipipe_max_reset,
+ .llseek = seq_lseek,
+ .release = seq_release,
+};
+
+static void *__ipipe_frozen_prtrace_start(struct seq_file *m, loff_t *pos)
+{
+ loff_t n = *pos;
+
+ down(&out_mutex);
+
+ if (!n) {
+ struct ipipe_trace_path *path;
+ int i;
+ unsigned long flags;
+
+ /* protect against max_path/frozen_path updates while we
+ * haven't locked our target path, also avoid recursively
+ * taking global_path_lock from NMI context */
+ flags = __ipipe_global_path_lock();
+
+ /* find the first of all per-cpu frozen paths */
+ print_path = NULL;
+ for_each_online_cpu(i) {
+ path = &trace_paths[i][frozen_path[i]];
+ if (path->end >= 0)
+ print_path = path;
+ }
+ if (print_path)
+ print_path->dump_lock = 1;
+
+ __ipipe_global_path_unlock(flags);
+
+ if (!print_path)
+ return NULL;
+
+ /* back- and post-tracing length, post-trace length was frozen
+ in __ipipe_trace, back-trace may have to be reduced due to
+ buffer overrun */
+ print_pre_trace = back_trace-1; /* substract freeze point */
+ print_post_trace = WRAP_POINT_NO(print_path->trace_pos -
+ print_path->end - 1);
+ if (1+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1)
+ print_pre_trace = IPIPE_TRACE_POINTS - 2 -
+ print_post_trace;
+
+ seq_printf(m, "I-pipe frozen back-tracing service on %s/ipipe-%s\n"
+ "------------------------------------------------------"
+ "------\n",
+ UTS_RELEASE, IPIPE_ARCH_STRING);
+ __ipipe_print_dbgwarning(m);
+ seq_printf(m, "Freeze: %lld cycles, Trace Points: %d (+%d)\n\n",
+ print_path->point[print_path->begin].timestamp,
+ print_pre_trace+1, print_post_trace);
+ __ipipe_print_headline(m);
+ }
+
+ /* check if we are inside the trace range */
+ if (n >= print_pre_trace + 1 + print_post_trace)
+ return NULL;
+
+ /* return the next point to be shown */
+ return &print_path->point[WRAP_POINT_NO(print_path->begin-
+ print_pre_trace+n)];
+}
+
+static struct seq_operations __ipipe_frozen_ptrace_ops = {
+ .start = __ipipe_frozen_prtrace_start,
+ .next = __ipipe_prtrace_next,
+ .stop = __ipipe_prtrace_stop,
+ .show = __ipipe_prtrace_show
+};
+
+static int __ipipe_frozen_prtrace_open(struct inode *inode, struct file *file)
+{
+ return seq_open(file, &__ipipe_frozen_ptrace_ops);
+}
+
+static ssize_t
+__ipipe_frozen_ctrl(struct file *file, const char __user *pbuffer,
+ size_t count, loff_t *data)
+{
+ char *end, buf[16];
+ int val;
+ int n;
+
+ n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
+
+ if (copy_from_user(buf, pbuffer, n))
+ return -EFAULT;
+
+ buf[n] = '\0';
+ val = simple_strtol(buf, &end, 0);
+
+ if (((*end != '\0') && !isspace(*end)) || (val < 0))
+ return -EINVAL;
+
+ down(&out_mutex);
+ ipipe_trace_frozen_reset();
+ if (val > 0)
+ ipipe_trace_freeze(-1);
+ up(&out_mutex);
+
+ return count;
+}
+
+struct file_operations __ipipe_frozen_prtrace_fops = {
+ .open = __ipipe_frozen_prtrace_open,
+ .read = seq_read,
+ .write = __ipipe_frozen_ctrl,
+ .llseek = seq_lseek,
+ .release = seq_release,
+};
+
+static int __ipipe_rd_proc_val(char *page, char **start, off_t off,
+ int count, int *eof, void *data)
+{
+ int len;
+
+ len = sprintf(page, "%u\n", *(int *)data);
+ len -= off;
+ if (len <= off + count)
+ *eof = 1;
+ *start = page + off;
+ if (len > count)
+ len = count;
+ if (len < 0)
+ len = 0;
+
+ return len;
+}
+
+static int __ipipe_wr_proc_val(struct file *file, const char __user *buffer,
+ unsigned long count, void *data)
+{
+ char *end, buf[16];
+ int val;
+ int n;
+
+ n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
+
+ if (copy_from_user(buf, buffer, n))
+ return -EFAULT;
+
+ buf[n] = '\0';
+ val = simple_strtol(buf, &end, 0);
+
+ if (((*end != '\0') && !isspace(*end)) || (val < 0))
+ return -EINVAL;
+
+ down(&out_mutex);
+ *(int *)data = val;
+ up(&out_mutex);
+
+ return count;
+}
+
+extern struct proc_dir_entry *ipipe_proc_root;
+
+static void __init
+__ipipe_create_trace_proc_val(struct proc_dir_entry *trace_dir,
+ const char *name, int *value_ptr)
+{
+ struct proc_dir_entry *entry;
+
+ entry = create_proc_entry(name, 0644, trace_dir);
+ if (entry) {
+ entry->data = value_ptr;
+ entry->read_proc = __ipipe_rd_proc_val;
+ entry->write_proc = __ipipe_wr_proc_val;
+ entry->owner = THIS_MODULE;
+ }
+}
+
+void __init __ipipe_init_tracer(void)
+{
+ struct proc_dir_entry *trace_dir;
+ struct proc_dir_entry *entry;
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
+ int cpu, path;
+
+ for (cpu = 0; cpu < NR_CPUS; cpu++) {
+ trace_paths[cpu] = vmalloc(
+ sizeof(struct ipipe_trace_path) * IPIPE_TRACE_PATHS);
+ if (!trace_paths) {
+ printk(KERN_ERR "I-pipe: "
+ "insufficient memory for trace buffer.\n");
+ return;
+ }
+ memset(trace_paths[cpu], 0,
+ sizeof(struct ipipe_trace_path) * IPIPE_TRACE_PATHS);
+ for (path = 0; path < IPIPE_TRACE_PATHS; path++) {
+ trace_paths[cpu][path].begin = -1;
+ trace_paths[cpu][path].end = -1;
+ }
+ }
+ ipipe_trace_enable = CONFIG_IPIPE_TRACE_ENABLE_VALUE;
+#endif /* CONFIG_IPIPE_TRACE_VMALLOC */
+
+ trace_dir = create_proc_entry("trace", S_IFDIR, ipipe_proc_root);
+
+ entry = create_proc_entry("max", 0644, trace_dir);
+ if (entry)
+ entry->proc_fops = &__ipipe_max_prtrace_fops;
+
+ entry = create_proc_entry("frozen", 0644, trace_dir);
+ if (entry)
+ entry->proc_fops = &__ipipe_frozen_prtrace_fops;
+
+ __ipipe_create_trace_proc_val(trace_dir, "pre_trace_points",
+ &pre_trace);
+ __ipipe_create_trace_proc_val(trace_dir, "post_trace_points",
+ &post_trace);
+ __ipipe_create_trace_proc_val(trace_dir, "back_trace_points",
+ &back_trace);
+ __ipipe_create_trace_proc_val(trace_dir, "verbose",
+ &verbose_trace);
+ __ipipe_create_trace_proc_val(trace_dir, "enable",
+ &ipipe_trace_enable);
+}
diff -Naur linux-2.6.19.orig/kernel/irq/handle.c linux-2.6.19.ipipe/kernel/irq/handle.c
--- linux-2.6.19.orig/kernel/irq/handle.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/kernel/irq/handle.c 2006-12-19 13:02:35.000000000 +0100
@@ -177,16 +177,21 @@
/*
* No locking required for CPU-local interrupts:
*/
+#ifndef CONFIG_IPIPE
if (desc->chip->ack)
desc->chip->ack(irq);
+#endif /* CONFIG_IPIPE */
action_ret = handle_IRQ_event(irq, desc->action);
desc->chip->end(irq);
return 1;
}
spin_lock(&desc->lock);
+#ifndef CONFIG_IPIPE
if (desc->chip->ack)
desc->chip->ack(irq);
+#endif /* CONFIG_IPIPE */
+
/*
* REPLAY is when Linux resends an IRQ that was dropped earlier
* WAITING is used by probe to mark irqs that are being tested
diff -Naur linux-2.6.19.orig/kernel/Makefile linux-2.6.19.ipipe/kernel/Makefile
--- linux-2.6.19.orig/kernel/Makefile 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/kernel/Makefile 2006-12-19 13:05:03.000000000 +0100
@@ -51,6 +51,7 @@
obj-$(CONFIG_UTS_NS) += utsname.o
obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o
+obj-$(CONFIG_IPIPE) += ipipe/
ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y)
# According to Alan Modra <alan@domain.hid>, the -fno-omit-frame-pointer is
diff -Naur linux-2.6.19.orig/kernel/printk.c linux-2.6.19.ipipe/kernel/printk.c
--- linux-2.6.19.orig/kernel/printk.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/kernel/printk.c 2006-12-19 13:02:35.000000000 +0100
@@ -489,6 +489,78 @@
* printf(3)
*/
+#ifdef CONFIG_IPIPE
+
+static ipipe_spinlock_t __ipipe_printk_lock = IPIPE_SPIN_LOCK_UNLOCKED;
+
+static int __ipipe_printk_fill;
+
+static char __ipipe_printk_buf[__LOG_BUF_LEN];
+
+void __ipipe_flush_printk (unsigned virq, void *cookie)
+{
+ char *p = __ipipe_printk_buf;
+ int len, lmax, out = 0;
+ unsigned long flags;
+
+ goto start;
+
+ do {
+ spin_unlock_irqrestore_hw(&__ipipe_printk_lock,flags);
+ start:
+ lmax = __ipipe_printk_fill;
+ while (out < lmax) {
+ len = strlen(p) + 1;
+ printk("%s",p);
+ p += len;
+ out += len;
+ }
+ spin_lock_irqsave_hw(&__ipipe_printk_lock,flags);
+ }
+ while (__ipipe_printk_fill != lmax);
+
+ __ipipe_printk_fill = 0;
+
+ spin_unlock_irqrestore_hw(&__ipipe_printk_lock,flags);
+}
+
+asmlinkage int printk(const char *fmt, ...)
+{
+ int r, fbytes, oldcount;
+ unsigned long flags;
+ va_list args;
+
+ va_start(args, fmt);
+
+ if (ipipe_current_domain == ipipe_root_domain ||
+ test_bit(IPIPE_SPRINTK_FLAG,&ipipe_current_domain->flags) ||
+ oops_in_progress) {
+ r = vprintk(fmt, args);
+ goto out;
+ }
+
+ spin_lock_irqsave_hw(&__ipipe_printk_lock,flags);
+
+ oldcount = __ipipe_printk_fill;
+ fbytes = __LOG_BUF_LEN - oldcount;
+
+ if (fbytes > 1) {
+ r = vscnprintf(__ipipe_printk_buf + __ipipe_printk_fill,
+ fbytes, fmt, args) + 1; /* account for the null byte */
+ __ipipe_printk_fill += r;
+ } else
+ r = 0;
+
+ spin_unlock_irqrestore_hw(&__ipipe_printk_lock,flags);
+
+ if (oldcount == 0)
+ ipipe_trigger_irq(__ipipe_printk_virq);
+out:
+ va_end(args);
+
+ return r;
+}
+#else /* !CONFIG_IPIPE */
asmlinkage int printk(const char *fmt, ...)
{
va_list args;
@@ -500,6 +572,7 @@
return r;
}
+#endif /* CONFIG_IPIPE */
/* cpu currently holding logbuf_lock */
static volatile unsigned int printk_cpu = UINT_MAX;
diff -Naur linux-2.6.19.orig/kernel/sched.c linux-2.6.19.ipipe/kernel/sched.c
--- linux-2.6.19.orig/kernel/sched.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/kernel/sched.c 2006-12-19 13:02:35.000000000 +0100
@@ -1391,7 +1391,7 @@
rq = task_rq_lock(p, &flags);
old_state = p->state;
- if (!(old_state & state))
+ if (!(old_state & state) || (old_state & TASK_NOWAKEUP))
goto out;
if (p->array)
@@ -1809,6 +1809,8 @@
#endif
if (current->set_child_tid)
put_user(current->pid, current->set_child_tid);
+
+ ipipe_init_notify(current);
}
/*
@@ -3324,12 +3326,16 @@
long *switch_count;
struct rq *rq;
+#ifdef CONFIG_IPIPE
+ if (unlikely(!ipipe_root_domain_p))
+ return;
+#endif /* CONFIG_IPIPE */
/*
* Test if we are atomic. Since do_exit() needs to call into
* schedule() atomically, we ignore that path for now.
* Otherwise, whine if we are scheduling when we should not be.
*/
- if (unlikely(in_atomic() && !current->exit_state)) {
+ if (unlikely(!(current->state & TASK_ATOMICSWITCH) && in_atomic() && !current->exit_state)) {
printk(KERN_ERR "BUG: scheduling while atomic: "
"%s/0x%08x/%d\n",
current->comm, preempt_count(), current->pid);
@@ -3337,8 +3343,13 @@
}
profile_hit(SCHED_PROFILING, __builtin_return_address(0));
+ if (unlikely(current->state & TASK_ATOMICSWITCH)) {
+ current->state &= ~TASK_ATOMICSWITCH;
+ goto need_resched_nodisable;
+ }
need_resched:
preempt_disable();
+need_resched_nodisable:
prev = current;
release_kernel_lock(prev);
need_resched_nonpreemptible:
@@ -3456,6 +3467,8 @@
prepare_task_switch(rq, next);
prev = context_switch(rq, prev, next);
barrier();
+ if (task_hijacked(prev))
+ return;
/*
* this_rq must be evaluated again because prev may have moved
* CPUs since it called schedule(), thus the 'rq' on its stack
@@ -3469,7 +3482,7 @@
if (unlikely(reacquire_kernel_lock(prev) < 0))
goto need_resched_nonpreemptible;
preempt_enable_no_resched();
- if (unlikely(test_thread_flag(TIF_NEED_RESCHED)))
+ if (unlikely(test_thread_flag(TIF_NEED_RESCHED) && ipipe_root_domain_p))
goto need_resched;
}
EXPORT_SYMBOL(schedule);
@@ -3487,6 +3500,11 @@
struct task_struct *task = current;
int saved_lock_depth;
#endif
+#ifdef CONFIG_IPIPE
+ /* Do not reschedule over non-Linux domains. */
+ if (unlikely(!ipipe_root_domain_p))
+ return;
+#endif /* CONFIG_IPIPE */
/*
* If there is a non-zero preempt_count or interrupts are disabled,
* we do not want to preempt the current task. Just return..
@@ -4185,6 +4203,7 @@
deactivate_task(p, rq);
oldprio = p->prio;
__setscheduler(p, policy, param->sched_priority);
+ ipipe_setsched_notify(p);
if (array) {
__activate_task(p, rq);
/*
@@ -6950,3 +6969,50 @@
}
#endif
+
+#ifdef CONFIG_IPIPE
+
+int ipipe_setscheduler_root (struct task_struct *p, int policy, int prio)
+{
+ struct prio_array *array;
+ unsigned long flags;
+ struct rq *rq;
+ int oldprio;
+
+ rq = task_rq_lock(p, &flags);
+ array = p->array;
+ if (array)
+ deactivate_task(p, rq);
+ oldprio = p->prio;
+ __setscheduler(p, policy, prio);
+ if (array) {
+ __activate_task(p, rq);
+ if (task_running(rq, p)) {
+ if (p->prio > oldprio)
+ resched_task(rq->curr);
+ } else if (TASK_PREEMPTS_CURR(p, rq))
+ resched_task(rq->curr);
+ }
+ task_rq_unlock(rq, &flags);
+
+ return 0;
+}
+
+EXPORT_SYMBOL(ipipe_setscheduler_root);
+
+int ipipe_reenter_root (struct task_struct *prev, int policy, int prio)
+{
+ finish_task_switch(this_rq(), prev);
+ if (reacquire_kernel_lock(current) < 0)
+ ;
+ preempt_enable_no_resched();
+
+ if (current->policy != policy || current->rt_priority != prio)
+ return ipipe_setscheduler_root(current,policy,prio);
+
+ return 0;
+}
+
+EXPORT_SYMBOL(ipipe_reenter_root);
+
+#endif /* CONFIG_IPIPE */
diff -Naur linux-2.6.19.orig/kernel/signal.c linux-2.6.19.ipipe/kernel/signal.c
--- linux-2.6.19.orig/kernel/signal.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/kernel/signal.c 2006-12-19 13:02:35.000000000 +0100
@@ -504,6 +504,7 @@
unsigned int mask;
set_tsk_thread_flag(t, TIF_SIGPENDING);
+ ipipe_sigwake_notify(t); /* TIF_SIGPENDING must be set first. */
/*
* For SIGKILL, we want to wake it up in the stopped/traced case.
diff -Naur linux-2.6.19.orig/lib/Kconfig.debug linux-2.6.19.ipipe/lib/Kconfig.debug
--- linux-2.6.19.orig/lib/Kconfig.debug 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/lib/Kconfig.debug 2006-12-19 13:02:36.000000000 +0100
@@ -46,6 +46,8 @@
you really need it, and what the merge plan to the mainline kernel for
your module is.
+source "kernel/ipipe/Kconfig.debug"
+
config DEBUG_KERNEL
bool "Kernel debugging"
help
diff -Naur linux-2.6.19.orig/lib/smp_processor_id.c linux-2.6.19.ipipe/lib/smp_processor_id.c
--- linux-2.6.19.orig/lib/smp_processor_id.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/lib/smp_processor_id.c 2006-12-19 13:02:36.000000000 +0100
@@ -13,6 +13,11 @@
int this_cpu = raw_smp_processor_id();
cpumask_t this_mask;
+#ifdef CONFIG_IPIPE
+ if (ipipe_current_domain != ipipe_root_domain)
+ return this_cpu;
+#endif /* CONFIG_IPIPE */
+
if (likely(preempt_count))
goto out;
diff -Naur linux-2.6.19.orig/Makefile linux-2.6.19.ipipe/Makefile
--- linux-2.6.19.orig/Makefile 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/Makefile 2006-12-19 13:02:15.000000000 +0100
@@ -491,6 +491,10 @@
include $(srctree)/arch/$(ARCH)/Makefile
+ifdef CONFIG_IPIPE_TRACE_MCOUNT
+CFLAGS += -pg
+endif
+
ifdef CONFIG_FRAME_POINTER
CFLAGS += -fno-omit-frame-pointer $(call cc-option,-fno-optimize-sibling-calls,)
else
diff -Naur linux-2.6.19.orig/mm/vmalloc.c linux-2.6.19.ipipe/mm/vmalloc.c
--- linux-2.6.19.orig/mm/vmalloc.c 2006-11-29 22:57:37.000000000 +0100
+++ linux-2.6.19.ipipe/mm/vmalloc.c 2006-12-19 13:02:36.000000000 +0100
@@ -19,6 +19,7 @@
#include <asm/uaccess.h>
#include <asm/tlbflush.h>
+#include <asm/pgalloc.h>
DEFINE_RWLOCK(vmlist_lock);
@@ -151,10 +152,14 @@
BUG_ON(addr >= end);
pgd = pgd_offset_k(addr);
do {
+ pgd_t oldpgd;
+ memcpy(&oldpgd,pgd,sizeof(pgd_t));
next = pgd_addr_end(addr, end);
err = vmap_pud_range(pgd, addr, next, prot, pages);
if (err)
break;
+ if (pgd_val(oldpgd) != pgd_val(*pgd))
+ set_pgdir(addr, *pgd);
} while (pgd++, addr = next, addr != end);
flush_cache_vmap((unsigned long) area->addr, end);
return err;
^ permalink raw reply [flat|nested] 5+ messages in thread