From mboxrd@z Thu Jan 1 00:00:00 1970 Message-ID: <449556F4.2010907@domain.hid> Date: Sun, 18 Jun 2006 15:36:52 +0200 From: Jan Kiszka MIME-Version: 1.0 Content-Type: multipart/signed; micalg=pgp-sha1; protocol="application/pgp-signature"; boundary="------------enig0B949EEF0D2EBE5A38BECB96" Sender: jan.kiszka@domain.hid Subject: [Xenomai-core] [PATCH] adeos-ipipe-2.6.17-i386-1.3.05 List-Id: "Xenomai life and development \(bug reports, patches, discussions\)" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: xenomai-core This is an OpenPGP/MIME signed message (RFC 2440 and 3156) --------------enig0B949EEF0D2EBE5A38BECB96 Content-Type: multipart/mixed; boundary="------------010006040407060908060502" This is a multi-part message in MIME format. --------------010006040407060908060502 Content-Type: text/plain; charset=ISO-8859-1 Content-Transfer-Encoding: quoted-printable Hi, this is a rebased i386 ipipe patch for the new 2.6.17 kernel. Again it was just about applying some failing hunks manually (and checking if that was legal). I also included my last two kgdb-over-ipipe patches, but they are of limited use as long as no kgdb is publicly available for the new kernel. All is working fine so far. This mail was, e.g., posted from a Xenomai box running the patch, switchtest, and latency. Jan --------------010006040407060908060502 Content-Type: text/plain; name="adeos-ipipe-2.6.17-i386-1.3.05" Content-Transfer-Encoding: quoted-printable Content-Disposition: inline; filename="adeos-ipipe-2.6.17-i386-1.3.05" Binary files 2.6.16/arch/ia64/hp/sim/boot/bootloader and 2.6.16-ipipe/arc= h/ia64/hp/sim/boot/bootloader differ Index: linux-2.6.17/drivers/pci/msi.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/drivers/pci/msi.c +++ linux-2.6.17/drivers/pci/msi.c @@ -151,6 +151,21 @@ static void unmask_MSI_irq(unsigned int=20 msi_set_mask_bit(vector, 0); } =20 +#ifdef CONFIG_IPIPE +static void ack_MSI_irq_w_maskbit(unsigned int vector) +{ + mask_MSI_irq(vector); + __ack_APIC_irq(); +} +static void ack_MSI_irq_wo_maskbit(unsigned int vector) +{ + __ack_APIC_irq(); +} +#else /* !CONFIG_IPIPE */ +#define ack_MSI_irq_wo_maskbit do_nothing +#define ack_MSI_irq_w_maskbit mask_MSI_irq +#endif /* CONFIG_IPIPE */ + static unsigned int startup_msi_irq_wo_maskbit(unsigned int vector) { struct msi_desc *entry; @@ -214,7 +229,7 @@ static struct hw_interrupt_type msix_irq .shutdown =3D shutdown_msi_irq, .enable =3D unmask_MSI_irq, .disable =3D mask_MSI_irq, - .ack =3D mask_MSI_irq, + .ack =3D ack_MSI_irq_w_maskbit, .end =3D end_msi_irq_w_maskbit, .set_affinity =3D set_msi_affinity }; @@ -230,7 +245,7 @@ static struct hw_interrupt_type msi_irq_ .shutdown =3D shutdown_msi_irq, .enable =3D unmask_MSI_irq, .disable =3D mask_MSI_irq, - .ack =3D mask_MSI_irq, + .ack =3D ack_MSI_irq_w_maskbit, .end =3D end_msi_irq_w_maskbit, .set_affinity =3D set_msi_affinity }; @@ -246,7 +261,7 @@ static struct hw_interrupt_type msi_irq_ .shutdown =3D shutdown_msi_irq, .enable =3D do_nothing, .disable =3D do_nothing, - .ack =3D do_nothing, + .ack =3D ack_MSI_irq_wo_maskbit, .end =3D end_msi_irq_wo_maskbit, .set_affinity =3D set_msi_affinity }; Index: linux-2.6.17/include/linux/hardirq.h =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/include/linux/hardirq.h +++ linux-2.6.17/include/linux/hardirq.h @@ -87,8 +87,21 @@ extern void synchronize_irq(unsigned int # define synchronize_irq(irq) barrier() #endif =20 +#ifdef CONFIG_IPIPE +#define nmi_enter() \ +do { \ + if (ipipe_current_domain =3D=3D ipipe_root_domain) \ + irq_enter(); \ +} while(0) +#define nmi_exit() \ +do { \ + if (ipipe_current_domain =3D=3D ipipe_root_domain) \ + sub_preempt_count(HARDIRQ_OFFSET); \ +} while(0) +#else /* !CONFIG_IPIPE */ #define nmi_enter() irq_enter() #define nmi_exit() sub_preempt_count(HARDIRQ_OFFSET) +#endif /* CONFIG_IPIPE */ =20 struct task_struct; =20 Index: linux-2.6.17/include/linux/ipipe.h =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- /dev/null +++ linux-2.6.17/include/linux/ipipe.h @@ -0,0 +1,726 @@ +/* -*- linux-c -*- + * include/linux/ipipe.h + * + * Copyright (C) 2002-2005 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,= + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,= USA. + */ + +#ifndef __LINUX_IPIPE_H +#define __LINUX_IPIPE_H + +#include +#include +#include +#include + +#ifdef CONFIG_IPIPE + +#define IPIPE_VERSION_STRING IPIPE_ARCH_STRING +#define IPIPE_RELEASE_NUMBER ((IPIPE_MAJOR_NUMBER << 16) | \ + (IPIPE_MINOR_NUMBER << 8) | \ + (IPIPE_PATCH_NUMBER)) + +#ifndef BROKEN_BUILTIN_RETURN_ADDRESS +#define __BUILTIN_RETURN_ADDRESS0 ((unsigned long)__builtin_return_addre= ss(0)) +#define __BUILTIN_RETURN_ADDRESS1 ((unsigned long)__builtin_return_addre= ss(1)) +#endif /* !BUILTIN_RETURN_ADDRESS */ + +#define IPIPE_ROOT_PRIO 100 +#define IPIPE_ROOT_ID 0 +#define IPIPE_ROOT_NPTDKEYS 4 /* Must be <=3D BITS_PER_LONG */ + +#define IPIPE_RESET_TIMER 0x1 +#define IPIPE_GRAB_TIMER 0x2 + +/* Global domain flags */ +#define IPIPE_SPRINTK_FLAG 0 /* Synchronous printk() allowed */ +#define IPIPE_AHEAD_FLAG 1 /* Domain always heads the pipeline */ + +#define IPIPE_STALL_FLAG 0 /* Stalls a pipeline stage -- guaranteed at b= it #0 */ +#define IPIPE_SYNC_FLAG 1 /* The interrupt syncer is running for the do= main */ +#define IPIPE_NOSTACK_FLAG 2 /* Domain currently runs on a foreign stack= */ + +#define IPIPE_SYNC_MASK (1 << IPIPE_SYNC_FLAG) + +#define IPIPE_HANDLE_FLAG 0 +#define IPIPE_PASS_FLAG 1 +#define IPIPE_ENABLE_FLAG 2 +#define IPIPE_DYNAMIC_FLAG IPIPE_HANDLE_FLAG +#define IPIPE_STICKY_FLAG 3 +#define IPIPE_SYSTEM_FLAG 4 +#define IPIPE_LOCK_FLAG 5 +#define IPIPE_SHARED_FLAG 6 +#define IPIPE_WIRED_FLAG 7 +#define IPIPE_EXCLUSIVE_FLAG 8 + +#define IPIPE_HANDLE_MASK (1 << IPIPE_HANDLE_FLAG) +#define IPIPE_PASS_MASK (1 << IPIPE_PASS_FLAG) +#define IPIPE_ENABLE_MASK (1 << IPIPE_ENABLE_FLAG) +#define IPIPE_DYNAMIC_MASK IPIPE_HANDLE_MASK +#define IPIPE_STICKY_MASK (1 << IPIPE_STICKY_FLAG) +#define IPIPE_SYSTEM_MASK (1 << IPIPE_SYSTEM_FLAG) +#define IPIPE_LOCK_MASK (1 << IPIPE_LOCK_FLAG) +#define IPIPE_SHARED_MASK (1 << IPIPE_SHARED_FLAG) +#define IPIPE_WIRED_MASK (1 << IPIPE_WIRED_FLAG) +#define IPIPE_EXCLUSIVE_MASK (1 << IPIPE_EXCLUSIVE_FLAG) + +#define IPIPE_DEFAULT_MASK (IPIPE_HANDLE_MASK|IPIPE_PASS_MASK) +#define IPIPE_STDROOT_MASK (IPIPE_HANDLE_MASK|IPIPE_PASS_MASK|IPIPE_SYST= EM_MASK) + +#define IPIPE_EVENT_SELF 0x80000000 + +/* Number of virtual IRQs */ +#define IPIPE_NR_VIRQS BITS_PER_LONG +/* First virtual IRQ # */ +#define IPIPE_VIRQ_BASE (((IPIPE_NR_XIRQS + BITS_PER_LONG - 1) / BITS_P= ER_LONG) * BITS_PER_LONG) +/* Total number of IRQ slots */ +#define IPIPE_NR_IRQS (IPIPE_VIRQ_BASE + IPIPE_NR_VIRQS) +/* Number of indirect words needed to map the whole IRQ space. */ +#define IPIPE_IRQ_IWORDS ((IPIPE_NR_IRQS + BITS_PER_LONG - 1) / BITS_PER= _LONG) +#define IPIPE_IRQ_IMASK (BITS_PER_LONG - 1) +#define IPIPE_IRQMASK_ANY (~0L) +#define IPIPE_IRQMASK_VIRT (IPIPE_IRQMASK_ANY << (IPIPE_VIRQ_BASE / BITS= _PER_LONG)) + +#ifdef CONFIG_SMP + +#define IPIPE_NR_CPUS NR_CPUS +#define ipipe_declare_cpuid int cpuid +#define ipipe_load_cpuid() do { \ + (cpuid) =3D ipipe_processor_id(); \ + } while(0) +#define ipipe_lock_cpu(flags) do { \ + local_irq_save_hw(flags); \ + (cpuid) =3D ipipe_processor_id(); \ + } while(0) +#define ipipe_unlock_cpu(flags) local_irq_restore_hw(flags) +#define ipipe_get_cpu(flags) ipipe_lock_cpu(flags) +#define ipipe_put_cpu(flags) ipipe_unlock_cpu(flags) +#define ipipe_current_domain (ipipe_percpu_domain[ipipe_processor_id()])= + +#else /* !CONFIG_SMP */ + +#define IPIPE_NR_CPUS 1 +#define ipipe_declare_cpuid const int cpuid =3D 0 +#define ipipe_load_cpuid() do { } while(0) +#define ipipe_lock_cpu(flags) local_irq_save_hw(flags) +#define ipipe_unlock_cpu(flags) local_irq_restore_hw(flags) +#define ipipe_get_cpu(flags) do { (void)(flags); } while(0) +#define ipipe_put_cpu(flags) do { } while(0) +#define ipipe_current_domain (ipipe_percpu_domain[0]) + +#endif /* CONFIG_SMP */ + +#define ipipe_virtual_irq_p(irq) ((irq) >=3D IPIPE_VIRQ_BASE && \ + (irq) < IPIPE_NR_IRQS) + +typedef void (*ipipe_irq_handler_t)(unsigned irq, + void *cookie); + +#define IPIPE_SAME_HANDLER ((ipipe_irq_handler_t)(-1)) + +typedef int (*ipipe_irq_ackfn_t)(unsigned irq); + +typedef int (*ipipe_event_handler_t)(unsigned event, + struct ipipe_domain *from, + void *data); +struct ipipe_domain { + + struct list_head p_link; /* Link in pipeline */ + + struct ipcpudata { + unsigned long status; + unsigned long irq_pending_hi; + unsigned long irq_pending_lo[IPIPE_IRQ_IWORDS]; + struct ipirqcnt { + unsigned long pending_hits; + unsigned long total_hits; + } irq_counters[IPIPE_NR_IRQS]; + } ____cacheline_aligned_in_smp cpudata[IPIPE_NR_CPUS]; + + struct { + unsigned long control; + ipipe_irq_ackfn_t acknowledge; + ipipe_irq_handler_t handler; + void *cookie; + } ____cacheline_aligned irqs[IPIPE_NR_IRQS]; + + ipipe_event_handler_t evhand[IPIPE_NR_EVENTS]; /* Event handlers. */ + unsigned long long evself; /* Self-monitored event bits. */ + unsigned long flags; + unsigned domid; + const char *name; + int priority; + void *pdd; +}; + +#define IPIPE_HEAD_PRIORITY (-1) /* For domains always heading the pipel= ine */ + +struct ipipe_domain_attr { + + unsigned domid; /* Domain identifier -- Magic value set by caller */ + const char *name; /* Domain name -- Warning: won't be dup'ed! */ + int priority; /* Priority in interrupt pipeline */ + void (*entry) (void); /* Domain entry point */ + void *pdd; /* Per-domain (opaque) data pointer */ +}; + +/* The following macros must be used hw interrupts off. */ + +#define __ipipe_irq_cookie(ipd,irq) (ipd)->irqs[irq].cookie +#define __ipipe_irq_handler(ipd,irq) (ipd)->irqs[irq].handler + +#define __ipipe_cpudata_irq_hits(ipd,cpuid,irq) ((ipd)->cpudata[cpuid].i= rq_counters[irq].total_hits) + +#define __ipipe_set_irq_bit(ipd,cpuid,irq) \ +do { \ + if (!test_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control)) { \ + __set_bit(irq & IPIPE_IRQ_IMASK,&(ipd)->cpudata[cpuid].irq_pending_lo[= irq >> IPIPE_IRQ_ISHIFT]); \ + __set_bit(irq >> IPIPE_IRQ_ISHIFT,&(ipd)->cpudata[cpuid].irq_pending_h= i); \ + } \ +} while(0) + +#define __ipipe_clear_pend(ipd,cpuid,irq) \ +do { \ + __clear_bit(irq & IPIPE_IRQ_IMASK,&(ipd)->cpudata[cpuid].irq_pending_lo= [irq >> IPIPE_IRQ_ISHIFT]); \ + if ((ipd)->cpudata[cpuid].irq_pending_lo[irq >> IPIPE_IRQ_ISHIFT] =3D=3D= 0) \ + __clear_bit(irq >> IPIPE_IRQ_ISHIFT,&(ipd)->cpudata[cpuid].irq_pending= _hi); \ +} while(0) + +#define __ipipe_lock_irq(ipd,cpuid,irq) \ +do { \ + if (!test_and_set_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control)) \ + __ipipe_clear_pend(ipd,cpuid,irq); \ +} while(0) + +#define __ipipe_unlock_irq(ipd,irq) \ +do { \ + int __cpuid, __nr_cpus =3D num_online_cpus(); \ + if (test_and_clear_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control)) \ + for (__cpuid =3D 0; __cpuid < __nr_cpus; __cpuid++) \ + if ((ipd)->cpudata[__cpuid].irq_counters[irq].pending_hits > 0) { /* = We need atomic ops next. */ \ + set_bit(irq & IPIPE_IRQ_IMASK,&(ipd)->cpudata[__cpuid].irq_pending_l= o[irq >> IPIPE_IRQ_ISHIFT]); \ + set_bit(irq >> IPIPE_IRQ_ISHIFT,&(ipd)->cpudata[__cpuid].irq_pending= _hi); \ + } \ +} while(0) + +#define __ipipe_clear_irq(ipd,irq) \ +do { \ + int __cpuid, __nr_cpus =3D num_online_cpus(); \ + clear_bit(IPIPE_LOCK_FLAG,&(ipd)->irqs[irq].control); \ + for (__cpuid =3D 0; __cpuid < __nr_cpus; __cpuid++) { \ + (ipd)->cpudata[__cpuid].irq_counters[irq].pending_hits =3D 0; \ + __ipipe_clear_pend(ipd,__cpuid,irq); \ + } \ +} while(0) + +#ifdef __RAW_SPIN_LOCK_UNLOCKED +#define spin_lock_hw(x) _raw_spin_lock(x) +#define spin_trylock_hw(x) _raw_spin_trylock(x) +#define spin_unlock_hw(x) _raw_spin_unlock(x) +#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) +#define write_lock_hw(x) _raw_write_lock(x) +#define write_trylock_hw(x) _raw_write_trylock(x) +#define write_unlock_hw(x) _raw_write_unlock(x) +#define read_lock_hw(x) _raw_read_lock(x) +#define read_trylock_hw(x) _raw_read_trylock(x) +#define read_unlock_hw(x) _raw_read_unlock(x) +#else /* UP non-debug */ +#define write_lock_hw(lock) do { (void)(lock); } while (0) +#define write_trylock_hw(lock) ({ (void)(lock); 1; }) +#define write_unlock_hw(lock) do { (void)(lock); } while (0) +#define read_lock_hw(lock) do { (void)(lock); } while (0) +#define read_trylock_hw(lock) ({ (void)(lock); 1; }) +#define read_unlock_hw(lock) do { (void)(lock); } while (0) +#endif /* CONFIG_SMP || CONFIG_DEBUG_SPINLOCK */ +#else /* !__RAW_SPIN_LOCK_UNLOCKED */ +#define spin_lock_hw(x) _spin_lock(x) +#define spin_unlock_hw(x) _spin_unlock(x) +#define spin_trylock_hw(x) _spin_trylock(x) +#define write_lock_hw(x) _write_lock(x) +#define write_unlock_hw(x) _write_unlock(x) +#define write_trylock_hw(x) _write_trylock(x) +#define read_lock_hw(x) _read_lock(x) +#define read_unlock_hw(x) _read_unlock(x) +#endif /* __RAW_SPIN_LOCK_UNLOCKED */ + +typedef spinlock_t ipipe_spinlock_t; +typedef rwlock_t ipipe_rwlock_t; +#define IPIPE_SPIN_LOCK_UNLOCKED SPIN_LOCK_UNLOCKED +#define IPIPE_RW_LOCK_UNLOCKED RW_LOCK_UNLOCKED + +#define spin_lock_irqsave_hw(x,flags) \ +do { \ + local_irq_save_hw(flags); \ + spin_lock_hw(x); \ +} while (0) + +#define spin_unlock_irqrestore_hw(x,flags) \ +do { \ + spin_unlock_hw(x); \ + local_irq_restore_hw(flags); \ +} while (0) + +#define spin_lock_irq_hw(x) \ +do { \ + local_irq_disable_hw(); \ + spin_lock_hw(x); \ +} while (0) + +#define spin_unlock_irq_hw(x) \ +do { \ + spin_unlock_hw(x); \ + local_irq_enable_hw(); \ +} while (0) + +#define read_lock_irqsave_hw(lock, flags) \ +do { \ + local_irq_save_hw(flags); \ + read_lock_hw(lock); \ +} while (0) + +#define read_unlock_irqrestore_hw(lock, flags) \ +do { \ + read_unlock_hw(lock); \ + local_irq_restore_hw(flags); \ +} while (0) + +#define write_lock_irqsave_hw(lock, flags) \ +do { \ + local_irq_save_hw(flags); \ + write_lock_hw(lock); \ +} while (0) + +#define write_unlock_irqrestore_hw(lock, flags) \ +do { \ + write_unlock_hw(lock); \ + local_irq_restore_hw(flags); \ +} while (0) + +extern struct ipipe_domain *ipipe_percpu_domain[], *ipipe_root_domain; + +extern unsigned __ipipe_printk_virq; + +extern unsigned long __ipipe_virtual_irq_map; + +extern struct list_head __ipipe_pipeline; + +extern ipipe_spinlock_t __ipipe_pipelock; + +extern int __ipipe_event_monitors[]; + +/* Private interface */ + +void ipipe_init(void); + +#ifdef CONFIG_PROC_FS +void ipipe_init_proc(void); + +#ifdef CONFIG_IPIPE_TRACE +void __ipipe_init_trace_proc(void); +#else /* !CONFIG_IPIPE_TRACE */ +#define __ipipe_init_trace_proc() do { } while(0) +#endif /* CONFIG_IPIPE_TRACE */ + +#else /* !CONFIG_PROC_FS */ +#define ipipe_init_proc() do { } while(0) +#endif /* CONFIG_PROC_FS */ + +void __ipipe_init_stage(struct ipipe_domain *ipd); + +void __ipipe_cleanup_domain(struct ipipe_domain *ipd); + +void __ipipe_add_domain_proc(struct ipipe_domain *ipd); + +void __ipipe_remove_domain_proc(struct ipipe_domain *ipd); + +void __ipipe_flush_printk(unsigned irq, void *cookie); + +void __ipipe_stall_root(void); + +void __ipipe_unstall_root(void); + +unsigned long __ipipe_test_root(void); + +unsigned long __ipipe_test_and_stall_root(void); + +void fastcall __ipipe_walk_pipeline(struct list_head *pos, int cpuid); + +void fastcall __ipipe_restore_root(unsigned long x); + +int fastcall __ipipe_schedule_irq(unsigned irq, struct list_head *head);= + +int fastcall __ipipe_dispatch_event(unsigned event, void *data); + +int fastcall __ipipe_dispatch_wired(struct ipipe_domain *head, unsigned = irq); + +void fastcall __ipipe_sync_stage(unsigned long syncmask); + +#ifndef __ipipe_sync_pipeline +#define __ipipe_sync_pipeline(syncmask) __ipipe_sync_stage(syncmask) +#endif + +#ifndef __ipipe_run_irqtail +#define __ipipe_run_irqtail() do { } while(0) +#endif + +#define __ipipe_pipeline_head_p(ipd) (&(ipd)->p_link =3D=3D __ipipe_pipe= line.next) + +/* + * Keep the following as a macro, so that client code could check for + * the support of the invariant pipeline head optimization. + */ +#define __ipipe_pipeline_head() list_entry(__ipipe_pipeline.next,struct = ipipe_domain,p_link) + +#define __ipipe_event_pipelined_p(ev) \ + (__ipipe_event_monitors[ev] > 0 || (ipipe_current_domain->evself & (1LL= << ev))) + +#ifdef CONFIG_SMP + +cpumask_t __ipipe_set_irq_affinity(unsigned irq, + cpumask_t cpumask); + +int fastcall __ipipe_send_ipi(unsigned ipi, + cpumask_t cpumask); + +#endif /* CONFIG_SMP */ + +/* Called with hw interrupts off. */ +static inline void __ipipe_switch_to(struct ipipe_domain *out, + struct ipipe_domain *in, int cpuid) +{ + void ipipe_suspend_domain(void); + + /* + * "in" is guaranteed to be closer than "out" from the head of the + * pipeline (and obviously different). + */ + + ipipe_percpu_domain[cpuid] =3D in; + + ipipe_suspend_domain(); /* Sync stage and propagate interrupts. */ + ipipe_load_cpuid(); /* Processor might have changed. */ + + if (ipipe_percpu_domain[cpuid] =3D=3D in) + /* + * Otherwise, something has changed the current domain under + * our feet recycling the register set; do not override. + */ + ipipe_percpu_domain[cpuid] =3D out; +} + +static inline void ipipe_sigwake_notify(struct task_struct *p) +{ + if (__ipipe_event_pipelined_p(IPIPE_EVENT_SIGWAKE)) + __ipipe_dispatch_event(IPIPE_EVENT_SIGWAKE,p); +} + +static inline void ipipe_setsched_notify(struct task_struct *p) +{ + if (__ipipe_event_pipelined_p(IPIPE_EVENT_SETSCHED)) + __ipipe_dispatch_event(IPIPE_EVENT_SETSCHED,p); +} + +static inline void ipipe_exit_notify(struct task_struct *p) +{ + if (__ipipe_event_pipelined_p(IPIPE_EVENT_EXIT)) + __ipipe_dispatch_event(IPIPE_EVENT_EXIT,p); +} + +static inline int ipipe_trap_notify(int ex, struct pt_regs *regs) +{ + return __ipipe_event_pipelined_p(ex) ? __ipipe_dispatch_event(ex,regs) = : 0; +} + +struct mm_struct; + +static inline void ipipe_cleanup_notify(struct mm_struct *mm) +{ + if (__ipipe_event_pipelined_p(IPIPE_EVENT_CLEANUP)) + __ipipe_dispatch_event(IPIPE_EVENT_CLEANUP,mm); +} + +/* Public interface */ + +int ipipe_register_domain(struct ipipe_domain *ipd, + struct ipipe_domain_attr *attr); + +int ipipe_unregister_domain(struct ipipe_domain *ipd); + +void ipipe_suspend_domain(void); + +int ipipe_virtualize_irq(struct ipipe_domain *ipd, + unsigned irq, + ipipe_irq_handler_t handler, + void *cookie, + ipipe_irq_ackfn_t acknowledge, + unsigned modemask); + +static inline int ipipe_share_irq(unsigned irq, + ipipe_irq_ackfn_t acknowledge) +{ + return ipipe_virtualize_irq(ipipe_current_domain, + irq, + IPIPE_SAME_HANDLER, + NULL, + acknowledge, + IPIPE_SHARED_MASK | IPIPE_HANDLE_MASK | + IPIPE_PASS_MASK); +} + +int ipipe_control_irq(unsigned irq, + unsigned clrmask, + unsigned setmask); + +unsigned ipipe_alloc_virq(void); + +int ipipe_free_virq(unsigned virq); + +int fastcall ipipe_trigger_irq(unsigned irq); + +static inline int ipipe_propagate_irq(unsigned irq) +{ + return __ipipe_schedule_irq(irq, ipipe_current_domain->p_link.next); +} + +static inline int ipipe_schedule_irq(unsigned irq) +{ + return __ipipe_schedule_irq(irq, &ipipe_current_domain->p_link); +} + +void fastcall ipipe_stall_pipeline_from(struct ipipe_domain *ipd); + +unsigned long fastcall ipipe_test_and_stall_pipeline_from(struct ipipe_d= omain *ipd); + +void fastcall ipipe_unstall_pipeline_from(struct ipipe_domain *ipd); + +unsigned long fastcall ipipe_test_and_unstall_pipeline_from(struct ipipe= _domain *ipd); + +void fastcall ipipe_restore_pipeline_from(struct ipipe_domain *ipd, + unsigned long x); + +static inline unsigned long ipipe_test_pipeline_from(struct ipipe_domain= *ipd) +{ + unsigned long flags, x; + ipipe_declare_cpuid; + + ipipe_get_cpu(flags); + x =3D test_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status); + ipipe_put_cpu(flags); + + return x; +} + +static inline void ipipe_restore_pipeline_nosync(struct ipipe_domain *ip= d, + unsigned long x, int cpuid) +{ + /* + * If cpuid is current, then it must be held on entry + * (ipipe_get_cpu/local_irq_save_hw/local_irq_disable_hw). + */ + + if (x) + __set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status); + else + __clear_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status); +} + +static inline void ipipe_stall_pipeline_head(void) +{ + ipipe_declare_cpuid; + unsigned long flags; + + ipipe_lock_cpu(flags); + __set_bit(IPIPE_STALL_FLAG, &__ipipe_pipeline_head()->cpudata[cpuid].st= atus); +} + +static inline unsigned long ipipe_test_and_stall_pipeline_head(void) +{ + unsigned long flags; + ipipe_declare_cpuid; + + ipipe_lock_cpu(flags); + return __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_pipeline_head()->c= pudata[cpuid].status); +} + +void ipipe_unstall_pipeline_head(void); + +void fastcall __ipipe_restore_pipeline_head(struct ipipe_domain *head, + unsigned long x); + +static inline void ipipe_restore_pipeline_head(unsigned long x) +{ + struct ipipe_domain *head =3D __ipipe_pipeline_head(); + /* On some archs, __test_and_set_bit() might return different + * truth value than test_bit(), so we test the exclusive OR of + * both statuses, assuming that the lowest bit is always set in + * the truth value (if this is wrong, the failed optimization will + * be caught in __ipipe_restore_pipeline_head() if + * CONFIG_DEBUG_KERNEL is set). */ + if ((x ^ test_bit(IPIPE_STALL_FLAG, &head->cpudata[ipipe_processor_id()= ].status)) & 1) + __ipipe_restore_pipeline_head(head,x); +} + +#define ipipe_unstall_pipeline() \ + ipipe_unstall_pipeline_from(ipipe_current_domain) + +#define ipipe_test_and_unstall_pipeline() \ + ipipe_test_and_unstall_pipeline_from(ipipe_current_domain) + +#define ipipe_test_pipeline() \ + ipipe_test_pipeline_from(ipipe_current_domain) + +#define ipipe_test_and_stall_pipeline() \ + ipipe_test_and_stall_pipeline_from(ipipe_current_domain) + +#define ipipe_stall_pipeline() \ + ipipe_stall_pipeline_from(ipipe_current_domain) + +#define ipipe_restore_pipeline(x) \ + ipipe_restore_pipeline_from(ipipe_current_domain, (x)) + +void ipipe_init_attr(struct ipipe_domain_attr *attr); + +int ipipe_get_sysinfo(struct ipipe_sysinfo *sysinfo); + +int ipipe_tune_timer(unsigned long ns, + int flags); + +unsigned long ipipe_critical_enter(void (*syncfn) (void)); + +void ipipe_critical_exit(unsigned long flags); + +static inline void ipipe_set_printk_sync(struct ipipe_domain *ipd) +{ + set_bit(IPIPE_SPRINTK_FLAG, &ipd->flags); +} + +static inline void ipipe_set_printk_async(struct ipipe_domain *ipd) +{ + clear_bit(IPIPE_SPRINTK_FLAG, &ipd->flags); +} + +static inline void ipipe_set_foreign_stack(struct ipipe_domain *ipd) +{ + /* Must be called hw interrupts off. */ + ipipe_declare_cpuid; + ipipe_load_cpuid(); + __set_bit(IPIPE_NOSTACK_FLAG, &ipd->cpudata[cpuid].status); +} + +static inline void ipipe_clear_foreign_stack(struct ipipe_domain *ipd) +{ + /* Must be called hw interrupts off. */ + ipipe_declare_cpuid; + ipipe_load_cpuid(); + __clear_bit(IPIPE_NOSTACK_FLAG, &ipd->cpudata[cpuid].status); +} + +#define ipipe_safe_current() \ +({ \ + ipipe_declare_cpuid; \ + struct task_struct *p; \ + ipipe_load_cpuid(); \ + p =3D test_bit(IPIPE_NOSTACK_FLAG, \ + &ipipe_percpu_domain[cpuid]->cpudata[cpuid].status) ? &init_task = : current; \ + p; \ +}) + +ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd, + unsigned event, + ipipe_event_handler_t handler); + +cpumask_t ipipe_set_irq_affinity(unsigned irq, + cpumask_t cpumask); + +int fastcall ipipe_send_ipi(unsigned ipi, + cpumask_t cpumask); + +int ipipe_setscheduler_root(struct task_struct *p, + int policy, + int prio); + +int ipipe_reenter_root(struct task_struct *prev, + int policy, + int prio); + +int ipipe_alloc_ptdkey(void); + +int ipipe_free_ptdkey(int key); + +int fastcall ipipe_set_ptd(int key, + void *value); + +void fastcall *ipipe_get_ptd(int key); + +#define local_irq_enable_hw_cond() local_irq_enable_hw() +#define local_irq_disable_hw_cond() local_irq_disable_hw() +#define local_irq_save_hw_cond(flags) local_irq_save_hw(flags) +#define local_irq_restore_hw_cond(flags) local_irq_restore_hw(flags) +#define spin_lock_irqsave_hw_cond(lock,flags) spin_lock_irqsave_hw(lock,= flags) +#define spin_unlock_irqrestore_hw_cond(lock,flags) spin_unlock_irqrestor= e_hw(lock,flags) +#define smp_processor_id_hw() ipipe_processor_id() + +#define ipipe_irq_lock(irq) \ + do { \ + ipipe_declare_cpuid; \ + ipipe_load_cpuid(); \ + __ipipe_lock_irq(ipipe_percpu_domain[cpuid], cpuid, irq);\ + } while(0) + +#define ipipe_irq_unlock(irq) \ + do { \ + ipipe_declare_cpuid; \ + ipipe_load_cpuid(); \ + __ipipe_unlock_irq(ipipe_percpu_domain[cpuid], irq); \ + } while(0) + +#define ipipe_root_domain_p (ipipe_current_domain =3D=3D ipipe_root_dom= ain) + +#else /* !CONFIG_IPIPE */ + +#define ipipe_init() do { } while(0) +#define ipipe_suspend_domain() do { } while(0) +#define ipipe_sigwake_notify(p) do { } while(0) +#define ipipe_setsched_notify(p) do { } while(0) +#define ipipe_exit_notify(p) do { } while(0) +#define ipipe_init_proc() do { } while(0) +#define ipipe_trap_notify(t,r) 0 +#define ipipe_cleanup_notify(mm) do { } while(0) + +#define spin_lock_hw(lock) spin_lock(lock) +#define spin_unlock_hw(lock) spin_unlock(lock) +#define spin_lock_irq_hw(lock) spin_lock_irq(lock) +#define spin_unlock_irq_hw(lock) spin_unlock_irq(lock) +#define spin_lock_irqsave_hw(lock,flags) spin_lock_irqsave(lock, flags) +#define spin_unlock_irqrestore_hw(lock,flags) spin_unlock_irqrestore(loc= k, flags) + +#define local_irq_enable_hw_cond() do { } while(0) +#define local_irq_disable_hw_cond() do { } while(0) +#define local_irq_save_hw_cond(flags) do { (void)(flags); } while(0) +#define local_irq_restore_hw_cond(flags) do { } while(0) +#define spin_lock_irqsave_hw_cond(lock,flags) do { (void)(flags); spin_l= ock(lock); } while(0) +#define spin_unlock_irqrestore_hw_cond(lock,flags) spin_unlock(lock) +#define smp_processor_id_hw() smp_processor_id() + +#define ipipe_irq_lock(irq) do { } while(0) +#define ipipe_irq_unlock(irq) do { } while(0) + +#define ipipe_root_domain_p 1 + +#endif /* CONFIG_IPIPE */ + +#endif /* !__LINUX_IPIPE_H */ Index: linux-2.6.17/include/linux/linkage.h =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/include/linux/linkage.h +++ linux-2.6.17/include/linux/linkage.h @@ -61,4 +61,8 @@ #define fastcall #endif =20 +#ifndef notrace +#define notrace __attribute__((no_instrument_function)) +#endif + #endif Index: linux-2.6.17/include/linux/preempt.h =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/include/linux/preempt.h +++ linux-2.6.17/include/linux/preempt.h @@ -27,29 +27,43 @@ =20 asmlinkage void preempt_schedule(void); =20 -#define preempt_disable() \ -do { \ - inc_preempt_count(); \ - barrier(); \ +#ifdef CONFIG_IPIPE +#include +extern struct ipipe_domain *ipipe_percpu_domain[], *ipipe_root_domain; +#define ipipe_preempt_guard() (ipipe_percpu_domain[ipipe_processor_id()]= =3D=3D ipipe_root_domain) +#else /* !CONFIG_IPIPE */ +#define ipipe_preempt_guard() 1 +#endif /* CONFIG_IPIPE */ + +#define preempt_disable() \ +do { \ + if (ipipe_preempt_guard()) { \ + inc_preempt_count(); \ + barrier(); \ + } \ } while (0) =20 -#define preempt_enable_no_resched() \ -do { \ - barrier(); \ - dec_preempt_count(); \ +#define preempt_enable_no_resched() \ +do { \ + if (ipipe_preempt_guard()) { \ + barrier(); \ + dec_preempt_count(); \ + } \ } while (0) =20 -#define preempt_check_resched() \ -do { \ - if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \ - preempt_schedule(); \ +#define preempt_check_resched() \ +do { \ + if (ipipe_preempt_guard()) { \ + if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \ + preempt_schedule(); \ + } \ } while (0) =20 -#define preempt_enable() \ -do { \ - preempt_enable_no_resched(); \ +#define preempt_enable() \ +do { \ + preempt_enable_no_resched(); \ barrier(); \ - preempt_check_resched(); \ + preempt_check_resched(); \ } while (0) =20 #else Index: linux-2.6.17/include/linux/sched.h =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/include/linux/sched.h +++ linux-2.6.17/include/linux/sched.h @@ -4,6 +4,7 @@ #include /* for HZ */ =20 #include +#include #include #include #include @@ -131,6 +132,11 @@ extern unsigned long nr_iowait(void); #define EXIT_DEAD 32 /* in tsk->state again */ #define TASK_NONINTERACTIVE 64 +#ifdef CONFIG_IPIPE +#define TASK_ATOMICSWITCH 512 +#else /* !CONFIG_IPIPE */ +#define TASK_ATOMICSWITCH 0 +#endif /* CONFIG_IPIPE */ =20 #define __set_task_state(tsk, state_value) \ do { (tsk)->state =3D (state_value); } while (0) @@ -883,6 +889,9 @@ struct task_struct { =20 atomic_t fs_excl; /* holding fs exclusive resources */ struct rcu_head rcu; +#ifdef CONFIG_IPIPE + void *ptd[IPIPE_ROOT_NPTDKEYS]; +#endif =20 /* * cache last used pipe for splice Index: linux-2.6.17/init/Kconfig =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/init/Kconfig +++ linux-2.6.17/init/Kconfig @@ -58,6 +58,7 @@ menu "General setup" =20 config LOCALVERSION string "Local version - append to kernel release" + default "-ipipe" help Append an extra string to the end of your kernel version. This will show up when you type uname, for example. Index: linux-2.6.17/init/main.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/init/main.c +++ linux-2.6.17/init/main.c @@ -490,6 +490,11 @@ asmlinkage void __init start_kernel(void hrtimers_init(); softirq_init(); time_init(); + /* + * We need to wait for the interrupt and time subsystems to be + * initialized before enabling the pipeline. + */ + ipipe_init(); =20 /* * HACK ALERT! This is early. We're enabling the console before @@ -623,6 +628,7 @@ static void __init do_basic_setup(void) #ifdef CONFIG_SYSCTL sysctl_init(); #endif + ipipe_init_proc(); =20 do_initcalls(); } Index: linux-2.6.17/kernel/Makefile =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/kernel/Makefile +++ linux-2.6.17/kernel/Makefile @@ -38,6 +38,7 @@ obj-$(CONFIG_GENERIC_HARDIRQS) +=3D irq/ obj-$(CONFIG_SECCOMP) +=3D seccomp.o obj-$(CONFIG_RCU_TORTURE_TEST) +=3D rcutorture.o obj-$(CONFIG_RELAY) +=3D relay.o +obj-$(CONFIG_IPIPE) +=3D ipipe/ =20 ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y) # According to Alan Modra , the -fno-omit-frame-p= ointer is Index: linux-2.6.17/kernel/exit.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/kernel/exit.c +++ linux-2.6.17/kernel/exit.c @@ -905,6 +905,7 @@ fastcall NORET_TYPE void do_exit(long co #endif if (unlikely(tsk->audit_context)) audit_free(tsk); + ipipe_exit_notify(tsk); exit_mm(tsk); =20 exit_sem(tsk); Index: linux-2.6.17/kernel/fork.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/kernel/fork.c +++ linux-2.6.17/kernel/fork.c @@ -369,6 +369,7 @@ void fastcall __mmdrop(struct mm_struct=20 void mmput(struct mm_struct *mm) { if (atomic_dec_and_test(&mm->mm_users)) { + ipipe_cleanup_notify(mm); exit_aio(mm); exit_mmap(mm); if (!list_empty(&mm->mmlist)) { @@ -1214,6 +1215,14 @@ static task_t *copy_process(unsigned lon spin_unlock(¤t->sighand->siglock); write_unlock_irq(&tasklist_lock); proc_fork_connector(p); +#ifdef CONFIG_IPIPE + { + int k; + + for (k =3D 0; k < IPIPE_ROOT_NPTDKEYS; k++) + p->ptd[k] =3D NULL; + } +#endif /* CONFIG_IPIPE */ return p; =20 bad_fork_cleanup_namespace: Index: linux-2.6.17/kernel/ipipe/Kconfig =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- /dev/null +++ linux-2.6.17/kernel/ipipe/Kconfig @@ -0,0 +1,6 @@ +config IPIPE + bool "Interrupt pipeline" + default y + ---help--- + Activate this option if you want the interrupt pipeline to be + compiled in. Index: linux-2.6.17/kernel/ipipe/Makefile =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- /dev/null +++ linux-2.6.17/kernel/ipipe/Makefile @@ -0,0 +1,3 @@ + +obj-$(CONFIG_IPIPE) +=3D core.o generic.o +obj-$(CONFIG_IPIPE_TRACE) +=3D tracer.o Index: linux-2.6.17/kernel/ipipe/core.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- /dev/null +++ linux-2.6.17/kernel/ipipe/core.c @@ -0,0 +1,1044 @@ +/* -*- linux-c -*- + * linux/kernel/ipipe/core.c + * + * Copyright (C) 2002-2005 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,= + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,= USA. + * + * Architecture-independent I-PIPE core support. + */ + +#include +#include +#include +#include +#ifdef CONFIG_PROC_FS +#include +#endif /* CONFIG_PROC_FS */ + +static struct ipipe_domain ipipe_root =3D + { .cpudata =3D {[0 ... IPIPE_NR_CPUS-1] =3D + { .status =3D (1<name =3D "Linux"; + ipd->domid =3D IPIPE_ROOT_ID; + ipd->priority =3D IPIPE_ROOT_PRIO; + + __ipipe_init_stage(ipd); + + INIT_LIST_HEAD(&ipd->p_link); + list_add_tail(&ipd->p_link, &__ipipe_pipeline); + + __ipipe_init_platform(); + +#ifdef CONFIG_PRINTK + __ipipe_printk_virq =3D ipipe_alloc_virq(); /* Cannot fail here. */ + ipd->irqs[__ipipe_printk_virq].handler =3D &__ipipe_flush_printk; + ipd->irqs[__ipipe_printk_virq].cookie =3D NULL; + ipd->irqs[__ipipe_printk_virq].acknowledge =3D NULL; + ipd->irqs[__ipipe_printk_virq].control =3D IPIPE_HANDLE_MASK; +#endif /* CONFIG_PRINTK */ + + __ipipe_enable_pipeline(); + + printk(KERN_INFO "I-pipe %s: pipeline enabled.\n", + IPIPE_VERSION_STRING); +} + +void __ipipe_init_stage(struct ipipe_domain *ipd) +{ + int cpuid, n; + + for (cpuid =3D 0; cpuid < IPIPE_NR_CPUS; cpuid++) { + ipd->cpudata[cpuid].irq_pending_hi =3D 0; + + for (n =3D 0; n < IPIPE_IRQ_IWORDS; n++) + ipd->cpudata[cpuid].irq_pending_lo[n] =3D 0; + + for (n =3D 0; n < IPIPE_NR_IRQS; n++) { + ipd->cpudata[cpuid].irq_counters[n].total_hits =3D 0; + ipd->cpudata[cpuid].irq_counters[n].pending_hits =3D 0; + } + } + + for (n =3D 0; n < IPIPE_NR_IRQS; n++) { + ipd->irqs[n].acknowledge =3D NULL; + ipd->irqs[n].handler =3D NULL; + ipd->irqs[n].control =3D IPIPE_PASS_MASK; /* Pass but don't handle */ + } + + for (n =3D 0; n < IPIPE_NR_EVENTS; n++) + ipd->evhand[n] =3D NULL; + + ipd->evself =3D 0; + +#ifdef CONFIG_SMP + ipd->irqs[IPIPE_CRITICAL_IPI].acknowledge =3D &__ipipe_ack_system_irq; + ipd->irqs[IPIPE_CRITICAL_IPI].handler =3D &__ipipe_do_critical_sync; + ipd->irqs[IPIPE_CRITICAL_IPI].cookie =3D NULL; + /* Immediately handle in the current domain but *never* pass */ + ipd->irqs[IPIPE_CRITICAL_IPI].control =3D + IPIPE_HANDLE_MASK|IPIPE_STICKY_MASK|IPIPE_SYSTEM_MASK; +#endif /* CONFIG_SMP */ +} + +void __ipipe_stall_root(void) +{ + ipipe_declare_cpuid; + unsigned long flags; + + ipipe_get_cpu(flags); /* Care for migration. */ + + set_bit(IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].status); + +#ifdef CONFIG_SMP + if (!__ipipe_pipeline_head_p(ipipe_root_domain)) + ipipe_put_cpu(flags); +#else /* CONFIG_SMP */ + if (__ipipe_pipeline_head_p(ipipe_root_domain)) + local_irq_disable_hw(); +#endif /* CONFIG_SMP */ +} + +void __ipipe_cleanup_domain(struct ipipe_domain *ipd) +{ + ipipe_unstall_pipeline_from(ipd); + +#ifdef CONFIG_SMP + { + int cpu; + + for_each_online_cpu(cpu) { + while (ipd->cpudata[cpu].irq_pending_hi !=3D 0) + cpu_relax(); + } + } +#endif /* CONFIG_SMP */ +} + +void __ipipe_unstall_root(void) +{ + ipipe_declare_cpuid; + + local_irq_disable_hw(); + + ipipe_load_cpuid(); + + __clear_bit(IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].status= ); + + if (ipipe_root_domain->cpudata[cpuid].irq_pending_hi !=3D 0) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + + local_irq_enable_hw(); +} + +unsigned long __ipipe_test_root(void) +{ + unsigned long flags, x; + ipipe_declare_cpuid; + + ipipe_get_cpu(flags); /* Care for migration. */ + x =3D test_bit(IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].sta= tus); + ipipe_put_cpu(flags); + + return x; +} + +unsigned long __ipipe_test_and_stall_root(void) +{ + unsigned long flags, x; + ipipe_declare_cpuid; + + ipipe_get_cpu(flags); /* Care for migration. */ + x =3D test_and_set_bit(IPIPE_STALL_FLAG, + &ipipe_root_domain->cpudata[cpuid].status); + ipipe_put_cpu(flags); + + return x; +} + +void fastcall __ipipe_restore_root(unsigned long x) +{ + if (x) + __ipipe_stall_root(); + else + __ipipe_unstall_root(); +} + +void fastcall ipipe_stall_pipeline_from(struct ipipe_domain *ipd) +{ + ipipe_declare_cpuid; +#ifdef CONFIG_SMP + unsigned long flags; + + ipipe_lock_cpu(flags); /* Care for migration. */ + + __set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status); + + if (!__ipipe_pipeline_head_p(ipd)) + ipipe_unlock_cpu(flags); +#else /* CONFIG_SMP */ + set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status); + + if (__ipipe_pipeline_head_p(ipd)) + local_irq_disable_hw(); +#endif /* CONFIG_SMP */ +} + +unsigned long fastcall ipipe_test_and_stall_pipeline_from(struct ipipe_d= omain *ipd) +{ + ipipe_declare_cpuid; + unsigned long s; +#ifdef CONFIG_SMP + unsigned long flags; + + ipipe_lock_cpu(flags); /* Care for migration. */ + + s =3D __test_and_set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status)= ; + + if (!__ipipe_pipeline_head_p(ipd)) + ipipe_unlock_cpu(flags); +#else /* CONFIG_SMP */ + s =3D test_and_set_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status); + + if (__ipipe_pipeline_head_p(ipd)) + local_irq_disable_hw(); +#endif /* CONFIG_SMP */ + + return s; +} + +/* + * ipipe_unstall_pipeline_from() -- Unstall the pipeline and + * synchronize pending interrupts for a given domain. See + * __ipipe_walk_pipeline() for more information. + */ +void fastcall ipipe_unstall_pipeline_from(struct ipipe_domain *ipd) +{ + struct list_head *pos; + unsigned long flags; + ipipe_declare_cpuid; + + ipipe_lock_cpu(flags); + + __clear_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status); + + if (ipd =3D=3D ipipe_percpu_domain[cpuid]) + pos =3D &ipd->p_link; + else + pos =3D __ipipe_pipeline.next; + + __ipipe_walk_pipeline(pos, cpuid); + + if (__ipipe_pipeline_head_p(ipd)) + local_irq_enable_hw(); + else + ipipe_unlock_cpu(flags); +} + +unsigned long fastcall ipipe_test_and_unstall_pipeline_from(struct ipipe= _domain *ipd) +{ + unsigned long flags, x; + ipipe_declare_cpuid; + + ipipe_get_cpu(flags); + x =3D test_bit(IPIPE_STALL_FLAG, &ipd->cpudata[cpuid].status); + ipipe_unstall_pipeline_from(ipd); + ipipe_put_cpu(flags); + + return x; +} + +void fastcall ipipe_restore_pipeline_from(struct ipipe_domain *ipd, + unsigned long x) +{ + if (x) + ipipe_stall_pipeline_from(ipd); + else + ipipe_unstall_pipeline_from(ipd); +} + +void ipipe_unstall_pipeline_head(void) +{ + struct ipipe_domain *head; + unsigned long flags; + ipipe_declare_cpuid; + + ipipe_lock_cpu(flags); + head =3D __ipipe_pipeline_head(); + __clear_bit(IPIPE_STALL_FLAG, &head->cpudata[cpuid].status); + + if (unlikely(head->cpudata[cpuid].irq_pending_hi !=3D 0)) { + if (likely(head =3D=3D ipipe_percpu_domain[cpuid])) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + else + __ipipe_walk_pipeline(&head->p_link, cpuid); + } + + local_irq_enable_hw(); +} + +void fastcall __ipipe_restore_pipeline_head(struct ipipe_domain *head, u= nsigned long x) +{ + ipipe_declare_cpuid; + unsigned long flags; + + ipipe_lock_cpu(flags); + + if (x) { +#ifdef CONFIG_DEBUG_KERNEL + static int warned; + if (!warned && test_and_set_bit(IPIPE_STALL_FLAG, &head->cpudata[cpuid= ].status)) { + /* + * Already stalled albeit ipipe_restore_pipeline_head() + * should have detected it? Send a warning once.\n"); + */ + warned =3D 1; + printk(KERN_WARNING + "I-pipe: ipipe_restore_pipeline_head() optimization failed.\n"); + dump_stack(); + } +#else /* !CONFIG_DEBUG_KERNEL */ + set_bit(IPIPE_STALL_FLAG, &head->cpudata[cpuid].status); +#endif /* CONFIG_DEBUG_KERNEL */ + } + else { + __clear_bit(IPIPE_STALL_FLAG, &head->cpudata[cpuid].status); + if (unlikely(head->cpudata[cpuid].irq_pending_hi !=3D 0)) { + if (likely(head =3D=3D ipipe_percpu_domain[cpuid])) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + else + __ipipe_walk_pipeline(&head->p_link, cpuid); + } + local_irq_enable_hw(); + } +} + +/* __ipipe_walk_pipeline(): Plays interrupts pending in the log. Must + be called with local hw interrupts disabled. */ + +void fastcall __ipipe_walk_pipeline(struct list_head *pos, int cpuid) +{ + struct ipipe_domain *this_domain =3D ipipe_percpu_domain[cpuid]; + + while (pos !=3D &__ipipe_pipeline) { + struct ipipe_domain *next_domain =3D + list_entry(pos, struct ipipe_domain, p_link); + + if (test_bit + (IPIPE_STALL_FLAG, &next_domain->cpudata[cpuid].status)) + break; /* Stalled stage -- do not go further. */ + + if (next_domain->cpudata[cpuid].irq_pending_hi !=3D 0) { + + if (next_domain =3D=3D this_domain) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + else { + __ipipe_switch_to(this_domain, next_domain, + cpuid); + + ipipe_load_cpuid(); /* Processor might have changed. */ + + if (this_domain->cpudata[cpuid]. + irq_pending_hi !=3D 0 + && !test_bit(IPIPE_STALL_FLAG, + &this_domain->cpudata[cpuid].status)) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + } + + break; + } else if (next_domain =3D=3D this_domain) + break; + + pos =3D next_domain->p_link.next; + } +} + +/* + * ipipe_suspend_domain() -- Suspend the current domain, switching to + * the next one which has pending work down the pipeline. + */ +void ipipe_suspend_domain(void) +{ + struct ipipe_domain *this_domain, *next_domain; + struct list_head *ln; + unsigned long flags; + ipipe_declare_cpuid; + + ipipe_lock_cpu(flags); + + this_domain =3D next_domain =3D ipipe_percpu_domain[cpuid]; + + __clear_bit(IPIPE_STALL_FLAG, &this_domain->cpudata[cpuid].status); + + if (this_domain->cpudata[cpuid].irq_pending_hi !=3D 0) + goto sync_stage; + + for (;;) { + ln =3D next_domain->p_link.next; + + if (ln =3D=3D &__ipipe_pipeline) + break; + + next_domain =3D list_entry(ln, struct ipipe_domain, p_link); + + if (test_bit(IPIPE_STALL_FLAG, + &next_domain->cpudata[cpuid].status)) + break; + + if (next_domain->cpudata[cpuid].irq_pending_hi =3D=3D 0) + continue; + + ipipe_percpu_domain[cpuid] =3D next_domain; + +sync_stage: + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + + ipipe_load_cpuid(); /* Processor might have changed. */ + + if (ipipe_percpu_domain[cpuid] !=3D next_domain) + /* + * Something has changed the current domain under our + * feet, recycling the register set; take note. + */ + this_domain =3D ipipe_percpu_domain[cpuid]; + } + + ipipe_percpu_domain[cpuid] =3D this_domain; + + ipipe_unlock_cpu(flags); +} + +/* ipipe_alloc_virq() -- Allocate a pipelined virtual/soft interrupt. + * Virtual interrupts are handled in exactly the same way than their + * hw-generated counterparts wrt pipelining. + */ +unsigned ipipe_alloc_virq(void) +{ + unsigned long flags, irq =3D 0; + int ipos; + + spin_lock_irqsave_hw(&__ipipe_pipelock, flags); + + if (__ipipe_virtual_irq_map !=3D ~0) { + ipos =3D ffz(__ipipe_virtual_irq_map); + set_bit(ipos, &__ipipe_virtual_irq_map); + irq =3D ipos + IPIPE_VIRQ_BASE; + } + + spin_unlock_irqrestore_hw(&__ipipe_pipelock, flags); + + return irq; +} + +/* ipipe_virtualize_irq() -- Attach a handler (and optionally a hw + acknowledge routine) to an interrupt for a given domain. */ + +int ipipe_virtualize_irq(struct ipipe_domain *ipd, + unsigned irq, + ipipe_irq_handler_t handler, + void *cookie, + ipipe_irq_ackfn_t acknowledge, + unsigned modemask) +{ + unsigned long flags; + int err; + + if (irq >=3D IPIPE_NR_IRQS) + return -EINVAL; + + if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK) + return -EPERM; + + if (!test_bit(IPIPE_AHEAD_FLAG, &ipd->flags)) + /* Silently unwire interrupts for non-heading domains. */ + modemask &=3D ~IPIPE_WIRED_MASK; + + spin_lock_irqsave_hw(&__ipipe_pipelock, flags); + + if (handler !=3D NULL) { + + if (handler =3D=3D IPIPE_SAME_HANDLER) { + handler =3D ipd->irqs[irq].handler; + cookie =3D ipd->irqs[irq].cookie; + + if (handler =3D=3D NULL) { + err =3D -EINVAL; + goto unlock_and_exit; + } + } else if ((modemask & IPIPE_EXCLUSIVE_MASK) !=3D 0 && + ipd->irqs[irq].handler !=3D NULL) { + err =3D -EBUSY; + goto unlock_and_exit; + } + + if ((modemask & (IPIPE_SHARED_MASK | IPIPE_PASS_MASK)) =3D=3D + IPIPE_SHARED_MASK) { + err =3D -EINVAL; + goto unlock_and_exit; + } + + /* Wired interrupts can only be delivered to domains + * always heading the pipeline. */ + + if ((modemask & IPIPE_WIRED_MASK) !=3D 0) { + if ((modemask & (IPIPE_SHARED_MASK | IPIPE_PASS_MASK | IPIPE_STICKY_M= ASK)) !=3D 0) { + err =3D -EINVAL; + goto unlock_and_exit; + } + modemask |=3D (IPIPE_HANDLE_MASK); + } + + if ((modemask & IPIPE_STICKY_MASK) !=3D 0) + modemask |=3D IPIPE_HANDLE_MASK; + } else + modemask &=3D + ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK | + IPIPE_SHARED_MASK | IPIPE_EXCLUSIVE_MASK | IPIPE_WIRED_MASK); + + if (acknowledge =3D=3D NULL) { + if ((modemask & IPIPE_SHARED_MASK) =3D=3D 0) { + if (!ipipe_virtual_irq_p(irq)) { + /* Acknowledge handler unspecified for a hw + interrupt -- this is ok in non-shared + management mode, but we will force the use + of the Linux-defined handler instead. */ + acknowledge =3D ipipe_root_domain->irqs[irq].acknowledge; + } + } + else { + /* A valid acknowledge handler to be called in shared mode + is required when declaring a shared IRQ. */ + err =3D -EINVAL; + goto unlock_and_exit; + } + } + + ipd->irqs[irq].handler =3D handler; + ipd->irqs[irq].cookie =3D cookie; + ipd->irqs[irq].acknowledge =3D acknowledge; + ipd->irqs[irq].control =3D modemask; + + if (irq < NR_IRQS && + handler !=3D NULL && + !ipipe_virtual_irq_p(irq) && (modemask & IPIPE_ENABLE_MASK) !=3D 0)= { + if (ipd !=3D ipipe_current_domain) { + /* IRQ enable/disable state is domain-sensitive, so we may + not change it for another domain. What is allowed + however is forcing some domain to handle an interrupt + source, by passing the proper 'ipd' descriptor which + thus may be different from ipipe_current_domain. */ + err =3D -EPERM; + goto unlock_and_exit; + } + + __ipipe_enable_irq(irq); + } + + err =3D 0; + + unlock_and_exit: + + spin_unlock_irqrestore_hw(&__ipipe_pipelock, flags); + + return err; +} + +/* ipipe_control_irq() -- Change modes of a pipelined interrupt for + * the current domain. */ + +int ipipe_control_irq(unsigned irq, unsigned clrmask, unsigned setmask) +{ + struct ipipe_domain *ipd; + unsigned long flags; + + if (irq >=3D IPIPE_NR_IRQS) + return -EINVAL; + + ipd =3D ipipe_current_domain; + + if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK) + return -EPERM; + + if (((setmask | clrmask) & IPIPE_SHARED_MASK) !=3D 0) + return -EINVAL; + + if (ipd->irqs[irq].handler =3D=3D NULL) + setmask &=3D ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK); + + if ((setmask & IPIPE_STICKY_MASK) !=3D 0) + setmask |=3D IPIPE_HANDLE_MASK; + + if ((clrmask & (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK)) !=3D 0) /* If o= ne goes, both go. */ + clrmask |=3D (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK); + + spin_lock_irqsave_hw(&__ipipe_pipelock, flags); + + ipd->irqs[irq].control &=3D ~clrmask; + ipd->irqs[irq].control |=3D setmask; + + if ((setmask & IPIPE_ENABLE_MASK) !=3D 0) + __ipipe_enable_irq(irq); + else if ((clrmask & IPIPE_ENABLE_MASK) !=3D 0) + __ipipe_disable_irq(irq); + + spin_unlock_irqrestore_hw(&__ipipe_pipelock, flags); + + return 0; +} + +/* __ipipe_dispatch_event() -- Low-level event dispatcher. */ + +int fastcall __ipipe_dispatch_event (unsigned event, void *data) +{ + struct ipipe_domain *start_domain, *this_domain, *next_domain; + struct list_head *pos, *npos; + unsigned long flags; + ipipe_declare_cpuid; + int propagate =3D 1; + + ipipe_lock_cpu(flags); + + start_domain =3D this_domain =3D ipipe_percpu_domain[cpuid]; + + list_for_each_safe(pos,npos,&__ipipe_pipeline) { + + next_domain =3D list_entry(pos,struct ipipe_domain,p_link); + + /* + * Note: Domain migration may occur while running + * event or interrupt handlers, in which case the + * current register set is going to be recycled for a + * different domain than the initiating one. We do + * care for that, always tracking the current domain + * descriptor upon return from those handlers. + */ + if (next_domain->evhand[event] !=3D NULL) { + ipipe_percpu_domain[cpuid] =3D next_domain; + ipipe_unlock_cpu(flags); + propagate =3D !next_domain->evhand[event](event,start_domain,data); + ipipe_lock_cpu(flags); + if (ipipe_percpu_domain[cpuid] !=3D next_domain) + this_domain =3D ipipe_percpu_domain[cpuid]; + } + + if (next_domain !=3D ipipe_root_domain && /* NEVER sync the root stage= here. */ + next_domain->cpudata[cpuid].irq_pending_hi !=3D 0 && + !test_bit(IPIPE_STALL_FLAG,&next_domain->cpudata[cpuid].status)) {= + ipipe_percpu_domain[cpuid] =3D next_domain; + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + ipipe_load_cpuid(); + if (ipipe_percpu_domain[cpuid] !=3D next_domain) + this_domain =3D ipipe_percpu_domain[cpuid]; + } + + ipipe_percpu_domain[cpuid] =3D this_domain; + + if (next_domain =3D=3D this_domain || !propagate) + break; + } + + ipipe_unlock_cpu(flags); + + return !propagate; +} + +/* + * __ipipe_dispatch_wired -- Wired interrupt dispatcher. Wired + * interrupts are immediately and unconditionally delivered to the + * domain heading the pipeline upon receipt, and such domain must have + * been registered as an invariant head for the system (priority =3D=3D + * IPIPE_HEAD_PRIORITY). The motivation for using wired interrupts is + * to get an extra-fast dispatching path for those IRQs, by relying on + * a straightforward logic based on assumptions that must always be + * true for invariant head domains. The following assumptions are + * made when dealing with such interrupts: + * + * 1- Wired interrupts are purely dynamic, i.e. the decision to + * propagate them down the pipeline must be done from the head domain + * ISR. + * 2- Wired interrupts cannot be shared or sticky. + * 3- The root domain cannot be an invariant pipeline head, in + * consequence of what the root domain cannot handle wired + * interrupts. + * 4- Wired interrupts must have a valid acknowledge handler for the + * head domain (if needed), and in any case, must not rely on handlers + * provided by lower priority domains during the acknowledge cycle + * (see __ipipe_handle_irq). + * + * Called with hw interrupts off. + */ + +int fastcall __ipipe_dispatch_wired(struct ipipe_domain *head, unsigned = irq) +{ + struct ipcpudata *cpudata; + struct ipipe_domain *old; + ipipe_declare_cpuid; + + ipipe_load_cpuid(); + cpudata =3D &head->cpudata[cpuid]; + cpudata->irq_counters[irq].total_hits++; + + if (test_bit(IPIPE_LOCK_FLAG, &head->irqs[irq].control)) { + /* If we can't process this IRQ right now, we must + * mark it as pending, so that it will get played + * during normal log sync when the corresponding + * interrupt source is eventually unlocked. */ + cpudata->irq_counters[irq].pending_hits++; + return 0; + } + + if (test_bit(IPIPE_STALL_FLAG, &cpudata->status)) { + cpudata->irq_counters[irq].pending_hits++; + __ipipe_set_irq_bit(head, cpuid, irq); + return 0; + } + + old =3D ipipe_percpu_domain[cpuid]; + ipipe_percpu_domain[cpuid] =3D head; /* Switch to the head domain. */ + + __set_bit(IPIPE_STALL_FLAG, &cpudata->status); + head->irqs[irq].handler(irq,head->irqs[irq].cookie); /* Call the ISR. *= / + __ipipe_run_irqtail(); + __clear_bit(IPIPE_STALL_FLAG, &cpudata->status); + + /* We expect the caller to start a complete pipeline walk upon + * return, so that propagated interrupts will get played. */ + + if (ipipe_percpu_domain[cpuid] =3D=3D head) + ipipe_percpu_domain[cpuid] =3D old; /* Back to the preempted domain. *= / + + return 1; +} + +/* + * __ipipe_sync_stage() -- Flush the pending IRQs for the current + * domain (and processor). This routine flushes the interrupt log + * (see "Optimistic interrupt protection" from D. Stodolsky et al. for + * more on the deferred interrupt scheme). Every interrupt that + * occurred while the pipeline was stalled gets played. WARNING: + * callers on SMP boxen should always check for CPU migration on + * return of this routine. One can control the kind of interrupts + * which are going to be sync'ed using the syncmask + * parameter. IPIPE_IRQMASK_ANY plays them all, IPIPE_IRQMASK_VIRT + * plays virtual interrupts only. + * + * This routine must be called with hw interrupts off. + */ +void fastcall __ipipe_sync_stage(unsigned long syncmask) +{ + unsigned long mask, submask; + struct ipcpudata *cpudata; + struct ipipe_domain *ipd; + ipipe_declare_cpuid; + int level, rank; + unsigned irq; + + ipipe_load_cpuid(); + ipd =3D ipipe_percpu_domain[cpuid]; + cpudata =3D &ipd->cpudata[cpuid]; + + if (__test_and_set_bit(IPIPE_SYNC_FLAG, &cpudata->status)) + return; + + /* + * The policy here is to keep the dispatching code interrupt-free + * by stalling the current stage. If the upper domain handler + * (which we call) wants to re-enable interrupts while in a safe + * portion of the code (e.g. SA_INTERRUPT flag unset for Linux's + * sigaction()), it will have to unstall (then stall again before + * returning to us!) the stage when it sees fit. + */ + while ((mask =3D (cpudata->irq_pending_hi & syncmask)) !=3D 0) { + level =3D __ipipe_ffnz(mask); + __clear_bit(level, &cpudata->irq_pending_hi); + + while ((submask =3D cpudata->irq_pending_lo[level]) !=3D 0) { + rank =3D __ipipe_ffnz(submask); + irq =3D (level << IPIPE_IRQ_ISHIFT) + rank; + + if (test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control)) { + __clear_bit(rank, &cpudata->irq_pending_lo[level]); + continue; + } + + if (--cpudata->irq_counters[irq].pending_hits =3D=3D 0) + __clear_bit(rank, &cpudata->irq_pending_lo[level]); + + __set_bit(IPIPE_STALL_FLAG, &cpudata->status); + __ipipe_run_isr(ipd, irq, cpuid); +#ifdef CONFIG_SMP + { + int _cpuid =3D ipipe_processor_id(); + + if (_cpuid !=3D cpuid) { /* Handle CPU migration. */ + /* + * We expect any domain to clear the SYNC bit each + * time it switches in a new task, so that preemptions + * and/or CPU migrations (in the SMP case) over the + * ISR do not lock out the log syncer for some + * indefinite amount of time. In the Linux case, + * schedule() handles this (see kernel/sched.c). For + * this reason, we don't bother clearing it here for + * the source CPU in the migration handling case, + * since it must have scheduled another task in by + * now. + */ + cpuid =3D _cpuid; + cpudata =3D &ipd->cpudata[cpuid]; + __set_bit(IPIPE_SYNC_FLAG, &cpudata->status); + } + } +#endif /* CONFIG_SMP */ + + __clear_bit(IPIPE_STALL_FLAG, &cpudata->status); + } + } + + __clear_bit(IPIPE_SYNC_FLAG, &cpudata->status); +} + +#ifdef CONFIG_PROC_FS + +#include + +struct proc_dir_entry *ipipe_proc_root; + +static int __ipipe_version_info_proc(char *page, + char **start, + off_t off, int count, int *eof, void *data) +{ + int len =3D sprintf(page, "%s\n", IPIPE_VERSION_STRING); + + len -=3D off; + + if (len <=3D off + count) + *eof =3D 1; + + *start =3D page + off; + + if(len > count) + len =3D count; + + if(len < 0) + len =3D 0; + + return len; +} + +static int __ipipe_common_info_proc(char *page, + char **start, + off_t off, int count, int *eof, void *data) +{ + struct ipipe_domain *ipd =3D (struct ipipe_domain *)data; + unsigned long ctlbits; + unsigned irq, _irq; + char *p =3D page; + int len; + + spin_lock(&__ipipe_pipelock); + + if (test_bit(IPIPE_AHEAD_FLAG,&ipd->flags)) + p +=3D sprintf(p, "Invariant head"); + else + p +=3D sprintf(p, "Priority=3D%d", ipd->priority); + + p +=3D sprintf(p, ", Id=3D0x%.8x\n", ipd->domid); + + irq =3D 0; + + while (irq < IPIPE_NR_IRQS) { + ctlbits =3D + (ipd->irqs[irq]. + control & (IPIPE_HANDLE_MASK | IPIPE_PASS_MASK | + IPIPE_STICKY_MASK | IPIPE_WIRED_MASK)); + if (irq >=3D IPIPE_NR_XIRQS && !ipipe_virtual_irq_p(irq)) { + /* + * There might be a hole between the last external + * IRQ and the first virtual one; skip it. + */ + irq++; + continue; + } + + if (ipipe_virtual_irq_p(irq) + && !test_bit(irq - IPIPE_VIRQ_BASE, + &__ipipe_virtual_irq_map)) { + /* Non-allocated virtual IRQ; skip it. */ + irq++; + continue; + } + + /* + * Attempt to group consecutive IRQ numbers having the + * same virtualization settings in a single line. + */ + + _irq =3D irq; + + while (++_irq < IPIPE_NR_IRQS) { + if (ipipe_virtual_irq_p(_irq) !=3D + ipipe_virtual_irq_p(irq) + || (ipipe_virtual_irq_p(_irq) + && !test_bit(_irq - IPIPE_VIRQ_BASE, + &__ipipe_virtual_irq_map)) + || ctlbits !=3D (ipd->irqs[_irq]. + control & (IPIPE_HANDLE_MASK | + IPIPE_PASS_MASK | + IPIPE_STICKY_MASK))) + break; + } + + if (_irq =3D=3D irq + 1) + p +=3D sprintf(p, "irq%u: ", irq); + else + p +=3D sprintf(p, "irq%u-%u: ", irq, _irq - 1); + + /* + * Statuses are as follows: + * o "accepted" means handled _and_ passed down the pipeline. + * o "grabbed" means handled, but the interrupt might be + * terminated _or_ passed down the pipeline depending on + * what the domain handler asks for to the I-pipe. + * o "wired" is basically the same as "grabbed", except that + * the interrupt is unconditionally delivered to an invariant + * pipeline head domain. + * o "passed" means unhandled by the domain but passed + * down the pipeline. + * o "discarded" means unhandled and _not_ passed down the + * pipeline. The interrupt merely disappears from the + * current domain down to the end of the pipeline. + */ + if (ctlbits & IPIPE_HANDLE_MASK) { + if (ctlbits & IPIPE_PASS_MASK) + p +=3D sprintf(p, "accepted"); + else if (ctlbits & IPIPE_WIRED_MASK) + p +=3D sprintf(p, "wired"); + else + p +=3D sprintf(p, "grabbed"); + } else if (ctlbits & IPIPE_PASS_MASK) + p +=3D sprintf(p, "passed"); + else + p +=3D sprintf(p, "discarded"); + + if (ctlbits & IPIPE_STICKY_MASK) + p +=3D sprintf(p, ", sticky"); + + if (ipipe_virtual_irq_p(irq)) + p +=3D sprintf(p, ", virtual"); + + p +=3D sprintf(p, "\n"); + + irq =3D _irq; + } + + spin_unlock(&__ipipe_pipelock); + + len =3D p - page; + + if (len <=3D off + count) + *eof =3D 1; + + *start =3D page + off; + + len -=3D off; + + if (len > count) + len =3D count; + + if (len < 0) + len =3D 0; + + return len; +} + +void __ipipe_add_domain_proc(struct ipipe_domain *ipd) +{ + create_proc_read_entry(ipd->name,0444,ipipe_proc_root,&__ipipe_common_i= nfo_proc,ipd); +} + +void __ipipe_remove_domain_proc(struct ipipe_domain *ipd) +{ + remove_proc_entry(ipd->name,ipipe_proc_root); +} + +void ipipe_init_proc(void) +{ + ipipe_proc_root =3D create_proc_entry("ipipe",S_IFDIR, 0); + create_proc_read_entry("version",0444,ipipe_proc_root,&__ipipe_version_= info_proc,NULL); + __ipipe_init_trace_proc(); + __ipipe_add_domain_proc(ipipe_root_domain); +} + +#endif /* CONFIG_PROC_FS */ + +EXPORT_SYMBOL(ipipe_virtualize_irq); +EXPORT_SYMBOL(ipipe_control_irq); +EXPORT_SYMBOL(ipipe_suspend_domain); +EXPORT_SYMBOL(ipipe_alloc_virq); +EXPORT_SYMBOL(ipipe_percpu_domain); +EXPORT_SYMBOL(ipipe_root_domain); +EXPORT_SYMBOL(ipipe_stall_pipeline_from); +EXPORT_SYMBOL(ipipe_test_and_stall_pipeline_from); +EXPORT_SYMBOL(ipipe_unstall_pipeline_from); +EXPORT_SYMBOL(ipipe_restore_pipeline_from); +EXPORT_SYMBOL(ipipe_test_and_unstall_pipeline_from); +EXPORT_SYMBOL(ipipe_unstall_pipeline_head); +EXPORT_SYMBOL(__ipipe_restore_pipeline_head); +EXPORT_SYMBOL(__ipipe_unstall_root); +EXPORT_SYMBOL(__ipipe_stall_root); +EXPORT_SYMBOL(__ipipe_restore_root); +EXPORT_SYMBOL(__ipipe_test_and_stall_root); +EXPORT_SYMBOL(__ipipe_test_root); +EXPORT_SYMBOL(__ipipe_dispatch_event); +EXPORT_SYMBOL(__ipipe_dispatch_wired); +EXPORT_SYMBOL(__ipipe_sync_stage); +EXPORT_SYMBOL(__ipipe_pipeline); +EXPORT_SYMBOL(__ipipe_pipelock); +EXPORT_SYMBOL(__ipipe_virtual_irq_map); Index: linux-2.6.17/kernel/ipipe/generic.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- /dev/null +++ linux-2.6.17/kernel/ipipe/generic.c @@ -0,0 +1,396 @@ +/* -*- linux-c -*- + * linux/kernel/ipipe/generic.c + * + * Copyright (C) 2002-2005 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,= + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,= USA. + * + * Architecture-independent I-PIPE services. + */ + +#include +#include +#include +#include +#include +#ifdef CONFIG_PROC_FS +#include +#endif /* CONFIG_PROC_FS */ + +MODULE_DESCRIPTION("I-pipe"); +MODULE_LICENSE("GPL"); + +static int __ipipe_ptd_key_count; + +static unsigned long __ipipe_ptd_key_map; + +/* ipipe_register_domain() -- Link a new domain to the pipeline. */ + +int ipipe_register_domain(struct ipipe_domain *ipd, + struct ipipe_domain_attr *attr) +{ + struct list_head *pos; + unsigned long flags; + + if (ipipe_current_domain !=3D ipipe_root_domain) { + printk(KERN_WARNING + "I-pipe: Only the root domain may register a new domain.\n"); + return -EPERM; + } + + if (attr->priority =3D=3D IPIPE_HEAD_PRIORITY && + test_bit(IPIPE_AHEAD_FLAG,&__ipipe_pipeline_head()->flags)) + return -EAGAIN; /* Cannot override current head. */ + + flags =3D ipipe_critical_enter(NULL); + + list_for_each(pos, &__ipipe_pipeline) { + struct ipipe_domain *_ipd =3D + list_entry(pos, struct ipipe_domain, p_link); + if (_ipd->domid =3D=3D attr->domid) + break; + } + + ipipe_critical_exit(flags); + + if (pos !=3D &__ipipe_pipeline) + /* A domain with the given id already exists -- fail. */ + return -EBUSY; + + ipd->name =3D attr->name; + ipd->domid =3D attr->domid; + ipd->pdd =3D attr->pdd; + ipd->flags =3D 0; + + if (attr->priority =3D=3D IPIPE_HEAD_PRIORITY) { + ipd->priority =3D INT_MAX; + __set_bit(IPIPE_AHEAD_FLAG,&ipd->flags); + } + else + ipd->priority =3D attr->priority; + + __ipipe_init_stage(ipd); + + INIT_LIST_HEAD(&ipd->p_link); + +#ifdef CONFIG_PROC_FS + __ipipe_add_domain_proc(ipd); +#endif /* CONFIG_PROC_FS */ + + flags =3D ipipe_critical_enter(NULL); + + list_for_each(pos, &__ipipe_pipeline) { + struct ipipe_domain *_ipd =3D + list_entry(pos, struct ipipe_domain, p_link); + if (ipd->priority > _ipd->priority) + break; + } + + list_add_tail(&ipd->p_link, pos); + + ipipe_critical_exit(flags); + + printk(KERN_WARNING "I-pipe: Domain %s registered.\n", ipd->name); + + /* + * Finally, allow the new domain to perform its initialization + * chores. + */ + + if (attr->entry !=3D NULL) { + ipipe_declare_cpuid; + + ipipe_lock_cpu(flags); + + ipipe_percpu_domain[cpuid] =3D ipd; + attr->entry(); + ipipe_percpu_domain[cpuid] =3D ipipe_root_domain; + + ipipe_load_cpuid(); /* Processor might have changed. */ + + if (ipipe_root_domain->cpudata[cpuid].irq_pending_hi !=3D 0 && + !test_bit(IPIPE_STALL_FLAG, + &ipipe_root_domain->cpudata[cpuid].status)) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + + ipipe_unlock_cpu(flags); + } + + return 0; +} + +/* ipipe_unregister_domain() -- Remove a domain from the pipeline. */ + +int ipipe_unregister_domain(struct ipipe_domain *ipd) +{ + unsigned long flags; + + if (ipipe_current_domain !=3D ipipe_root_domain) { + printk(KERN_WARNING + "I-pipe: Only the root domain may unregister a domain.\n"); + return -EPERM; + } + + if (ipd =3D=3D ipipe_root_domain) { + printk(KERN_WARNING + "I-pipe: Cannot unregister the root domain.\n"); + return -EPERM; + } +#ifdef CONFIG_SMP + { + int nr_cpus =3D num_online_cpus(), _cpuid; + unsigned irq; + + /* + * In the SMP case, wait for the logged events to drain on + * other processors before eventually removing the domain + * from the pipeline. + */ + + ipipe_unstall_pipeline_from(ipd); + + flags =3D ipipe_critical_enter(NULL); + + for (irq =3D 0; irq < IPIPE_NR_IRQS; irq++) { + clear_bit(IPIPE_HANDLE_FLAG, &ipd->irqs[irq].control); + clear_bit(IPIPE_STICKY_FLAG, &ipd->irqs[irq].control); + set_bit(IPIPE_PASS_FLAG, &ipd->irqs[irq].control); + } + + ipipe_critical_exit(flags); + + for (_cpuid =3D 0; _cpuid < nr_cpus; _cpuid++) + for (irq =3D 0; irq < IPIPE_NR_IRQS; irq++) + while (ipd->cpudata[_cpuid].irq_counters[irq].pending_hits > 0) + cpu_relax(); + } +#endif /* CONFIG_SMP */ + +#ifdef CONFIG_PROC_FS + __ipipe_remove_domain_proc(ipd); +#endif /* CONFIG_PROC_FS */ + + /* + * Simply remove the domain from the pipeline and we are almost done. + */ + + flags =3D ipipe_critical_enter(NULL); + list_del_init(&ipd->p_link); + ipipe_critical_exit(flags); + + __ipipe_cleanup_domain(ipd); + + printk(KERN_WARNING "I-pipe: Domain %s unregistered.\n", ipd->name); + + return 0; +} + +/* + * ipipe_propagate_irq() -- Force a given IRQ propagation on behalf of + * a running interrupt handler to the next domain down the pipeline. + * ipipe_schedule_irq() -- Does almost the same as above, but attempts + * to pend the interrupt for the current domain first. + */ +int fastcall __ipipe_schedule_irq(unsigned irq, struct list_head *head) +{ + struct list_head *ln; + unsigned long flags; + ipipe_declare_cpuid; + + if (irq >=3D IPIPE_NR_IRQS || + (ipipe_virtual_irq_p(irq) + && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map))) + return -EINVAL; + + ipipe_lock_cpu(flags); + + ln =3D head; + + while (ln !=3D &__ipipe_pipeline) { + struct ipipe_domain *ipd =3D + list_entry(ln, struct ipipe_domain, p_link); + + if (test_bit(IPIPE_HANDLE_FLAG, &ipd->irqs[irq].control)) { + ipd->cpudata[cpuid].irq_counters[irq].total_hits++; + ipd->cpudata[cpuid].irq_counters[irq].pending_hits++; + __ipipe_set_irq_bit(ipd, cpuid, irq); + ipipe_unlock_cpu(flags); + return 1; + } + + ln =3D ipd->p_link.next; + } + + ipipe_unlock_cpu(flags); + + return 0; +} + +/* ipipe_free_virq() -- Release a virtual/soft interrupt. */ + +int ipipe_free_virq(unsigned virq) +{ + if (!ipipe_virtual_irq_p(virq)) + return -EINVAL; + + clear_bit(virq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map); + + return 0; +} + +void ipipe_init_attr(struct ipipe_domain_attr *attr) +{ + attr->name =3D "anon"; + attr->domid =3D 1; + attr->entry =3D NULL; + attr->priority =3D IPIPE_ROOT_PRIO; + attr->pdd =3D NULL; +} + +/* + * ipipe_catch_event() -- Interpose or remove an event handler for a + * given domain. + */ +ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd, + unsigned event, + ipipe_event_handler_t handler) +{ + ipipe_event_handler_t old_handler; + int self =3D 0; + + if (event & IPIPE_EVENT_SELF) { + event &=3D ~IPIPE_EVENT_SELF; + self =3D 1; + } + + if (event >=3D IPIPE_NR_EVENTS) + return NULL; + + if (!(old_handler =3D xchg(&ipd->evhand[event],handler))) { + if (handler) { + if (self) + ipd->evself |=3D (1LL << event); + else + __ipipe_event_monitors[event]++; + } + } + else if (!handler) { + if (ipd->evself & (1LL << event)) + ipd->evself &=3D ~(1LL << event); + else + __ipipe_event_monitors[event]--; + } else if ((ipd->evself & (1LL << event)) && !self) { + __ipipe_event_monitors[event]++; + ipd->evself &=3D ~(1LL << event); + } else if (!(ipd->evself & (1LL << event)) && self) { + __ipipe_event_monitors[event]--; + ipd->evself |=3D (1LL << event); + } + + return old_handler; +} + +cpumask_t ipipe_set_irq_affinity (unsigned irq, cpumask_t cpumask) +{ +#ifdef CONFIG_SMP + if (irq >=3D IPIPE_NR_XIRQS) + /* Allow changing affinity of external IRQs only. */ + return CPU_MASK_NONE; + + if (num_online_cpus() > 1) + return __ipipe_set_irq_affinity(irq,cpumask); +#endif /* CONFIG_SMP */ + + return CPU_MASK_NONE; +} + +int fastcall ipipe_send_ipi (unsigned ipi, cpumask_t cpumask) + +{ +#ifdef CONFIG_SMP + return __ipipe_send_ipi(ipi,cpumask); +#else /* !CONFIG_SMP */ + return -EINVAL; +#endif /* CONFIG_SMP */ +} + +int ipipe_alloc_ptdkey (void) +{ + unsigned long flags; + int key =3D -1; + + spin_lock_irqsave_hw(&__ipipe_pipelock,flags); + + if (__ipipe_ptd_key_count < IPIPE_ROOT_NPTDKEYS) { + key =3D ffz(__ipipe_ptd_key_map); + set_bit(key,&__ipipe_ptd_key_map); + __ipipe_ptd_key_count++; + } + + spin_unlock_irqrestore_hw(&__ipipe_pipelock,flags); + + return key; +} + +int ipipe_free_ptdkey (int key) +{ + unsigned long flags; + + if (key < 0 || key >=3D IPIPE_ROOT_NPTDKEYS) + return -EINVAL; + + spin_lock_irqsave_hw(&__ipipe_pipelock,flags); + + if (test_and_clear_bit(key,&__ipipe_ptd_key_map)) + __ipipe_ptd_key_count--; + + spin_unlock_irqrestore_hw(&__ipipe_pipelock,flags); + + return 0; +} + +int fastcall ipipe_set_ptd (int key, void *value) + +{ + if (key < 0 || key >=3D IPIPE_ROOT_NPTDKEYS) + return -EINVAL; + + current->ptd[key] =3D value; + + return 0; +} + +void fastcall *ipipe_get_ptd (int key) + +{ + if (key < 0 || key >=3D IPIPE_ROOT_NPTDKEYS) + return NULL; + + return current->ptd[key]; +} + +EXPORT_SYMBOL(ipipe_register_domain); +EXPORT_SYMBOL(ipipe_unregister_domain); +EXPORT_SYMBOL(ipipe_free_virq); +EXPORT_SYMBOL(ipipe_init_attr); +EXPORT_SYMBOL(ipipe_catch_event); +EXPORT_SYMBOL(ipipe_alloc_ptdkey); +EXPORT_SYMBOL(ipipe_free_ptdkey); +EXPORT_SYMBOL(ipipe_set_ptd); +EXPORT_SYMBOL(ipipe_get_ptd); +EXPORT_SYMBOL(ipipe_set_irq_affinity); +EXPORT_SYMBOL(ipipe_send_ipi); +EXPORT_SYMBOL(__ipipe_schedule_irq); Index: linux-2.6.17/kernel/irq/handle.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/kernel/irq/handle.c +++ linux-2.6.17/kernel/irq/handle.c @@ -81,6 +81,17 @@ fastcall int handle_IRQ_event(unsigned i { int ret, retval =3D 0, status =3D 0; =20 +#ifdef CONFIG_IPIPE + /* + * If processing a timer tick, pass the original regs as + * collected during preemption and not our phony - always + * kernel-originated - frame, so that we don't wreck the + * profiling code. + */ + if (__ipipe_tick_irq =3D=3D irq) + regs =3D __ipipe_tick_regs + smp_processor_id(); +#endif /* CONFIG_IPIPE */ + if (!(action->flags & SA_INTERRUPT)) local_irq_enable(); =20 @@ -117,16 +128,20 @@ fastcall unsigned int __do_IRQ(unsigned=20 /* * No locking required for CPU-local interrupts: */ +#ifndef CONFIG_IPIPE if (desc->handler->ack) desc->handler->ack(irq); +#endif /* !CONFIG_IPIPE */ action_ret =3D handle_IRQ_event(irq, regs, desc->action); desc->handler->end(irq); return 1; } =20 spin_lock(&desc->lock); +#ifndef CONFIG_IPIPE if (desc->handler->ack) desc->handler->ack(irq); +#endif /* !CONFIG_IPIPE */ /* * REPLAY is when Linux resends an IRQ that was dropped earlier * WAITING is used by probe to mark irqs that are being tested Index: linux-2.6.17/kernel/printk.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/kernel/printk.c +++ linux-2.6.17/kernel/printk.c @@ -472,6 +472,78 @@ __attribute__((weak)) unsigned long long * printf(3) */ =20 +#ifdef CONFIG_IPIPE + +static ipipe_spinlock_t __ipipe_printk_lock =3D IPIPE_SPIN_LOCK_UNLOCKED= ; + +static int __ipipe_printk_fill; + +static char __ipipe_printk_buf[__LOG_BUF_LEN]; + +void __ipipe_flush_printk (unsigned virq, void *cookie) +{ + char *p =3D __ipipe_printk_buf; + int len, lmax, out =3D 0; + unsigned long flags; + + goto start; + + do { + spin_unlock_irqrestore_hw(&__ipipe_printk_lock,flags); + start: + lmax =3D __ipipe_printk_fill; + while (out < lmax) { + len =3D strlen(p) + 1; + printk("%s",p); + p +=3D len; + out +=3D len; + } + spin_lock_irqsave_hw(&__ipipe_printk_lock,flags); + } + while (__ipipe_printk_fill !=3D lmax); + + __ipipe_printk_fill =3D 0; + + spin_unlock_irqrestore_hw(&__ipipe_printk_lock,flags); +} + +asmlinkage int printk(const char *fmt, ...) +{ + int r, fbytes, oldcount; + unsigned long flags; + va_list args; + + va_start(args, fmt); + + if (ipipe_current_domain =3D=3D ipipe_root_domain || + test_bit(IPIPE_SPRINTK_FLAG,&ipipe_current_domain->flags) || + oops_in_progress) { + r =3D vprintk(fmt, args); + goto out; + } + + spin_lock_irqsave_hw(&__ipipe_printk_lock,flags); + + oldcount =3D __ipipe_printk_fill; + fbytes =3D __LOG_BUF_LEN - oldcount; + + if (fbytes > 1) { + r =3D vscnprintf(__ipipe_printk_buf + __ipipe_printk_fill, + fbytes, fmt, args) + 1; /* account for the null byte */ + __ipipe_printk_fill +=3D r; + } else + r =3D 0; + + spin_unlock_irqrestore_hw(&__ipipe_printk_lock,flags); + + if (oldcount =3D=3D 0) + ipipe_trigger_irq(__ipipe_printk_virq); +out: + va_end(args); + + return r; +} +#else /* !CONFIG_IPIPE */ asmlinkage int printk(const char *fmt, ...) { va_list args; @@ -483,6 +555,7 @@ asmlinkage int printk(const char *fmt, . =20 return r; } +#endif /* CONFIG_IPIPE */ =20 /* cpu currently holding logbuf_lock */ static volatile unsigned int printk_cpu =3D UINT_MAX; Index: linux-2.6.17/kernel/sched.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/kernel/sched.c +++ linux-2.6.17/kernel/sched.c @@ -2903,12 +2903,17 @@ asmlinkage void __sched schedule(void) unsigned long run_time; int cpu, idx, new_prio; =20 +#ifdef CONFIG_IPIPE + if (unlikely(ipipe_current_domain !=3D ipipe_root_domain)) + goto need_resched; +#endif /* CONFIG_IPIPE */ /* * Test if we are atomic. Since do_exit() needs to call into * schedule() atomically, we ignore that path for now. * Otherwise, whine if we are scheduling when we should not be. */ - if (unlikely(in_atomic() && !current->exit_state)) { + if (unlikely(!(current->state & TASK_ATOMICSWITCH) && in_atomic() && + !current->exit_state)) { printk(KERN_ERR "BUG: scheduling while atomic: " "%s/0x%08x/%d\n", current->comm, preempt_count(), current->pid); @@ -2916,8 +2921,19 @@ asmlinkage void __sched schedule(void) } profile_hit(SCHED_PROFILING, __builtin_return_address(0)); =20 + if (unlikely(current->state & TASK_ATOMICSWITCH)) { + current->state &=3D ~TASK_ATOMICSWITCH; + goto preemption_off; + } need_resched: preempt_disable(); +preemption_off: +#ifdef CONFIG_IPIPE + if (unlikely(ipipe_current_domain !=3D ipipe_root_domain)) { + preempt_enable(); + return; + } +#endif /* CONFIG_IPIPE */ prev =3D current; release_kernel_lock(prev); need_resched_nonpreemptible: @@ -3055,6 +3071,8 @@ switch_tasks: prepare_task_switch(rq, next); prev =3D context_switch(rq, prev, next); barrier(); + if (task_hijacked(prev)) + return; /* * this_rq must be evaluated again because prev may have moved * CPUs since it called schedule(), thus the 'rq' on its stack @@ -3087,6 +3105,11 @@ asmlinkage void __sched preempt_schedule struct task_struct *task =3D current; int saved_lock_depth; #endif +#ifdef CONFIG_IPIPE + /* Do not reschedule over non-Linux domains. */ + if (ipipe_current_domain !=3D ipipe_root_domain) + return; +#endif /* CONFIG_IPIPE */ /* * If there is a non-zero preempt_count or interrupts are disabled, * we do not want to preempt the current task. Just return.. @@ -3725,6 +3748,7 @@ recheck: deactivate_task(p, rq); oldprio =3D p->prio; __setscheduler(p, policy, param->sched_priority); + ipipe_setsched_notify(p); if (array) { __activate_task(p, rq); /* @@ -6252,3 +6276,50 @@ void set_curr_task(int cpu, task_t *p) } =20 #endif + +#ifdef CONFIG_IPIPE + +int ipipe_setscheduler_root (struct task_struct *p, int policy, int prio= ) +{ + prio_array_t *array; + unsigned long flags; + runqueue_t *rq; + int oldprio; + + rq =3D task_rq_lock(p, &flags); + array =3D p->array; + if (array) + deactivate_task(p, rq); + oldprio =3D p->prio; + __setscheduler(p, policy, prio); + if (array) { + __activate_task(p, rq); + if (task_running(rq, p)) { + if (p->prio > oldprio) + resched_task(rq->curr); + } else if (TASK_PREEMPTS_CURR(p, rq)) + resched_task(rq->curr); + } + task_rq_unlock(rq, &flags); + + return 0; +} + +EXPORT_SYMBOL(ipipe_setscheduler_root); + +int ipipe_reenter_root (struct task_struct *prev, int policy, int prio) +{ + finish_task_switch(this_rq(), prev); + if (reacquire_kernel_lock(current) < 0) + ; + preempt_enable_no_resched(); + + if (current->policy !=3D policy || current->rt_priority !=3D prio) + return ipipe_setscheduler_root(current,policy,prio); + + return 0; +} + +EXPORT_SYMBOL(ipipe_reenter_root); + +#endif /* CONFIG_IPIPE */ Index: linux-2.6.17/kernel/signal.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/kernel/signal.c +++ linux-2.6.17/kernel/signal.c @@ -500,6 +500,7 @@ void signal_wake_up(struct task_struct * unsigned int mask; =20 set_tsk_thread_flag(t, TIF_SIGPENDING); + ipipe_sigwake_notify(t); /* TIF_SIGPENDING must be set first. */ =20 /* * For SIGKILL, we want to wake it up in the stopped/traced case. Index: linux-2.6.17/lib/smp_processor_id.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/lib/smp_processor_id.c +++ linux-2.6.17/lib/smp_processor_id.c @@ -13,6 +13,11 @@ unsigned int debug_smp_processor_id(void int this_cpu =3D raw_smp_processor_id(); cpumask_t this_mask; =20 +#ifdef CONFIG_IPIPE + if (ipipe_current_domain !=3D ipipe_root_domain) + return this_cpu; +#endif /* CONFIG_IPIPE */ + if (likely(preempt_count)) goto out; =20 Index: linux-2.6.17/lib/spinlock_debug.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/lib/spinlock_debug.c +++ linux-2.6.17/lib/spinlock_debug.c @@ -10,6 +10,7 @@ #include #include #include +#include =20 static void spin_bug(spinlock_t *lock, const char *msg) { @@ -96,6 +97,8 @@ void _raw_spin_lock(spinlock_t *lock) debug_spin_lock_after(lock); } =20 +EXPORT_SYMBOL(_raw_spin_lock); + int _raw_spin_trylock(spinlock_t *lock) { int ret =3D __raw_spin_trylock(&lock->raw_lock); @@ -111,12 +114,16 @@ int _raw_spin_trylock(spinlock_t *lock) return ret; } =20 +EXPORT_SYMBOL(_raw_spin_trylock); + void _raw_spin_unlock(spinlock_t *lock) { debug_spin_unlock(lock); __raw_spin_unlock(&lock->raw_lock); } =20 +EXPORT_SYMBOL(_raw_spin_unlock); + static void rwlock_bug(rwlock_t *lock, const char *msg) { static long print_once =3D 1; @@ -167,6 +174,8 @@ void _raw_read_lock(rwlock_t *lock) __read_lock_debug(lock); } =20 +EXPORT_SYMBOL(_raw_read_lock); + int _raw_read_trylock(rwlock_t *lock) { int ret =3D __raw_read_trylock(&lock->raw_lock); @@ -180,12 +189,16 @@ int _raw_read_trylock(rwlock_t *lock) return ret; } =20 +EXPORT_SYMBOL(_raw_read_trylock); + void _raw_read_unlock(rwlock_t *lock) { RWLOCK_BUG_ON(lock->magic !=3D RWLOCK_MAGIC, lock, "bad magic"); __raw_read_unlock(&lock->raw_lock); } =20 +EXPORT_SYMBOL(_raw_read_unlock); + static inline void debug_write_lock_before(rwlock_t *lock) { RWLOCK_BUG_ON(lock->magic !=3D RWLOCK_MAGIC, lock, "bad magic"); @@ -241,6 +254,8 @@ void _raw_write_lock(rwlock_t *lock) debug_write_lock_after(lock); } =20 +EXPORT_SYMBOL(_raw_write_lock); + int _raw_write_trylock(rwlock_t *lock) { int ret =3D __raw_write_trylock(&lock->raw_lock); @@ -256,8 +271,12 @@ int _raw_write_trylock(rwlock_t *lock) return ret; } =20 +EXPORT_SYMBOL(_raw_write_trylock); + void _raw_write_unlock(rwlock_t *lock) { debug_write_unlock(lock); __raw_write_unlock(&lock->raw_lock); } + +EXPORT_SYMBOL(_raw_write_unlock); Index: linux-2.6.17/mm/vmalloc.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/mm/vmalloc.c +++ linux-2.6.17/mm/vmalloc.c @@ -19,6 +19,7 @@ =20 #include #include +#include =20 =20 DEFINE_RWLOCK(vmlist_lock); @@ -148,10 +149,14 @@ int map_vm_area(struct vm_struct *area,=20 BUG_ON(addr >=3D end); pgd =3D pgd_offset_k(addr); do { + pgd_t oldpgd; + memcpy(&oldpgd,pgd,sizeof(pgd_t)); next =3D pgd_addr_end(addr, end); err =3D vmap_pud_range(pgd, addr, next, prot, pages); if (err) break; + if (pgd_val(oldpgd) !=3D pgd_val(*pgd)) + set_pgdir(addr, *pgd); } while (pgd++, addr =3D next, addr !=3D end); flush_cache_vmap((unsigned long) area->addr, end); return err; Index: linux-2.6.17/arch/i386/Kconfig =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/arch/i386/Kconfig +++ linux-2.6.17/arch/i386/Kconfig @@ -242,6 +242,8 @@ config SCHED_MC =20 source "kernel/Kconfig.preempt" =20 +source "kernel/ipipe/Kconfig" + config X86_UP_APIC bool "Local APIC support on uniprocessors" depends on !SMP && !(X86_VISWS || X86_VOYAGER) @@ -765,7 +767,6 @@ config HOTPLUG_CPU =20 endmenu =20 - menu "Power management options (ACPI, APM)" depends on !X86_VOYAGER =20 Index: linux-2.6.17/arch/i386/kernel/Makefile =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/arch/i386/kernel/Makefile +++ linux-2.6.17/arch/i386/kernel/Makefile @@ -31,6 +31,8 @@ obj-$(CONFIG_X86_SUMMIT_NUMA) +=3D summit. obj-$(CONFIG_KPROBES) +=3D kprobes.o obj-$(CONFIG_MODULES) +=3D module.o obj-y +=3D sysenter.o vsyscall.o +obj-$(CONFIG_IPIPE) +=3D ipipe-core.o ipipe-root.o +obj-$(CONFIG_IPIPE_TRACE) +=3D ipipe-mcount.o obj-$(CONFIG_ACPI_SRAT) +=3D srat.o obj-$(CONFIG_HPET_TIMER) +=3D time_hpet.o obj-$(CONFIG_EFI) +=3D efi.o efi_stub.o Index: linux-2.6.17/arch/i386/kernel/apic.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/arch/i386/kernel/apic.c +++ linux-2.6.17/arch/i386/kernel/apic.c @@ -91,7 +91,7 @@ void ack_bad_irq(unsigned int irq) * But only ack when the APIC is enabled -AK */ if (cpu_has_apic) - ack_APIC_irq(); + __ack_APIC_irq(); } =20 void __init apic_intr_init(void) @@ -1203,6 +1203,9 @@ inline void smp_local_timer_interrupt(st fastcall void smp_apic_timer_interrupt(struct pt_regs *regs) { int cpu =3D smp_processor_id(); +#ifdef CONFIG_IPIPE + regs =3D __ipipe_tick_regs + cpu; +#endif /* CONFIG_IPIPE */ =20 /* * the NMI deadlock-detector uses this. @@ -1276,7 +1279,7 @@ fastcall void smp_spurious_interrupt(str */ v =3D apic_read(APIC_ISR + ((SPURIOUS_APIC_VECTOR & ~0x1f) >> 1)); if (v & (1 << (SPURIOUS_APIC_VECTOR & 0x1f))) - ack_APIC_irq(); + __ack_APIC_irq(); =20 /* see sw-dev-man vol 3, chapter 7.4.13.5 */ printk(KERN_INFO "spurious APIC interrupt on CPU#%d, should never happe= n.\n", Index: linux-2.6.17/arch/i386/kernel/entry.S =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/arch/i386/kernel/entry.S +++ linux-2.6.17/arch/i386/kernel/entry.S @@ -48,6 +48,7 @@ #include #include #include +#include #include "irq_vectors.h" =20 #define nr_syscalls ((syscall_table_size)/4) @@ -75,11 +76,81 @@ DF_MASK =3D 0x00000400=20 NT_MASK =3D 0x00004000 VM_MASK =3D 0x00020000 =20 +#ifdef CONFIG_IPIPE +#define CLI call __ipipe_stall_root ; sti +#define STI call __ipipe_unstall_root +#define STI_COND_HW sti +#define EMULATE_ROOT_IRET(bypass) \ + call __ipipe_unstall_iret_root ; \ + bypass: \ + movl EAX(%esp),%eax +#define TEST_PREEMPTIBLE(regs) call __ipipe_kpreempt_root ; testl %eax,= %eax +#define restore_nocheck unstall_and_restore_nocheck +#define restore_nmi restore_raw +#define CATCH_ROOT_SYSCALL(bypass1,bypass2) \ + call __ipipe_syscall_root ; \ + testl %eax,%eax ; \ + js bypass1 ; \ + jne bypass2 ; \ + movl ORIG_EAX(%esp),%eax +#define PUSH_XCODE(v) pushl $ ex_/**/v +#define HANDLE_EXCEPTION(code) movl %code,%ecx ; \ + call __ipipe_handle_exception ; \ + testl %eax,%eax ; \ + jnz restore_raw +#define DIVERT_EXCEPTION(code) movl $(__USER_DS), %ecx ; \ + movl %ecx, %ds ; \ + movl %ecx, %es ; \ + movl %esp, %eax ; \ + movl $ex_/**/code,%edx ; \ + call __ipipe_divert_exception ; \ + testl %eax,%eax ; \ + jnz restore_raw + +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF +# ifdef CONFIG_REGPARM +# define LOAD_ARG +# define REMOVE_ARG +# else /* !CONFIG_REGPARM */ +# define LOAD_ARG pushl %eax +# define REMOVE_ARG addl $4, %esp +# endif /* CONFIG_REGPARM */ +# define IPIPE_TRACE_IRQ_ENTER \ + lea EIP-4(%esp), %ebp; \ + movl ORIG_EAX(%esp), %eax; \ + LOAD_ARG; \ + call ipipe_trace_begin; \ + REMOVE_ARG +# define IPIPE_TRACE_IRQ_EXIT \ + pushl %eax; \ + movl ORIG_EAX+4(%esp), %eax; \ + LOAD_ARG; \ + call ipipe_trace_end; \ + REMOVE_ARG; \ + popl %eax +#else /* !CONFIG_IPIPE_TRACE_IRQSOFF */ +# define IPIPE_TRACE_IRQ_ENTER +# define IPIPE_TRACE_IRQ_EXIT +#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */ + +#else /* !CONFIG_IPIPE */ +#define CLI cli +#define STI sti +#define STI_COND_HW +#define EMULATE_ROOT_IRET(bypass) +#define TEST_PREEMPTIBLE(regs) testl $IF_MASK,EFLAGS(regs) +#define restore_nmi restore_all +#define CATCH_ROOT_SYSCALL(bypass1,bypass2) +#define PUSH_XCODE(v) pushl $v +#define HANDLE_EXCEPTION(code) call *%code +#define DIVERT_EXCEPTION(code) +#endif /* CONFIG_IPIPE */ + #ifdef CONFIG_PREEMPT -#define preempt_stop cli +#define preempt_stop CLI #else #define preempt_stop -#define resume_kernel restore_nocheck +#define resume_kernel restore_nocheck #endif =20 #define SAVE_ALL \ @@ -124,6 +195,7 @@ VM_MASK =3D 0x00020000 =20 =20 ENTRY(ret_from_fork) + STI_COND_HW pushl %eax call schedule_tail GET_THREAD_INFO(%ebp) @@ -141,14 +213,14 @@ ENTRY(ret_from_fork) ALIGN ret_from_exception: preempt_stop -ret_from_intr: +ENTRY(ret_from_intr) GET_THREAD_INFO(%ebp) movl EFLAGS(%esp), %eax # mix EFLAGS and CS movb CS(%esp), %al testl $(VM_MASK | 3), %eax jz resume_kernel ENTRY(resume_userspace) - cli # make sure we don't miss an interrupt + CLI # make sure we don't miss an interrupt # setting need_resched or sigpending # between sampling and the iret movl TI_flags(%ebp), %ecx @@ -166,7 +238,7 @@ need_resched: movl TI_flags(%ebp), %ecx # need_resched set ? testb $_TIF_NEED_RESCHED, %cl jz restore_all - testl $IF_MASK,EFLAGS(%esp) # interrupts off (exception path) ? + TEST_PREEMPTIBLE(%esp) # interrupts off (exception path) ? jz restore_all call preempt_schedule_irq jmp need_resched @@ -201,6 +273,7 @@ sysenter_past_esp: pushl %eax SAVE_ALL GET_THREAD_INFO(%ebp) + CATCH_ROOT_SYSCALL(sysenter_tail,sysenter_exit) =20 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not te= stb */ testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_A= UDIT),TI_flags(%ebp) @@ -209,11 +282,13 @@ sysenter_past_esp: jae syscall_badsys call *sys_call_table(,%eax,4) movl %eax,EAX(%esp) - cli +sysenter_tail: + CLI movl TI_flags(%ebp), %ecx testw $_TIF_ALLWORK_MASK, %cx jne syscall_exit_work /* if something modifies registers it must also disable sysexit */ + EMULATE_ROOT_IRET(sysenter_exit) movl EIP(%esp), %edx movl OLDESP(%esp), %ecx xorl %ebp,%ebp @@ -226,6 +301,7 @@ ENTRY(system_call) pushl %eax # save orig_eax SAVE_ALL GET_THREAD_INFO(%ebp) + CATCH_ROOT_SYSCALL(syscall_exit,restore_raw) testl $TF_MASK,EFLAGS(%esp) jz no_singlestep orl $_TIF_SINGLESTEP,TI_flags(%ebp) @@ -240,7 +316,7 @@ syscall_call: call *sys_call_table(,%eax,4) movl %eax,EAX(%esp) # store the return value syscall_exit: - cli # make sure we don't miss an interrupt + CLI # make sure we don't miss an interrupt # setting need_resched or sigpending # between sampling and the iret movl TI_flags(%ebp), %ecx @@ -257,7 +333,15 @@ restore_all: andl $(VM_MASK | (4 << 8) | 3), %eax cmpl $((4 << 8) | 3), %eax je ldt_ss # returning to user-space with LDT SS +#ifdef CONFIG_IPIPE +unstall_and_restore_nocheck: + call __ipipe_unstall_iret_root +restore_raw: + # FIXME: we need to check for a return to + # user-space on a 16bit stack even in the NMI case +#else /* !CONFIG_IPIPE */ restore_nocheck: +#endif /* CONFIG_IPIPE */ RESTORE_REGS addl $4, %esp 1: iret @@ -265,7 +349,7 @@ restore_nocheck: iret_exc: sti pushl $0 # no error code - pushl $do_iret_error + PUSH_XCODE(do_iret_error) jmp error_code .previous .section __ex_table,"a" @@ -304,8 +388,9 @@ work_pending: testb $_TIF_NEED_RESCHED, %cl jz work_notifysig work_resched: + STI_COND_HW call schedule - cli # make sure we don't miss an interrupt + CLI # make sure we don't miss an interrupt # setting need_resched or sigpending # between sampling and the iret movl TI_flags(%ebp), %ecx @@ -357,7 +442,7 @@ syscall_trace_entry: syscall_exit_work: testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl jz work_pending - sti # could let do_syscall_trace() call + STI # could let do_syscall_trace() call # schedule() instead movl %esp, %eax movl $1, %edx @@ -408,7 +493,7 @@ ENTRY(interrupt) =20 vector=3D0 ENTRY(irq_entries_start) -.rept NR_IRQS +.rept NR_XIRQS ALIGN 1: pushl $vector-256 jmp common_interrupt @@ -418,6 +503,32 @@ ENTRY(irq_entries_start) vector=3Dvector+1 .endr =20 +#ifdef CONFIG_IPIPE + ALIGN +common_interrupt: + SAVE_ALL + IPIPE_TRACE_IRQ_ENTER; \ + call __ipipe_handle_irq + IPIPE_TRACE_IRQ_EXIT; \ + testl %eax,%eax + jnz ret_from_intr + RESTORE_REGS + addl $4, %esp + iret + +#define BUILD_INTERRUPT(name, nr) \ +ENTRY(name) \ + pushl $nr-288; /* nr - (256 + FIRST_EXTERNAL_VECTOR) */ \ + SAVE_ALL; \ + IPIPE_TRACE_IRQ_ENTER; \ + call __ipipe_handle_irq; \ + IPIPE_TRACE_IRQ_EXIT; \ + testl %eax,%eax; \ + jnz ret_from_intr; \ + RESTORE_REGS; \ + addl $4, %esp; \ + iret +#else /* CONFIG_IPIPE */ ALIGN common_interrupt: SAVE_ALL @@ -432,13 +543,14 @@ ENTRY(name) \ movl %esp,%eax; \ call smp_/**/name; \ jmp ret_from_intr; +#endif /* CONFIG_IPIPE */ =20 /* The include is where all of the SMP etc. interrupts come from */ #include "entry_arch.h" =20 ENTRY(divide_error) pushl $0 # no error code - pushl $do_divide_error + PUSH_XCODE(do_divide_error) ALIGN error_code: pushl %ds @@ -463,22 +575,23 @@ error_code: movl %ecx, %ds movl %ecx, %es movl %esp,%eax # pt_regs pointer - call *%edi + HANDLE_EXCEPTION(edi) jmp ret_from_exception =20 ENTRY(coprocessor_error) pushl $0 - pushl $do_coprocessor_error + PUSH_XCODE(do_coprocessor_error) jmp error_code =20 ENTRY(simd_coprocessor_error) pushl $0 - pushl $do_simd_coprocessor_error + PUSH_XCODE(do_simd_coprocessor_error) jmp error_code =20 ENTRY(device_not_available) pushl $-1 # mark this as an int SAVE_ALL + DIVERT_EXCEPTION(device_not_available) movl %cr0, %eax testl $0x4, %eax # EM (math emulation bit) jne device_not_available_emulate @@ -520,6 +633,7 @@ KPROBE_ENTRY(debug) debug_stack_correct: pushl $-1 # mark this as an int SAVE_ALL + DIVERT_EXCEPTION(do_debug) xorl %edx,%edx # error code 0 movl %esp,%eax # pt_regs pointer call do_debug @@ -558,7 +672,7 @@ nmi_stack_correct: xorl %edx,%edx # zero error code movl %esp,%eax # pt_regs pointer call do_nmi - jmp restore_all + jmp restore_nmi =20 nmi_stack_fixup: FIX_STACK(12,nmi_stack_correct, 1) @@ -599,6 +713,7 @@ nmi_16bit_stack: KPROBE_ENTRY(int3) pushl $-1 # mark this as an int SAVE_ALL + DIVERT_EXCEPTION(do_int3) xorl %edx,%edx # zero error code movl %esp,%eax # pt_regs pointer call do_int3 @@ -607,60 +722,60 @@ KPROBE_ENTRY(int3) =20 ENTRY(overflow) pushl $0 - pushl $do_overflow + PUSH_XCODE(do_overflow) jmp error_code =20 ENTRY(bounds) pushl $0 - pushl $do_bounds + PUSH_XCODE(do_bounds) jmp error_code =20 ENTRY(invalid_op) pushl $0 - pushl $do_invalid_op + PUSH_XCODE(do_invalid_op) jmp error_code =20 ENTRY(coprocessor_segment_overrun) pushl $0 - pushl $do_coprocessor_segment_overrun + PUSH_XCODE(do_coprocessor_segment_overrun) jmp error_code =20 ENTRY(invalid_TSS) - pushl $do_invalid_TSS + PUSH_XCODE(do_invalid_TSS) jmp error_code =20 ENTRY(segment_not_present) - pushl $do_segment_not_present + PUSH_XCODE(do_segment_not_present) jmp error_code =20 ENTRY(stack_segment) - pushl $do_stack_segment + PUSH_XCODE(do_stack_segment) jmp error_code =20 KPROBE_ENTRY(general_protection) - pushl $do_general_protection + PUSH_XCODE(do_general_protection) jmp error_code .previous .text =20 ENTRY(alignment_check) - pushl $do_alignment_check + PUSH_XCODE(do_alignment_check) jmp error_code =20 KPROBE_ENTRY(page_fault) - pushl $do_page_fault + PUSH_XCODE(do_page_fault) jmp error_code .previous .text =20 #ifdef CONFIG_X86_MCE ENTRY(machine_check) pushl $0 - pushl machine_check_vector + PUSH_XCODE(machine_check_vector) jmp error_code #endif =20 ENTRY(spurious_interrupt_bug) pushl $0 - pushl $do_spurious_interrupt_bug + PUSH_XCODE(do_spurious_interrupt_bug) jmp error_code =20 .section .rodata,"a" Index: linux-2.6.17/arch/i386/kernel/i8259.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/arch/i386/kernel/i8259.c +++ linux-2.6.17/arch/i386/kernel/i8259.c @@ -39,8 +39,9 @@ DEFINE_SPINLOCK(i8259A_lock); =20 static void end_8259A_irq (unsigned int irq) { - if (!(irq_desc[irq].status & (IRQ_DISABLED|IRQ_INPROGRESS)) && - irq_desc[irq].action) + if (!ipipe_root_domain_p || + (!(irq_desc[irq].status & (IRQ_DISABLED|IRQ_INPROGRESS)) && + irq_desc[irq].action)) enable_8259A_irq(irq); } =20 @@ -89,13 +90,14 @@ void disable_8259A_irq(unsigned int irq) unsigned int mask =3D 1 << irq; unsigned long flags; =20 - spin_lock_irqsave(&i8259A_lock, flags); + spin_lock_irqsave_hw(&i8259A_lock, flags); + ipipe_irq_lock(irq); cached_irq_mask |=3D mask; if (irq & 8) outb(cached_slave_mask, PIC_SLAVE_IMR); else outb(cached_master_mask, PIC_MASTER_IMR); - spin_unlock_irqrestore(&i8259A_lock, flags); + spin_unlock_irqrestore_hw(&i8259A_lock, flags); } =20 void enable_8259A_irq(unsigned int irq) @@ -103,13 +105,14 @@ void enable_8259A_irq(unsigned int irq) unsigned int mask =3D ~(1 << irq); unsigned long flags; =20 - spin_lock_irqsave(&i8259A_lock, flags); + spin_lock_irqsave_hw(&i8259A_lock, flags); cached_irq_mask &=3D mask; if (irq & 8) outb(cached_slave_mask, PIC_SLAVE_IMR); else outb(cached_master_mask, PIC_MASTER_IMR); - spin_unlock_irqrestore(&i8259A_lock, flags); + ipipe_irq_unlock(irq); + spin_unlock_irqrestore_hw(&i8259A_lock, flags); } =20 int i8259A_irq_pending(unsigned int irq) @@ -118,12 +121,12 @@ int i8259A_irq_pending(unsigned int irq) unsigned long flags; int ret; =20 - spin_lock_irqsave(&i8259A_lock, flags); + spin_lock_irqsave_hw(&i8259A_lock, flags); if (irq < 8) ret =3D inb(PIC_MASTER_CMD) & mask; else ret =3D inb(PIC_SLAVE_CMD) & (mask >> 8); - spin_unlock_irqrestore(&i8259A_lock, flags); + spin_unlock_irqrestore_hw(&i8259A_lock, flags); =20 return ret; } @@ -170,7 +173,7 @@ static void mask_and_ack_8259A(unsigned=20 unsigned int irqmask =3D 1 << irq; unsigned long flags; =20 - spin_lock_irqsave(&i8259A_lock, flags); + spin_lock_irqsave_hw(&i8259A_lock, flags); /* * Lightweight spurious IRQ detection. We do not want * to overdo spurious IRQ handling - it's usually a sign @@ -188,6 +191,15 @@ static void mask_and_ack_8259A(unsigned=20 */ if (cached_irq_mask & irqmask) goto spurious_8259A_irq; +#ifdef CONFIG_IPIPE + if (irq =3D=3D 0) { + /* Fast timer ack -- don't mask (unless supposedly + spurious) */ + outb(0x20,PIC_MASTER_CMD); + spin_unlock_irqrestore_hw(&i8259A_lock, flags); + return; + } +#endif /* CONFIG_IPIPE */ cached_irq_mask |=3D irqmask; =20 handle_real_irq: @@ -201,7 +213,7 @@ handle_real_irq: outb(cached_master_mask, PIC_MASTER_IMR); outb(0x60+irq,PIC_MASTER_CMD); /* 'Specific EOI to master */ } - spin_unlock_irqrestore(&i8259A_lock, flags); + spin_unlock_irqrestore_hw(&i8259A_lock, flags); return; =20 spurious_8259A_irq: @@ -302,7 +314,7 @@ void init_8259A(int auto_eoi) { unsigned long flags; =20 - spin_lock_irqsave(&i8259A_lock, flags); + spin_lock_irqsave_hw(&i8259A_lock, flags); =20 outb(0xff, PIC_MASTER_IMR); /* mask all of 8259A-1 */ outb(0xff, PIC_SLAVE_IMR); /* mask all of 8259A-2 */ @@ -336,7 +348,7 @@ void init_8259A(int auto_eoi) outb(cached_master_mask, PIC_MASTER_IMR); /* restore master IRQ mask */= outb(cached_slave_mask, PIC_SLAVE_IMR); /* restore slave IRQ mask */ =20 - spin_unlock_irqrestore(&i8259A_lock, flags); + spin_unlock_irqrestore_hw(&i8259A_lock, flags); } =20 /* @@ -410,7 +422,7 @@ void __init init_IRQ(void) */ for (i =3D 0; i < (NR_VECTORS - FIRST_EXTERNAL_VECTOR); i++) { int vector =3D FIRST_EXTERNAL_VECTOR + i; - if (i >=3D NR_IRQS) + if (i >=3D NR_XIRQS) break; if (vector !=3D SYSCALL_VECTOR)=20 set_intr_gate(vector, interrupt[i]); Index: linux-2.6.17/arch/i386/kernel/io_apic.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/arch/i386/kernel/io_apic.c +++ linux-2.6.17/arch/i386/kernel/io_apic.c @@ -182,18 +182,20 @@ static void mask_IO_APIC_irq (unsigned i { unsigned long flags; =20 - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); + ipipe_irq_lock(irq); __mask_IO_APIC_irq(irq); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); } =20 static void unmask_IO_APIC_irq (unsigned int irq) { unsigned long flags; =20 - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); + ipipe_irq_unlock(irq); __unmask_IO_APIC_irq(irq); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); } =20 static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin) @@ -202,10 +204,10 @@ static void clear_IO_APIC_pin(unsigned i unsigned long flags; =09 /* Check delivery_mode to be sure we're not clearing an SMI pin */ - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); *(((int*)&entry) + 0) =3D io_apic_read(apic, 0x10 + 2 * pin); *(((int*)&entry) + 1) =3D io_apic_read(apic, 0x11 + 2 * pin); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); if (entry.delivery_mode =3D=3D dest_SMI) return; =20 @@ -214,10 +216,10 @@ static void clear_IO_APIC_pin(unsigned i */ memset(&entry, 0, sizeof(entry)); entry.mask =3D 1; - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); io_apic_write(apic, 0x10 + 2 * pin, *(((int *)&entry) + 0)); io_apic_write(apic, 0x11 + 2 * pin, *(((int *)&entry) + 1)); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); } =20 static void clear_IO_APIC (void) @@ -247,7 +249,7 @@ static void set_ioapic_affinity_irq(unsi apicid_value =3D cpu_mask_to_apicid(cpumask); /* Prepare to do the io_apic_write */ apicid_value =3D apicid_value << 24; - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); for (;;) { pin =3D entry->pin; if (pin =3D=3D -1) @@ -258,7 +260,7 @@ static void set_ioapic_affinity_irq(unsi entry =3D irq_2_pin + entry->next; } set_irq_info(irq, cpumask); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); } =20 #if defined(CONFIG_IRQBALANCE) @@ -1266,11 +1268,11 @@ static void __init setup_IO_APIC_irqs(vo if (!apic && (irq < 16)) disable_8259A_irq(irq); } - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); io_apic_write(apic, 0x11+2*pin, *(((int *)&entry)+1)); io_apic_write(apic, 0x10+2*pin, *(((int *)&entry)+0)); set_native_irq_info(irq, TARGET_CPUS); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); } } =20 @@ -1314,10 +1316,10 @@ static void __init setup_ExtINT_IRQ0_pin /* * Add it to the IO-APIC irq-routing table: */ - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); io_apic_write(apic, 0x11+2*pin, *(((int *)&entry)+1)); io_apic_write(apic, 0x10+2*pin, *(((int *)&entry)+0)); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); =20 enable_8259A_irq(0); } @@ -1351,14 +1353,14 @@ void __init print_IO_APIC(void) =20 for (apic =3D 0; apic < nr_ioapics; apic++) { =20 - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); reg_00.raw =3D io_apic_read(apic, 0); reg_01.raw =3D io_apic_read(apic, 1); if (reg_01.bits.version >=3D 0x10) reg_02.raw =3D io_apic_read(apic, 2); if (reg_01.bits.version >=3D 0x20) reg_03.raw =3D io_apic_read(apic, 3); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); =20 printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].mpc_apicid); printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw); @@ -1427,10 +1429,10 @@ void __init print_IO_APIC(void) for (i =3D 0; i <=3D reg_01.bits.entries; i++) { struct IO_APIC_route_entry entry; =20 - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); *(((int *)&entry)+0) =3D io_apic_read(apic, 0x10+i*2); *(((int *)&entry)+1) =3D io_apic_read(apic, 0x11+i*2); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); =20 printk(KERN_DEBUG " %02x %03X %02X ", i, @@ -1596,7 +1598,7 @@ void /*__init*/ print_PIC(void) =20 printk(KERN_DEBUG "\nprinting PIC contents\n"); =20 - spin_lock_irqsave(&i8259A_lock, flags); + spin_lock_irqsave_hw(&i8259A_lock, flags); =20 v =3D inb(0xa1) << 8 | inb(0x21); printk(KERN_DEBUG "... PIC IMR: %04x\n", v); @@ -1610,7 +1612,7 @@ void /*__init*/ print_PIC(void) outb(0x0a,0xa0); outb(0x0a,0x20); =20 - spin_unlock_irqrestore(&i8259A_lock, flags); + spin_unlock_irqrestore_hw(&i8259A_lock, flags); =20 printk(KERN_DEBUG "... PIC ISR: %04x\n", v); =20 @@ -1639,9 +1641,9 @@ static void __init enable_IO_APIC(void) * The number of IO-APIC IRQ registers (=3D=3D #pins): */ for (apic =3D 0; apic < nr_ioapics; apic++) { - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); reg_01.raw =3D io_apic_read(apic, 1); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); nr_ioapic_registers[apic] =3D reg_01.bits.entries+1; } for(apic =3D 0; apic < nr_ioapics; apic++) { @@ -1649,10 +1651,10 @@ static void __init enable_IO_APIC(void) /* See if any of the pins is in ExtINT mode */ for (pin =3D 0; pin < nr_ioapic_registers[apic]; pin++) { struct IO_APIC_route_entry entry; - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); *(((int *)&entry) + 0) =3D io_apic_read(apic, 0x10 + 2 * pin); *(((int *)&entry) + 1) =3D io_apic_read(apic, 0x11 + 2 * pin); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); =20 =20 /* If the interrupt line is enabled and in ExtInt mode @@ -1726,12 +1728,12 @@ void disable_IO_APIC(void) /* * Add it to the IO-APIC irq-routing table: */ - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); io_apic_write(ioapic_i8259.apic, 0x11+2*ioapic_i8259.pin, *(((int *)&entry)+1)); io_apic_write(ioapic_i8259.apic, 0x10+2*ioapic_i8259.pin, *(((int *)&entry)+0)); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); } disconnect_bsp_APIC(ioapic_i8259.pin !=3D -1); } @@ -1772,9 +1774,9 @@ static void __init setup_ioapic_ids_from for (apic =3D 0; apic < nr_ioapics; apic++) { =20 /* Read the register 0 value */ - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); reg_00.raw =3D io_apic_read(apic, 0); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); =09 old_id =3D mp_ioapics[apic].mpc_apicid; =20 @@ -1833,16 +1835,16 @@ static void __init setup_ioapic_ids_from mp_ioapics[apic].mpc_apicid); =20 reg_00.bits.ID =3D mp_ioapics[apic].mpc_apicid; - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); io_apic_write(apic, 0, reg_00.raw); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); =20 /* * Sanity check */ - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); reg_00.raw =3D io_apic_read(apic, 0); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); if (reg_00.bits.ID !=3D mp_ioapics[apic].mpc_apicid) printk("could not set ID!\n"); else @@ -1909,14 +1911,15 @@ static unsigned int startup_edge_ioapic_ int was_pending =3D 0; unsigned long flags; =20 - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); if (irq < 16) { disable_8259A_irq(irq); if (i8259A_irq_pending(irq)) was_pending =3D 1; } __unmask_IO_APIC_irq(irq); - spin_unlock_irqrestore(&ioapic_lock, flags); + ipipe_irq_unlock(irq); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); =20 return was_pending; } @@ -1926,6 +1929,32 @@ static unsigned int startup_edge_ioapic_ * interrupt for real. This prevents IRQ storms from unhandled * devices. */ + +#ifdef CONFIG_IPIPE + +static void ack_edge_ioapic_irq (unsigned irq) + +{ + if ((irq_desc[irq].status & (IRQ_PENDING | IRQ_DISABLED)) + =3D=3D (IRQ_PENDING | IRQ_DISABLED)) { + unsigned long flags; + spin_lock_irqsave_hw(&ioapic_lock,flags); + __mask_IO_APIC_irq(irq); + spin_unlock_irqrestore_hw(&ioapic_lock,flags); + } + + __ack_APIC_irq(); +} + +#ifdef CONFIG_IRQBALANCE +static void end_edge_ioapic_irq (unsigned irq) +{ + move_irq(irq); +} +#endif /* CONFIG_IRQBALANCE */ + +#else /* !CONFIG_IPIPE */ + static void ack_edge_ioapic_irq(unsigned int irq) { move_irq(irq); @@ -1935,6 +1964,8 @@ static void ack_edge_ioapic_irq(unsigned ack_APIC_irq(); } =20 +#endif /* CONFIG_IPIPE */ + /* * Level triggered interrupts can just be masked, * and shutting down and starting up the interrupt @@ -1956,6 +1987,82 @@ static unsigned int startup_level_ioapic return 0; /* don't check for pending */ } =20 +#ifdef CONFIG_IPIPE + +/* Prevent low priority IRQs grabbed by high priority domains from + being delayed, waiting for a high priority interrupt handler + running in a low priority domain to complete. */ + +static unsigned long bugous_edge_triggers; + +static void end_level_ioapic_irq (unsigned irq) + +{ + unsigned long flags; + + move_irq(irq); + + spin_lock_irqsave_hw(&ioapic_lock, flags); + + if (test_and_clear_bit(irq,&bugous_edge_triggers)) { + atomic_inc(&irq_mis_count); + __unmask_and_level_IO_APIC_irq(irq); + } + else + __unmask_IO_APIC_irq(irq); + + ipipe_irq_unlock(irq); + + spin_unlock_irqrestore_hw(&ioapic_lock, flags); +} + +static void mask_and_ack_level_ioapic_irq (unsigned irq) + +{ + unsigned long flags, v; + int i; + + i =3D IO_APIC_VECTOR(irq); + v =3D apic_read(APIC_TMR + ((i & ~0x1f) >> 1)); + + spin_lock_irqsave_hw(&ioapic_lock, flags); + + if (!(v & (1 << (i & 0x1f)))) { + set_bit(irq,&bugous_edge_triggers); + __mask_and_edge_IO_APIC_irq(irq); + } + else + __mask_IO_APIC_irq(irq); + + spin_unlock_irqrestore_hw(&ioapic_lock, flags); + + __ack_APIC_irq(); +} + +#ifdef CONFIG_PCI_MSI + +#ifdef CONFIG_IRQBALANCE +static inline void end_edge_ioapic_vector(unsigned int vector) + +{ + int irq =3D vector_to_irq(vector); + + end_edge_ioapic_irq(irq); +} +#endif /* CONFIG_IRQBALANCE */ + +static inline void mask_and_ack_level_ioapic_vector(unsigned int vector)= + +{ + int irq =3D vector_to_irq(vector); + + mask_and_ack_level_ioapic_irq(irq); +} + +#endif /* CONFIG_PCI_MSI */ + +#else /* !CONFIG_IPIPE */ + static void end_level_ioapic_irq (unsigned int irq) { unsigned long v; @@ -1996,6 +2103,8 @@ static void end_level_ioapic_irq (unsign } } =20 +#endif /* CONFIG_IPIPE */ + #ifdef CONFIG_PCI_MSI static unsigned int startup_edge_ioapic_vector(unsigned int vector) { @@ -2142,7 +2251,7 @@ static void disable_lapic_irq (unsigned=20 =20 static void ack_lapic_irq (unsigned int irq) { - ack_APIC_irq(); + __ack_APIC_irq(); } =20 static void end_lapic_irq (unsigned int i) { /* nothing */ } @@ -2194,10 +2303,10 @@ static inline void unlock_ExtINT_logic(v if (pin =3D=3D -1) return; =20 - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); *(((int *)&entry0) + 1) =3D io_apic_read(apic, 0x11 + 2 * pin); *(((int *)&entry0) + 0) =3D io_apic_read(apic, 0x10 + 2 * pin); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); clear_IO_APIC_pin(apic, pin); =20 memset(&entry1, 0, sizeof(entry1)); @@ -2210,10 +2319,10 @@ static inline void unlock_ExtINT_logic(v entry1.trigger =3D 0; entry1.vector =3D 0; =20 - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); io_apic_write(apic, 0x11 + 2 * pin, *(((int *)&entry1) + 1)); io_apic_write(apic, 0x10 + 2 * pin, *(((int *)&entry1) + 0)); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); =20 save_control =3D CMOS_READ(RTC_CONTROL); save_freq_select =3D CMOS_READ(RTC_FREQ_SELECT); @@ -2232,10 +2341,10 @@ static inline void unlock_ExtINT_logic(v CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT); clear_IO_APIC_pin(apic, pin); =20 - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); io_apic_write(apic, 0x11 + 2 * pin, *(((int *)&entry0) + 1)); io_apic_write(apic, 0x10 + 2 * pin, *(((int *)&entry0) + 0)); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); } =20 int timer_uses_ioapic_pin_0; @@ -2440,12 +2549,12 @@ static int ioapic_suspend(struct sys_dev =09 data =3D container_of(dev, struct sysfs_ioapic_data, dev); entry =3D data->entry; - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); for (i =3D 0; i < nr_ioapic_registers[dev->id]; i ++, entry ++ ) { *(((int *)entry) + 1) =3D io_apic_read(dev->id, 0x11 + 2 * i); *(((int *)entry) + 0) =3D io_apic_read(dev->id, 0x10 + 2 * i); } - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); =20 return 0; } @@ -2461,7 +2570,7 @@ static int ioapic_resume(struct sys_devi data =3D container_of(dev, struct sysfs_ioapic_data, dev); entry =3D data->entry; =20 - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); reg_00.raw =3D io_apic_read(dev->id, 0); if (reg_00.bits.ID !=3D mp_ioapics[dev->id].mpc_apicid) { reg_00.bits.ID =3D mp_ioapics[dev->id].mpc_apicid; @@ -2471,7 +2580,7 @@ static int ioapic_resume(struct sys_devi io_apic_write(dev->id, 0x11+2*i, *(((int *)entry)+1)); io_apic_write(dev->id, 0x10+2*i, *(((int *)entry)+0)); } - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); =20 return 0; } @@ -2543,9 +2652,9 @@ int __init io_apic_get_unique_id (int io if (physids_empty(apic_id_map)) apic_id_map =3D ioapic_phys_id_map(phys_cpu_present_map); =20 - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); reg_00.raw =3D io_apic_read(ioapic, 0); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); =20 if (apic_id >=3D get_physical_broadcast()) { printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying " @@ -2579,10 +2688,10 @@ int __init io_apic_get_unique_id (int io if (reg_00.bits.ID !=3D apic_id) { reg_00.bits.ID =3D apic_id; =20 - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); io_apic_write(ioapic, 0, reg_00.raw); reg_00.raw =3D io_apic_read(ioapic, 0); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); =20 /* Sanity check */ if (reg_00.bits.ID !=3D apic_id) { @@ -2603,9 +2712,9 @@ int __init io_apic_get_version (int ioap union IO_APIC_reg_01 reg_01; unsigned long flags; =20 - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); reg_01.raw =3D io_apic_read(ioapic, 1); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); =20 return reg_01.bits.version; } @@ -2616,9 +2725,9 @@ int __init io_apic_get_redir_entries (in union IO_APIC_reg_01 reg_01; unsigned long flags; =20 - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); reg_01.raw =3D io_apic_read(ioapic, 1); - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); =20 return reg_01.bits.entries; } @@ -2668,11 +2777,11 @@ int io_apic_set_pci_routing (int ioapic, if (!ioapic && (irq < 16)) disable_8259A_irq(irq); =20 - spin_lock_irqsave(&ioapic_lock, flags); + spin_lock_irqsave_hw(&ioapic_lock, flags); io_apic_write(ioapic, 0x11+2*pin, *(((int *)&entry)+1)); io_apic_write(ioapic, 0x10+2*pin, *(((int *)&entry)+0)); set_native_irq_info(use_pci_vector() ? entry.vector : irq, TARGET_CPUS)= ; - spin_unlock_irqrestore(&ioapic_lock, flags); + spin_unlock_irqrestore_hw(&ioapic_lock, flags); =20 return 0; } Index: linux-2.6.17/arch/i386/kernel/ipipe-core.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- /dev/null +++ linux-2.6.17/arch/i386/kernel/ipipe-core.c @@ -0,0 +1,306 @@ +/* -*- linux-c -*- + * linux/arch/i386/kernel/ipipe-core.c + * + * Copyright (C) 2002-2005 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modif= y + * it under the terms of the GNU General Public License as published b= y + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 0213= 9, + * USA; either version 2 of the License, or (at your option) any later= + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-130= 7, USA. + * + * Architecture-dependent I-PIPE core support for x86. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_X86_LOCAL_APIC +#include +#include +#include +#ifdef CONFIG_X86_IO_APIC +#include +#endif /* CONFIG_X86_IO_APIC */ +#include +#include +#endif /* CONFIG_X86_LOCAL_APIC */ + +struct pt_regs __ipipe_tick_regs[IPIPE_NR_CPUS]; + +int __ipipe_tick_irq; + +#ifdef CONFIG_SMP + +static cpumask_t __ipipe_cpu_sync_map; + +static cpumask_t __ipipe_cpu_lock_map; + +static ipipe_spinlock_t __ipipe_cpu_barrier =3D IPIPE_SPIN_LOCK_UNLOCKED= ; + +static atomic_t __ipipe_critical_count =3D ATOMIC_INIT(0); + +static void (*__ipipe_cpu_sync) (void); + +#endif /* CONFIG_SMP */ + +int __ipipe_ack_system_irq(unsigned irq) +{ +#ifdef CONFIG_X86_LOCAL_APIC + __ack_APIC_irq(); +#endif /* CONFIG_X86_LOCAL_APIC */ + return 1; +} + +#ifdef CONFIG_SMP + +/* Always called with hw interrupts off. */ + +void __ipipe_do_critical_sync(unsigned irq, void *cookie) +{ + ipipe_declare_cpuid; + + ipipe_load_cpuid(); + + cpu_set(cpuid, __ipipe_cpu_sync_map); + + /* Now we are in sync with the lock requestor running on another + CPU. Enter a spinning wait until he releases the global + lock. */ + spin_lock_hw(&__ipipe_cpu_barrier); + + /* Got it. Now get out. */ + + if (__ipipe_cpu_sync) + /* Call the sync routine if any. */ + __ipipe_cpu_sync(); + + spin_unlock_hw(&__ipipe_cpu_barrier); + + cpu_clear(cpuid, __ipipe_cpu_sync_map); +} + +#endif /* CONFIG_SMP */ + +/* ipipe_critical_enter() -- Grab the superlock excluding all CPUs + but the current one from a critical section. This lock is used when + we must enforce a global critical section for a single CPU in a + possibly SMP system whichever context the CPUs are running. */ + +unsigned long ipipe_critical_enter(void (*syncfn) (void)) +{ + unsigned long flags; + + local_irq_save_hw(flags); + +#ifdef CONFIG_SMP + if (num_online_cpus() > 1) { /* We might be running a SMP-kernel on a U= P box... */ + ipipe_declare_cpuid; + cpumask_t lock_map; + + ipipe_load_cpuid(); + + if (!cpu_test_and_set(cpuid, __ipipe_cpu_lock_map)) { + while (cpu_test_and_set + (BITS_PER_LONG - 1, __ipipe_cpu_lock_map)) { + int n =3D 0; + do { + cpu_relax(); + } while (++n < cpuid); + } + + spin_lock_hw(&__ipipe_cpu_barrier); + + __ipipe_cpu_sync =3D syncfn; + + /* Send the sync IPI to all processors but the current one. */ + send_IPI_allbutself(IPIPE_CRITICAL_VECTOR); + + cpus_andnot(lock_map, cpu_online_map, + __ipipe_cpu_lock_map); + + while (!cpus_equal(__ipipe_cpu_sync_map, lock_map)) + cpu_relax(); + } + + atomic_inc(&__ipipe_critical_count); + } +#endif /* CONFIG_SMP */ + + return flags; +} + +/* ipipe_critical_exit() -- Release the superlock. */ + +void ipipe_critical_exit(unsigned long flags) +{ +#ifdef CONFIG_SMP + if (num_online_cpus() > 1) { /* We might be running a SMP-kernel on a U= P box... */ + ipipe_declare_cpuid; + + ipipe_load_cpuid(); + + if (atomic_dec_and_test(&__ipipe_critical_count)) { + spin_unlock_hw(&__ipipe_cpu_barrier); + + while (!cpus_empty(__ipipe_cpu_sync_map)) + cpu_relax(); + + cpu_clear(cpuid, __ipipe_cpu_lock_map); + cpu_clear(BITS_PER_LONG - 1, __ipipe_cpu_lock_map); + } + } +#endif /* CONFIG_SMP */ + + local_irq_restore_hw(flags); +} + +/* ipipe_trigger_irq() -- Push the interrupt at front of the pipeline + just like if it has been actually received from a hw source. Also + works for virtual interrupts. */ + +int fastcall ipipe_trigger_irq(unsigned irq) +{ + struct pt_regs regs; + unsigned long flags; + + if (irq >=3D IPIPE_NR_IRQS || + (ipipe_virtual_irq_p(irq) && + !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map))) + return -EINVAL; + + local_irq_save_hw(flags); + + regs.orig_eax =3D irq; /* Won't be acked */ + regs.xcs =3D __KERNEL_CS; + regs.eflags =3D flags; + + __ipipe_handle_irq(regs); + + local_irq_restore_hw(flags); + + return 1; +} + +#ifdef CONFIG_SMP + +cpumask_t __ipipe_set_irq_affinity (unsigned irq, cpumask_t cpumask) + +{ + cpumask_t oldmask =3D irq_affinity[irq]; + + if (irq_desc[irq].handler->set_affinity =3D=3D NULL) + return CPU_MASK_NONE; + + if (cpus_empty(cpumask)) + return oldmask; /* Return mask value -- no change. */ + + cpus_and(cpumask,cpumask,cpu_online_map); + + if (cpus_empty(cpumask)) + return CPU_MASK_NONE; /* Error -- bad mask value or non-routable IRQ. = */ + + irq_affinity[irq] =3D cpumask; + irq_desc[irq].handler->set_affinity(irq,cpumask); + return oldmask; +} + +int fastcall __ipipe_send_ipi (unsigned ipi, cpumask_t cpumask) + +{ + unsigned long flags; + ipipe_declare_cpuid; + int self; + + if (ipi !=3D IPIPE_SERVICE_IPI0 && + ipi !=3D IPIPE_SERVICE_IPI1 && + ipi !=3D IPIPE_SERVICE_IPI2 && + ipi !=3D IPIPE_SERVICE_IPI3) + return -EINVAL; + + ipipe_lock_cpu(flags); + + self =3D cpu_isset(cpuid,cpumask); + cpu_clear(cpuid,cpumask); + + if (!cpus_empty(cpumask)) + send_IPI_mask(cpumask,ipi + FIRST_EXTERNAL_VECTOR); + + if (self) + ipipe_trigger_irq(ipi); + + ipipe_unlock_cpu(flags); + + return 0; +} + +#endif /* CONFIG_SMP */ + +int ipipe_get_sysinfo(struct ipipe_sysinfo *info) +{ + info->ncpus =3D num_online_cpus(); + info->cpufreq =3D ipipe_cpu_freq(); + info->archdep.tmirq =3D __ipipe_tick_irq; +#ifdef CONFIG_X86_TSC + info->archdep.tmfreq =3D ipipe_cpu_freq(); +#else /* !CONFIG_X86_TSC */ + info->archdep.tmfreq =3D CLOCK_TICK_RATE; +#endif /* CONFIG_X86_TSC */ + + return 0; +} + +int ipipe_tune_timer (unsigned long ns, int flags) + +{ + unsigned hz, latch; + unsigned long x; + + if (flags & IPIPE_RESET_TIMER) + latch =3D LATCH; + else { + hz =3D 1000000000 / ns; + + if (hz < HZ) + return -EINVAL; + + latch =3D (CLOCK_TICK_RATE + hz/2) / hz; + } + + x =3D ipipe_critical_enter(NULL); /* Sync with all CPUs */ + + /* Shamelessly lifted from init_IRQ() in i8259.c */ + outb_p(0x34,0x43); /* binary, mode 2, LSB/MSB, ch 0 */ + outb_p(latch & 0xff,0x40); /* LSB */ + outb(latch >> 8,0x40); /* MSB */ + + ipipe_critical_exit(x); + + return 0; +} + +EXPORT_SYMBOL(__ipipe_tick_irq); +EXPORT_SYMBOL(ipipe_critical_enter); +EXPORT_SYMBOL(ipipe_critical_exit); +EXPORT_SYMBOL(ipipe_trigger_irq); +EXPORT_SYMBOL(ipipe_get_sysinfo); +EXPORT_SYMBOL(ipipe_tune_timer); Index: linux-2.6.17/arch/i386/kernel/ipipe-root.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- /dev/null +++ linux-2.6.17/arch/i386/kernel/ipipe-root.c @@ -0,0 +1,642 @@ +/* -*- linux-c -*- + * linux/arch/i386/kernel/ipipe-root.c + * + * Copyright (C) 2002-2005 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modif= y + * it under the terms of the GNU General Public License as published b= y + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 0213= 9, + * USA; either version 2 of the License, or (at your option) any later= + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-130= 7, USA. + * + * Architecture-dependent I-PIPE support for x86. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_X86_LOCAL_APIC +#include +#include +#include +#include +#ifdef CONFIG_X86_IO_APIC +#include +#endif /* CONFIG_X86_IO_APIC */ +#include +#include + +static int __ipipe_noack_irq(unsigned irq) +{ + return 1; +} +#endif /* CONFIG_X86_LOCAL_APIC */ + +fastcall unsigned int do_IRQ(struct pt_regs *regs); +fastcall void smp_apic_timer_interrupt(struct pt_regs *regs); +fastcall void smp_spurious_interrupt(struct pt_regs *regs); +fastcall void smp_error_interrupt(struct pt_regs *regs); +fastcall void smp_thermal_interrupt(struct pt_regs *regs); +fastcall void smp_reschedule_interrupt(struct pt_regs *regs); +fastcall void smp_invalidate_interrupt(struct pt_regs *regs); +fastcall void smp_call_function_interrupt(struct pt_regs *regs); + +static int __ipipe_ack_common_irq(unsigned irq) +{ + irq_desc_t *desc =3D irq_desc + irq; + unsigned long flags; + ipipe_declare_cpuid; + + ipipe_load_cpuid(); /* hw interrupts are off. */ + flags =3D ipipe_test_and_stall_pipeline(); + preempt_disable(); + desc->handler->ack(irq); + preempt_enable_no_resched(); + ipipe_restore_pipeline_nosync(ipipe_percpu_domain[cpuid], flags, cpuid)= ; + + return 1; +} + +#ifdef CONFIG_X86_LOCAL_APIC + +static void __ipipe_null_handler(unsigned irq, void *cookie) +{ + /* Nop. */ +} + +#ifdef CONFIG_SMP + +static notrace int __ipipe_boot_cpuid(void) +{ + return 0; +} + +u8 __ipipe_apicid_2_cpu[IPIPE_NR_CPUS]; + +static notrace int __ipipe_hard_cpuid(void) +{ + unsigned long flags; + int cpu; + + local_irq_save_hw_notrace(flags); + cpu =3D __ipipe_apicid_2_cpu[GET_APIC_ID(apic_read(APIC_ID))]; + local_irq_restore_hw_notrace(flags); + return cpu; +} + +int (*__ipipe_logical_cpuid)(void) =3D &__ipipe_boot_cpuid; + +EXPORT_SYMBOL(__ipipe_logical_cpuid); + +#endif /* CONFIG_SMP */ + +#endif /* CONFIG_X86_LOCAL_APIC */ + +/* __ipipe_enable_pipeline() -- We are running on the boot CPU, hw + interrupts are off, and secondary CPUs are still lost in space. */ + +void __init __ipipe_enable_pipeline(void) +{ + unsigned irq; + +#ifdef CONFIG_X86_LOCAL_APIC + + /* Map the APIC system vectors. */ + + ipipe_virtualize_irq(ipipe_root_domain, + LOCAL_TIMER_VECTOR - FIRST_EXTERNAL_VECTOR, + (ipipe_irq_handler_t)&smp_apic_timer_interrupt, + NULL, + &__ipipe_ack_system_irq, + IPIPE_STDROOT_MASK); + + ipipe_virtualize_irq(ipipe_root_domain, + SPURIOUS_APIC_VECTOR - FIRST_EXTERNAL_VECTOR, + (ipipe_irq_handler_t)&smp_spurious_interrupt, + NULL, + &__ipipe_noack_irq, + IPIPE_STDROOT_MASK); + + ipipe_virtualize_irq(ipipe_root_domain, + ERROR_APIC_VECTOR - FIRST_EXTERNAL_VECTOR, + (ipipe_irq_handler_t)&smp_error_interrupt, + NULL, + &__ipipe_ack_system_irq, + IPIPE_STDROOT_MASK); + + ipipe_virtualize_irq(ipipe_root_domain, + IPIPE_SERVICE_VECTOR0 - FIRST_EXTERNAL_VECTOR, + &__ipipe_null_handler, + NULL, + &__ipipe_ack_system_irq, + IPIPE_STDROOT_MASK); + + ipipe_virtualize_irq(ipipe_root_domain, + IPIPE_SERVICE_VECTOR1 - FIRST_EXTERNAL_VECTOR, + &__ipipe_null_handler, + NULL, + &__ipipe_ack_system_irq, + IPIPE_STDROOT_MASK); + + ipipe_virtualize_irq(ipipe_root_domain, + IPIPE_SERVICE_VECTOR2 - FIRST_EXTERNAL_VECTOR, + &__ipipe_null_handler, + NULL, + &__ipipe_ack_system_irq, + IPIPE_STDROOT_MASK); + + ipipe_virtualize_irq(ipipe_root_domain, + IPIPE_SERVICE_VECTOR3 - FIRST_EXTERNAL_VECTOR, + &__ipipe_null_handler, + NULL, + &__ipipe_ack_system_irq, + IPIPE_STDROOT_MASK); + +#ifdef CONFIG_X86_MCE_P4THERMAL + ipipe_virtualize_irq(ipipe_root_domain, + THERMAL_APIC_VECTOR - FIRST_EXTERNAL_VECTOR, + (ipipe_irq_handler_t)&smp_thermal_interrupt, + NULL, + &__ipipe_ack_system_irq, + IPIPE_STDROOT_MASK); +#endif /* CONFIG_X86_MCE_P4THERMAL */ + + __ipipe_tick_irq =3D + using_apic_timer ? LOCAL_TIMER_VECTOR - FIRST_EXTERNAL_VECTOR : 0; + +#else /* !CONFIG_X86_LOCAL_APIC */ + + __ipipe_tick_irq =3D 0; + +#endif /* CONFIG_X86_LOCAL_APIC */ + +#ifdef CONFIG_SMP + + ipipe_virtualize_irq(ipipe_root_domain, + RESCHEDULE_VECTOR - FIRST_EXTERNAL_VECTOR, + (ipipe_irq_handler_t)&smp_reschedule_interrupt, + NULL, + &__ipipe_ack_system_irq, + IPIPE_STDROOT_MASK); + + ipipe_virtualize_irq(ipipe_root_domain, + INVALIDATE_TLB_VECTOR - FIRST_EXTERNAL_VECTOR, + (ipipe_irq_handler_t)&smp_invalidate_interrupt, + NULL, + &__ipipe_ack_system_irq, + IPIPE_STDROOT_MASK); + + ipipe_virtualize_irq(ipipe_root_domain, + CALL_FUNCTION_VECTOR - FIRST_EXTERNAL_VECTOR, + (ipipe_irq_handler_t)&smp_call_function_interrupt, + NULL, + &__ipipe_ack_system_irq, + IPIPE_STDROOT_MASK); + + /* Some guest O/S may run tasks over non-Linux stacks, so we + * cannot rely on the regular definition of smp_processor_id() + * on x86 to fetch the logical cpu id. We fix this by using + * our own private physical apicid -> logicial cpuid mapping + * as soon as the pipeline is enabled, so that + * ipipe_processor_id() always do the right thing, regardless + * of the current stack setup. Also note that the pipeline is + * enabled after the APIC space has been mapped in + * trap_init(), so it's safe to use it. */ + + __ipipe_logical_cpuid =3D &__ipipe_hard_cpuid; + +#endif /* CONFIG_SMP */ + + /* Finally, virtualize the remaining ISA and IO-APIC + * interrupts. Interrupts which have already been virtualized + * will just beget a silent -EPERM error since + * IPIPE_SYSTEM_MASK has been passed for them, that's ok. */ + + for (irq =3D 0; irq < NR_IRQS; irq++) { + /* Fails for IPIPE_CRITICAL_IPI but that's ok. */ + ipipe_virtualize_irq(ipipe_root_domain, + irq, + (ipipe_irq_handler_t)&do_IRQ, + NULL, + &__ipipe_ack_common_irq, + IPIPE_STDROOT_MASK); + } + +#ifdef CONFIG_X86_LOCAL_APIC + /* Eventually allow these vectors to be reprogrammed. */ + ipipe_root_domain->irqs[IPIPE_SERVICE_IPI0].control &=3D ~IPIPE_SYSTEM_= MASK; + ipipe_root_domain->irqs[IPIPE_SERVICE_IPI1].control &=3D ~IPIPE_SYSTEM_= MASK; + ipipe_root_domain->irqs[IPIPE_SERVICE_IPI2].control &=3D ~IPIPE_SYSTEM_= MASK; + ipipe_root_domain->irqs[IPIPE_SERVICE_IPI3].control &=3D ~IPIPE_SYSTEM_= MASK; +#endif /* CONFIG_X86_LOCAL_APIC */ +} + +static inline void __fixup_if(struct pt_regs *regs) +{ + ipipe_declare_cpuid; + unsigned long flags; + + ipipe_get_cpu(flags); + + if (ipipe_percpu_domain[cpuid] =3D=3D ipipe_root_domain) { + /* Have the saved hw state look like the domain stall bit, so + that __ipipe_unstall_iret_root() restores the proper + pipeline state for the root stage upon exit. */ + + if (test_bit + (IPIPE_STALL_FLAG, + &ipipe_root_domain->cpudata[cpuid].status)) + regs->eflags &=3D ~X86_EFLAGS_IF; + else + regs->eflags |=3D X86_EFLAGS_IF; + } + + ipipe_put_cpu(flags); +} + +/* Check the stall bit of the root domain to make sure the existing + preemption opportunity upon in-kernel resumption could be + exploited. In case a rescheduling could take place, the root stage + is stalled before the hw interrupts are re-enabled. This routine + must be called with hw interrupts off. */ + +asmlinkage int __ipipe_kpreempt_root(struct pt_regs regs) +{ + ipipe_declare_cpuid; + unsigned long flags; + + ipipe_get_cpu(flags); + + if (test_bit + (IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].status)) { + ipipe_put_cpu(flags); + return 0; /* Root stage is stalled: rescheduling denied. */ + } + + __ipipe_stall_root(); + local_irq_enable_hw_notrace(); + + return 1; /* Ok, may reschedule now. */ +} + +asmlinkage void __ipipe_unstall_iret_root(struct pt_regs regs) +{ + ipipe_declare_cpuid; + + /* Emulate IRET's handling of the interrupt flag. */ + + local_irq_disable_hw(); + + ipipe_load_cpuid(); + + /* Restore the software state as it used to be on kernel + entry. CAUTION: NMIs must *not* return through this + emulation. */ + + if (!(regs.eflags & X86_EFLAGS_IF)) { + __set_bit(IPIPE_STALL_FLAG, + &ipipe_root_domain->cpudata[cpuid].status); + regs.eflags |=3D X86_EFLAGS_IF; + } else { + __clear_bit(IPIPE_STALL_FLAG, + &ipipe_root_domain->cpudata[cpuid].status); + + /* Only sync virtual IRQs here, so that we don't recurse + indefinitely in case of an external interrupt flood. */ + + if ((ipipe_root_domain->cpudata[cpuid]. + irq_pending_hi & IPIPE_IRQMASK_VIRT) !=3D 0) + __ipipe_sync_pipeline(IPIPE_IRQMASK_VIRT); + } +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + ipipe_trace_end(0x8000000D); +#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */ +} + +asmlinkage int __ipipe_syscall_root(struct pt_regs regs) +{ + ipipe_declare_cpuid; + unsigned long flags; + + __fixup_if(®s); + + /* This routine either returns: + 0 -- if the syscall is to be passed to Linux; + >0 -- if the syscall should not be passed to Linux, and no + tail work should be performed; + <0 -- if the syscall should not be passed to Linux but the + tail work has to be performed (for handling signals etc). */ + + if (__ipipe_event_pipelined_p(IPIPE_EVENT_SYSCALL) && + __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL,®s) > 0) { + /* We might enter here over a non-root domain and exit + * over the root one as a result of the syscall + * (i.e. by recycling the register set of the current + * context across the migration), so we need to fixup + * the interrupt flag upon return too, so that + * __ipipe_unstall_iret_root() resets the correct + * stall bit on exit. */ + __fixup_if(®s); + + if (ipipe_current_domain =3D=3D ipipe_root_domain && !in_atomic()) { + /* Sync pending VIRQs before _TIF_NEED_RESCHED + * is tested. */ + ipipe_lock_cpu(flags); + if ((ipipe_root_domain->cpudata[cpuid].irq_pending_hi & IPIPE_IRQMASK= _VIRT) !=3D 0) + __ipipe_sync_pipeline(IPIPE_IRQMASK_VIRT); + ipipe_unlock_cpu(flags); + return -1; + } + return 1; + } + + return 0; +} + +static fastcall void do_machine_check_vector(struct pt_regs *regs, long = error_code) +{ +#ifdef CONFIG_X86_MCE + extern fastcall void (*machine_check_vector)(struct pt_regs *, long); + machine_check_vector(regs,error_code); +#endif /* CONFIG_X86_MCE */ +} + +fastcall void do_divide_error(struct pt_regs *regs, long error_code); +fastcall void do_overflow(struct pt_regs *regs, long error_code); +fastcall void do_bounds(struct pt_regs *regs, long error_code); +fastcall void do_invalid_op(struct pt_regs *regs, long error_code); +fastcall void do_coprocessor_segment_overrun(struct pt_regs *regs, long = error_code); +fastcall void do_invalid_TSS(struct pt_regs *regs, long error_code); +fastcall void do_segment_not_present(struct pt_regs *regs, long error_co= de); +fastcall void do_stack_segment(struct pt_regs *regs, long error_code); +fastcall void do_general_protection(struct pt_regs *regs, long error_cod= e); +fastcall void do_page_fault(struct pt_regs *regs, long error_code); +fastcall void do_spurious_interrupt_bug(struct pt_regs *regs, long error= _code); +fastcall void do_coprocessor_error(struct pt_regs *regs, long error_code= ); +fastcall void do_alignment_check(struct pt_regs *regs, long error_code);= +fastcall void do_simd_coprocessor_error(struct pt_regs *regs, long error= _code); +fastcall void do_iret_error(struct pt_regs *regs, long error_code); + +/* Work around genksyms's issue with over-qualification in decls. */ + +typedef fastcall void __ipipe_exhandler(struct pt_regs *, long); + +typedef __ipipe_exhandler *__ipipe_exptr; + +static __ipipe_exptr __ipipe_std_extable[] =3D { + + [ex_do_divide_error] =3D &do_divide_error, + [ex_do_overflow] =3D &do_overflow, + [ex_do_bounds] =3D &do_bounds, + [ex_do_invalid_op] =3D &do_invalid_op, + [ex_do_coprocessor_segment_overrun] =3D &do_coprocessor_segment_overrun= , + [ex_do_invalid_TSS] =3D &do_invalid_TSS, + [ex_do_segment_not_present] =3D &do_segment_not_present, + [ex_do_stack_segment] =3D &do_stack_segment, + [ex_do_general_protection] =3D do_general_protection, + [ex_do_page_fault] =3D &do_page_fault, + [ex_do_spurious_interrupt_bug] =3D &do_spurious_interrupt_bug, + [ex_do_coprocessor_error] =3D &do_coprocessor_error, + [ex_do_alignment_check] =3D &do_alignment_check, + [ex_machine_check_vector] =3D &do_machine_check_vector, + [ex_do_simd_coprocessor_error] =3D &do_simd_coprocessor_error, + [ex_do_iret_error] =3D &do_iret_error, +}; + +#ifdef CONFIG_KGDB +#include + +static int __ipipe_xlate_signo[] =3D { + + [ex_do_divide_error] =3D SIGFPE, + [ex_do_debug] =3D SIGTRAP, + [2] =3D -1, + [ex_do_int3] =3D SIGTRAP, + [ex_do_overflow] =3D SIGSEGV, + [ex_do_bounds] =3D SIGSEGV, + [ex_do_invalid_op] =3D SIGILL, + [ex_device_not_available] =3D -1, + [8] =3D -1, + [ex_do_coprocessor_segment_overrun] =3D SIGFPE, + [ex_do_invalid_TSS] =3D SIGSEGV, + [ex_do_segment_not_present] =3D SIGBUS, + [ex_do_stack_segment] =3D SIGBUS, + [ex_do_general_protection] =3D SIGSEGV, + [ex_do_page_fault] =3D SIGSEGV, + [ex_do_spurious_interrupt_bug] =3D -1, + [ex_do_coprocessor_error] =3D -1, + [ex_do_alignment_check] =3D SIGBUS, + [ex_machine_check_vector] =3D -1, + [ex_do_simd_coprocessor_error] =3D -1, + [20 ... 31] =3D -1, + [ex_do_iret_error] =3D SIGSEGV, +}; +#endif /* CONFIG_KGDB */ + +fastcall int __ipipe_handle_exception(struct pt_regs *regs, long error_c= ode, int vector) +{ +#ifdef CONFIG_KGDB + /* catch exception KGDB is interested in over non-root domains */ + if (ipipe_current_domain !=3D ipipe_root_domain && + __ipipe_xlate_signo[vector] >=3D 0 && + !kgdb_handle_exception(vector, __ipipe_xlate_signo[vector], error_c= ode, regs)) + return 1; +#endif /* CONFIG_KGDB */ + + if (!__ipipe_event_pipelined_p(vector) || + __ipipe_dispatch_event(vector,regs) =3D=3D 0) { + __ipipe_exptr handler =3D __ipipe_std_extable[vector]; + handler(regs,error_code); + __fixup_if(regs); + return 0; + } + + return 1; +} + +fastcall int __ipipe_divert_exception(struct pt_regs *regs, int vector) +{ +#ifdef CONFIG_KGDB + /* catch int1 and int3 over non-root domains */ + if ((ipipe_current_domain !=3D ipipe_root_domain) && + (vector !=3D ex_device_not_available)) { + unsigned int condition =3D 0; + + if (vector =3D=3D 1) + get_debugreg(condition, 6); + if (!kgdb_handle_exception(vector, SIGTRAP, condition, regs)) + return 1; + } +#endif /* CONFIG_KGDB */ + + if (__ipipe_event_pipelined_p(vector) && + __ipipe_dispatch_event(vector,regs) !=3D 0) + return 1; + + __fixup_if(regs); + + return 0; +} + +/* __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic + interrupt protection log is maintained here for each domain. Hw + interrupts are off on entry. */ + +int __ipipe_handle_irq(struct pt_regs regs) +{ + struct ipipe_domain *this_domain, *next_domain; + unsigned irq =3D regs.orig_eax; + struct list_head *head, *pos; + ipipe_declare_cpuid; + int m_ack, s_ack; + + ipipe_load_cpuid(); + + if (regs.orig_eax < 0) { + irq &=3D 0xff; + m_ack =3D 0; + } else + m_ack =3D 1; + + this_domain =3D ipipe_percpu_domain[cpuid]; + + if (test_bit(IPIPE_STICKY_FLAG, &this_domain->irqs[irq].control)) + head =3D &this_domain->p_link; + else { + head =3D __ipipe_pipeline.next; + next_domain =3D list_entry(head, struct ipipe_domain, p_link); + if (likely(test_bit(IPIPE_WIRED_FLAG, &next_domain->irqs[irq].control)= )) { + if (!m_ack && next_domain->irqs[irq].acknowledge !=3D NULL) + next_domain->irqs[irq].acknowledge(irq); + if (likely(__ipipe_dispatch_wired(next_domain, irq))) + goto finalize; + else + goto finalize_nosync; + } + } + + /* Ack the interrupt. */ + + s_ack =3D m_ack; + pos =3D head; + + while (pos !=3D &__ipipe_pipeline) { + next_domain =3D list_entry(pos, struct ipipe_domain, p_link); + + /* For each domain handling the incoming IRQ, mark it as + pending in its log. */ + + if (test_bit + (IPIPE_HANDLE_FLAG, &next_domain->irqs[irq].control)) { + /* Domains that handle this IRQ are polled for + acknowledging it by decreasing priority order. The + interrupt must be made pending _first_ in the domain's + status flags before the PIC is unlocked. */ + + next_domain->cpudata[cpuid].irq_counters[irq].total_hits++; + next_domain->cpudata[cpuid].irq_counters[irq].pending_hits++; + __ipipe_set_irq_bit(next_domain, cpuid, irq); + + /* Always get the first master acknowledge available. Once + we've got it, allow slave acknowledge handlers to run + (until one of them stops us). */ + + if (!m_ack) + m_ack =3D next_domain->irqs[irq].acknowledge(irq); + else if (test_bit + (IPIPE_SHARED_FLAG, + &next_domain->irqs[irq].control) && !s_ack) + s_ack =3D next_domain->irqs[irq].acknowledge(irq); + } + + /* If the domain does not want the IRQ to be passed down the + interrupt pipe, exit the loop now. */ + + if (!test_bit(IPIPE_PASS_FLAG, &next_domain->irqs[irq].control)) + break; + + pos =3D next_domain->p_link.next; + } + + if (irq =3D=3D __ipipe_tick_irq && + __ipipe_pipeline_head_p(ipipe_root_domain) && + ipipe_root_domain->cpudata[cpuid].irq_counters[irq].pending_hits > = 1) + /* + * Emulate a loss of clock ticks if Linux is owning + * the time source. The drift will be compensated by + * the timer support code. + */ + ipipe_root_domain->cpudata[cpuid].irq_counters[irq].pending_hits =3D 1= ; + +finalize: + + if (irq =3D=3D __ipipe_tick_irq) { + __ipipe_tick_regs[cpuid].eflags =3D regs.eflags; + __ipipe_tick_regs[cpuid].eip =3D regs.eip; + __ipipe_tick_regs[cpuid].xcs =3D regs.xcs; +#if defined(CONFIG_SMP) && defined(CONFIG_FRAME_POINTER) + /* Linux profiling code needs this. */ + __ipipe_tick_regs[cpuid].ebp =3D regs.ebp; +#endif /* CONFIG_SMP && CONFIG_FRAME_POINTER */ + } + + /* Now walk the pipeline, yielding control to the highest + priority domain that has pending interrupt(s) or + immediately to the current domain if the interrupt has been + marked as 'sticky'. This search does not go beyond the + current domain in the pipeline. */ + + __ipipe_walk_pipeline(head, cpuid); + +finalize_nosync: + + ipipe_load_cpuid(); + + if (ipipe_percpu_domain[cpuid] !=3D ipipe_root_domain || + test_bit(IPIPE_STALL_FLAG, + &ipipe_root_domain->cpudata[cpuid].status)) + return 0; + +#ifdef CONFIG_SMP + /* Prevent a spurious rescheduling from being triggered on + preemptible kernels along the way out through + ret_from_intr. */ + if (regs.orig_eax < 0) + __set_bit(IPIPE_STALL_FLAG, &ipipe_root_domain->cpudata[cpuid].status)= ; +#endif /* CONFIG_SMP */ + + return 1; +} + +EXPORT_SYMBOL_GPL(irq_desc); +EXPORT_SYMBOL_GPL(default_ldt); +EXPORT_SYMBOL_GPL(__switch_to); +EXPORT_SYMBOL_GPL(show_stack); +EXPORT_PER_CPU_SYMBOL_GPL(init_tss); +#ifdef CONFIG_SMP +EXPORT_PER_CPU_SYMBOL_GPL(cpu_tlbstate); +#endif /* CONFIG_SMP */ + +#ifdef CONFIG_IPIPE_TRACE +void notrace mcount(void); +EXPORT_SYMBOL(mcount); +#endif /* CONFIG_IPIPE_TRACE */ Index: linux-2.6.17/arch/i386/kernel/nmi.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/arch/i386/kernel/nmi.c +++ linux-2.6.17/arch/i386/kernel/nmi.c @@ -38,6 +38,7 @@ static unsigned int nmi_hz =3D HZ; static unsigned int nmi_perfctr_msr; /* the MSR to reset in NMI handler = */ static unsigned int nmi_p4_cccr_val; extern void show_registers(struct pt_regs *regs); +static void default_nmi_watchdog_tick (struct pt_regs * regs); =20 /* * lapic_nmi_owner tracks the ownership of the lapic NMI hardware: @@ -184,6 +185,7 @@ static int __init setup_nmi_watchdog(cha =20 if (nmi >=3D NMI_INVALID) return 0; + nmi_watchdog_tick =3D default_nmi_watchdog_tick; if (nmi =3D=3D NMI_NONE) nmi_watchdog =3D nmi; /* @@ -519,9 +521,7 @@ void touch_nmi_watchdog (void) touch_softlockup_watchdog(); } =20 -extern void die_nmi(struct pt_regs *, const char *msg); - -void nmi_watchdog_tick (struct pt_regs * regs) +static void default_nmi_watchdog_tick (struct pt_regs * regs) { =20 /* @@ -530,7 +530,11 @@ void nmi_watchdog_tick (struct pt_regs * * smp_processor_id(). */ unsigned int sum; +#ifdef CONFIG_IPIPE + int cpu =3D ipipe_processor_id(); +#else /* !CONFIG_IPIPE */ int cpu =3D smp_processor_id(); +#endif /* !CONFIG_IPIPE */ =20 sum =3D per_cpu(irq_stat, cpu).apic_timer_irqs; =20 @@ -620,3 +624,4 @@ EXPORT_SYMBOL(reserve_lapic_nmi); EXPORT_SYMBOL(release_lapic_nmi); EXPORT_SYMBOL(disable_timer_nmi_watchdog); EXPORT_SYMBOL(enable_timer_nmi_watchdog); +EXPORT_SYMBOL(touch_nmi_watchdog); Index: linux-2.6.17/arch/i386/kernel/process.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/arch/i386/kernel/process.c +++ linux-2.6.17/arch/i386/kernel/process.c @@ -194,6 +194,7 @@ void cpu_idle(void) play_dead(); =20 __get_cpu_var(irq_stat).idle_timestamp =3D jiffies; + ipipe_suspend_domain(); idle(); } preempt_enable_no_resched(); Index: linux-2.6.17/arch/i386/kernel/smp.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/arch/i386/kernel/smp.c +++ linux-2.6.17/arch/i386/kernel/smp.c @@ -132,6 +132,9 @@ void __send_IPI_shortcut(unsigned int sh * to the APIC. */ unsigned int cfg; + unsigned long flags; + + local_irq_save_hw_cond(flags); =20 /* * Wait for idle. @@ -147,6 +150,8 @@ void __send_IPI_shortcut(unsigned int sh * Send the IPI. The write to APIC_ICR fires this off. */ apic_write_around(APIC_ICR, cfg); + + local_irq_restore_hw_cond(flags); } =20 void fastcall send_IPI_self(int vector) @@ -163,7 +168,7 @@ void send_IPI_mask_bitmask(cpumask_t cpu unsigned long cfg; unsigned long flags; =20 - local_irq_save(flags); + local_irq_save_hw(flags); WARN_ON(mask & ~cpus_addr(cpu_online_map)[0]); /* * Wait for idle. @@ -186,7 +191,7 @@ void send_IPI_mask_bitmask(cpumask_t cpu */ apic_write_around(APIC_ICR, cfg); =20 - local_irq_restore(flags); + local_irq_restore_hw(flags); } =20 void send_IPI_mask_sequence(cpumask_t mask, int vector) @@ -200,7 +205,7 @@ void send_IPI_mask_sequence(cpumask_t ma * should be modified to do 1 message per cluster ID - mbligh */=20 =20 - local_irq_save(flags); + local_irq_save_hw(flags); =20 for (query_cpu =3D 0; query_cpu < NR_CPUS; ++query_cpu) { if (cpu_isset(query_cpu, mask)) { @@ -227,7 +232,7 @@ void send_IPI_mask_sequence(cpumask_t ma apic_write_around(APIC_ICR, cfg); } } - local_irq_restore(flags); + local_irq_restore_hw(flags); } =20 #include /* must come after the send_IPI functions above fo= r inlining */ @@ -311,7 +316,9 @@ static inline void leave_mm (unsigned lo =20 fastcall void smp_invalidate_interrupt(struct pt_regs *regs) { - unsigned long cpu; + unsigned long cpu, flags; + + local_irq_save_hw_cond(flags); =20 cpu =3D get_cpu(); =20 @@ -341,6 +348,7 @@ fastcall void smp_invalidate_interrupt(s smp_mb__after_clear_bit(); out: put_cpu_no_resched(); + local_irq_restore_hw_cond(flags); } =20 static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm, @@ -401,14 +409,19 @@ void flush_tlb_current_task(void) { struct mm_struct *mm =3D current->mm; cpumask_t cpu_mask; + unsigned long flags; =20 preempt_disable(); + local_irq_save_hw_cond(flags); + cpu_mask =3D mm->cpu_vm_mask; cpu_clear(smp_processor_id(), cpu_mask); =20 local_flush_tlb(); if (!cpus_empty(cpu_mask)) flush_tlb_others(cpu_mask, mm, FLUSH_ALL); + + local_irq_restore_hw_cond(flags); preempt_enable(); } =20 @@ -436,8 +449,11 @@ void flush_tlb_page(struct vm_area_struc { struct mm_struct *mm =3D vma->vm_mm; cpumask_t cpu_mask; + unsigned long flags; =20 preempt_disable(); + local_irq_save_hw_cond(flags); + cpu_mask =3D mm->cpu_vm_mask; cpu_clear(smp_processor_id(), cpu_mask); =20 @@ -448,6 +464,8 @@ void flush_tlb_page(struct vm_area_struc leave_mm(smp_processor_id()); } =20 + local_irq_restore_hw_cond(flags); + if (!cpus_empty(cpu_mask)) flush_tlb_others(cpu_mask, mm, va); =20 @@ -623,4 +641,3 @@ fastcall void smp_call_function_interrup atomic_inc(&call_data->finished); } } - Index: linux-2.6.17/arch/i386/kernel/smpboot.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/arch/i386/kernel/smpboot.c +++ linux-2.6.17/arch/i386/kernel/smpboot.c @@ -925,6 +925,7 @@ static int __devinit do_boot_cpu(int api unsigned short nmi_high =3D 0, nmi_low =3D 0; =20 ++cpucount; + ipipe_note_apicid(apicid, cpu); alternatives_smp_switch(1); =20 /* @@ -1139,6 +1140,7 @@ static void __init smp_boot_cpus(unsigne boot_cpu_physical_apicid =3D GET_APIC_ID(apic_read(APIC_ID)); boot_cpu_logical_apicid =3D logical_smp_processor_id(); x86_cpu_to_apicid[0] =3D boot_cpu_physical_apicid; + ipipe_note_apicid(boot_cpu_physical_apicid,0); =20 current_thread_info()->cpu =3D 0; smp_tune_scheduling(); Index: linux-2.6.17/arch/i386/kernel/time.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/arch/i386/kernel/time.c +++ linux-2.6.17/arch/i386/kernel/time.c @@ -255,11 +255,12 @@ static inline void do_timer_interrupt(in * This will also deassert NMI lines for the watchdog if run * on an 82489DX-based system. */ - spin_lock(&i8259A_lock); + unsigned long flags; + spin_lock_irqsave_hw_cond(&i8259A_lock,flags); outb(0x0c, PIC_MASTER_OCW3); /* Ack the IRQ; AEOI will end it automatically. */ inb(PIC_MASTER_POLL); - spin_unlock(&i8259A_lock); + spin_unlock_irqrestore_hw_cond(&i8259A_lock,flags); } #endif =20 Index: linux-2.6.17/arch/i386/kernel/timers/timer_pit.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/arch/i386/kernel/timers/timer_pit.c +++ linux-2.6.17/arch/i386/kernel/timers/timer_pit.c @@ -99,6 +99,10 @@ static unsigned long get_offset_pit(void */ unsigned long jiffies_t; =20 +#ifdef CONFIG_IPIPE + if (!__ipipe_pipeline_head_p(ipipe_root_domain)) + return 0; /* We don't really own the PIT. */ +#endif /* CONFIG_IPIPE */ spin_lock_irqsave(&i8253_lock, flags); /* timer count may underflow right here */ outb_p(0x00, PIT_MODE); /* latch the count ASAP */ Index: linux-2.6.17/arch/i386/kernel/timers/timer_tsc.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/arch/i386/kernel/timers/timer_tsc.c +++ linux-2.6.17/arch/i386/kernel/timers/timer_tsc.c @@ -389,6 +389,20 @@ static void mark_offset_tsc(void) =20 rdtsc(last_tsc_low, last_tsc_high); =20 +#ifdef CONFIG_IPIPE + if (!__ipipe_pipeline_head_p(ipipe_root_domain)) { + /* If Linux does not actually own the timer, clock + ticks will be posted by some higher level domain to + us, and we expect it to do this right and never + lose any of them; so we just need to update the + monotonic base here. */ + this_offset =3D ((unsigned long long)last_tsc_high<<32)|last_tsc_low; + monotonic_base +=3D cycles_2_ns(this_offset - last_offset); + write_sequnlock(&monotonic_lock); + return; + } +#endif /* CONFIG_IPIPE */ + spin_lock(&i8253_lock); outb_p(0x00, PIT_MODE); /* latch the count ASAP */ =20 Index: linux-2.6.17/arch/i386/kernel/traps.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/arch/i386/kernel/traps.c +++ linux-2.6.17/arch/i386/kernel/traps.c @@ -94,6 +94,9 @@ asmlinkage void machine_check(void); static int kstack_depth_to_print =3D 24; ATOMIC_NOTIFIER_HEAD(i386die_chain); =20 +void (*nmi_watchdog_tick) (struct pt_regs * regs); +EXPORT_SYMBOL(nmi_watchdog_tick); + int register_die_notifier(struct notifier_block *nb) { vmalloc_sync_all(); @@ -268,6 +271,11 @@ void show_registers(struct pt_regs *regs regs->esi, regs->edi, regs->ebp, esp); printk(KERN_EMERG "ds: %04x es: %04x ss: %04x\n", regs->xds & 0xffff, regs->xes & 0xffff, ss); +#ifdef CONFIG_IPIPE + if (ipipe_current_domain !=3D ipipe_root_domain) + printk("I-pipe domain %s",ipipe_current_domain->name); + else +#endif /* CONFIG_IPIPE */ printk(KERN_EMERG "Process %s (pid: %d, threadinfo=3D%p task=3D%p)", current->comm, current->pid, current_thread_info(), current); /* @@ -670,13 +678,18 @@ void die_nmi (struct pt_regs *regs, cons =20 do_exit(SIGSEGV); } +EXPORT_SYMBOL(die_nmi); =20 static void default_do_nmi(struct pt_regs * regs) { unsigned char reason =3D 0; =20 /* Only the BSP gets external NMIs from the system. */ +#ifdef CONFIG_IPIPE + if (!ipipe_processor_id()) +#else /* !CONFIG_IPIPE */ if (!smp_processor_id()) +#endif /* !CONFIG_IPIPE */ reason =3D get_nmi_reason(); =20 if (!(reason & 0xc0)) { @@ -722,7 +735,11 @@ fastcall void do_nmi(struct pt_regs * re =20 nmi_enter(); =20 +#ifdef CONFIG_IPIPE + cpu =3D ipipe_processor_id(); +#else /* !CONFIG_IPIPE */ cpu =3D smp_processor_id(); +#endif /* !CONFIG_IPIPE */ =20 ++nmi_count(cpu); =20 @@ -1050,12 +1067,15 @@ asmlinkage void math_state_restore(struc { struct thread_info *thread =3D current_thread_info(); struct task_struct *tsk =3D thread->task; + unsigned long flags; =20 + local_irq_save_hw_cond(flags); clts(); /* Allow maths ops (or we recurse) */ if (!tsk_used_math(tsk)) init_fpu(tsk); restore_fpu(tsk); thread->status |=3D TS_USEDFPU; /* So we fnsave on switch_to() */ + local_irq_restore_hw_cond(flags); } =20 #ifndef CONFIG_MATH_EMULATION Index: linux-2.6.17/arch/i386/mach-visws/visws_apic.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/arch/i386/mach-visws/visws_apic.c +++ linux-2.6.17/arch/i386/mach-visws/visws_apic.c @@ -198,7 +198,7 @@ static irqreturn_t piix4_master_intr(int irq_desc_t *desc; unsigned long flags; =20 - spin_lock_irqsave(&i8259A_lock, flags); + spin_lock_irqsave_hw(&i8259A_lock, flags); =20 /* Find out what's interrupting in the PIIX4 master 8259 */ outb(0x0c, 0x20); /* OCW3 Poll command */ @@ -235,7 +235,7 @@ static irqreturn_t piix4_master_intr(int outb(0x60 + realirq, 0x20); } =20 - spin_unlock_irqrestore(&i8259A_lock, flags); + spin_unlock_irqrestore_hw(&i8259A_lock, flags); =20 desc =3D irq_desc + realirq; =20 @@ -253,7 +253,7 @@ static irqreturn_t piix4_master_intr(int return IRQ_HANDLED; =20 out_unlock: - spin_unlock_irqrestore(&i8259A_lock, flags); + spin_unlock_irqrestore_hw(&i8259A_lock, flags); return IRQ_NONE; } =20 Index: linux-2.6.17/arch/i386/mm/fault.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/arch/i386/mm/fault.c +++ linux-2.6.17/arch/i386/mm/fault.c @@ -331,6 +331,8 @@ fastcall void __kprobes do_page_fault(st goto bad_area_nosemaphore; } =20 + local_irq_enable_hw_cond(); + if (notify_die(DIE_PAGE_FAULT, "page fault", regs, error_code, 14, SIGSEGV) =3D=3D NOTIFY_STOP) return; Index: linux-2.6.17/arch/i386/mm/ioremap.c =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/arch/i386/mm/ioremap.c +++ linux-2.6.17/arch/i386/mm/ioremap.c @@ -17,6 +17,7 @@ #include #include #include +#include =20 #define ISA_START_ADDRESS 0xa0000 #define ISA_END_ADDRESS 0x100000 @@ -92,6 +93,7 @@ static int ioremap_page_range(unsigned l err =3D ioremap_pud_range(pgd, addr, next, phys_addr+addr, flags); if (err) break; + set_pgdir(addr, *pgd); } while (pgd++, addr =3D next, addr !=3D end); flush_tlb_all(); return err; Index: linux-2.6.17/include/asm-i386/apic.h =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/include/asm-i386/apic.h +++ linux-2.6.17/include/asm-i386/apic.h @@ -82,7 +82,13 @@ int get_physical_broadcast(void); # define apic_write_around(x,y) apic_write_atomic((x),(y)) #endif =20 +#ifdef CONFIG_IPIPE +#define ack_APIC_irq() do { } while(0) +static inline void __ack_APIC_irq(void) +#else /* !CONFIG_IPIPE */ +#define __ack_APIC_irq() ack_APIC_irq() static inline void ack_APIC_irq(void) +#endif /* CONFIG_IPIPE */ { /* * ack_APIC_irq() actually gets compiled as a single instruction: @@ -117,7 +123,7 @@ extern int reserve_lapic_nmi(void); extern void release_lapic_nmi(void); extern void disable_timer_nmi_watchdog(void); extern void enable_timer_nmi_watchdog(void); -extern void nmi_watchdog_tick (struct pt_regs * regs); +extern void (*nmi_watchdog_tick) (struct pt_regs * regs); extern int APIC_init_uniprocessor (void); extern void disable_APIC_timer(void); extern void enable_APIC_timer(void); Index: linux-2.6.17/include/asm-i386/io_apic.h =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/include/asm-i386/io_apic.h +++ linux-2.6.17/include/asm-i386/io_apic.h @@ -16,8 +16,14 @@ #ifdef CONFIG_PCI_MSI static inline int use_pci_vector(void) {return 1;} static inline void disable_edge_ioapic_vector(unsigned int vector) { } +#ifdef CONFIG_IPIPE +#ifndef CONFIG_IRQBALANCE +static inline void end_edge_ioapic_vector (unsigned int vector) { } +#endif /* !CONFIG_IRQBALANCE */ +#else /* !CONFIG_IPIPE */ static inline void mask_and_ack_level_ioapic_vector(unsigned int vector)= { } static inline void end_edge_ioapic_vector (unsigned int vector) { } +#endif /* CONFIG_IPIPE */ #define startup_level_ioapic startup_level_ioapic_vector #define shutdown_level_ioapic mask_IO_APIC_vector #define enable_level_ioapic unmask_IO_APIC_vector @@ -35,8 +41,14 @@ static inline void end_edge_ioapic_vecto #else static inline int use_pci_vector(void) {return 0;} static inline void disable_edge_ioapic_irq(unsigned int irq) { } +#ifdef CONFIG_IPIPE +#ifndef CONFIG_IRQBALANCE +static inline void end_edge_ioapic_irq (unsigned int irq) { } +#endif /* !CONFIG_IRQBALANCE */ +#else /* CONFIG_IPIPE */ static inline void mask_and_ack_level_ioapic_irq(unsigned int irq) { } static inline void end_edge_ioapic_irq (unsigned int irq) { } +#endif /* CONFIG_IPIPE */ #define startup_level_ioapic startup_level_ioapic_irq #define shutdown_level_ioapic mask_IO_APIC_irq #define enable_level_ioapic unmask_IO_APIC_irq Index: linux-2.6.17/include/asm-i386/ipipe.h =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- /dev/null +++ linux-2.6.17/include/asm-i386/ipipe.h @@ -0,0 +1,281 @@ +/* -*- linux-c -*- + * include/asm-i386/ipipe.h + * + * Copyright (C) 2002-2005 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modif= y + * it under the terms of the GNU General Public License as published b= y + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 0213= 9, + * USA; either version 2 of the License, or (at your option) any later= + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-130= 7, USA. + */ + +#ifndef __I386_IPIPE_H +#define __I386_IPIPE_H + +#include + +#ifdef CONFIG_IPIPE + +#include +#include + +#define IPIPE_ARCH_STRING "1.3-05" +#define IPIPE_MAJOR_NUMBER 1 +#define IPIPE_MINOR_NUMBER 3 +#define IPIPE_PATCH_NUMBER 5 + +#ifdef CONFIG_X86_LOCAL_APIC +/* We want to cover the whole IRQ space when the APIC is enabled. */ +#ifdef CONFIG_PCI_MSI +#define IPIPE_NR_XIRQS NR_IRQS +#else /* CONFIG_PCI_MSI */ +#define IPIPE_NR_XIRQS 224 +#endif /* CONFIG_PCI_MSI */ +/* If the APIC is enabled, then we expose four service vectors in the + APIC space which are freely available to domains. */ +#define IPIPE_SERVICE_VECTOR0 0xf5 +#define IPIPE_SERVICE_IPI0 (IPIPE_SERVICE_VECTOR0 - FIRST_EXTERNAL_VECTO= R) +#define IPIPE_SERVICE_VECTOR1 0xf6 +#define IPIPE_SERVICE_IPI1 (IPIPE_SERVICE_VECTOR1 - FIRST_EXTERNAL_VECTO= R) +#define IPIPE_SERVICE_VECTOR2 0xf7 +#define IPIPE_SERVICE_IPI2 (IPIPE_SERVICE_VECTOR2 - FIRST_EXTERNAL_VECTO= R) +#define IPIPE_SERVICE_VECTOR3 0xf8 +#define IPIPE_SERVICE_IPI3 (IPIPE_SERVICE_VECTOR3 - FIRST_EXTERNAL_VECTO= R) +#else /* !CONFIG_X86_LOCAL_APIC */ +#define IPIPE_NR_XIRQS NR_IRQS +#endif /* CONFIG_X86_LOCAL_APIC */ + +#define IPIPE_IRQ_ISHIFT 5 /* 2^5 for 32bits arch. */ +#define NR_XIRQS IPIPE_NR_XIRQS + +#define ex_do_divide_error 0 +#define ex_do_debug 1 +/* NMI not pipelined. */ +#define ex_do_int3 3 +#define ex_do_overflow 4 +#define ex_do_bounds 5 +#define ex_do_invalid_op 6 +#define ex_device_not_available 7 +/* Double fault not pipelined. */ +#define ex_do_coprocessor_segment_overrun 9 +#define ex_do_invalid_TSS 10 +#define ex_do_segment_not_present 11 +#define ex_do_stack_segment 12 +#define ex_do_general_protection 13 +#define ex_do_page_fault 14 +#define ex_do_spurious_interrupt_bug 15 +#define ex_do_coprocessor_error 16 +#define ex_do_alignment_check 17 +#define ex_machine_check_vector 18 +#define ex_do_simd_coprocessor_error 19 +#define ex_do_iret_error 32 + +#ifndef __ASSEMBLY__ + +#include +#include +#include +#include + +#ifdef CONFIG_SMP + +#include +#include +#include +#include + +#define IPIPE_CRITICAL_VECTOR 0xf9 /* Used by ipipe_critical_enter/exit= () */ +#define IPIPE_CRITICAL_IPI (IPIPE_CRITICAL_VECTOR - FIRST_EXTERNAL_V= ECTOR) + +extern int (*__ipipe_logical_cpuid)(void); + +#define ipipe_processor_id() __ipipe_logical_cpuid() + +extern u8 __ipipe_apicid_2_cpu[]; + +#define ipipe_note_apicid(apicid,cpu) \ +do { \ + __ipipe_apicid_2_cpu[apicid] =3D cpu; \ +} while(0) + +#else /* !CONFIG_SMP */ + +#define ipipe_note_apicid(apicid,cpu) do { } while(0) +#define ipipe_processor_id() 0 + +#endif /* CONFIG_SMP */ + +#define prepare_arch_switch(next) \ +do { \ + __ipipe_dispatch_event(IPIPE_EVENT_SCHEDULE,next); \ + local_irq_disable_hw(); \ +} while(0) + +#define task_hijacked(p) \ + ({ int x =3D ipipe_current_domain !=3D ipipe_root_domain; \ + __clear_bit(IPIPE_SYNC_FLAG,&ipipe_root_domain->cpudata[task_cpu(p)= ].status); \ + local_irq_enable_hw(); x; }) + +/* IDT fault vectors */ +#define IPIPE_NR_FAULTS 33 /* 32 from IDT + iret_error */ +/* Pseudo-vectors used for kernel events */ +#define IPIPE_FIRST_EVENT IPIPE_NR_FAULTS +#define IPIPE_EVENT_SYSCALL (IPIPE_FIRST_EVENT) +#define IPIPE_EVENT_SCHEDULE (IPIPE_FIRST_EVENT + 1) +#define IPIPE_EVENT_SIGWAKE (IPIPE_FIRST_EVENT + 2) +#define IPIPE_EVENT_SETSCHED (IPIPE_FIRST_EVENT + 3) +#define IPIPE_EVENT_EXIT (IPIPE_FIRST_EVENT + 4) +#define IPIPE_EVENT_CLEANUP (IPIPE_FIRST_EVENT + 5) +#define IPIPE_LAST_EVENT IPIPE_EVENT_CLEANUP +#define IPIPE_NR_EVENTS (IPIPE_LAST_EVENT + 1) + +struct ipipe_domain; + +struct ipipe_sysinfo { + + int ncpus; /* Number of CPUs on board */ + u64 cpufreq; /* CPU frequency (in Hz) */ + + /* Arch-dependent block */ + + struct { + unsigned tmirq; /* Timer tick IRQ */ + u64 tmfreq; /* Timer frequency */ + } archdep; +}; + +#define ipipe_read_tsc(t) __asm__ __volatile__("rdtsc" : "=3DA" (t)) +#define ipipe_cpu_freq() ({ unsigned long long __freq =3D cpu_has_tsc?(1= 000LL * cpu_khz):CLOCK_TICK_RATE; __freq; }) + +#define ipipe_tsc2ns(t) \ +({ \ + unsigned long long delta =3D (t)*1000; \ + do_div(delta, cpu_khz/1000+1); \ + (unsigned long)delta; \ +}) + +#define ipipe_tsc2us(t) \ +({ \ + unsigned long long delta =3D (t); \ + do_div(delta, cpu_khz/1000+1); \ + (unsigned long)delta; \ +}) + +/* Private interface -- Internal use only */ + +#define __ipipe_check_platform() do { } while(0) + +#define __ipipe_init_platform() do { } while(0) + +#define __ipipe_enable_irq(irq) irq_desc[irq].handler->enable(irq) + +#define __ipipe_disable_irq(irq) irq_desc[irq].handler->disable(irq) + +void __ipipe_enable_pipeline(void); + +int __ipipe_ack_system_irq(unsigned irq); + +int __ipipe_handle_irq(struct pt_regs regs); + +void __ipipe_do_critical_sync(unsigned irq, void *cookie); + +extern struct pt_regs __ipipe_tick_regs[]; + +extern int __ipipe_tick_irq; + +#define __ipipe_call_root_xirq_handler(ipd,irq) \ + __asm__ __volatile__ ("pushfl\n\t" \ + "pushl %%cs\n\t" \ + "pushl $1f\n\t" \ + "pushl %%eax\n\t" \ + "pushl %%es\n\t" \ + "pushl %%ds\n\t" \ + "pushl %%eax\n\t" \ + "pushl %%ebp\n\t" \ + "pushl %%edi\n\t" \ + "pushl %%esi\n\t" \ + "pushl %%edx\n\t" \ + "pushl %%ecx\n\t" \ + "pushl %%ebx\n\t" \ + "movl %%esp,%%eax\n\t" \ + "call *%1\n\t" \ + "jmp ret_from_intr\n\t" \ + "1:\n" \ + : /* no output */ \ + : "a" (irq-256), "m" ((ipd)->irqs[irq].handler)) + +#define __ipipe_call_root_virq_handler(ipd,irq) \ + __asm__ __volatile__ ("pushfl\n\t" \ + "pushl %%cs\n\t" \ + "pushl $1f\n\t" \ + "pushl $-1\n\t" \ + "pushl %%es\n\t" \ + "pushl %%ds\n\t" \ + "pushl %%eax\n\t" \ + "pushl %%ebp\n\t" \ + "pushl %%edi\n\t" \ + "pushl %%esi\n\t" \ + "pushl %%edx\n\t" \ + "pushl %%ecx\n\t" \ + "pushl %%ebx\n\t" \ + "pushl %2\n\t" \ + "pushl %%eax\n\t" \ + "call *%1\n\t" \ + "addl $8,%%esp\n\t" \ + "jmp ret_from_intr\n\t" \ + "1:\n" \ + : /* no output */ \ + : "a" (irq), "m" ((ipd)->irqs[irq].handler), "r" ((ipd)->irqs[irq].c= ookie)) + +static inline unsigned long __ipipe_ffnz(unsigned long ul) +{ + __asm__("bsrl %1, %0":"=3Dr"(ul) + : "r"(ul)); + return ul; +} + +#define __ipipe_run_isr(ipd, irq, cpuid) \ +do { \ + if (ipd =3D=3D ipipe_root_domain) { \ + /* Linux handlers are called hw interrupts on so that \ + they could not defer interrupts for higher priority \ + domains. */ \ + local_irq_enable_hw(); \ + if (likely(!ipipe_virtual_irq_p(irq))) { \ + __ipipe_call_root_xirq_handler(ipd,irq); \ + } else { \ + irq_enter(); \ + __ipipe_call_root_virq_handler(ipd,irq); \ + irq_exit(); \ + } \ + local_irq_disable_hw(); \ + } else { \ + __clear_bit(IPIPE_SYNC_FLAG, &cpudata->status); \ + ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie); \ + __set_bit(IPIPE_SYNC_FLAG, &cpudata->status); \ + } \ +} while(0) + +#endif /* __ASSEMBLY__ */ + +#else /* !CONFIG_IPIPE */ + +#define task_hijacked(p) 0 + +#define NR_XIRQS NR_IRQS + +#define ipipe_note_apicid(apicid,cpu) do { } while(0) + +#endif /* CONFIG_IPIPE */ + +#endif /* !__I386_IPIPE_H */ Index: linux-2.6.17/include/asm-i386/mach-default/do_timer.h =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/include/asm-i386/mach-default/do_timer.h +++ linux-2.6.17/include/asm-i386/mach-default/do_timer.h @@ -50,14 +50,15 @@ static inline void do_timer_interrupt_ho static inline int do_timer_overflow(int count) { int i; + unsigned long flags; =20 - spin_lock(&i8259A_lock); + spin_lock_irqsave_hw_cond(&i8259A_lock, flags); /* * This is tricky when I/O APICs are used; * see do_timer_interrupt(). */ i =3D inb(0x20); - spin_unlock(&i8259A_lock); + spin_unlock_irqrestore_hw_cond(&i8259A_lock, flags); =09 /* assumption about timer being IRQ0 */ if (i & 0x01) { Index: linux-2.6.17/include/asm-i386/mach-visws/do_timer.h =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/include/asm-i386/mach-visws/do_timer.h +++ linux-2.6.17/include/asm-i386/mach-visws/do_timer.h @@ -29,14 +29,15 @@ static inline void do_timer_interrupt_ho static inline int do_timer_overflow(int count) { int i; + unsigned long flags; =20 - spin_lock(&i8259A_lock); + spin_lock_irqsave_hw_cond(&i8259A_lock, flags); /* * This is tricky when I/O APICs are used; * see do_timer_interrupt(). */ i =3D inb(0x20); - spin_unlock(&i8259A_lock); + spin_unlock_irqrestore_hw_cond(&i8259A_lock, flags); =09 /* assumption about timer being IRQ0 */ if (i & 0x01) { Index: linux-2.6.17/include/asm-i386/mmu_context.h =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/include/asm-i386/mmu_context.h +++ linux-2.6.17/include/asm-i386/mmu_context.h @@ -17,7 +17,7 @@ void destroy_context(struct mm_struct *m static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_stru= ct *tsk) { #ifdef CONFIG_SMP - unsigned cpu =3D smp_processor_id(); + unsigned cpu =3D smp_processor_id_hw(); if (per_cpu(cpu_tlbstate, cpu).state =3D=3D TLBSTATE_OK) per_cpu(cpu_tlbstate, cpu).state =3D TLBSTATE_LAZY; #endif @@ -27,7 +27,7 @@ static inline void switch_mm(struct mm_s struct mm_struct *next, struct task_struct *tsk) { - int cpu =3D smp_processor_id(); + int cpu =3D smp_processor_id_hw(); =20 if (likely(prev !=3D next)) { /* stop flush ipis for the previous mm */ @@ -67,6 +67,11 @@ static inline void switch_mm(struct mm_s asm("movl %0,%%fs ; movl %0,%%gs": :"r" (0)) =20 #define activate_mm(prev, next) \ - switch_mm((prev),(next),NULL) +do { \ + unsigned long flags; \ + local_irq_save_hw_cond(flags); \ + switch_mm((prev),(next),NULL); \ + local_irq_restore_hw_cond(flags); \ +} while(0) =20 #endif Index: linux-2.6.17/include/asm-i386/nmi.h =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/include/asm-i386/nmi.h +++ linux-2.6.17/include/asm-i386/nmi.h @@ -25,4 +25,6 @@ void set_nmi_callback(nmi_callback_t cal */ void unset_nmi_callback(void); =20 +void die_nmi(struct pt_regs *, const char *msg); + #endif /* ASM_NMI_H */ Index: linux-2.6.17/include/asm-i386/pgalloc.h =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/include/asm-i386/pgalloc.h +++ linux-2.6.17/include/asm-i386/pgalloc.h @@ -47,4 +47,27 @@ static inline void pte_free(struct page=20 =20 #define check_pgt_cache() do { } while (0) =20 +static inline void set_pgdir(unsigned long address, pgd_t entry) +{ +#ifdef CONFIG_IPIPE + struct task_struct * p; + struct page *page; + pgd_t *pgd; + + read_lock(&tasklist_lock); + + for_each_process(p) { + if(p->mm) + *pgd_offset(p->mm,address) =3D entry; + } + + read_unlock(&tasklist_lock); + + for (page =3D pgd_list; page; page =3D (struct page *)page->index) { + pgd =3D (pgd_t *)page_address(page); + pgd[address >> PGDIR_SHIFT] =3D entry; + } +#endif /* CONFIG_IPIPE */ +} + #endif /* _I386_PGALLOC_H */ Index: linux-2.6.17/include/asm-i386/spinlock.h =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/include/asm-i386/spinlock.h +++ linux-2.6.17/include/asm-i386/spinlock.h @@ -32,6 +32,9 @@ "jmp 1b\n" \ "3:\n\t" =20 +#ifdef CONFIG_IPIPE +#define __raw_spin_lock_string_flags __raw_spin_lock_string +#else /* !CONFIG_IPIPE */ #define __raw_spin_lock_string_flags \ "\n1:\t" \ "lock ; decb %0\n\t" \ @@ -52,6 +55,7 @@ "jg 1b\n\t" \ "jmp 4b\n" \ "5:\n\t" +#endif /* CONFIG_IPIPE */ =20 #define __raw_spin_lock_string_up \ "\n\tdecb %0" Index: linux-2.6.17/include/asm-i386/system.h =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- linux-2.6.17.orig/include/asm-i386/system.h +++ linux-2.6.17/include/asm-i386/system.h @@ -458,6 +458,109 @@ static inline unsigned long long __cmpxc #define set_wmb(var, value) do { var =3D value; wmb(); } while (0) =20 /* interrupt control.. */ +#ifdef CONFIG_IPIPE + +#include + +void __ipipe_stall_root(void); + +void __ipipe_unstall_root(void); + +unsigned long __ipipe_test_root(void); + +unsigned long __ipipe_test_and_stall_root(void); + +void fastcall __ipipe_restore_root(unsigned long flags); + +#define local_save_flags(x) ((x) =3D (!__ipipe_test_root()) << 9) +#define local_irq_save(x) ((x) =3D (!__ipipe_test_and_stall_root()) << 9= ) +#define local_irq_restore(x) __ipipe_restore_root(!(x & 0x200)) +#define local_irq_disable() __ipipe_stall_root() +#define local_irq_enable() __ipipe_unstall_root() + +#define irqs_disabled() __ipipe_test_root() + +#define halt() __asm__ __volatile__("hlt": : :"memory") + +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + +#include + +#define safe_halt() do { \ + __ipipe_unstall_root(); \ + ipipe_trace_end(0x8000000E); \ + __asm__ __volatile__("sti; hlt": : :"memory"); \ +} while(0) + +#define ipipe_hw_save_flags_and_sti(x) do { \ + ipipe_trace_end(0x8000000F); \ + __asm__ __volatile__("pushfl ; popl %0 ; sti":"=3Dg" (x): /* no input *= / :"memory"); \ +} while (0) +#define local_irq_disable_hw() do { \ + if (!irqs_disabled_hw()) { \ + __asm__ __volatile__("cli": : :"memory"); \ + ipipe_trace_begin(0x80000000); \ + } \ +} while (0) +#define local_irq_enable_hw() do { \ + if (irqs_disabled_hw()) { \ + ipipe_trace_end(0x80000000); \ + __asm__ __volatile__("sti": : :"memory"); \ + } \ +} while (0) +#define local_irq_save_hw(x) do { \ + local_save_flags_hw(x); \ + if (local_test_iflag_hw(x)) { \ + __asm__ __volatile__("cli": : :"memory"); \ + ipipe_trace_begin(0x80000001); \ + } \ +} while (0) +#define local_irq_restore_hw(x) do { \ + if (local_test_iflag_hw(x)) \ + ipipe_trace_end(0x80000001); \ + __asm__ __volatile__("pushl %0 ; popfl": /* no output */ :"g" (x):"memo= ry", "cc"); \ +} while (0) + +#define local_irq_disable_hw_notrace() \ + __asm__ __volatile__("cli": : :"memory") +#define local_irq_enable_hw_notrace() \ + __asm__ __volatile__("sti": : :"memory") +#define local_irq_save_hw_notrace(x) \ + __asm__ __volatile__("pushfl ; popl %0 ; cli":"=3Dg" (x): /* no input *= / :"memory") +#define local_irq_restore_hw_notrace(x) \ + __asm__ __volatile__("pushl %0 ; popfl": /* no output */ :"g" (x):"memo= ry", "cc") + +#else /* !CONFIG_IPIPE_TRACE_IRQSOFF */ + +#define safe_halt() do { \ + __ipipe_unstall_root(); \ + __asm__ __volatile__("sti; hlt": : :"memory"); \ +} while(0) + +#define ipipe_hw_save_flags_and_sti(x) __asm__ __volatile__("pushfl ; po= pl %0 ; sti":"=3Dg" (x): /* no input */ :"memory") +#define local_irq_disable_hw() __asm__ __volatile__("cli": : :"memory= ") +#define local_irq_enable_hw() __asm__ __volatile__("sti": : :"memory")= +#define local_irq_save_hw(x) __asm__ __volatile__("pushfl ; popl %0 ;= cli":"=3Dg" (x): /* no input */ :"memory") +#define local_irq_restore_hw(x) __asm__ __volatile__("pushl %0 ; popfl":= /* no output */ :"g" (x):"memory", "cc") + +#define local_irq_disable_hw_notrace local_irq_disable_hw +#define local_irq_enable_hw_notrace local_irq_enable_hw +#define local_irq_save_hw_notrace local_irq_save_hw +#define local_irq_restore_hw_notrace local_irq_restore_hw + +#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */ + +#define local_save_flags_hw(x) __asm__ __volatile__("pushfl ; popl %0"= :"=3Dg" (x): /* no input */) +#define local_test_iflag_hw(x) ((x) & (1<<9)) +#define irqs_disabled_hw() \ +({ \ + unsigned long flags; \ + local_save_flags_hw(flags); \ + !local_test_iflag_hw(flags); \ +}) + +#else /* !CONFIG_IPIPE */ + #define local_save_flags(x) do { typecheck(unsigned long,x); __asm__ __v= olatile__("pushfl ; popl %0":"=3Dg" (x): /* no input */); } while (0) #define local_irq_restore(x) do { typecheck(unsigned long,x); __asm__ _= _volatile__("pushl %0 ; popfl": /* no output */ :"g" (x):"memory", "cc");= } while (0) #define local_irq_disable() __asm__ __volatile__("cli": : :"memory") @@ -477,6 +580,13 @@ static inline unsigned long long __cmpxc /* For spinlocks etc */ #define local_irq_save(x) __asm__ __volatile__("pushfl ; popl %0 ; cli":= "=3Dg" (x): /* no input */ :"memory") =20 +#define local_irq_save_hw(flags) local_irq_save(flags) +#define local_irq_restore_hw(flags) local_irq_restore(flags) +#define local_irq_enable_hw() local_irq_enable() +#define local_irq_disable_hw(flags) local_irq_disable() + +#endif /* CONFIG_IPIPE */ + /* * disable hlt during certain critical i/o operations */ --------------010006040407060908060502-- --------------enig0B949EEF0D2EBE5A38BECB96 Content-Type: application/pgp-signature; name="signature.asc" Content-Description: OpenPGP digital signature Content-Disposition: attachment; filename="signature.asc" -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.2 (GNU/Linux) Comment: Using GnuPG with SUSE - http://enigmail.mozdev.org iD8DBQFElVb4niDOoMHTA+kRAghlAJ4yzam3sINW2A9BtqYpT+E6inerkACfVybn OIBmRJbNmFmkH+JdCqMdcdg= =IFSD -----END PGP SIGNATURE----- --------------enig0B949EEF0D2EBE5A38BECB96--