public inbox for linux-kernel@vger.kernel.org
 help / color / mirror / Atom feed
* Performance Stats: Kernel patch
@ 2007-04-09 14:22 Maxim Uvarov
  2007-04-10  8:21 ` Eric Dumazet
  2007-04-15  9:47 ` Pavel Machek
  0 siblings, 2 replies; 28+ messages in thread
From: Maxim Uvarov @ 2007-04-09 14:22 UTC (permalink / raw)
  To: linux-kernel; +Cc: davidsen, randy.dunlap, dada1, Valdis.Kletnieks, jesper.juhl

[-- Attachment #1: Type: text/plain, Size: 544 bytes --]

Hello collogues,

Patches make available to the user the following new per-process 
(thread) performance statistics:

    * Involuntary Context Switches
    * Voluntary Context Switches
    * Number of system calls

This data is useful for detecting hyperactivity patterns between processes.

I have attached 3 patches  perf_stat.patch, perf_stat_x86_64.patch and 
perf_stat_ppc.patch for linux-2.6.21-rc5
and corrected all notes which were before.  Do you have any objections? 
I need your opinion again.

Best regards,
Maxim Uvarov.










[-- Attachment #2: perf_stat.patch --]
[-- Type: text/plain, Size: 5368 bytes --]

Patch adds Process Performance Statistics.

 arch/i386/kernel/asm-offsets.c |    3 +++
 arch/i386/kernel/entry.S       |    6 ++++++
 fs/proc/array.c                |   17 +++++++++++++++++
 include/asm-i386/thread_info.h |    5 +++--
 kernel/fork.c                  |    4 ++++
 lib/Kconfig.debug              |   15 +++++++++++++++
 6 files changed, 48 insertions(+), 2 deletions(-)

Index: linux-2.6.21-rc5/fs/proc/array.c
===================================================================
--- linux-2.6.21-rc5.orig/fs/proc/array.c
+++ linux-2.6.21-rc5/fs/proc/array.c
@@ -291,6 +291,20 @@ static inline char *task_cap(struct task
 			    cap_t(p->cap_effective));
 }
 
+#ifdef CONFIG_THREAD_PERF_STAT
+static inline char *task_perf(struct task_struct *p, char *buffer)
+{
+#ifdef CONFIG_THREAD_PERF_STAT_SYSC
+       buffer += sprintf(buffer, "Syscalls:\t%lu\n", p->thread_info->sysc_cnt);
+#endif /* CONFIG_THREAD_PERF_STAT_SYSC */
+
+       return buffer + sprintf(buffer, "Nvcsw:\t%lu\n"
+                           "Nivcsw:\t%lu\n",
+                           p->nvcsw,
+                           p->nivcsw);
+}
+#endif /* CONFIG_THREAD_PERF_STAT */
+
 int proc_pid_status(struct task_struct *task, char * buffer)
 {
 	char * orig = buffer;
@@ -309,6 +323,9 @@ int proc_pid_status(struct task_struct *
 #if defined(CONFIG_S390)
 	buffer = task_show_regs(task, buffer);
 #endif
+#ifdef CONFIG_THREAD_PERF_STAT
+	buffer = task_perf(task, buffer);
+#endif /* CONFIG_THREAD_PERF_STAT */
 	return buffer - orig;
 }
 
Index: linux-2.6.21-rc5/kernel/fork.c
===================================================================
--- linux-2.6.21-rc5.orig/kernel/fork.c
+++ linux-2.6.21-rc5/kernel/fork.c
@@ -1044,6 +1044,10 @@ static struct task_struct *copy_process(
 	p->syscr = 0;		/* I/O counter: read syscalls */
 	p->syscw = 0;		/* I/O counter: write syscalls */
 #endif
+#ifdef CONFIG_THREAD_PERF_STAT_SYSC
+        p->thread_info->sysc_cnt = 0;   /* Syscall counter: total numbers of syscalls */
+#endif /* CONFIG_THREAD_PERF_STAT_SYSC */
+
 	task_io_accounting_init(p);
 	acct_clear_integrals(p);
 
Index: linux-2.6.21-rc5/lib/Kconfig.debug
===================================================================
--- linux-2.6.21-rc5.orig/lib/Kconfig.debug
+++ linux-2.6.21-rc5/lib/Kconfig.debug
@@ -446,3 +446,18 @@ config FAULT_INJECTION_STACKTRACE_FILTER
 	select FRAME_POINTER
 	help
 	  Provide stacktrace filter for fault-injection capabilities
+
+config THREAD_PERF_STAT
+       bool "Per-process (thread) performance statistics"
+       help
+         Make available to the user the following per-process (thread) performance statistics:
+            * Number of involuntary context switches
+            * Number of voluntary context switches
+            * Number of system calls (optional)
+         This information is available via /proc/PID/status.
+
+config THREAD_PERF_STAT_SYSC
+       bool "Enable syscall counter"
+       depends on THREAD_PERF_STAT && X86_32
+       help
+         This option adds a syscall counter to /proc/PID/status.
Index: linux-2.6.21-rc5/arch/i386/kernel/entry.S
===================================================================
--- linux-2.6.21-rc5.orig/arch/i386/kernel/entry.S
+++ linux-2.6.21-rc5/arch/i386/kernel/entry.S
@@ -342,6 +342,9 @@ sysenter_past_esp:
 	jae syscall_badsys
 	call *sys_call_table(,%eax,4)
 	movl %eax,PT_EAX(%esp)
+#ifdef CONFIG_THREAD_PERF_STAT_SYSC
+        incl    TI_sysc_cnt(%ebp)       # Increment syscalls counter
+#endif /* CONFIG_THREAD_PERF_STAT_SYSC */
 	DISABLE_INTERRUPTS(CLBR_ECX|CLBR_EDX)
 	TRACE_IRQS_OFF
 	movl TI_flags(%ebp), %ecx
@@ -385,6 +388,9 @@ syscall_call:
 	call *sys_call_table(,%eax,4)
 	movl %eax,PT_EAX(%esp)		# store the return value
 syscall_exit:
+#ifdef CONFIG_THREAD_PERF_STAT_SYSC
+        incl    TI_sysc_cnt(%ebp)       # Increment syscalls counter 
+#endif /* CONFIG_THREAD_PERF_STAT_SYSC */
 	DISABLE_INTERRUPTS(CLBR_ANY)	# make sure we don't miss an interrupt
 					# setting need_resched or sigpending
 					# between sampling and the iret
Index: linux-2.6.21-rc5/arch/i386/kernel/asm-offsets.c
===================================================================
--- linux-2.6.21-rc5.orig/arch/i386/kernel/asm-offsets.c
+++ linux-2.6.21-rc5/arch/i386/kernel/asm-offsets.c
@@ -56,6 +56,9 @@ void foo(void)
 	OFFSET(TI_addr_limit, thread_info, addr_limit);
 	OFFSET(TI_restart_block, thread_info, restart_block);
 	OFFSET(TI_sysenter_return, thread_info, sysenter_return);
+#ifdef CONFIG_THREAD_PERF_STAT_SYSC
+        OFFSET(TI_sysc_cnt, thread_info, sysc_cnt);
+#endif /* CONFIG_THREAD_PERF_STAT_SYSC */
 	BLANK();
 
 	OFFSET(GDS_size, Xgt_desc_struct, size);
Index: linux-2.6.21-rc5/include/asm-i386/thread_info.h
===================================================================
--- linux-2.6.21-rc5.orig/include/asm-i386/thread_info.h
+++ linux-2.6.21-rc5/include/asm-i386/thread_info.h
@@ -31,8 +31,9 @@ struct thread_info {
 	unsigned long		status;		/* thread-synchronous flags */
 	__u32			cpu;		/* current CPU */
 	int			preempt_count;	/* 0 => preemptable, <0 => BUG */
-
-
+#ifdef CONFIG_THREAD_PERF_STAT_SYSC
+	unsigned long		sysc_cnt;	/* Syscall counter */
+#endif /* CONFIG_THREAD_PERF_STAT_SYSC */
 	mm_segment_t		addr_limit;	/* thread address space:
 					 	   0-0xBFFFFFFF for user-thead
 						   0-0xFFFFFFFF for kernel-thread

[-- Attachment #3: perf_stat_x86_64.patch --]
[-- Type: text/plain, Size: 2671 bytes --]

Index: linux-2.6.21-rc5/lib/Kconfig.debug
===================================================================
--- linux-2.6.21-rc5.orig/lib/Kconfig.debug
+++ linux-2.6.21-rc5/lib/Kconfig.debug
@@ -458,6 +458,6 @@ config THREAD_PERF_STAT
 
 config THREAD_PERF_STAT_SYSC
        bool "Enable syscall counter"
-       depends on THREAD_PERF_STAT && X86_32
+       depends on THREAD_PERF_STAT && X86
        help
          This option adds a syscall counter to /proc/PID/status.
Index: linux-2.6.21-rc5/include/asm-x86_64/thread_info.h
===================================================================
--- linux-2.6.21-rc5.orig/include/asm-x86_64/thread_info.h
+++ linux-2.6.21-rc5/include/asm-x86_64/thread_info.h
@@ -30,7 +30,9 @@ struct thread_info {
 	__u32			status;		/* thread synchronous flags */
 	__u32			cpu;		/* current CPU */
 	int 			preempt_count;	/* 0 => preemptable, <0 => BUG */
-
+#ifdef CONFIG_THREAD_PERF_STAT_SYSC
+	unsigned long		sysc_cnt;	/* Syscall counter */
+#endif /* CONFIG_THREAD_PERF_STAT_SYSC */
 	mm_segment_t		addr_limit;	
 	struct restart_block    restart_block;
 };
Index: linux-2.6.21-rc5/arch/x86_64/kernel/entry.S
===================================================================
--- linux-2.6.21-rc5.orig/arch/x86_64/kernel/entry.S
+++ linux-2.6.21-rc5/arch/x86_64/kernel/entry.S
@@ -236,6 +236,11 @@ ENTRY(system_call)
 	movq %r10,%rcx
 	call *sys_call_table(,%rax,8)  # XXX:	 rip relative
 	movq %rax,RAX-ARGOFFSET(%rsp)
+#ifdef CONFIG_THREAD_PERF_STAT_SYSC
+	GET_THREAD_INFO(%rcx)
+	addq $1, threadinfo_sysc_cnt(%rcx)   # Increment syscalls counter
+#endif /* CONFIG_THREAD_PERF_STAT_SYSC */
+
 /*
  * Syscall return path ending with SYSRET (fast path)
  * Has incomplete stack frame and undefined top of stack. 
@@ -319,6 +324,10 @@ tracesys:			 
 	call *sys_call_table(,%rax,8)
 1:	movq %rax,RAX-ARGOFFSET(%rsp)
 	/* Use IRET because user could have changed frame */
+#ifdef CONFIG_THREAD_PERF_STAT_SYSC
+	GET_THREAD_INFO(%rcx)
+	addq $1, threadinfo_sysc_cnt(%rcx)   # Increment syscalls counter
+#endif /* CONFIG_THREAD_PERF_STAT_SYSC */
 		
 /* 
  * Syscall return path ending with IRET.
Index: linux-2.6.21-rc5/arch/x86_64/kernel/asm-offsets.c
===================================================================
--- linux-2.6.21-rc5.orig/arch/x86_64/kernel/asm-offsets.c
+++ linux-2.6.21-rc5/arch/x86_64/kernel/asm-offsets.c
@@ -35,6 +35,9 @@ int main(void)
 	ENTRY(addr_limit);
 	ENTRY(preempt_count);
 	ENTRY(status);
+#ifdef CONFIG_THREAD_PERF_STAT_SYSC 
+	ENTRY(sysc_cnt);
+#endif /* CONFIG_THREAD_PERF_STAT_SYSC */
 	BLANK();
 #undef ENTRY
 #define ENTRY(entry) DEFINE(pda_ ## entry, offsetof(struct x8664_pda, entry))

[-- Attachment #4: perf_stat_ppc.patch --]
[-- Type: text/plain, Size: 2501 bytes --]

Index: linux-2.6.21-rc5/include/asm-powerpc/thread_info.h
===================================================================
--- linux-2.6.21-rc5.orig/include/asm-powerpc/thread_info.h
+++ linux-2.6.21-rc5/include/asm-powerpc/thread_info.h
@@ -35,6 +35,9 @@ struct thread_info {
 	int		cpu;			/* cpu we're on */
 	int		preempt_count;		/* 0 => preemptable,
 						   <0 => BUG */
+#ifdef CONFIG_THREAD_PERF_STAT_SYSC
+	unsigned long	sysc_cnt;		/* Syscall counter */
+#endif /* CONFIG_THREAD_PERF_STAT_SYSC */
 	struct restart_block restart_block;
 	unsigned long	local_flags;		/* private flags for thread */
 
Index: linux-2.6.21-rc5/arch/powerpc/kernel/asm-offsets.c
===================================================================
--- linux-2.6.21-rc5.orig/arch/powerpc/kernel/asm-offsets.c
+++ linux-2.6.21-rc5/arch/powerpc/kernel/asm-offsets.c
@@ -94,6 +94,10 @@ int main(void)
 	DEFINE(TI_LOCAL_FLAGS, offsetof(struct thread_info, local_flags));
 	DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count));
 	DEFINE(TI_TASK, offsetof(struct thread_info, task));
+#ifdef CONFIG_THREAD_PERF_STAT_SYSC
+        DEFINE(TI_SYSC_CNT, offsetof(struct thread_info, sysc_cnt)); 
+#endif /* CONFIG_THREAD_PERF_STAT_SYSC */
+		
 #ifdef CONFIG_PPC32
 	DEFINE(TI_EXECDOMAIN, offsetof(struct thread_info, exec_domain));
 	DEFINE(TI_CPU, offsetof(struct thread_info, cpu));
Index: linux-2.6.21-rc5/arch/powerpc/kernel/entry_32.S
===================================================================
--- linux-2.6.21-rc5.orig/arch/powerpc/kernel/entry_32.S
+++ linux-2.6.21-rc5/arch/powerpc/kernel/entry_32.S
@@ -223,6 +223,11 @@ ret_from_syscall:
 #endif
 	mr	r6,r3
 	rlwinm	r12,r1,0,0,(31-THREAD_SHIFT)	/* current_thread_info() */
+#ifdef CONFIG_THREAD_PERF_STAT_SYSC
+	lwz r9,TI_SYSC_CNT(r12)
+	addi r9,r9,1
+	stw r9,TI_SYSC_CNT(r12)
+#endif /* CONFIG_THREAD_PERF_STAT_SYSC */
 	/* disable interrupts so current_thread_info()->flags can't change */
 	LOAD_MSR_KERNEL(r10,MSR_KERNEL)	/* doesn't include MSR_EE */
 	SYNC
Index: linux-2.6.21-rc5/lib/Kconfig.debug
===================================================================
--- linux-2.6.21-rc5.orig/lib/Kconfig.debug
+++ linux-2.6.21-rc5/lib/Kconfig.debug
@@ -458,6 +458,6 @@ config THREAD_PERF_STAT
 
 config THREAD_PERF_STAT_SYSC
        bool "Enable syscall counter"
-       depends on THREAD_PERF_STAT && X86
+       depends on THREAD_PERF_STAT && (X86 || PPC32)
        help
          This option adds a syscall counter to /proc/PID/status.

^ permalink raw reply	[flat|nested] 28+ messages in thread
* Performance Stats: Kernel patch
@ 2007-04-03 12:54 Maxim Uvarov
  2007-04-03 23:01 ` Valdis.Kletnieks
  0 siblings, 1 reply; 28+ messages in thread
From: Maxim Uvarov @ 2007-04-03 12:54 UTC (permalink / raw)
  To: linux-kernel

Hello all,

I have idea to include to the user the following new per-process 
(thread) performance statistics:

    * Involuntary Context Switches
    * Voluntary Context Switches
    * Number of system calls

What do you think about it? Patch is bellow.

Best regards,
Max Uvarov.


Description:
    Patch adds Process Performance Statistics.

 arch/i386/kernel/entry.S       |    3 +++
 arch/mips/kernel/entry.S       |    3 +++
 arch/powerpc/kernel/entry_32.S |    3 +++
 arch/x86_64/kernel/entry.S     |    3 +++
 fs/proc/array.c                |   13 +++++++++++++
 include/linux/sched.h          |    3 +++
 kernel/fork.c                  |    3 +++
 kernel/sys.c                   |    8 ++++++++
 lib/Kconfig.debug              |   10 ++++++++++
 9 files changed, 49 insertions(+)

Index: linux-2.6.18/fs/proc/array.c
===================================================================
--- linux-2.6.18.orig/fs/proc/array.c
+++ linux-2.6.18/fs/proc/array.c
@@ -295,6 +295,18 @@ static inline char *task_cap(struct task
                            cap_t(p->cap_effective));
 }

+#ifdef THREAD_PERF_STAT
+static inline char *task_perf(struct task_struct *p, char *buffer)
+{
+    return buffer + sprintf(buffer, "Nvcsw:\t%lu\n"
+                           "Nivcsw:\t%lu\n"
+                           "Syscalls:\t%d\n",
+                           cap_t(p->nvcsw),
+                           cap_t(p->nivcsw),
+                           cap_t(p->sysc_cnt));
+}
+#endif /* THREAD_PERF_STAT */
+
 #define get_blocked_on(t)      (-1)

 static char *show_blocked_on(struct task_struct *task, char *buffer)
@@ -327,6 +339,7 @@ int proc_pid_status(struct task_struct *
        buffer = task_show_regs(task, buffer);
 #endif
        buffer = show_blocked_on(task,buffer);
+       buffer = task_perf(task, buffer);
        return buffer - orig;
 }

Index: linux-2.6.18/include/linux/sched.h
===================================================================
--- linux-2.6.18.orig/include/linux/sched.h
+++ linux-2.6.18/include/linux/sched.h
@@ -1193,6 +1193,9 @@ struct task_struct {
        ltt_facility_t ltt_facilities[LTT_FAC_PER_PROCESS];
 #endif //CONFIG_LTT_USERSPACE_GENERIC

+#ifdef THREAD_PERF_STAT
+       int sysc_cnt;           /* Syscall counter */
+#endif /* THREAD_PERF_STAT */
        /*
         * task-specic data
         * XXX: for a future release, MontaVista-only
Index: linux-2.6.18/kernel/sys.c
===================================================================
--- linux-2.6.18.orig/kernel/sys.c
+++ linux-2.6.18/kernel/sys.c
@@ -2071,3 +2071,11 @@ asmlinkage long sys_prctl(int option, un
        }
        return error;
 }
+#ifdef THREAD_PERF_STAT
+asmlinkage void inc_syscallcnt(void)
+{
+       current->sysc_cnt += 1;
+       return;
+}
+#endif /* THREAD_PERF_STAT */
+
Index: linux-2.6.18/kernel/fork.c
===================================================================
--- linux-2.6.18.orig/kernel/fork.c
+++ linux-2.6.18/kernel/fork.c
@@ -1079,6 +1079,9 @@ static struct task_struct *copy_process(
        p->wchar = 0;           /* I/O counter: bytes written */
        p->syscr = 0;           /* I/O counter: read syscalls */
        p->syscw = 0;           /* I/O counter: write syscalls */
+#ifdef THREAD_PERF_STAT
+       p->sysc_cnt = 0;        /* Syscall counter: total numbers of 
syscalls */
+#endif /* THREAD_PERF_STAT */
        acct_clear_integrals(p);

        p->it_virt_expires = cputime_zero;
Index: linux-2.6.18/arch/i386/kernel/entry.S
===================================================================
--- linux-2.6.18.orig/arch/i386/kernel/entry.S
+++ linux-2.6.18/arch/i386/kernel/entry.S
@@ -394,6 +394,9 @@ syscall_exit:
        cli                             # make sure we don't miss an 
interrupt
                                        # setting need_resched or sigpending
                                        # between sampling and the iret
+#ifdef THREAD_PERF_STAT
+       call inc_syscallcnt             # Increment syscalls counter 
current->sysc_cnt
+#endif /* THREAD_PERF_STAT */
        TRACE_IRQS_OFF
        movl TI_flags(%ebp), %ecx
        testw $_TIF_ALLWORK_MASK, %cx   # current->work
Index: linux-2.6.18/arch/x86_64/kernel/entry.S
===================================================================
--- linux-2.6.18.orig/arch/x86_64/kernel/entry.S
+++ linux-2.6.18/arch/x86_64/kernel/entry.S
@@ -388,6 +388,9 @@ ENTRY(int_ret_from_sys_call)
        /* edi: mask to check */
 int_with_check:
        GET_THREAD_INFO(%rcx)
+#ifdef THREAD_PERF_STAT
+       call inc_syscallcnt     # Increment syscalls counter 
current->sysc_cnt
+#endif /* THREAD_PERF_STAT */
        movl threadinfo_flags(%rcx),%edx
        andl %edi,%edx
        jnz   int_careful
Index: linux-2.6.18/arch/powerpc/kernel/entry_32.S
===================================================================
--- linux-2.6.18.orig/arch/powerpc/kernel/entry_32.S
+++ linux-2.6.18/arch/powerpc/kernel/entry_32.S
@@ -236,6 +236,9 @@ ret_from_syscall:
 #ifdef SHOW_SYSCALLS
        bl      do_show_syscall_exit
 #endif
+#ifdef THREAD_PERF_STAT
+       bl      inc_syscallcnt  /* Increment syscalls counter 
current->sysc_cnt */
+#endif /* THREAD_PERF_STAT */
 #ifdef CONFIG_MICROSTATE
        /* FIXME: this is too early since this might not return to
           user space just yet because of syscall_exit_work */
Index: linux-2.6.18/arch/mips/kernel/entry.S
===================================================================
--- linux-2.6.18.orig/arch/mips/kernel/entry.S
+++ linux-2.6.18/arch/mips/kernel/entry.S
@@ -75,6 +75,9 @@ FEXPORT(syscall_exit)
        raw_local_irq_disable   # make sure need_resched and
                                        # signals dont change between
                                        # sampling and return
+#ifdef THREAD_PERF_STAT
+       jal     inc_syscallcnt          # Increment syscalls counter 
current->sysc_cnt
+#endif /* THREAD_PERF_STAT */
        LONG_L  a2, TI_FLAGS($28)       # current->work
        li      t0, _TIF_ALLWORK_MASK
        and     t0, a2, t0
Index: linux-2.6.18/lib/Kconfig.debug
===================================================================
--- linux-2.6.18.orig/lib/Kconfig.debug
+++ linux-2.6.18/lib/Kconfig.debug
@@ -539,4 +539,14 @@ config RCU_TORTURE_TEST
          Say M if you want the RCU torture tests to build as a module.
          Say N if you are unsure.

+config THREAD_PERF_STAT
+       bool "Per-process (thread) performance statistics"
+       depends on (X86 || PPC || MIPS)
+       help
+         Make available to the user the following new per-process 
(thread) performance statistics:
+           * Involuntary Context Switches
+           * Voluntary Context Switches
+           * Number of system calls
+         This information is available via /proc/PID/status.
+
 source "lib/Kconfig.kgdb"

^ permalink raw reply	[flat|nested] 28+ messages in thread

end of thread, other threads:[~2007-04-16  9:24 UTC | newest]

Thread overview: 28+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2007-04-09 14:22 Performance Stats: Kernel patch Maxim Uvarov
2007-04-10  8:21 ` Eric Dumazet
2007-04-11 11:59   ` Maxim Uvarov
2007-04-11 12:26     ` Eric Dumazet
2007-04-11 13:15       ` Maxim Uvarov
2007-04-11 14:15     ` Eric Dumazet
2007-04-11 15:33       ` Bill Davidsen
2007-04-11 15:57         ` Maxim Uvarov
2007-04-11 15:53       ` Maxim Uvarov
2007-04-11 19:22         ` Eric Dumazet
2007-04-12 13:46           ` Maxim Uvarov
2007-04-15  9:47 ` Pavel Machek
2007-04-15 10:21   ` William Lee Irwin III
2007-04-15 20:10     ` Pavel Machek
2007-04-16  1:04       ` William Lee Irwin III
2007-04-16  9:24         ` Maxim Uvarov
  -- strict thread matches above, loose matches on Subject: below --
2007-04-03 12:54 Maxim Uvarov
2007-04-03 23:01 ` Valdis.Kletnieks
2007-04-04 13:15   ` Maxim Uvarov
2007-04-04 13:46     ` Eric Dumazet
2007-04-04 16:52       ` Maxim Uvarov
2007-04-04 18:04         ` Eric Dumazet
2007-04-04 21:54       ` Valdis.Kletnieks
2007-04-04 13:59     ` Jesper Juhl
2007-04-04 21:50     ` Valdis.Kletnieks
2007-04-04 22:03       ` Randy Dunlap
2007-04-06 21:50       ` Bill Davidsen
2007-04-08 16:58     ` Pavel Machek

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox