[PATCH] IA64 audit support

[PATCH] IA64 audit support

[Peter Martuccelli]

Hello Andrew,

Here is one of the audit tasks we talked about.  This patch adds 
ia64 support to the audit subsystem.  I have tested the ia64 audit 
support without any issues to report. The original patch was from
from Ray Lanza.  Requesting patch to be merged in the next version. 

Please CC me with any comments, problems, changes, etc.

Regards,

Peter


diffstat:
 arch/ia64/ia32/ia32_entry.S    |    8 ++++---
 arch/ia64/kernel/entry.S       |   10 +++++----
 arch/ia64/kernel/ivt.S         |    7 ++++--
 arch/ia64/kernel/ptrace.c      |   43 +++++++++++++++++++++++++++++++++++++++--
 include/asm-ia64/thread_info.h |    5 +++-
 init/Kconfig                   |    2 -
 6 files changed, 62 insertions(+), 13 deletions(-)

diff -Naurp linux-2.6.7-pristine/arch/ia64/ia32/ia32_entry.S linux-2.6.7/arch/ia64/ia32/ia32_entry.S
--- linux-2.6.7-pristine/arch/ia64/ia32/ia32_entry.S	2004-06-16 16:32:05.000000000 -0400
+++ linux-2.6.7/arch/ia64/ia32/ia32_entry.S	2004-06-16 16:34:10.000000000 -0400
@@ -110,7 +110,9 @@ GLOBAL_ENTRY(ia32_ret_from_clone)
 	ld4 r2=[r2]
 	;;
 	mov r8=0
-	tbit.nz p6,p0=r2,TIF_SYSCALL_TRACE
+	and r2=_TIF_SYSCALL_TRACEAUDIT,r2
+	;; 
+	cmp.ne p6,p0=r2,r0
 (p6)	br.cond.spnt .ia32_strace_check_retval
 	;;					// prevent RAW on r8
 END(ia32_ret_from_clone)
@@ -142,7 +144,7 @@ GLOBAL_ENTRY(ia32_trace_syscall)
 	adds r2=IA64_PT_REGS_R8_OFFSET+16,sp
 	;;
 	st8 [r2]=r3				// initialize return code to -ENOSYS
-	br.call.sptk.few rp=syscall_trace	// give parent a chance to catch syscall args
+	br.call.sptk.few rp=syscall_trace_enter	// give parent a chance to catch syscall args
 .ret2:	// Need to reload arguments (they may be changed by the tracing process)
 	adds r2=IA64_PT_REGS_R1_OFFSET+16,sp	// r2 = &pt_regs.r1
 	adds r3=IA64_PT_REGS_R13_OFFSET+16,sp	// r3 = &pt_regs.r13
@@ -170,7 +172,7 @@ GLOBAL_ENTRY(ia32_trace_syscall)
 	adds r2=IA64_PT_REGS_R8_OFFSET+16,sp	// r2 = &pt_regs.r8
 	;;
 	st8.spill [r2]=r8			// store return value in slot for r8
-	br.call.sptk.few rp=syscall_trace	// give parent a chance to catch return value
+	br.call.sptk.few rp=syscall_trace_leave	// give parent a chance to catch return value
 .ret4:	alloc r2=ar.pfs,0,0,0,0			// drop the syscall argument frame
 	br.cond.sptk.many ia64_leave_kernel
 END(ia32_trace_syscall)
diff -Naurp linux-2.6.7-pristine/arch/ia64/kernel/entry.S linux-2.6.7/arch/ia64/kernel/entry.S
--- linux-2.6.7-pristine/arch/ia64/kernel/entry.S	2004-06-16 16:32:05.000000000 -0400
+++ linux-2.6.7/arch/ia64/kernel/entry.S	2004-06-16 16:34:10.000000000 -0400
@@ -506,7 +506,7 @@ GLOBAL_ENTRY(ia64_trace_syscall)
 	;;
  	stf.spill [r16]=f10
  	stf.spill [r17]=f11
-	br.call.sptk.many rp=syscall_trace // give parent a chance to catch syscall args
+	br.call.sptk.many rp=syscall_trace_enter // give parent a chance to catch syscall args
 	adds r16=PT(F6)+16,sp
 	adds r17=PT(F7)+16,sp
 	;;
@@ -546,7 +546,7 @@ GLOBAL_ENTRY(ia64_trace_syscall)
 .strace_save_retval:
 .mem.offset 0,0; st8.spill [r2]=r8		// store return value in slot for r8
 .mem.offset 8,0; st8.spill [r3]=r10		// clear error indication in slot for r10
-	br.call.sptk.many rp=syscall_trace // give parent a chance to catch return value
+	br.call.sptk.many rp=syscall_trace_leave // give parent a chance to catch return value
 .ret3:	br.cond.sptk ia64_leave_syscall
 
 strace_error:
@@ -573,7 +573,7 @@ GLOBAL_ENTRY(ia64_strace_leave_kernel)
 	 */
 	nop.m 0
 	nop.i 0
-	br.call.sptk.many rp=syscall_trace // give parent a chance to catch return value
+	br.call.sptk.many rp=syscall_trace_leave // give parent a chance to catch return value
 }
 .ret4:	br.cond.sptk ia64_leave_kernel
 END(ia64_strace_leave_kernel)
@@ -599,7 +599,9 @@ GLOBAL_ENTRY(ia64_ret_from_clone)
 	ld4 r2=[r2]
 	;;
 	mov r8=0
-	tbit.nz p6,p0=r2,TIF_SYSCALL_TRACE
+	and r2=_TIF_SYSCALL_TRACEAUDIT,r2
+	;;
+	cmp.ne p6,p0=r2,r0
 (p6)	br.cond.spnt .strace_check_retval
 	;;					// added stop bits to prevent r8 dependency
 END(ia64_ret_from_clone)
diff -Naurp linux-2.6.7-pristine/arch/ia64/kernel/ivt.S linux-2.6.7/arch/ia64/kernel/ivt.S
--- linux-2.6.7-pristine/arch/ia64/kernel/ivt.S	2004-06-16 16:32:05.000000000 -0400
+++ linux-2.6.7/arch/ia64/kernel/ivt.S	2004-06-16 16:42:44.000000000 -0400
@@ -752,7 +752,9 @@ ENTRY(break_fault)
 	;;
 	ld4 r2=[r2]				// r2 = current_thread_info()->flags
 	;;
-	tbit.z p8,p0=r2,TIF_SYSCALL_TRACE
+	and r2=_TIF_SYSCALL_TRACEAUDIT,r2	// mask trace or audit
+	;; 
+	cmp.eq p8,p0=r2,r0
 	mov b6=r20
 	;;
 (p8)	br.call.sptk.many b6=b6			// ignore this return addr
@@ -1573,10 +1575,11 @@ ENTRY(dispatch_to_ia32_handler)
 	ld4 r2=[r2]		// r2 = current_thread_info()->flags
 	;;
 	ld8 r16=[r16]
-	tbit.z p8,p0=r2,TIF_SYSCALL_TRACE
+	and r2=_TIF_SYSCALL_TRACEAUDIT,r2	// mask trace or audit
 	;;
 	mov b6=r16
 	movl r15=ia32_ret_from_syscall
+	cmp.eq p8,p0=r2,r0
 	;;
 	mov rp=r15
 (p8)	br.call.sptk.many b6=b6
diff -Naurp linux-2.6.7-pristine/arch/ia64/kernel/ptrace.c linux-2.6.7/arch/ia64/kernel/ptrace.c
--- linux-2.6.7-pristine/arch/ia64/kernel/ptrace.c	2004-06-16 16:32:05.000000000 -0400
+++ linux-2.6.7/arch/ia64/kernel/ptrace.c	2004-06-16 16:34:10.000000000 -0400
@@ -1447,9 +1447,8 @@ sys_ptrace (long request, pid_t pid, uns
 	return ret;
 }
 
-/* "asmlinkage" so the input arguments are preserved... */
 
-asmlinkage void
+void
 syscall_trace (void)
 {
 	if (!test_thread_flag(TIF_SYSCALL_TRACE))
@@ -1472,3 +1471,43 @@ syscall_trace (void)
 		current->exit_code = 0;
 	}
 }
+
+/* "asmlinkage" so the input arguments are preserved... */
+
+asmlinkage void 
+syscall_trace_enter (long arg0, long arg1, long arg2, long arg3,
+	  long arg4, long arg5, long arg6, long arg7, long stack)
+{
+	struct pt_regs *regs = (struct pt_regs *) &stack;
+	long syscall;
+	unsigned long 	psr = regs->cr_ipsr;	/* process status word */
+
+	if (unlikely(current->audit_context)) {
+		if (psr & IA64_PSR_IS) 
+			syscall = regs->r1;
+		else
+			syscall = regs->r15;
+
+		audit_syscall_entry(current, syscall, arg0, arg1, arg2, arg3);
+	}
+
+	if (test_thread_flag(TIF_SYSCALL_TRACE)
+	    && (current->ptrace & PT_PTRACED))
+		syscall_trace();
+}
+
+/* "asmlinkage" so the input arguments are preserved... */
+
+asmlinkage void 
+syscall_trace_leave (long arg0, long arg1, long arg2, long arg3,
+	  long arg4, long arg5, long arg6, long arg7, long stack)
+{
+	struct pt_regs *regs = (struct pt_regs *) &stack;
+
+	if (unlikely(current->audit_context))
+		audit_syscall_exit(current, regs->r8);
+
+	if (test_thread_flag(TIF_SYSCALL_TRACE)
+	    && (current->ptrace & PT_PTRACED))
+		syscall_trace();
+}
diff -Naurp linux-2.6.7-pristine/include/asm-ia64/thread_info.h linux-2.6.7/include/asm-ia64/thread_info.h
--- linux-2.6.7-pristine/include/asm-ia64/thread_info.h	2004-06-16 16:32:09.000000000 -0400
+++ linux-2.6.7/include/asm-ia64/thread_info.h	2004-06-16 16:34:10.000000000 -0400
@@ -73,12 +73,15 @@ struct thread_info {
 #define TIF_SIGPENDING		1	/* signal pending */
 #define TIF_NEED_RESCHED	2	/* rescheduling necessary */
 #define TIF_SYSCALL_TRACE	3	/* syscall trace active */
+#define TIF_SYSCALL_AUDIT	4	/* syscall auditing active */
 #define TIF_POLLING_NRFLAG	16	/* true if poll_idle() is polling TIF_NEED_RESCHED */
 
 #define TIF_WORK_MASK		0x7	/* like TIF_ALLWORK_BITS but sans TIF_SYSCALL_TRACE */
-#define TIF_ALLWORK_MASK	0xf	/* bits 0..3 are "work to do on user-return" bits */
+#define TIF_ALLWORK_MASK	0x1f	/* bits 0..4 are "work to do on user-return" bits */
 
 #define _TIF_SYSCALL_TRACE	(1 << TIF_SYSCALL_TRACE)
+#define _TIF_SYSCALL_AUDIT	(1 << TIF_SYSCALL_AUDIT)
+#define _TIF_SYSCALL_TRACEAUDIT	(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT)
 #define _TIF_NOTIFY_RESUME	(1 << TIF_NOTIFY_RESUME)
 #define _TIF_SIGPENDING		(1 << TIF_SIGPENDING)
 #define _TIF_NEED_RESCHED	(1 << TIF_NEED_RESCHED)
diff -Naurp linux-2.6.7-pristine/init/Kconfig linux-2.6.7/init/Kconfig
--- linux-2.6.7-pristine/init/Kconfig	2004-06-16 16:33:52.000000000 -0400
+++ linux-2.6.7/init/Kconfig	2004-06-16 16:34:10.000000000 -0400
@@ -149,7 +149,7 @@ config AUDIT
 
 config AUDITSYSCALL
 	bool "Enable system-call auditing support"
-	depends on AUDIT && (X86 || PPC64 || ARCH_S390)
+	depends on AUDIT && (X86 || PPC64 || ARCH_S390 || IA64)
 	default y if SECURITY_SELINUX
 	default n
 	help

Re: [PATCH] IA64 audit support

[Andrew Morton]

Peter Martuccelli <peterm@redhat.com> wrote:
>
>  Here is one of the audit tasks we talked about.  This patch adds 
>  ia64 support to the audit subsystem.  I have tested the ia64 audit 
>  support without any issues to report. The original patch was from
>  from Ray Lanza.

It's nice and simple.

> Requesting patch to be merged in the next version. 

Via davidm, please.

Re: [PATCH] IA64 audit support

[Karim Yaghmour]

Andrew Morton wrote:
> It's nice and simple.

Andrew, I have got to ask:

I, and quite a few other folks, have been trying to get the Linux Trace Toolkit
in the kernel for the past 5 years and the code being added is almost identical to what the audit patch adds, yet
we've

Karim
-- 
Author, Speaker, Developer, Consultant
Pushing Embedded and Real-Time Linux Systems Beyond the Limits
http://www.opersys.com || karim@opersys.com || 1-866-677-4546

Re: [PATCH] IA64 audit support

[Karim Yaghmour]

Sorry, sent before completion ... will send full mail shortly.

Karim Yaghmour wrote:
> 
> Andrew Morton wrote:
> 
>> It's nice and simple.
> 
> 
> Andrew, I have got to ask:
> 
> I, and quite a few other folks, have been trying to get the Linux Trace 
> Toolkit
> in the kernel for the past 5 years and the code being added is almost 
> identical to what the audit patch adds, yet
> we've
> 
> Karim

-- 
Author, Speaker, Developer, Consultant
Pushing Embedded and Real-Time Linux Systems Beyond the Limits
http://www.opersys.com || karim@opersys.com || 1-866-677-4546

Re: [PATCH] IA64 audit support

[Karim Yaghmour]

Here's what I was trying to say:

Andrew Morton wrote:
> It's nice and simple.

I, and quite a few other folks, have been trying to get the Linux Trace Toolkit
in the kernel for the past 5 years and the code being added is almost identical
to what the audit patch adds, yet we've always got reponses such "this is
bloated" and Linus told us that he didn't see the use of this kind of stuff.

Have we simply not figured out the secret handshake?

I'd really like to have some advice here since I believe we have tried every
trick in the book: posting the patches for review, asking kernel developers for
input, porting the patches to multiple architectures, modulirizing the system,
etc.

Thanks,

Karim
-- 
Author, Speaker, Developer, Consultant
Pushing Embedded and Real-Time Linux Systems Beyond the Limits
http://www.opersys.com || karim@opersys.com || 1-866-677-4546

Re: [PATCH] IA64 audit support

[Andrew Morton]

Karim Yaghmour <karim@opersys.com> wrote:
>
> 
> Here's what I was trying to say:
> 
> Andrew Morton wrote:
> > It's nice and simple.
> 
> I, and quite a few other folks, have been trying to get the Linux Trace Toolkit
> in the kernel for the past 5 years and the code being added is almost identical
> to what the audit patch adds, yet we've always got reponses such "this is
> bloated" and Linus told us that he didn't see the use of this kind of stuff.

The auditing framework was quite unintrusive:

 arch/i386/kernel/entry.S         |    6 
 arch/i386/kernel/ptrace.c        |   10 
 arch/ppc64/kernel/entry.S        |   15 
 arch/ppc64/kernel/ptrace.c       |   29 +
 arch/x86_64/ia32/ia32entry.S     |   18 
 arch/x86_64/kernel/entry.S       |   21 
 arch/x86_64/kernel/ptrace.c      |   30 +
 fs/namei.c                       |   15 
 include/asm-i386/thread_info.h   |    6 
 include/asm-ppc64/thread_info.h  |    3 
 include/asm-x86_64/thread_info.h |    5 
 include/linux/audit.h            |  211 ++++++++
 include/linux/fs.h               |   14 
 include/linux/netlink.h          |    1 
 include/linux/sched.h            |    3 
 init/Kconfig                     |   20 
 kernel/Makefile                  |    2 
 kernel/audit.c                   |  825 ++++++++++++++++++++++++++++++++++
 kernel/auditsc.c                 |  922 +++++++++++++++++++++++++++++++++++++++
 kernel/fork.c                    |   10 
 security/selinux/avc.c           |  168 +++----
 security/selinux/include/avc.h   |    7 
 security/selinux/ss/services.c   |    2 
 23 files changed, 2199 insertions(+), 144 deletions(-)

It treads on fork.c and namei.c only, really.

Whereas LTT:

 Makefile                     |    1 
 arch/i386/config.in          |    2 
 arch/i386/kernel/entry.S     |   10 
 arch/i386/kernel/irq.c       |    6 
 arch/i386/kernel/process.c   |    6 
 arch/i386/kernel/sys_i386.c  |    4 
 arch/i386/kernel/traps.c     |  106 +++
 arch/i386/mm/fault.c         |   11 
 arch/mips/config.in          |    2 
 arch/mips/ddb5476/irq.c      |   10 
 arch/mips/kernel/irq.c       |   10 
 arch/mips/kernel/scall_o32.S |   38 +
 arch/mips/kernel/time.c      |    4 
 arch/mips/kernel/traps.c     |  131 +++-
 arch/mips/kernel/unaligned.c |   13 
 arch/mips/mm/fault.c         |   15 
 arch/ppc/config.in           |    2 
 arch/ppc/kernel/entry.S      |   38 +
 arch/ppc/kernel/irq.c        |    6 
 arch/ppc/kernel/misc.S       |    4 
 arch/ppc/kernel/process.c    |   15 
 arch/ppc/kernel/syscalls.c   |    4 
 arch/ppc/kernel/time.c       |    6 
 arch/ppc/kernel/traps.c      |   87 ++
 arch/ppc/mm/fault.c          |   16 
 arch/s390/config.in          |    2 
 arch/s390/kernel/entry.S     |   18 
 arch/s390/kernel/process.c   |    5 
 arch/s390/kernel/sys_s390.c  |    3 
 arch/s390/kernel/traps.c     |  118 +++
 arch/s390/mm/fault.c         |   12 
 arch/sh/config.in            |    3 
 arch/sh/kernel/entry.S       |   42 +
 arch/sh/kernel/irq.c         |   14 
 arch/sh/kernel/process.c     |   13 
 arch/sh/kernel/sys_sh.c      |    4 
 arch/sh/kernel/traps.c       |   88 ++
 arch/sh/lib/checksum.S       |    1 
 arch/sh/mm/fault.c           |   15 
 drivers/Makefile             |    1 
 drivers/input/mousedev.c     |    1 
 drivers/s390/s390io.c        |    3 
 drivers/s390/s390mach.c      |   12 
 drivers/trace/Config.help    |   36 +
 drivers/trace/Config.in      |    4 
 drivers/trace/Makefile       |   17 
 drivers/trace/tracer.c       | 1382 +++++++++++++++++++++++++++++++++++++++++++
 drivers/trace/tracer.h       |  275 ++++++++
 fs/buffer.c                  |    4 
 fs/exec.c                    |    7 
 fs/ioctl.c                   |    6 
 fs/open.c                    |   10 
 fs/read_write.c              |   44 +
 fs/select.c                  |   10 
 include/linux/trace.h        |  433 +++++++++++++
 ipc/msg.c                    |    3 
 ipc/sem.c                    |    2 
 ipc/shm.c                    |    3 
 kernel/Makefile              |    6 
 kernel/exit.c                |    6 
 kernel/fork.c                |    5 
 kernel/itimer.c              |    5 
 kernel/sched.c               |    6 
 kernel/signal.c              |    4 
 kernel/softirq.c             |   16 
 kernel/timer.c               |    4 
 kernel/trace.c               |  692 +++++++++++++++++++++
 mm/filemap.c                 |    4 
 mm/memory.c                  |    4 
 mm/page_alloc.c              |    6 
 mm/swap_state.c              |    5 
 net/core/dev.c               |    7 
 net/socket.c                 |   10 
 73 files changed, 3901 insertions(+), 17 deletions(-)

adds hooks all over the place.

The security code adds hooks everywhere too, but those deliver end-user
functionality rather than being purely a developer support tool.

Developer support tools are good, but are not as persuasive as end-user
features.  Because the audience is smaller, and developers know how to
apply patches and rebuild stuff.

> Have we simply not figured out the secret handshake?

It's a balance between (ongoing maintenance cost multiplied by the number of
impacted developers) versus (additional functionality multiplied by the
number of users who benefit from it).  To my mind, LTT (and kgdb and various
other developer-support things) don't offer good ratios here.

> I'd really like to have some advice here since I believe we have tried every
> trick in the book: posting the patches for review, asking kernel developers for
> input, porting the patches to multiple architectures, modulirizing the system,
> etc.

If it could use kprobes hooks that'd be neat.  kprobes is low-impact.

Re: [PATCH] IA64 audit support

[Karim Yaghmour]

[-- Attachment #1: Type: text/plain, Size: 13072 bytes --]


First of all thanks for the detailed answer, I think this is an opportunity
to clarify a few things regarding LTT.

For the patch comparison to be useful we should only keep one instance of
the architecture-dependent portions of each patch as LTT has support for
6 architectures while audit only has 3.

Andrew Morton wrote:
>  arch/i386/kernel/entry.S         |    6 
>  arch/i386/kernel/ptrace.c        |   10 
...
>  fs/namei.c                       |   15 
>  include/asm-i386/thread_info.h   |    6 
...
>  include/linux/audit.h            |  211 ++++++++
>  include/linux/fs.h               |   14 
>  include/linux/netlink.h          |    1 
>  include/linux/sched.h            |    3 
>  init/Kconfig                     |   20 
>  kernel/Makefile                  |    2 
>  kernel/audit.c                   |  825 ++++++++++++++++++++++++++++++++++
>  kernel/auditsc.c                 |  922 +++++++++++++++++++++++++++++++++++++++
>  kernel/fork.c                    |   10 
>  security/selinux/avc.c           |  168 +++----
>  security/selinux/include/avc.h   |    7 
>  security/selinux/ss/services.c   |    2 
>  23 files changed, 2199 insertions(+), 144 deletions(-)

NOTE: The LTT patch you analyzed is fairly old. Here's a more recent
patch against 2.6.3 (attached) using the same criteria as above:
   MAINTAINERS                    |    7
...
   arch/i386/kernel/entry.S       |   21
   arch/i386/kernel/irq.c         |    6
   arch/i386/kernel/process.c     |    9
   arch/i386/kernel/sys_i386.c    |    3
   arch/i386/kernel/traps.c       |  103 +
   arch/i386/mm/fault.c           |   21
...
   fs/buffer.c                    |    3
   fs/exec.c                      |    6
   fs/ioctl.c                     |    6
   fs/open.c                      |   10
   fs/read_write.c                |   36
   fs/select.c                    |    9
...
   include/asm-i386/ltt.h         |   15
...
   include/linux/ltt-core.h       |  428 ++++++
   include/linux/ltt-events.h     |  424 ++++++
   init/Kconfig                   |   32
   ipc/msg.c                      |    2
   ipc/sem.c                      |    3
   ipc/shm.c                      |    3
   kernel/Makefile                |    1
   kernel/exit.c                  |    5
   kernel/fork.c                  |    3
   kernel/itimer.c                |    4
   kernel/ltt-core.c              | 2557 +++++++++++++++++++++++++++++++++++++++++
   kernel/sched.c                 |    4
   kernel/signal.c                |    3
   kernel/softirq.c               |   11
   kernel/time.c                  |    1
   kernel/timer.c                 |    4
   mm/filemap.c                   |    3
   mm/memory.c                    |    4
   mm/page_alloc.c                |    4
   mm/page_io.c                   |    2
   net/core/dev.c                 |    5
   net/socket.c                   |    9
   71 files changed, 4442 insertions(+), 16 deletions(-)

> adds hooks all over the place.

You're right. However, it's worth looking at what's being added. So here's
an example from the scheduler:
@@ -1709,6 +1712,7 @@ switch_tasks:
   		++*switch_count;

   		prepare_arch_switch(rq, next);
+		TRACE_SCHEDCHANGE(prev, next);
   		prev = context_switch(rq, prev, next);
   		barrier();

If tracing is disabled, here's what this macro resolves to:
#define TRACE_SCHEDCHANGE(OUT, IN)

All the trace points added in fs/* ipc/* kernel/* mm/* net/* are the
same type of one-liners. Also, please note the number of lines changed
for each of the files in those directories. Typically, for each file,
there is one #include added and a trace statement such as the above.

As far as the architecture-dependent changes, they are of the same nature.
The only exception is the tracing of the system call entries and exits
which requires changes to entry.S and some C code that calls on the tracing
function that records the system call details, which in the case of LTT is
added to traps.c (hence the number of changes to that file as can be seen
in the above diffstat.)

Overall, the trace statements added could be found in any other Unix kernel.
They are not specific to the Linux kernel, but rather to the architecture of
the OS being analyzed by the end-user.

> The security code adds hooks everywhere too, but those deliver end-user
> functionality rather than being purely a developer support tool.
> 
> Developer support tools are good, but are not as persuasive as end-user
> features.  Because the audience is smaller, and developers know how to
> apply patches and rebuild stuff.

This is probably one of the biggest misconception about LTT amongst kernel
developers. So let me present this once more: LTT is _NOT_ for kernel
developers, it has never been developed with this crowd in mind. LTT is and
has _ALWAYS_ been intended for the end user.

The fact of the matter is that the events recorded by LTT are far too little
in detail to help in any sort of kernel debugging. Don't take my word for it:
I met Marcelo at OLS once and he recounted attempting to use LTT to track
things in the kernel and how he found it NOT to be good enough for what he
was doing. Ditto with Andrea.

How is this tool useful for the end user? Here's an excerpt from an e-mail I
sent to Andrea and a few other SuSE folks explaining this some time ago:
> What LTT is really good at, however, is to provide non-kernel gurus with an
> understanding of kernel dynamics. It is not reasonable to expect that every
> sysadmin will understand exactly how the kernel behaves and then rely on
> ktrace to isolate a problem (as I expect most kernel developers to be able
> to do). On the other hand, it is quite reasonable to expect sysadmins to be
> able to fire-up a tool which gives them a good idea of what's going on in a
> system. This may not help them find kernel bugs, but it will most certainly
> help them track down transient performance problems, and all other kernel-
> behavior-related bugs which are simply invisible to /proc, ps, and their
> friends.
> 
> The same goes for developers tracking synchronization problems. gdb won't
> help, strace won't help, etc. because they rely on ptrace() which itself
> modifies application behavior ... same applies to printf() etc.etc.etc.
> There's an entire category of problems for which current user-space tools
> are not adapted for and kernel debugging tools (ktrace including) are
> simply overkill.

Generally speaking, there isn't a single tool out there that currently
exists that enables any end-user to understand the complex dynamic behavior
between the Linux kernel, his applications and the outside world. And as
you personally noted in the forward to Robert Love's book, the kernel is
only getting more complicated. Using the trace points added by the LTT
patch, the user-space utilities can provide a wealth of information to the
end-user that he cannot possibly collect in any other way. Here's some
example output from the user tools:

## Event type   Time-stamp              PID     Description
Syscall entry   1,086,989,312,690,339 	20 	SYSCALL : setpgid; IP : 0x1000422C
Syscall exit    1,086,989,312,690,341 	20
Trap entry      1,086,989,312,690,343 	20	TRAP : Data Access; IP : 0x100097AC
Trap exit       1,086,989,312,690,400 	20 	
Trap entry      1,086,989,312,690,406 	20 	TRAP : Data Access; IP : 0x1000A0DC
Trap exit       1,086,989,312,690,462 	20 	
Trap entry      1,086,989,312,690,472 	20 	TRAP : Data Access; IP : 0x0FF21F70
Trap exit       1,086,989,312,690,533 	20	
Trap entry      1,086,989,312,690,540 	20	TRAP : Data Access; IP : 0x1000A10C
Trap exit       1,086,989,312,690,595 	20	
Syscall entry   1,086,989,312,690,606 	20	SYSCALL : getpgrp; IP : 0x10004654
Syscall exit    1,086,989,312,690,607 	20	
Syscall entry   1,086,989,312,690,610 	20	SYSCALL : wait4; IP : 0x100099C0
Process         1,086,989,312,690,611 	20	WAIT PID : -1
Sched change    1,086,989,312,690,615 	29	IN : 29; OUT : 20; STATE : 1
Trap entry      1,086,989,312,690,624 	29	TRAP : Data Access; IP : 0x10009584
Trap exit       1,086,989,312,690,628 	29
Trap entry      1,086,989,312,690,631 	29	TRAP : Data Access; IP : 0x100052EC
Trap exit       1,086,989,312,690,634 	29
Syscall entry   1,086,989,312,690,644 	29	SYSCALL : getpid; IP : 0x1000422C
Syscall exit    1,086,989,312,690,645 	29
Syscall entry   1,086,989,312,690,647 	29	SYSCALL : setpgid; IP : 0x1000422C
Syscall exit    1,086,989,312,690,648 	29
Syscall entry   1,086,989,312,690,652 	29	SYSCALL : ioctl; IP : 0x1000960C
File system     1,086,989,312,690,654 	29	IOCTL : 2; COMMAND : 0x80047476

As you can see, the granularity of the details is not refined enough for any
sort of kernel debugging, yet it is clear that an end-user or an application
developer can benefit immensly from such information. Given the ever
increasing complexity of the kernel, the ever increasing number of applications
run on servers and workstations, and the ever increasing use of Linux in
time-sensitive applications such as embedded systems, it seems to me that this
type of capability is no less necessary then ptrace().

I'll conceed that LTT may be of some benefit for some driver developers in some
cases and that it may help consolidate the slew of tracing mechanisms
already included in the kernel as part of various drivers and subsystems, but
the fact of the matter is that it is of little use for kernel developers. If
a kernel developer needs tracing, he should be using ktrace.

> It's a balance between (ongoing maintenance cost multiplied by the number of
> impacted developers) versus (additional functionality multiplied by the
> number of users who benefit from it).  To my mind, LTT (and kgdb and various
> other developer-support things) don't offer good ratios here.

Again, LTT is of marginal use to kernel developers, the benefits all go
to the end users' ability to understand what's going on in their system (see
above for examples.)

On the topic of maintenance cost, I fail to see how one-liners such as the
above can be of any burden to any kernel developer, they have remained
virtually unchanged for the past 5 years and any look throughout the LTT
archives or the kernel mailing list archive for LTT patches will readily
show this.

> If it could use kprobes hooks that'd be neat.  kprobes is low-impact.

The issues about the spread of trace points across the source code are
exactly the same, you still need to mark the code-paths (and maintain
these markings for each version) regardless of the mechanism being used.
Not to mention that the whole idea of LTT is not to modify the kernel
behavior while, to the best of my understanding, kprobes relies on the
debug int ... Not to mention (#2) that once disabled, there is zero code
added to the kernel.

I submit to you that LTT's trace statements have not changed for the
past 5 years (since 2.2.x). They have not increased in number, nor changed
in their semantics. Here's a diffstat on 2.2.13 dated 18th of November 1999
(also attached):
   Documentation/Configure.help |   37 +
   Makefile                     |    4
   arch/i386/config.in          |    5
   arch/i386/kernel/entry.S     |   21
   arch/i386/kernel/irq.c       |    6
   arch/i386/kernel/process.c   |    6
   arch/i386/kernel/sys_i386.c  |    4
   arch/i386/kernel/traps.c     |  105 ++++
   arch/i386/mm/fault.c         |   10
   drivers/Makefile             |   10
   drivers/trace/Makefile       |   30 +
   drivers/trace/tracer.c       |  948 +++++++++++++++++++++++++++++++++++++++++++
   drivers/trace/tracer.h       |   99 ++++
   fs/buffer.c                  |    4
   fs/exec.c                    |    7
   fs/ioctl.c                   |    6
   fs/open.c                    |   10
   fs/read_write.c              |   34 +
   fs/select.c                  |   10
   include/linux/trace.h        |  293 +++++++++++++
   init/main.c                  |   10
   ipc/msg.c                    |    3
   ipc/sem.c                    |    3
   ipc/shm.c                    |    3
   kernel/Makefile              |    4
   kernel/exit.c                |    6
   kernel/fork.c                |    6
   kernel/itimer.c              |    4
   kernel/sched.c               |    9
   kernel/signal.c              |    4
   kernel/softirq.c             |    6
   kernel/trace.c               |  189 ++++++++
   mm/filemap.c                 |    4
   mm/page_alloc.c              |    7
   mm/vmscan.c                  |    4
   net/core/dev.c               |    9
   net/socket.c                 |    9
   37 files changed, 1928 insertions(+), 1 deletion(-)

I believe that this is a very strong argument regarding the maintainability
of such statements, and it is my hope that, based on this evidence, the
kernel development community will recognize that the persistent fears
regarding the maintainbility of the LTT trace statements are unfounded.

If I overlooked something, please let me know.

Karim
-- 
Author, Speaker, Developer, Consultant
Pushing Embedded and Real-Time Linux Systems Beyond the Limits
http://www.opersys.com || karim@opersys.com || 1-866-677-4546

[-- Attachment #2: patch-ltt-linux-2.6.3-relayfs-040315-2.2.bz2 --]
[-- Type: application/x-bzip2, Size: 35013 bytes --]

[-- Attachment #3: patch-ltt-2.2.13-991118.bz2 --]
[-- Type: application/x-bzip2, Size: 19088 bytes --]

Re: [PATCH] IA64 audit support

[Andrew Morton]

Karim Yaghmour <karim@opersys.com> wrote:
>
> > Developer support tools are good, but are not as persuasive as end-user
> > features.  Because the audience is smaller, and developers know how to
> > apply patches and rebuild stuff.
> 
> This is probably one of the biggest misconception about LTT amongst kernel
> developers. So let me present this once more: LTT is _NOT_ for kernel
> developers, it has never been developed with this crowd in mind. LTT is and
> has _ALWAYS_ been intended for the end user.

Note I said "developer", not "kernel developer".  If the audience for a
feature is kernel developers, userspace developers and perhaps the most
sophisticated sysadmins then that's a small audience.  It's certainly an
_important_ audience, but the feature is not as important as those
codepaths which Uncle Tillie needs to run his applications.

> Again, LTT is of marginal use to kernel developers, the benefits all go
> to the end users' ability to understand what's going on in their system (see
> above for examples.)

To me, an "end user" is one who is capable of identifying the power switch
and the ANY key, not an application programmer!

> On the topic of maintenance cost, I fail to see how one-liners such as the
> above can be of any burden to any kernel developer, they have remained
> virtually unchanged for the past 5 years and any look throughout the LTT
> archives or the kernel mailing list archive for LTT patches will readily
> show this.

Fair enough.

> > If it could use kprobes hooks that'd be neat.  kprobes is low-impact.
> 
> The issues about the spread of trace points across the source code are
> exactly the same, you still need to mark the code-paths (and maintain
> these markings for each version) regardless of the mechanism being used.

Nope, kprobes allows a kernel module to patch hooks into the running
binary.  That's all it does, really.   See
http://www-124.ibm.com/linux/projects/kprobes/

LTT kernel inclusion (was Re: [PATCH] IA64 audit support)

[Karim Yaghmour]

Andrew Morton wrote:
> Note I said "developer", not "kernel developer".  If the audience for a
> feature is kernel developers, userspace developers and perhaps the most
> sophisticated sysadmins then that's a small audience.  It's certainly an
> _important_ audience, but the feature is not as important as those
> codepaths which Uncle Tillie needs to run his applications.
...
> To me, an "end user" is one who is capable of identifying the power switch
> and the ANY key, not an application programmer!

I must admit that I'm confused. In the context of how this thread started, I
have to ask what use Uncle Tillie has for kernel auditing. For that matter,
what does Uncle Tillie and the evil horde of ANY key worshipers have to do
with all the features that were added for power users and application
developers while LTT was still waiting in line?

If features such as UML, oprofile, audit, security hooks, vserver, etc.,
that are targeted at the same category of users that LTT is targeted at
can make it in the kernel, then I have a hard time understanding how
there could be any justification for refusing LTT's inclusion simply on
the basis that it doesn't benefit the least computer-literate of Linux
users.

If the inclusion algorithm is based on the following three priorities:
1- Uncle tillie and the evil horde of ANY key worshippers
2- Carnivorous pepsi-can super-users
3- Beaver on steroids kernel developers
with features in category N being more important than those of N+1,
shouldn't all features useful for category N be considered based on an
equal footing? I mean, there is a total and complete absence of any tool
to category 2 users that even closely resembles LTT and addresses the
needs I outlined in my earlier email, and yet features that are useful to
an even narrower group than LTT are routinely included in the kernel.

Given that there is a regular set of patches that are being included in
the kernel to cater for the same audience LTT is meant for, given the
importance of having such a tool for the purposes I outlined earlier,
and given that you've agreed that there is no issue with maintenance,
what else remains for us (the LTT development team) to do in order to
get LTT in the kernel? Should we just send you an updated set of patches
for review and inclusion?

>>On the topic of maintenance cost, I fail to see how one-liners such as the
>>above can be of any burden to any kernel developer, they have remained
>>virtually unchanged for the past 5 years and any look throughout the LTT
>>archives or the kernel mailing list archive for LTT patches will readily
>>show this.
> 
> Fair enough.

Great. Glad to see you agree. At least this one is crossed out.

> Nope, kprobes allows a kernel module to patch hooks into the running
> binary.  That's all it does, really.   See
> http://www-124.ibm.com/linux/projects/kprobes/

I've double-checked what I already knew about kprobes and have looked again
at the site and the patch, and unless there's some feature of kprobes I don't
know about that allows using something else than the debug interrupt to add
hooks, you're wrong about this. In order to do what you describe, kprobes has
to hook itself onto the debug interrupt. Don't take my word for it, here's
what their web site says:
> With each probe, a corresponding probe event handler address is specified.
> Probe event handlers run as extensions to the system breakpoint interrupt
> handler ...
Generating new interrupts is simply unacceptable for LTT's functionality.
Not to mention that it breaks LTT because tracing something will generate
events of its own, which will generating tracing events of their own ...
recursion.

We have, however, contemplated using kernel hooks, which BTW could be used
by the auditing code and quite a few other things too:
http://oss.software.ibm.com/linux/projects/kernelhooks/

Karim
-- 
Author, Speaker, Developer, Consultant
Pushing Embedded and Real-Time Linux Systems Beyond the Limits
http://www.opersys.com || karim@opersys.com || 1-866-677-4546

Re: [PATCH] IA64 audit support

[Roman Zippel]

Hi,

On Thu, 1 Jul 2004, Andrew Morton wrote:

> > This is probably one of the biggest misconception about LTT amongst kernel
> > developers. So let me present this once more: LTT is _NOT_ for kernel
> > developers, it has never been developed with this crowd in mind. LTT is and
> > has _ALWAYS_ been intended for the end user.
> 
> Note I said "developer", not "kernel developer".  If the audience for a
> feature is kernel developers, userspace developers and perhaps the most
> sophisticated sysadmins then that's a small audience.  It's certainly an
> _important_ audience, but the feature is not as important as those
> codepaths which Uncle Tillie needs to run his applications.

And if one this applications misbehaves, he might want to know what is 
actually going wrong. A tracing mechanism could tell him e.g. why xmms 
sometimes stops and with the right visualization tools even an user with 
basic technical knowledge could give some more precise feedback. You 
wouldn't just get reports with "if I do X, the system feels weird".

I'd really like to see some tracing infrastructure in the kernel, as it's 
also a very useful tool during kernel development, when trying to find 
some weird problems or finding perfomance bottlenecks, where printk or 
kgdb isn't much help.

> > The issues about the spread of trace points across the source code are
> > exactly the same, you still need to mark the code-paths (and maintain
> > these markings for each version) regardless of the mechanism being used.
> 
> Nope, kprobes allows a kernel module to patch hooks into the running
> binary.  That's all it does, really.   See
> http://www-124.ibm.com/linux/projects/kprobes/

kprobes are a nice option, but it would be a nightmare (especially 
regarding portability) if any project had to rely on them.
kprobes are already very complex conceptually, it may be a requirement to 
debug a binary kernel, but otherwise some simple hooks are preferable.

bye, Roman

Re: [PATCH] IA64 audit support

[David Mosberger]

The patch is mostly fine with me (it looks identical to me as the last
version I saw from Ray; but my memory may be fuzzy).

There are two minor things I'd like to see changed, though:

 - Use IS_IA32_PROCESS() instead of testing psr.is directly (the macro
   gets defined by system.h and using it ensures that the
   IA-32-specific code will go away if the ia32 subsystem is not
   compiled into the kernel).

 - Remove trailing white-space.

Thanks,

	--david