[PATCH 0/3] ftrace: overrun accounting and trace_pipe headers

[PATCH 0/3] ftrace: overrun accounting and trace_pipe headers

[Steven Rostedt]

The first patch in this series adds accounting to record overruns. That is
where the writing catches up to the reading of the buffer. This really
only matters for trace_pipe since that's a consumer / producer output file.

The next patch adds new methods for the plugins to hook into the
open_pipe and open_read, to let a plugin produce a header.
The open_read method can also be used to do something when overruns
are detected.

The last patch is a test patch AND SHOULD NOT BE APPLIED. It is just an
example in how to use the new methods to produce a header. It simply
makes the ftrace tracer produce a simple "Test header" before any
output.

-- Steve

[PATCH 1/3] ftrace: add logic to record overruns

[Steven Rostedt]

[-- Attachment #1: ftrace-account-overruns.patch --]
[-- Type: text/plain, Size: 3161 bytes --]

This patch sets up the infrastructure to record overruns of the tracing
buffer.

Signed-off-by: Steven Rostedt <srostedt@redhat.com>
---
 kernel/trace/trace.c |   16 +++++++++++-----
 kernel/trace/trace.h |    6 +++++-
 2 files changed, 16 insertions(+), 6 deletions(-)

Index: linux-sched-devel.git/kernel/trace/trace.c
===================================================================
--- linux-sched-devel.git.orig/kernel/trace/trace.c	2008-04-21 13:47:30.000000000 -0400
+++ linux-sched-devel.git/kernel/trace/trace.c	2008-04-21 14:39:43.000000000 -0400
@@ -612,6 +612,7 @@ void unregister_tracer(struct tracer *ty
 void tracing_reset(struct trace_array_cpu *data)
 {
 	data->trace_idx = 0;
+	data->overrun = 0;
 	data->trace_head = data->trace_tail = head_page(data);
 	data->trace_head_idx = 0;
 	data->trace_tail_idx = 0;
@@ -753,6 +754,7 @@ tracing_get_trace_entry(struct trace_arr
 	if (data->trace_head == data->trace_tail &&
 	    idx_next == data->trace_tail_idx) {
 		/* overrun */
+		data->overrun++;
 		data->trace_tail_idx++;
 		if (data->trace_tail_idx >= ENTRIES_PER_PAGE) {
 			data->trace_tail =
@@ -2398,8 +2400,6 @@ tracing_read_pipe(struct file *filp, cha
 {
 	struct trace_iterator *iter = filp->private_data;
 	struct trace_array_cpu *data;
-	struct trace_array *tr = iter->tr;
-	struct tracer *tracer = iter->trace;
 	static cpumask_t mask;
 	static int start;
 	unsigned long flags;
@@ -2478,10 +2478,11 @@ tracing_read_pipe(struct file *filp, cha
 	if (cnt >= PAGE_SIZE)
 		cnt = PAGE_SIZE - 1;
 
-	memset(iter, 0, sizeof(*iter));
-	iter->tr = tr;
-	iter->trace = tracer;
+	/* reset all but tr, trace, and overruns */
 	iter->pos = -1;
+	memset(&iter->seq, 0,
+	       sizeof(struct trace_iterator) -
+	       offsetof(struct trace_iterator, seq));
 
 	/*
 	 * We need to stop all tracing on all CPUS to read the
@@ -2510,6 +2511,11 @@ tracing_read_pipe(struct file *filp, cha
 	for_each_cpu_mask(cpu, mask) {
 		data = iter->tr->data[cpu];
 		__raw_spin_lock(&data->lock);
+
+		if (data->overrun > iter->last_overrun[cpu])
+			iter->overrun[cpu] +=
+				data->overrun - iter->last_overrun[cpu];
+		iter->last_overrun[cpu] = data->overrun;
 	}
 
 	while (find_next_entry_inc(iter) != NULL) {
Index: linux-sched-devel.git/kernel/trace/trace.h
===================================================================
--- linux-sched-devel.git.orig/kernel/trace/trace.h	2008-04-21 13:47:30.000000000 -0400
+++ linux-sched-devel.git/kernel/trace/trace.h	2008-04-21 14:38:09.000000000 -0400
@@ -102,6 +102,7 @@ struct trace_array_cpu {
 	void			*trace_head; /* producer */
 	void			*trace_tail; /* consumer */
 	unsigned long		trace_idx;
+	unsigned long		overrun;
 	unsigned long		saved_latency;
 	unsigned long		critical_start;
 	unsigned long		critical_end;
@@ -162,10 +163,13 @@ struct trace_seq {
  * results to users and which routines might sleep, etc:
  */
 struct trace_iterator {
-	struct trace_seq	seq;
 	struct trace_array	*tr;
 	struct tracer		*trace;
+	long			last_overrun[NR_CPUS];
+	long			overrun[NR_CPUS];
 
+	/* The below is zeroed out in pipe_read */
+	struct trace_seq	seq;
 	struct trace_entry	*ent;
 	int			cpu;
 

-- 

Re: [PATCH 1/3] ftrace: add logic to record overruns

[Pekka Paalanen]

On Mon, 21 Apr 2008 17:09:36 -0400
Steven Rostedt <rostedt@goodmis.org> wrote:

> This patch sets up the infrastructure to record overruns of the tracing
> buffer.
> 
> Signed-off-by: Steven Rostedt <srostedt@redhat.com>
> ---
>  kernel/trace/trace.c |   16 +++++++++++-----
>  kernel/trace/trace.h |    6 +++++-
>  2 files changed, 16 insertions(+), 6 deletions(-)
> 
> Index: linux-sched-devel.git/kernel/trace/trace.c
> ===================================================================
> --- linux-sched-devel.git.orig/kernel/trace/trace.c	2008-04-21 13:47:30.000000000 -0400
> +++ linux-sched-devel.git/kernel/trace/trace.c	2008-04-21 14:39:43.000000000 -0400
> @@ -612,6 +612,7 @@ void unregister_tracer(struct tracer *ty
>  void tracing_reset(struct trace_array_cpu *data)
>  {
>  	data->trace_idx = 0;
> +	data->overrun = 0;
>  	data->trace_head = data->trace_tail = head_page(data);
>  	data->trace_head_idx = 0;
>  	data->trace_tail_idx = 0;
> @@ -753,6 +754,7 @@ tracing_get_trace_entry(struct trace_arr
>  	if (data->trace_head == data->trace_tail &&
>  	    idx_next == data->trace_tail_idx) {
>  		/* overrun */
> +		data->overrun++;
>  		data->trace_tail_idx++;
>  		if (data->trace_tail_idx >= ENTRIES_PER_PAGE) {
>  			data->trace_tail =
...
> @@ -2510,6 +2511,11 @@ tracing_read_pipe(struct file *filp, cha
>  	for_each_cpu_mask(cpu, mask) {
>  		data = iter->tr->data[cpu];
>  		__raw_spin_lock(&data->lock);
> +
> +		if (data->overrun > iter->last_overrun[cpu])
> +			iter->overrun[cpu] +=
> +				data->overrun - iter->last_overrun[cpu];
> +		iter->last_overrun[cpu] = data->overrun;

Option 1: move this code earlier.
(Could also remove the `if' and make it unconditional.)

>  	}
>  
>  	while (find_next_entry_inc(iter) != NULL) {
> Index: linux-sched-devel.git/kernel/trace/trace.h
> ===================================================================
> --- linux-sched-devel.git.orig/kernel/trace/trace.h	2008-04-21 13:47:30.000000000 -0400
> +++ linux-sched-devel.git/kernel/trace/trace.h	2008-04-21 14:38:09.000000000 -0400
> @@ -102,6 +102,7 @@ struct trace_array_cpu {
>  	void			*trace_head; /* producer */
>  	void			*trace_tail; /* consumer */
>  	unsigned long		trace_idx;
> +	unsigned long		overrun;

Option 2: Change this into atomic_t.

>  	unsigned long		saved_latency;
>  	unsigned long		critical_start;
>  	unsigned long		critical_end;
> @@ -162,10 +163,13 @@ struct trace_seq {
>   * results to users and which routines might sleep, etc:
>   */
>  struct trace_iterator {
> -	struct trace_seq	seq;
>  	struct trace_array	*tr;
>  	struct tracer		*trace;
> +	long			last_overrun[NR_CPUS];
> +	long			overrun[NR_CPUS];

The problem with these fields is that they are updated only after the read
hook has been called in tracing_read_pipe(), which means all "N events
lost!" entries will be one read call behind. This can be any number of
events.

If trace_array_cpu::overrun was atomic_t, I could process that in
the read callback. But, trace_iterator::overrun would be good, because
I could just reset it to zero every time I encounter a non-zero value.

I can think of two options, the ones noted above: moving the update
earlier, or using atomic_t. Or maybe do both, so that we don't have to
lock the trace_array_cpu structs early and also avoid duplicating the
overrun/last_overrun logic in a tracer.

What shall we do?


Thanks.

-- 
Pekka Paalanen
http://www.iki.fi/pq/

Re: [PATCH 1/3] ftrace: add logic to record overruns

[Steven Rostedt]

On Sat, 26 Apr 2008, Pekka Paalanen wrote:

> On Mon, 21 Apr 2008 17:09:36 -0400
> Steven Rostedt <rostedt@goodmis.org> wrote:
>
> ....
> > @@ -2510,6 +2511,11 @@ tracing_read_pipe(struct file *filp, cha
> >  	for_each_cpu_mask(cpu, mask) {
> >  		data = iter->tr->data[cpu];
> >  		__raw_spin_lock(&data->lock);
> > +
> > +		if (data->overrun > iter->last_overrun[cpu])
> > +			iter->overrun[cpu] +=
> > +				data->overrun - iter->last_overrun[cpu];
> > +		iter->last_overrun[cpu] = data->overrun;
>
> Option 1: move this code earlier.
> (Could also remove the `if' and make it unconditional.)

I'll move it. The condition was there because I could have sworn that
there's some way (perhaps just via an error) that lastrun could be greater
than the overrun. But perhaps it doesn't matter.

> > +++ linux-sched-devel.git/kernel/trace/trace.h	2008-04-21 14:38:09.000000000 -0400
> > @@ -102,6 +102,7 @@ struct trace_array_cpu {
> >  	void			*trace_head; /* producer */
> >  	void			*trace_tail; /* consumer */
> >  	unsigned long		trace_idx;
> > +	unsigned long		overrun;
>
> Option 2: Change this into atomic_t.

I could do that too.

> >   * results to users and which routines might sleep, etc:
> >   */
> >  struct trace_iterator {
> > -	struct trace_seq	seq;
> >  	struct trace_array	*tr;
> >  	struct tracer		*trace;
> > +	long			last_overrun[NR_CPUS];
> > +	long			overrun[NR_CPUS];
>
> The problem with these fields is that they are updated only after the read
> hook has been called in tracing_read_pipe(), which means all "N events
> lost!" entries will be one read call behind. This can be any number of
> events.
>
> If trace_array_cpu::overrun was atomic_t, I could process that in
> the read callback. But, trace_iterator::overrun would be good, because
> I could just reset it to zero every time I encounter a non-zero value.
>
> I can think of two options, the ones noted above: moving the update
> earlier, or using atomic_t. Or maybe do both, so that we don't have to
> lock the trace_array_cpu structs early and also avoid duplicating the
> overrun/last_overrun logic in a tracer.
>
> What shall we do?

I think you're right that we should move it before calling the read hook.
I didn't want to make the one value atomic because that would just add
another atomic operation in a fast path. I may wait on making that atomic.

-- Steve

[PATCH 2/3] ftrace: add trace pipe header pluggin

[Steven Rostedt]

[-- Attachment #1: ftrace-header-by-method.patch --]
[-- Type: text/plain, Size: 3552 bytes --]

This patch adds a method for open_pipe and open_read to the pluggins
so that they can add a header to the trace pipe call.

Signed-off-by: Steven Rostedt <srostedt@redhat.com>
---
 kernel/trace/trace.c |   38 +++++++++++++++++++++++++++++++-------
 kernel/trace/trace.h |    5 +++++
 2 files changed, 36 insertions(+), 7 deletions(-)

Index: linux-sched-devel.git/kernel/trace/trace.c
===================================================================
--- linux-sched-devel.git.orig/kernel/trace/trace.c	2008-04-21 14:39:43.000000000 -0400
+++ linux-sched-devel.git/kernel/trace/trace.c	2008-04-21 16:50:49.000000000 -0400
@@ -2352,11 +2352,15 @@ static int tracing_open_pipe(struct inod
 	if (!iter)
 		return -ENOMEM;
 
+	mutex_lock(&trace_types_lock);
 	iter->tr = &global_trace;
 	iter->trace = current_trace;
-
 	filp->private_data = iter;
 
+	if (iter->trace->pipe_open)
+		iter->trace->pipe_open(iter);
+	mutex_unlock(&trace_types_lock);
+
 	return 0;
 }
 
@@ -2425,13 +2429,24 @@ tracing_read_pipe(struct file *filp, cha
 		return cnt;
 	}
 
+	mutex_lock(&trace_types_lock);
+	if (iter->trace->read) {
+		ret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
+		if (ret) {
+			read = ret;
+			goto out;
+		}
+	}
+
 	trace_seq_reset(&iter->seq);
 	start = 0;
 
 	while (trace_empty(iter)) {
 
-		if ((filp->f_flags & O_NONBLOCK))
-			return -EAGAIN;
+		if ((filp->f_flags & O_NONBLOCK)) {
+			read = -EAGAIN;
+			goto out;
+		}
 
 		/*
 		 * This is a make-shift waitqueue. The reason we don't use
@@ -2445,16 +2460,22 @@ tracing_read_pipe(struct file *filp, cha
 		set_current_state(TASK_INTERRUPTIBLE);
 		iter->tr->waiter = current;
 
+		mutex_unlock(&trace_types_lock);
+
 		/* sleep for one second, and try again. */
 		schedule_timeout(HZ);
 
+		mutex_lock(&trace_types_lock);
+
 		iter->tr->waiter = NULL;
 
-		if (signal_pending(current))
-			return -EINTR;
+		if (signal_pending(current)) {
+			read = -EINTR;
+			goto out;
+		}
 
 		if (iter->trace != current_trace)
-			return 0;
+			goto out;
 
 		/*
 		 * We block until we read something and tracing is disabled.
@@ -2473,7 +2494,7 @@ tracing_read_pipe(struct file *filp, cha
 
 	/* stop when tracing is finished */
 	if (trace_empty(iter))
-		return 0;
+		goto out;
 
 	if (cnt >= PAGE_SIZE)
 		cnt = PAGE_SIZE - 1;
@@ -2563,6 +2584,9 @@ tracing_read_pipe(struct file *filp, cha
 	if (ret)
 		read = -EFAULT;
 
+out:
+	mutex_unlock(&trace_types_lock);
+
 	return read;
 }
 
Index: linux-sched-devel.git/kernel/trace/trace.h
===================================================================
--- linux-sched-devel.git.orig/kernel/trace/trace.h	2008-04-21 14:38:09.000000000 -0400
+++ linux-sched-devel.git/kernel/trace/trace.h	2008-04-21 15:23:47.000000000 -0400
@@ -140,9 +140,13 @@ struct tracer {
 	void			(*init)(struct trace_array *tr);
 	void			(*reset)(struct trace_array *tr);
 	void			(*open)(struct trace_iterator *iter);
+	void			(*pipe_open)(struct trace_iterator *iter);
 	void			(*close)(struct trace_iterator *iter);
 	void			(*start)(struct trace_iterator *iter);
 	void			(*stop)(struct trace_iterator *iter);
+	ssize_t			(*read)(struct trace_iterator *iter,
+					struct file *filp, char __user *ubuf,
+					size_t cnt, loff_t *ppos);
 	void			(*ctrl_update)(struct trace_array *tr);
 #ifdef CONFIG_FTRACE_STARTUP_TEST
 	int			(*selftest)(struct tracer *trace,
@@ -165,6 +169,7 @@ struct trace_seq {
 struct trace_iterator {
 	struct trace_array	*tr;
 	struct tracer		*trace;
+	void			*private;
 	long			last_overrun[NR_CPUS];
 	long			overrun[NR_CPUS];
 

-- 

Re: [PATCH 2/3] ftrace: add trace pipe header pluggin

[Pekka Paalanen]

On Mon, 21 Apr 2008 17:09:37 -0400
Steven Rostedt <rostedt@goodmis.org> wrote:

> This patch adds a method for open_pipe and open_read to the pluggins
> so that they can add a header to the trace pipe call.
> 
> Signed-off-by: Steven Rostedt <srostedt@redhat.com>

In addition to this, I think I need tracer::close method called on
tracing_release_pipe(). Use case: the reader closes the pipe before
I have printed all of my header, i.e. walked through all PCI devices,
so I need to properly destroy trace_iterator::private.

I also intend to use trace_iterator::seq for printing my header, so I
am looking if I can reuse code from tracing_read_pipe(). Looks like I'll
extract the "return any leftover data" part into a separate function.

To check trace_iterator::overrun, I'd like to be able to use
for_each_tracing_cpu(), but that is local to trace.c.

What happens when the tracing cpu mask changes? Mmiotrace does not take
that into account and will continue to record with all CPUs, will this
lead to problems, when mmiotrace starts to support SMP properly?

Now that I think of it, I probably want to use for_each_online_cpu()
in checking for overruns... but that's not good either, if someone
enables/disables a cpu during tracing. I guess you will have to / had to
deal with this case in tracing core, too.


Thanks.

-- 
Pekka Paalanen
http://www.iki.fi/pq/

Re: [PATCH 2/3] ftrace: add trace pipe header pluggin

[Steven Rostedt]

On Sat, 26 Apr 2008, Pekka Paalanen wrote:

> On Mon, 21 Apr 2008 17:09:37 -0400
> Steven Rostedt <rostedt@goodmis.org> wrote:
>
> > This patch adds a method for open_pipe and open_read to the pluggins
> > so that they can add a header to the trace pipe call.
> >
> > Signed-off-by: Steven Rostedt <srostedt@redhat.com>
>
> In addition to this, I think I need tracer::close method called on
> tracing_release_pipe(). Use case: the reader closes the pipe before
> I have printed all of my header, i.e. walked through all PCI devices,
> so I need to properly destroy trace_iterator::private.

Good idea, I'll add that.

>
> I also intend to use trace_iterator::seq for printing my header, so I
> am looking if I can reuse code from tracing_read_pipe(). Looks like I'll
> extract the "return any leftover data" part into a separate function.
>
> To check trace_iterator::overrun, I'd like to be able to use
> for_each_tracing_cpu(), but that is local to trace.c.

I'll look into that.

>
> What happens when the tracing cpu mask changes? Mmiotrace does not take
> that into account and will continue to record with all CPUs, will this
> lead to problems, when mmiotrace starts to support SMP properly?

Good question. The tracing_cpu_mask isn't mine. So I'll need to know that
answer too.

>
> Now that I think of it, I probably want to use for_each_online_cpu()
> in checking for overruns... but that's not good either, if someone
> enables/disables a cpu during tracing. I guess you will have to / had to
> deal with this case in tracing core, too.

Good point.


-- Steve

[PATCH 3/3] TEST PATCH - ftrace example patch for use of trace pipe headers

[Steven Rostedt]

[-- Attachment #1: ftrace-test-pipe-open.patch --]
[-- Type: text/plain, Size: 1706 bytes --]

DO NOT APPLY THIS PATCH!!!!!

This is an example of implementing a plugin for header for the trace_pipe
file.

DO NOT APPLY THIS PATCH!!!!! 

Signed-off-by: Steven Rostedt <srostedt@redhat.com>
---
 kernel/trace/trace_functions.c |   41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

Index: linux-sched-devel.git/kernel/trace/trace_functions.c
===================================================================
--- linux-sched-devel.git.orig/kernel/trace/trace_functions.c	2008-04-21 16:50:49.000000000 -0400
+++ linux-sched-devel.git/kernel/trace/trace_functions.c	2008-04-21 16:54:55.000000000 -0400
@@ -16,6 +16,45 @@
 
 #include "trace.h"
 
+static void function_pipe_open(struct trace_iterator *iter)
+{
+	int *traceme;
+
+	traceme = kzalloc(sizeof(int), GFP_KERNEL);
+	if (!traceme)
+		return;
+
+	iter->private = traceme;
+}
+
+static ssize_t function_read(struct trace_iterator *iter,
+			     struct file *filp, char __user *ubuf,
+			     size_t cnt, loff_t *ppos)
+{
+	unsigned char head[] = "Test header\n";
+	int *traced;
+	int ret;
+
+	if (!iter->private)
+		return 0;
+
+	traced = iter->private;
+	if (*traced >= sizeof(head)-1)
+		return 0;
+
+	if (cnt > sizeof(head)-1)
+		cnt = sizeof(head)-1;
+
+	ret = copy_to_user(ubuf, head + *traced, cnt);
+
+	*traced += cnt;
+
+	if (ret)
+		cnt = -EFAULT;
+
+	return cnt;
+}
+
 static void function_reset(struct trace_array *tr)
 {
 	int cpu;
@@ -68,6 +107,8 @@ static struct tracer function_trace __re
 #ifdef CONFIG_FTRACE_SELFTEST
 	.selftest    = trace_selftest_startup_function,
 #endif
+	.pipe_open   = function_pipe_open,
+	.read        = function_read,
 };
 
 static __init int init_function_trace(void)

-- 

Re: [PATCH 0/3] ftrace: overrun accounting and trace_pipe headers

[Ingo Molnar]

* Steven Rostedt <rostedt@goodmis.org> wrote:

> The first patch in this series adds accounting to record overruns. 
> That is where the writing catches up to the reading of the buffer. 
> This really only matters for trace_pipe since that's a consumer / 
> producer output file.
> 
> The next patch adds new methods for the plugins to hook into the 
> open_pipe and open_read, to let a plugin produce a header. The 
> open_read method can also be used to do something when overruns are 
> detected.

thanks, applied.

> The last patch is a test patch AND SHOULD NOT BE APPLIED. It is just 
> an example in how to use the new methods to produce a header. It 
> simply makes the ftrace tracer produce a simple "Test header" before 
> any output.

thanks, not applied ;-)

	Ingo

Re: [PATCH 0/3] ftrace: overrun accounting and trace_pipe headers

[Pekka Paalanen]

Excellent!

On Mon, 21 Apr 2008 17:09:35 -0400
Steven Rostedt <rostedt@goodmis.org> wrote:

> The first patch in this series adds accounting to record overruns. That is
> where the writing catches up to the reading of the buffer. This really
> only matters for trace_pipe since that's a consumer / producer output file.

This takes me some thinking how to make the best of it, but looks like it
allows me to relay the overrun events into the trace log. Very good.

> The next patch adds new methods for the plugins to hook into the
> open_pipe and open_read, to let a plugin produce a header.
> The open_read method can also be used to do something when overruns
> are detected.

You mean pipe_open() and read()? :-)
And read() is pipe specific?
Ah, now I see what you mean by read() can be used to notice overruns:
it is called for every read syscall, not just in the beginning.

> The last patch is a test patch AND SHOULD NOT BE APPLIED. It is just an
> example in how to use the new methods to produce a header. It simply
> makes the ftrace tracer produce a simple "Test header" before any
> output.

In function_read(), shouldn't I be doing something with filp and/or ppos?


Thanks.

-- 
Pekka Paalanen
http://www.iki.fi/pq/

Re: [PATCH 0/3] ftrace: overrun accounting and trace_pipe headers

[Steven Rostedt]

On Tue, 22 Apr 2008, Pekka Paalanen wrote:

> Excellent!

Thanks ;-)

>
> On Mon, 21 Apr 2008 17:09:35 -0400
> Steven Rostedt <rostedt@goodmis.org> wrote:
>
> > The first patch in this series adds accounting to record overruns. That is
> > where the writing catches up to the reading of the buffer. This really
> > only matters for trace_pipe since that's a consumer / producer output file.
>
> This takes me some thinking how to make the best of it, but looks like it
> allows me to relay the overrun events into the trace log. Very good.
>
> > The next patch adds new methods for the plugins to hook into the
> > open_pipe and open_read, to let a plugin produce a header.
> > The open_read method can also be used to do something when overruns
> > are detected.
>
> You mean pipe_open() and read()? :-)
> And read() is pipe specific?

Yeah the read API is pipe specific (for now). I'll have to think about
whether or not we should have a generic "read" and a "pipe_read"

> Ah, now I see what you mean by read() can be used to notice overruns:
> it is called for every read syscall, not just in the beginning.

Right!

>
> > The last patch is a test patch AND SHOULD NOT BE APPLIED. It is just an
> > example in how to use the new methods to produce a header. It simply
> > makes the ftrace tracer produce a simple "Test header" before any
> > output.
>
> In function_read(), shouldn't I be doing something with filp and/or ppos?

Probably, that's why I said not to apply it ;-)

Thanks,

-- Steve