[RFC][PATCH] 0/3] [V2] Update xentrace to be compatable with PPC.

[RFC][PATCH] 0/3] [V2] Update xentrace to be compatable with PPC.

[Tony Breeds]

Hello All,
        This Patch series updates xentrace such that it will compile and work
correctly (assuming underlying hypervisor support) on all architectures.
Also allowing trace files captured on one architecture to be portable to other
systems with no changes to tools.

Essentially the patches do:
 1. Use unit64_t instead of unsigned long.
    I did attempt to leave "data" as unsigned long, but in 64bit mode this will
    not compile on ppc.  Using explicitly sized types in shared hypervisor/userspace
    header files is arguably cleaner IMO.
 2. Make the xentrace tool (xentrace copies the xentrace buffers from xen
    memory to disk) write data in network byte order.
 3. Various cleanups.
    This patch is optional.

These patches have been tested on ppc64 and x86_32, and datafiles transported
between said hosts.

[RFC][PATCH] 1/3] [XEN] Use explicit bit sized fields for exported xentrace data.

[Tony Breeds]

Signed-off-by: Tony Breeds <tony@bakeyournoodle.com>
---

 xen/common/trace.c         |    6 +++---
 xen/include/public/trace.h |    2 +-
 xen/include/xen/trace.h    |   14 +++++++-------
 3 files changed, 11 insertions(+), 11 deletions(-)

Index: xen-unstable.hg-mainline.xentrace/xen/common/trace.c
===================================================================
--- xen-unstable.hg-mainline.xentrace.orig/xen/common/trace.c
+++ xen-unstable.hg-mainline.xentrace/xen/common/trace.c
@@ -46,7 +46,7 @@ static int nr_recs;
 static int t_buf_highwater;
 
 /* Number of records lost due to per-CPU trace buffer being full. */
-static DEFINE_PER_CPU(unsigned long, lost_records);
+static DEFINE_PER_CPU(uint64_t, lost_records);
 
 /* a flag recording whether initialization has been done */
 /* or more properly, if the tbuf subsystem is enabled right now */
@@ -228,8 +228,8 @@ int tb_control(xen_sysctl_tbuf_op_t *tbc
  * failure, otherwise 0.  Failure occurs only if the trace buffers are not yet
  * initialised.
  */
-void trace(u32 event, unsigned long d1, unsigned long d2,
-           unsigned long d3, unsigned long d4, unsigned long d5)
+void trace(uint32_t event, uint64_t d1, uint64_t d2, uint64_t d3, uint64_t d4,
+                           uint64_t d5)
 {
     struct t_buf *buf;
     struct t_rec *rec;
Index: xen-unstable.hg-mainline.xentrace/xen/include/public/trace.h
===================================================================
--- xen-unstable.hg-mainline.xentrace.orig/xen/include/public/trace.h
+++ xen-unstable.hg-mainline.xentrace/xen/include/public/trace.h
@@ -76,7 +76,7 @@
 struct t_rec {
     uint64_t cycles;          /* cycle counter timestamp */
     uint32_t event;           /* event ID                */
-    unsigned long data[5];    /* event data items        */
+    uint64_t data[5];         /* event data items        */
 };
 
 /*
Index: xen-unstable.hg-mainline.xentrace/xen/include/xen/trace.h
===================================================================
--- xen-unstable.hg-mainline.xentrace.orig/xen/include/xen/trace.h
+++ xen-unstable.hg-mainline.xentrace/xen/include/xen/trace.h
@@ -33,19 +33,19 @@ void init_trace_bufs(void);
 /* used to retrieve the physical address of the trace buffers */
 int tb_control(struct xen_sysctl_tbuf_op *tbc);
 
-void trace(u32 event, unsigned long d1, unsigned long d2,
-           unsigned long d3, unsigned long d4, unsigned long d5);
+void trace(uint32_t event, uint64_t d1, uint64_t d2, uint64_t d3, uint64_t d4,
+                           uint64_t d5);
 
 /* Avoids troubling the caller with casting their arguments to a trace macro */
 #define trace_do_casts(e,d1,d2,d3,d4,d5) \
     do {                                 \
         if ( unlikely(tb_init_done) )    \
             trace(e,                     \
-                 (unsigned long)d1,      \
-                 (unsigned long)d2,      \
-                 (unsigned long)d3,      \
-                 (unsigned long)d4,      \
-                 (unsigned long)d5);     \
+                 (uint64_t)d1,           \
+                 (uint64_t)d2,           \
+                 (uint64_t)d3,           \
+                 (uint64_t)d4,           \
+                 (uint64_t)d5);          \
     } while ( 0 )
 
 /* Convenience macros for calling the trace function. */

Re: [RFC][PATCH] 1/3] [XEN] Use explicit bit sized fields for exported xentrace data.

[Mark Williamson]

It's worth noting that this patch will (roughly) halve the number of trace 
records.  Nobody should probably be relying on trace buffer size anymore, so 
this shouldn't be an issue.

No objections here.

Cheers,
Mark

On Thursday 30 November 2006 05:59, Tony Breeds wrote:
> Signed-off-by: Tony Breeds <tony@bakeyournoodle.com>
> ---
>
>  xen/common/trace.c         |    6 +++---
>  xen/include/public/trace.h |    2 +-
>  xen/include/xen/trace.h    |   14 +++++++-------
>  3 files changed, 11 insertions(+), 11 deletions(-)
>
> Index: xen-unstable.hg-mainline.xentrace/xen/common/trace.c
> ===================================================================
> --- xen-unstable.hg-mainline.xentrace.orig/xen/common/trace.c
> +++ xen-unstable.hg-mainline.xentrace/xen/common/trace.c
> @@ -46,7 +46,7 @@ static int nr_recs;
>  static int t_buf_highwater;
>
>  /* Number of records lost due to per-CPU trace buffer being full. */
> -static DEFINE_PER_CPU(unsigned long, lost_records);
> +static DEFINE_PER_CPU(uint64_t, lost_records);
>
>  /* a flag recording whether initialization has been done */
>  /* or more properly, if the tbuf subsystem is enabled right now */
> @@ -228,8 +228,8 @@ int tb_control(xen_sysctl_tbuf_op_t *tbc
>   * failure, otherwise 0.  Failure occurs only if the trace buffers are not
> yet * initialised.
>   */
> -void trace(u32 event, unsigned long d1, unsigned long d2,
> -           unsigned long d3, unsigned long d4, unsigned long d5)
> +void trace(uint32_t event, uint64_t d1, uint64_t d2, uint64_t d3, uint64_t
> d4, +                           uint64_t d5)
>  {
>      struct t_buf *buf;
>      struct t_rec *rec;
> Index: xen-unstable.hg-mainline.xentrace/xen/include/public/trace.h
> ===================================================================
> --- xen-unstable.hg-mainline.xentrace.orig/xen/include/public/trace.h
> +++ xen-unstable.hg-mainline.xentrace/xen/include/public/trace.h
> @@ -76,7 +76,7 @@
>  struct t_rec {
>      uint64_t cycles;          /* cycle counter timestamp */
>      uint32_t event;           /* event ID                */
> -    unsigned long data[5];    /* event data items        */
> +    uint64_t data[5];         /* event data items        */
>  };
>
>  /*
> Index: xen-unstable.hg-mainline.xentrace/xen/include/xen/trace.h
> ===================================================================
> --- xen-unstable.hg-mainline.xentrace.orig/xen/include/xen/trace.h
> +++ xen-unstable.hg-mainline.xentrace/xen/include/xen/trace.h
> @@ -33,19 +33,19 @@ void init_trace_bufs(void);
>  /* used to retrieve the physical address of the trace buffers */
>  int tb_control(struct xen_sysctl_tbuf_op *tbc);
>
> -void trace(u32 event, unsigned long d1, unsigned long d2,
> -           unsigned long d3, unsigned long d4, unsigned long d5);
> +void trace(uint32_t event, uint64_t d1, uint64_t d2, uint64_t d3, uint64_t
> d4, +                           uint64_t d5);
>
>  /* Avoids troubling the caller with casting their arguments to a trace
> macro */ #define trace_do_casts(e,d1,d2,d3,d4,d5) \
>      do {                                 \
>          if ( unlikely(tb_init_done) )    \
>              trace(e,                     \
> -                 (unsigned long)d1,      \
> -                 (unsigned long)d2,      \
> -                 (unsigned long)d3,      \
> -                 (unsigned long)d4,      \
> -                 (unsigned long)d5);     \
> +                 (uint64_t)d1,           \
> +                 (uint64_t)d2,           \
> +                 (uint64_t)d3,           \
> +                 (uint64_t)d4,           \
> +                 (uint64_t)d5);          \
>      } while ( 0 )
>
>  /* Convenience macros for calling the trace function. */
>
>
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel

-- 
Dave: Just a question. What use is a unicyle with no seat?  And no pedals!
Mark: To answer a question with a question: What use is a skateboard?
Dave: Skateboards have wheels.
Mark: My wheel has a wheel!

Re: [RFC][PATCH] 1/3] [XEN] Use explicit bit sized fields for exported xentrace data.

[George Dunlap]

Hmm... this has the unfortunate side-effect of doubling the size of
the trace, and effectively halving the effectiveness of the trace
buffer in avoiding drops.  My moderate-length traces are already in
the gigabyte range, and I occasionally lose trace records even with a
buffer size of 256.  It would be really nice if we could avoid that.

I happen to be using the VMENTER/VMEXIT tracing, which could be
consolidated into one record if we went to a 64-bit trace.  Is anyone
else doing high-bandwidth tracing that this would affect in a
significantly negative way?

 -George

On 11/30/06, Tony Breeds <tony@bakeyournoodle.com> wrote:
> Signed-off-by: Tony Breeds <tony@bakeyournoodle.com>
> ---
>
>  xen/common/trace.c         |    6 +++---
>  xen/include/public/trace.h |    2 +-
>  xen/include/xen/trace.h    |   14 +++++++-------
>  3 files changed, 11 insertions(+), 11 deletions(-)
>
> Index: xen-unstable.hg-mainline.xentrace/xen/common/trace.c
> ===================================================================
> --- xen-unstable.hg-mainline.xentrace.orig/xen/common/trace.c
> +++ xen-unstable.hg-mainline.xentrace/xen/common/trace.c
> @@ -46,7 +46,7 @@ static int nr_recs;
>  static int t_buf_highwater;
>
>  /* Number of records lost due to per-CPU trace buffer being full. */
> -static DEFINE_PER_CPU(unsigned long, lost_records);
> +static DEFINE_PER_CPU(uint64_t, lost_records);
>
>  /* a flag recording whether initialization has been done */
>  /* or more properly, if the tbuf subsystem is enabled right now */
> @@ -228,8 +228,8 @@ int tb_control(xen_sysctl_tbuf_op_t *tbc
>   * failure, otherwise 0.  Failure occurs only if the trace buffers are not yet
>   * initialised.
>   */
> -void trace(u32 event, unsigned long d1, unsigned long d2,
> -           unsigned long d3, unsigned long d4, unsigned long d5)
> +void trace(uint32_t event, uint64_t d1, uint64_t d2, uint64_t d3, uint64_t d4,
> +                           uint64_t d5)
>  {
>      struct t_buf *buf;
>      struct t_rec *rec;
> Index: xen-unstable.hg-mainline.xentrace/xen/include/public/trace.h
> ===================================================================
> --- xen-unstable.hg-mainline.xentrace.orig/xen/include/public/trace.h
> +++ xen-unstable.hg-mainline.xentrace/xen/include/public/trace.h
> @@ -76,7 +76,7 @@
>  struct t_rec {
>      uint64_t cycles;          /* cycle counter timestamp */
>      uint32_t event;           /* event ID                */
> -    unsigned long data[5];    /* event data items        */
> +    uint64_t data[5];         /* event data items        */
>  };
>
>  /*
> Index: xen-unstable.hg-mainline.xentrace/xen/include/xen/trace.h
> ===================================================================
> --- xen-unstable.hg-mainline.xentrace.orig/xen/include/xen/trace.h
> +++ xen-unstable.hg-mainline.xentrace/xen/include/xen/trace.h
> @@ -33,19 +33,19 @@ void init_trace_bufs(void);
>  /* used to retrieve the physical address of the trace buffers */
>  int tb_control(struct xen_sysctl_tbuf_op *tbc);
>
> -void trace(u32 event, unsigned long d1, unsigned long d2,
> -           unsigned long d3, unsigned long d4, unsigned long d5);
> +void trace(uint32_t event, uint64_t d1, uint64_t d2, uint64_t d3, uint64_t d4,
> +                           uint64_t d5);
>
>  /* Avoids troubling the caller with casting their arguments to a trace macro */
>  #define trace_do_casts(e,d1,d2,d3,d4,d5) \
>      do {                                 \
>          if ( unlikely(tb_init_done) )    \
>              trace(e,                     \
> -                 (unsigned long)d1,      \
> -                 (unsigned long)d2,      \
> -                 (unsigned long)d3,      \
> -                 (unsigned long)d4,      \
> -                 (unsigned long)d5);     \
> +                 (uint64_t)d1,           \
> +                 (uint64_t)d2,           \
> +                 (uint64_t)d3,           \
> +                 (uint64_t)d4,           \
> +                 (uint64_t)d5);          \
>      } while ( 0 )
>
>  /* Convenience macros for calling the trace function. */
>
>
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel
>

Re: [RFC][PATCH] 1/3] [XEN] Use explicit bit sized fields for exported xentrace data.

[Keir Fraser]

On 30/11/06 16:58, "George Dunlap" <dunlapg@umich.edu> wrote:

> Hmm... this has the unfortunate side-effect of doubling the size of
> the trace, and effectively halving the effectiveness of the trace
> buffer in avoiding drops.  My moderate-length traces are already in
> the gigabyte range, and I occasionally lose trace records even with a
> buffer size of 256.  It would be really nice if we could avoid that.
> 
> I happen to be using the VMENTER/VMEXIT tracing, which could be
> consolidated into one record if we went to a 64-bit trace.  Is anyone
> else doing high-bandwidth tracing that this would affect in a
> significantly negative way?

As we move increasingly towards x86/64 this is an issue that will need to be
addressed even if we leave the tracing fields as longs.

 -- Keir

Re: [RFC][PATCH] 1/3] [XEN] Use explicit bit sized fields for exported xentrace data.

[Mark Williamson]

I guess one possibility would be to continue using unsigned longs but store 
the machine word size and endianness in a header in the trace file.  This 
gets us platform independence.

This avoids adding extra overhead on the fast path, the extra processing can 
happen offline (and probably not at all in the common case that you're on the 
same endianness / word size as the trace was collected on).

Another alternative would be to allow some combination of 32-bit or (fewer) 
64-bit words in the record.  This would let us keep the same record size, but 
have a bit more flexibility.

Going the whole hog, we could even make the trace data opaque to trace.c - 
have a char[] for the data, and deal with the semantics in terms 
of "longs" "u64" etc in macros in the traced code, and in xentrace_format.

If we did this, the logical extension would be to have variable length trace 
records with a fixed-size header giving the full length.  I think this would 
be a good direction to go in, and would ensure that we maximise use of the 
trace buffer space.  It shouldn't be that hard to modify the system to do 
this - most of the work may even be in making it nice to use!

Cheers,
Mark

On Thursday 30 November 2006 17:03, Keir Fraser wrote:
> On 30/11/06 16:58, "George Dunlap" <dunlapg@umich.edu> wrote:
> > Hmm... this has the unfortunate side-effect of doubling the size of
> > the trace, and effectively halving the effectiveness of the trace
> > buffer in avoiding drops.  My moderate-length traces are already in
> > the gigabyte range, and I occasionally lose trace records even with a
> > buffer size of 256.  It would be really nice if we could avoid that.
> >
> > I happen to be using the VMENTER/VMEXIT tracing, which could be
> > consolidated into one record if we went to a 64-bit trace.  Is anyone
> > else doing high-bandwidth tracing that this would affect in a
> > significantly negative way?
>
> As we move increasingly towards x86/64 this is an issue that will need to
> be addressed even if we leave the tracing fields as longs.
>
>  -- Keir
>
>
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel

-- 
Dave: Just a question. What use is a unicyle with no seat?  And no pedals!
Mark: To answer a question with a question: What use is a skateboard?
Dave: Skateboards have wheels.
Mark: My wheel has a wheel!

Re: [RFC][PATCH] 1/3] [XEN] Use explicit bit sized fields for exported xentrace data.

[Jimi Xenidis]

I'll go one (huge) step further...
should we adopt LTTng? http://ltt.polymtl.ca/
The trace stanzas are well defined and comes with a bunch of  
visualization and analysis tools.
I know a lot of people in at IBM have been using it for performance  
studies of various operating systems and I have been asked how Xen  
could support if for Xen but for domains as well.
(http://ltt.polymtl.ca/svn/ltt/branches/poly/doc/developer/xen-port.txt)

-JX

On Nov 30, 2006, at 9:29 PM, Mark Williamson wrote:

> I guess one possibility would be to continue using unsigned longs  
> but store
> the machine word size and endianness in a header in the trace  
> file.  This
> gets us platform independence.
>
> This avoids adding extra overhead on the fast path, the extra  
> processing can
> happen offline (and probably not at all in the common case that  
> you're on the
> same endianness / word size as the trace was collected on).
>
> Another alternative would be to allow some combination of 32-bit or  
> (fewer)
> 64-bit words in the record.  This would let us keep the same record  
> size, but
> have a bit more flexibility.
>
> Going the whole hog, we could even make the trace data opaque to  
> trace.c -
> have a char[] for the data, and deal with the semantics in terms
> of "longs" "u64" etc in macros in the traced code, and in  
> xentrace_format.
>
> If we did this, the logical extension would be to have variable  
> length trace
> records with a fixed-size header giving the full length.  I think  
> this would
> be a good direction to go in, and would ensure that we maximise use  
> of the
> trace buffer space.  It shouldn't be that hard to modify the system  
> to do
> this - most of the work may even be in making it nice to use!
>
> Cheers,
> Mark
>
> On Thursday 30 November 2006 17:03, Keir Fraser wrote:
>> On 30/11/06 16:58, "George Dunlap" <dunlapg@umich.edu> wrote:
>>> Hmm... this has the unfortunate side-effect of doubling the size of
>>> the trace, and effectively halving the effectiveness of the trace
>>> buffer in avoiding drops.  My moderate-length traces are already in
>>> the gigabyte range, and I occasionally lose trace records even  
>>> with a
>>> buffer size of 256.  It would be really nice if we could avoid that.
>>>
>>> I happen to be using the VMENTER/VMEXIT tracing, which could be
>>> consolidated into one record if we went to a 64-bit trace.  Is  
>>> anyone
>>> else doing high-bandwidth tracing that this would affect in a
>>> significantly negative way?
>>
>> As we move increasingly towards x86/64 this is an issue that will  
>> need to
>> be addressed even if we leave the tracing fields as longs.
>>
>>  -- Keir
>>
>>
>> _______________________________________________
>> Xen-devel mailing list
>> Xen-devel@lists.xensource.com
>> http://lists.xensource.com/xen-devel
>
> -- 
> Dave: Just a question. What use is a unicyle with no seat?  And no  
> pedals!
> Mark: To answer a question with a question: What use is a skateboard?
> Dave: Skateboards have wheels.
> Mark: My wheel has a wheel!
>
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel

Re: [RFC][PATCH] 1/3] [XEN] Use explicit bit sized fields for exported xentrace data.

[George Dunlap]

Variable-length trace buffers would certainly be nice.  Right now, for
some records, I'm squeezing bits into things, and just *one* more byte
would be great... but for other records, I'm storing 4 words of 0, and
there just doesn't seem to be any other useful information to add.

Keir, is the simpler, "one trace size fits all" method just because it
was easier to implement originally, or is the simplicity expected to
greatly reduce overhead and/or bugginess?  If the former, then there
seems enough interest in making the tracing more flexible to be worth
changing; if the latter, then we should probably chose something and
live with it, or perhaps a compromise (i.e., two record sizes).

 -George

On 11/30/06, Mark Williamson <mark.williamson@cl.cam.ac.uk> wrote:
> I guess one possibility would be to continue using unsigned longs but store
> the machine word size and endianness in a header in the trace file.  This
> gets us platform independence.
>
> This avoids adding extra overhead on the fast path, the extra processing can
> happen offline (and probably not at all in the common case that you're on the
> same endianness / word size as the trace was collected on).
>
> Another alternative would be to allow some combination of 32-bit or (fewer)
> 64-bit words in the record.  This would let us keep the same record size, but
> have a bit more flexibility.
>
> Going the whole hog, we could even make the trace data opaque to trace.c -
> have a char[] for the data, and deal with the semantics in terms
> of "longs" "u64" etc in macros in the traced code, and in xentrace_format.
>
> If we did this, the logical extension would be to have variable length trace
> records with a fixed-size header giving the full length.  I think this would
> be a good direction to go in, and would ensure that we maximise use of the
> trace buffer space.  It shouldn't be that hard to modify the system to do
> this - most of the work may even be in making it nice to use!
>
> Cheers,
> Mark
>
> On Thursday 30 November 2006 17:03, Keir Fraser wrote:
> > On 30/11/06 16:58, "George Dunlap" <dunlapg@umich.edu> wrote:
> > > Hmm... this has the unfortunate side-effect of doubling the size of
> > > the trace, and effectively halving the effectiveness of the trace
> > > buffer in avoiding drops.  My moderate-length traces are already in
> > > the gigabyte range, and I occasionally lose trace records even with a
> > > buffer size of 256.  It would be really nice if we could avoid that.
> > >
> > > I happen to be using the VMENTER/VMEXIT tracing, which could be
> > > consolidated into one record if we went to a 64-bit trace.  Is anyone
> > > else doing high-bandwidth tracing that this would affect in a
> > > significantly negative way?
> >
> > As we move increasingly towards x86/64 this is an issue that will need to
> > be addressed even if we leave the tracing fields as longs.
> >
> >  -- Keir
> >
> >
> > _______________________________________________
> > Xen-devel mailing list
> > Xen-devel@lists.xensource.com
> > http://lists.xensource.com/xen-devel
>
> --
> Dave: Just a question. What use is a unicyle with no seat?  And no pedals!
> Mark: To answer a question with a question: What use is a skateboard?
> Dave: Skateboards have wheels.
> Mark: My wheel has a wheel!
>

Re: [RFC][PATCH] 1/3] [XEN] Use explicit bit sized fields for exported xentrace data.

[Keir Fraser]

On 1/12/06 5:54 pm, "George Dunlap" <dunlapg@umich.edu> wrote:

> Variable-length trace buffers would certainly be nice.  Right now, for
> some records, I'm squeezing bits into things, and just *one* more byte
> would be great... but for other records, I'm storing 4 words of 0, and
> there just doesn't seem to be any other useful information to add.
> 
> Keir, is the simpler, "one trace size fits all" method just because it
> was easier to implement originally, or is the simplicity expected to
> greatly reduce overhead and/or bugginess?  If the former, then there
> seems enough interest in making the tracing more flexible to be worth
> changing; if the latter, then we should probably chose something and
> live with it, or perhaps a compromise (i.e., two record sizes).

There's no reason not to make the trace format more flexible. There's a
question about how you represent trace points in the Xen code though, when
the format is no longer a list of fixed size integers.

 -- Keir

Re: [RFC][PATCH] 1/3] [XEN] Use explicit bit sized fields for exported xentrace data.

[Mark Williamson]

> There's no reason not to make the trace format more flexible. There's a
> question about how you represent trace points in the Xen code though, when
> the format is no longer a list of fixed size integers.

I can see two main possibilities.  One involving a variadic function and one 
involving mega macros of doom.

One possibility would be a trace() function taking a variable number of 
arguments, i.e.

void trace(type, unsigned char data1, unsigned char data2, ... etc)

And a set of arch-defined macros (or at least bitness / endian defined 
macros).  Eg. on x86 we might have:

#define TRACE_U16(d) ((unsigned char)(d & 255)), ((unsigned char)(d >> 8))

We'd need to verify whether the extra processing had a measurable performance 
impact, however.

Another alternative would be to make the array of trace buffers globally 
accessible and then use a set of macros for the trace() instead of an inline 
function.  The macros could then look something like (pseudocode):

struct trace_record {
u32 type;
u32 data_len;
char data[]
};

char *trace_buffer[NR_CPUS]

#define open_trace(type)      \
                              do { \
                                  disable local irqs                     \
                                  struct trace_record *record =          \
                                         &trace_buffer[cpu][producer_idx]; \
                                  record->type = (u32)type               \
                                  record->data_len = 0;

#define trace_u16(data)       *(u16 *)record->data[record->data_len] = data \
                              record->data_len += sizeof(u16)

... etc for different data types, with appropriate variations if necessary for 
different platforms ...

#define close_trace()       \
                            inc producer counter by sizeof(struct          \
                            trace_record) + record->data_len for userspace \
                            to see \
                            re-enable local irqs                         \
                            } while(0)
                            

Things become unhappy here because there'd need to be some kind of bounds 
checking in here to determine whether we need to wrap to the beginning of the 
trace buffer again.  The alternatives as I see them would be either:

a) include code in each data macro to check if we'd reached the end of the 
buffer and wrap the data appropriately, or
b) include code that'll simply copy everything we've built so far to the 
beginning of the trace buffer and start again.

Either way is going to be ugly and unpleasant.  Also, we have the problem of 
not knowing whether we're going to wrap OR run out of space until we're part 
way through the trace record, although in this instance, I guess we could 
just change to create a "missed data" record.

I think the first approach (variadic function) above is probably nicer.  We 
can always make a few macros to make common cases (e.g. recording a type and 
a single u64 of data) less verbose.

Any thoughts?

Cheers,
Mark

-- 
Dave: Just a question. What use is a unicyle with no seat?  And no pedals!
Mark: To answer a question with a question: What use is a skateboard?
Dave: Skateboards have wheels.
Mark: My wheel has a wheel!

Re: [RFC][PATCH] 1/3] [XEN] Use explicit bit sized fields for exported xentrace data.

[George Dunlap]

I see two other options:
 * Pre-allocate a block of data and fill it in
 * Allocate a struct on the stack, and copy it all at once.

In the first case you'd do something like the following:
  struct {
     [trace layout]
  } *trec;

  trec=trace_var(TRC_TYPE, sizeof(*trec), [maybe some other info]);

  /* Fill in trec->* */

The second case looks similar:
 struct {
     [trace layout]
 } trec;

 /* Fill in trec.*  */

 trace_var(TRC_TYPE, &trec, sizeof(trec), [maybe other info]);

The second case involves an extra copy, but that shouldn't be a big
deal.  It has the advantage of being self-contained, and the trace
code can make the record "wrap around" transparently.

The first means no copying, but it also means no "wrap around"; if
there's not enough room at the end of a buffer, the space would just
have to be left empty.  That's not probably such a big deal, though.
The bigger problem comes if several "open" trace records happen at
once.  It's technically possible that the trace buffer will wrap
around before a function is done writing to its original buffer.

In both of these cases, the common "TRACE_nD" macros can be left, I
think.  We might want to add "TRACE_nDL" for 64-bit values, and then
let those who need more flexible trace structures call trace_var()
directly.

This way of doing things also has the advantage that the trace record
can be defined in a public header somewhere, and used by user-space
analysis tools as well as the hypervisor tracing code.

Thoughts?

 -George

On 12/5/06, Mark Williamson <mark.williamson@cl.cam.ac.uk> wrote:
> > There's no reason not to make the trace format more flexible. There's a
> > question about how you represent trace points in the Xen code though, when
> > the format is no longer a list of fixed size integers.
>
> I can see two main possibilities.  One involving a variadic function and one
> involving mega macros of doom.
>
> One possibility would be a trace() function taking a variable number of
> arguments, i.e.
>
> void trace(type, unsigned char data1, unsigned char data2, ... etc)
>
> And a set of arch-defined macros (or at least bitness / endian defined
> macros).  Eg. on x86 we might have:
>
> #define TRACE_U16(d) ((unsigned char)(d & 255)), ((unsigned char)(d >> 8))
>
> We'd need to verify whether the extra processing had a measurable performance
> impact, however.
>
> Another alternative would be to make the array of trace buffers globally
> accessible and then use a set of macros for the trace() instead of an inline
> function.  The macros could then look something like (pseudocode):
>
> struct trace_record {
> u32 type;
> u32 data_len;
> char data[]
> };
>
> char *trace_buffer[NR_CPUS]
>
> #define open_trace(type)      \
>                               do { \
>                                   disable local irqs                     \
>                                   struct trace_record *record =          \
>                                          &trace_buffer[cpu][producer_idx]; \
>                                   record->type = (u32)type               \
>                                   record->data_len = 0;
>
> #define trace_u16(data)       *(u16 *)record->data[record->data_len] = data \
>                               record->data_len += sizeof(u16)
>
> ... etc for different data types, with appropriate variations if necessary for
> different platforms ...
>
> #define close_trace()       \
>                             inc producer counter by sizeof(struct          \
>                             trace_record) + record->data_len for userspace \
>                             to see \
>                             re-enable local irqs                         \
>                             } while(0)
>
>
> Things become unhappy here because there'd need to be some kind of bounds
> checking in here to determine whether we need to wrap to the beginning of the
> trace buffer again.  The alternatives as I see them would be either:
>
> a) include code in each data macro to check if we'd reached the end of the
> buffer and wrap the data appropriately, or
> b) include code that'll simply copy everything we've built so far to the
> beginning of the trace buffer and start again.
>
> Either way is going to be ugly and unpleasant.  Also, we have the problem of
> not knowing whether we're going to wrap OR run out of space until we're part
> way through the trace record, although in this instance, I guess we could
> just change to create a "missed data" record.
>
> I think the first approach (variadic function) above is probably nicer.  We
> can always make a few macros to make common cases (e.g. recording a type and
> a single u64 of data) less verbose.
>
> Any thoughts?
>
> Cheers,
> Mark
>
> --
> Dave: Just a question. What use is a unicyle with no seat?  And no pedals!
> Mark: To answer a question with a question: What use is a skateboard?
> Dave: Skateboards have wheels.
> Mark: My wheel has a wheel!
>

RE: [RFC][PATCH] 1/3] [XEN] Use explicit bit sized fieldsfor exported xentrace data.

[Ian Pratt]

> I see two other options:
>  * Pre-allocate a block of data and fill it in
>  * Allocate a struct on the stack, and copy it all at once.
> 
> In the first case you'd do something like the following:
>   struct {
>      [trace layout]
>   } *trec;
> 
>   trec=trace_var(TRC_TYPE, sizeof(*trec), [maybe some other info]);
> 
>   /* Fill in trec->* */
> 
> The second case looks similar:
>  struct {
>      [trace layout]
>  } trec;
> 
>  /* Fill in trec.*  */
> 
>  trace_var(TRC_TYPE, &trec, sizeof(trec), [maybe other info]);

The ultimate best way of doing it would be to have trace functions that
took a format string and a variable number of arguments. The actual
trace record written in the buffer would just contain the record type
and the length of the record, followed by the variable data. The format
string would be written out in an a separate segment, enabling it to be
extracted and used by the trace post-process tool to pretty print the
records.

Ian
 
> The second case involves an extra copy, but that shouldn't be a big
> deal.  It has the advantage of being self-contained, and the trace
> code can make the record "wrap around" transparently.
> 
> The first means no copying, but it also means no "wrap around"; if
> there's not enough room at the end of a buffer, the space would just
> have to be left empty.  That's not probably such a big deal, though.
> The bigger problem comes if several "open" trace records happen at
> once.  It's technically possible that the trace buffer will wrap
> around before a function is done writing to its original buffer.
> 
> In both of these cases, the common "TRACE_nD" macros can be left, I
> think.  We might want to add "TRACE_nDL" for 64-bit values, and then
> let those who need more flexible trace structures call trace_var()
> directly.
> 
> This way of doing things also has the advantage that the trace record
> can be defined in a public header somewhere, and used by user-space
> analysis tools as well as the hypervisor tracing code.
> 
> Thoughts?
> 
>  -George
> 
> On 12/5/06, Mark Williamson <mark.williamson@cl.cam.ac.uk> wrote:
> > > There's no reason not to make the trace format more flexible.
There's a
> > > question about how you represent trace points in the Xen code
though,
> when
> > > the format is no longer a list of fixed size integers.
> >
> > I can see two main possibilities.  One involving a variadic function
and
> one
> > involving mega macros of doom.
> >
> > One possibility would be a trace() function taking a variable number
of
> > arguments, i.e.
> >
> > void trace(type, unsigned char data1, unsigned char data2, ... etc)
> >
> > And a set of arch-defined macros (or at least bitness / endian
defined
> > macros).  Eg. on x86 we might have:
> >
> > #define TRACE_U16(d) ((unsigned char)(d & 255)), ((unsigned char)(d
>>
> 8))
> >
> > We'd need to verify whether the extra processing had a measurable
> performance
> > impact, however.
> >
> > Another alternative would be to make the array of trace buffers
globally
> > accessible and then use a set of macros for the trace() instead of
an
> inline
> > function.  The macros could then look something like (pseudocode):
> >
> > struct trace_record {
> > u32 type;
> > u32 data_len;
> > char data[]
> > };
> >
> > char *trace_buffer[NR_CPUS]
> >
> > #define open_trace(type)      \
> >                               do { \
> >                                   disable local irqs
> \
> >                                   struct trace_record *record =
> \
> >
> &trace_buffer[cpu][producer_idx]; \
> >                                   record->type = (u32)type
> \
> >                                   record->data_len = 0;
> >
> > #define trace_u16(data)       *(u16 *)record->data[record->data_len]
=
> data \
> >                               record->data_len += sizeof(u16)
> >
> > ... etc for different data types, with appropriate variations if
> necessary for
> > different platforms ...
> >
> > #define close_trace()       \
> >                             inc producer counter by sizeof(struct
> \
> >                             trace_record) + record->data_len for
> userspace \
> >                             to see \
> >                             re-enable local irqs
> \
> >                             } while(0)
> >
> >
> > Things become unhappy here because there'd need to be some kind of
bounds
> > checking in here to determine whether we need to wrap to the
beginning of
> the
> > trace buffer again.  The alternatives as I see them would be either:
> >
> > a) include code in each data macro to check if we'd reached the end
of
> the
> > buffer and wrap the data appropriately, or
> > b) include code that'll simply copy everything we've built so far to
the
> > beginning of the trace buffer and start again.
> >
> > Either way is going to be ugly and unpleasant.  Also, we have the
problem
> of
> > not knowing whether we're going to wrap OR run out of space until
we're
> part
> > way through the trace record, although in this instance, I guess we
could
> > just change to create a "missed data" record.
> >
> > I think the first approach (variadic function) above is probably
nicer.
> We
> > can always make a few macros to make common cases (e.g. recording a
type
> and
> > a single u64 of data) less verbose.
> >
> > Any thoughts?
> >
> > Cheers,
> > Mark
> >
> > --
> > Dave: Just a question. What use is a unicyle with no seat?  And no
> pedals!
> > Mark: To answer a question with a question: What use is a
skateboard?
> > Dave: Skateboards have wheels.
> > Mark: My wheel has a wheel!
> >
> 
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel

Re: [RFC][PATCH] 1/3] [XEN] Use explicit bit sized fieldsfor exported xentrace data.

[Mark Williamson]

> The ultimate best way of doing it would be to have trace functions that
> took a format string and a variable number of arguments.

Agreed.

> The actual 
> trace record written in the buffer would just contain the record type
> and the length of the record, followed by the variable data. The format
> string would be written out in an a separate segment, enabling it to be
> extracted and used by the trace post-process tool to pretty print the
> records.

Cool.

It would be fairly trivial to declare a name for each "type" and allow 
userspace to query the type name <-> type ID mapping.  This then removes all 
responsibility for determining output formatting from userspace, and should 
make everything neater.

Nice.  All we need is a coder - anyone interested?

Cheers,
Mark

> Ian
>
> > The second case involves an extra copy, but that shouldn't be a big
> > deal.  It has the advantage of being self-contained, and the trace
> > code can make the record "wrap around" transparently.
> >
> > The first means no copying, but it also means no "wrap around"; if
> > there's not enough room at the end of a buffer, the space would just
> > have to be left empty.  That's not probably such a big deal, though.
> > The bigger problem comes if several "open" trace records happen at
> > once.  It's technically possible that the trace buffer will wrap
> > around before a function is done writing to its original buffer.
> >
> > In both of these cases, the common "TRACE_nD" macros can be left, I
> > think.  We might want to add "TRACE_nDL" for 64-bit values, and then
> > let those who need more flexible trace structures call trace_var()
> > directly.
> >
> > This way of doing things also has the advantage that the trace record
> > can be defined in a public header somewhere, and used by user-space
> > analysis tools as well as the hypervisor tracing code.
> >
> > Thoughts?
> >
> >  -George
> >
> > On 12/5/06, Mark Williamson <mark.williamson@cl.cam.ac.uk> wrote:
> > > > There's no reason not to make the trace format more flexible.
>
> There's a
>
> > > > question about how you represent trace points in the Xen code
>
> though,
>
> > when
> >
> > > > the format is no longer a list of fixed size integers.
> > >
> > > I can see two main possibilities.  One involving a variadic function
>
> and
>
> > one
> >
> > > involving mega macros of doom.
> > >
> > > One possibility would be a trace() function taking a variable number
>
> of
>
> > > arguments, i.e.
> > >
> > > void trace(type, unsigned char data1, unsigned char data2, ... etc)
> > >
> > > And a set of arch-defined macros (or at least bitness / endian
>
> defined
>
> > > macros).  Eg. on x86 we might have:
> > >
> > > #define TRACE_U16(d) ((unsigned char)(d & 255)), ((unsigned char)(d
> >
> > 8))
> >
> > > We'd need to verify whether the extra processing had a measurable
> >
> > performance
> >
> > > impact, however.
> > >
> > > Another alternative would be to make the array of trace buffers
>
> globally
>
> > > accessible and then use a set of macros for the trace() instead of
>
> an
>
> > inline
> >
> > > function.  The macros could then look something like (pseudocode):
> > >
> > > struct trace_record {
> > > u32 type;
> > > u32 data_len;
> > > char data[]
> > > };
> > >
> > > char *trace_buffer[NR_CPUS]
> > >
> > > #define open_trace(type)      \
> > >                               do { \
> > >                                   disable local irqs
> >
> > \
> >
> > >                                   struct trace_record *record =
> >
> > \
> >
> > &trace_buffer[cpu][producer_idx]; \
> >
> > >                                   record->type = (u32)type
> >
> > \
> >
> > >                                   record->data_len = 0;
> > >
> > > #define trace_u16(data)       *(u16 *)record->data[record->data_len]
>
> =
>
> > data \
> >
> > >                               record->data_len += sizeof(u16)
> > >
> > > ... etc for different data types, with appropriate variations if
> >
> > necessary for
> >
> > > different platforms ...
> > >
> > > #define close_trace()       \
> > >                             inc producer counter by sizeof(struct
> >
> > \
> >
> > >                             trace_record) + record->data_len for
> >
> > userspace \
> >
> > >                             to see \
> > >                             re-enable local irqs
> >
> > \
> >
> > >                             } while(0)
> > >
> > >
> > > Things become unhappy here because there'd need to be some kind of
>
> bounds
>
> > > checking in here to determine whether we need to wrap to the
>
> beginning of
>
> > the
> >
> > > trace buffer again.  The alternatives as I see them would be either:
> > >
> > > a) include code in each data macro to check if we'd reached the end
>
> of
>
> > the
> >
> > > buffer and wrap the data appropriately, or
> > > b) include code that'll simply copy everything we've built so far to
>
> the
>
> > > beginning of the trace buffer and start again.
> > >
> > > Either way is going to be ugly and unpleasant.  Also, we have the
>
> problem
>
> > of
> >
> > > not knowing whether we're going to wrap OR run out of space until
>
> we're
>
> > part
> >
> > > way through the trace record, although in this instance, I guess we
>
> could
>
> > > just change to create a "missed data" record.
> > >
> > > I think the first approach (variadic function) above is probably
>
> nicer.
>
> > We
> >
> > > can always make a few macros to make common cases (e.g. recording a
>
> type
>
> > and
> >
> > > a single u64 of data) less verbose.
> > >
> > > Any thoughts?
> > >
> > > Cheers,
> > > Mark
> > >
> > > --
> > > Dave: Just a question. What use is a unicyle with no seat?  And no
> >
> > pedals!
> >
> > > Mark: To answer a question with a question: What use is a
>
> skateboard?
>
> > > Dave: Skateboards have wheels.
> > > Mark: My wheel has a wheel!
> >
> > _______________________________________________
> > Xen-devel mailing list
> > Xen-devel@lists.xensource.com
> > http://lists.xensource.com/xen-devel

-- 
Dave: Just a question. What use is a unicyle with no seat?  And no pedals!
Mark: To answer a question with a question: What use is a skateboard?
Dave: Skateboards have wheels.
Mark: My wheel has a wheel!

Re: [RFC][PATCH] 1/3] [XEN] Use explicit bit sized fieldsfor exported xentrace data.

[Tony Breeds]

On Thu, Dec 07, 2006 at 02:43:58AM +0000, Mark Williamson wrote:
> > The ultimate best way of doing it would be to have trace functions that
> > took a format string and a variable number of arguments.
> 
> Agreed.
> 
> > The actual 
> > trace record written in the buffer would just contain the record type
> > and the length of the record, followed by the variable data. The format
> > string would be written out in an a separate segment, enabling it to be
> > extracted and used by the trace post-process tool to pretty print the
> > records.
> 
> Cool.
> 
> It would be fairly trivial to declare a name for each "type" and allow 
> userspace to query the type name <-> type ID mapping.  This then removes all 
> responsibility for determining output formatting from userspace, and should 
> make everything neater.
> 
> Nice.  All we need is a coder - anyone interested?

I'll hack something together. to match /my/ understanding of the current
proposals.  We can go from there.

Briefly I think we're talking about using a variable length structure,
with tracepoints looking like a printf() style function.

so
--
	TRACE_4D(TRC_SCHED_SWITCH,
	         prev->domain->domain_id, prev->vcpu_id,
	         next->domain->domain_id, next->vcpu_id);
--
would become something like:
--
	trace(TRC_SCHED_SWITCH, "%d%d%d%d", 
	      prev->domain->domain_id, prev->vcpu_id,
	      next->domain->domain_id, next->vcpu_id);
--

Trace would store a lookup table for TRC_SCHED_SWITCH => "%d%d%d%d".
The data would get stored in a fashion similar to Mark's suggestion in
http://lists.xensource.com/archives/html/xen-devel/2006-12/msg00179.html

The xentrace tool would write the record and format specifier to disk,
in network byte order (until then data remains in native byte order).
There would fo course be implicit 64bit tsc and 32bit event-type
prefixed to each record.

xentrace_format will pretty print the datafile.

I like the idea of stepping away from printf codes and using explicit
size args (%8, %16, %32 and %64), does anyone disagree?

Yours Tony

   linux.conf.au       http://linux.conf.au/ || http://lca2007.linux.org.au/
   Jan 15-20 2007      The Australian Linux Technical Conference!

Re: [RFC][PATCH] 1/3] [XEN] Use explicit bit sized fieldsfor exported xentrace data.

[Keir Fraser]

On 6/12/06 16:10, "Ian Pratt" <m+Ian.Pratt@cl.cam.ac.uk> wrote:

> The ultimate best way of doing it would be to have trace functions that
> took a format string and a variable number of arguments. The actual
> trace record written in the buffer would just contain the record type
> and the length of the record, followed by the variable data. The format
> string would be written out in an a separate segment, enabling it to be
> extracted and used by the trace post-process tool to pretty print the
> records.

I agree this is a good way to go. We certainly don't want lots of little
single use trace-record structures, one per trace point in Xen! That would
be overkill -- format strings are a good middle ground.

 -- Keir

RE: [RFC][PATCH] 1/3] [XEN] Use explicit bit sized fieldsfor exported xentrace data.

[Ian Pratt]

> On 6/12/06 16:10, "Ian Pratt" <m+Ian.Pratt@cl.cam.ac.uk> wrote:
> 
> > The ultimate best way of doing it would be to have trace functions
that
> > took a format string and a variable number of arguments. The actual
> > trace record written in the buffer would just contain the record
type
> > and the length of the record, followed by the variable data. The
format
> > string would be written out in an a separate segment, enabling it to
be
> > extracted and used by the trace post-process tool to pretty print
the
> > records.
> 
> I agree this is a good way to go. We certainly don't want lots of
little
> single use trace-record structures, one per trace point in Xen! That
would
> be overkill -- format strings are a good middle ground.

If the trace record writing function was inlined, I wander whether gcc
would be smart enough to skip all the var args stuff and just write the
appropriate parameters into the trace buffer? I think that's probably
being optimistic.  It might be possible to do something horrendous with
the C pre-processor, matching various static format strings for common
patterns (e.g. all %x), else falling back to a trivial parser. 

Ideally the format string would include the 'pretty print' format for
the post processing tools (e.g. put the strings as literals in a
separate segment and extract them later in the build process. That's not
attractive if we have to parse the strings at run time, though. It could
easily be fixed by pre-processing all source files with M4 or perl, but
I don't think we want to go there.

Ian 

Re: [RFC][PATCH] 1/3] [XEN] Use explicit bit sized fieldsfor exported xentrace data.

[George Dunlap]

On 12/7/06, Ian Pratt <m+Ian.Pratt@cl.cam.ac.uk> wrote:
> Ideally the format string would include the 'pretty print' format for
> the post processing tools (e.g. put the strings as literals in a
> separate segment and extract them later in the build process. That's not
> attractive if we have to parse the strings at run time, though. It could
> easily be fixed by pre-processing all source files with M4 or perl, but
> I don't think we want to go there.

I should think that parsing the strings at runtime should be pretty
quick -- we don't have to actually make text out of them, we just need
to scan through looking for % tokens, determine the size, and copy to
the trace buffer.

We should be able to make more optimized tracing functions for hot
paths that copy data directly into the trace buffer.

Just for the record, my idea wan't that every single trace record
would have a struct associated with it.  I think that for most trace
records, a list of u32 or u64 would work just fine.  I was envisioning
a class of functions:
  trace_1d(int type, u32 val1);
  trace_2d(int type, u32 val1, u32 val2);
  /* etc */
  trace_nd(int type, int n, ...); /* Take N unsigned longs */
  trace_1l(int type, u64 val1);
  /* etc */
  trace_v(int type, size_t bytes, char *data); /* Take a struct of
size 'bytes' */

The final one would only be used in cases where layout is important,
mainly where space is critical.  An advantage of this method is that
we can use sub-byte-sized fields (e.g., 3 bits) and let the compiler
do the work of shifting things around.

If we adopt the format-string model, I suspect that in most cases, no
one is going to worry about bits anyway, but just put up "%d" (which
will probably translate to u32) even for data that only need 3 bits.

If we adopt the format-string model, we need to make sure it's
unambiguous.  For instnace, does "%lx" translate to u32 on both 32-
and 64-bit builds?  How do I specify u16 and u8?  Can I print strings?

Pretty-printing might be nice, but I plan on using the binary format
anyway for my analysis tools, so it's not that big a deal to me.

 -George

RE: [RFC][PATCH] 1/3] [XEN] Use explicit bit sizedfieldsfor exported xentrace data.

[Ian Pratt]

> On 12/7/06, Ian Pratt <m+Ian.Pratt@cl.cam.ac.uk> wrote:
> > Ideally the format string would include the 'pretty print' format
for
> > the post processing tools (e.g. put the strings as literals in a
> > separate segment and extract them later in the build process. That's
not
> > attractive if we have to parse the strings at run time, though. It
could
> > easily be fixed by pre-processing all source files with M4 or perl,
but
> > I don't think we want to go there.
> 
> I should think that parsing the strings at runtime should be pretty
> quick -- we don't have to actually make text out of them, we just need
> to scan through looking for % tokens, determine the size, and copy to
> the trace buffer.

It's still a bit gross given that the string doesn't even need to be in
the executable and we knew all this stuff at compile time anyhow. 

If we did this with macros we could use sizeof(_x) to determine how many
bytes to write to the trace buffer. Os is the intention to deliberately
truncate some 64b quantities to 32b save trace buffer space? 

It may be ugly, but it may actually be possible to do this with the C
pre-processor. 

Ian

Re: [RFC][PATCH] 1/3] [XEN] Use explicit bit sizedfieldsfor exported xentrace data.

[George Dunlap]

On 12/8/06, Ian Pratt <m+Ian.Pratt@cl.cam.ac.uk> wrote:
> If we did this with macros we could use sizeof(_x) to determine how many
> bytes to write to the trace buffer. Os is the intention to deliberately
> truncate some 64b quantities to 32b save trace buffer space?

Yes.  What initiated this discussion was that changing the trace
buffers to u64 would double the size of traces.  My typical traces are
already 1GB+, with longer ones reaching 10G+.  Since at this point I'm
looking mainly at 32-bit VMX code, it would be nice if I could avoid
that.

However, as long as the /number/ of 64b quantities was variable, I
could make do with either size.

 -George

Re: [RFC][PATCH] 1/3] [XEN] Use explicit bit sized fields for exported xentrace data.

[Mark Williamson]

> Variable-length trace buffers would certainly be nice.  Right now, for
> some records, I'm squeezing bits into things, and just *one* more byte
> would be great... but for other records, I'm storing 4 words of 0, and
> there just doesn't seem to be any other useful information to add.

Yep, this is really something we should look about improving now.

> Keir, is the simpler, "one trace size fits all" method just because it
> was easier to implement originally, or is the simplicity expected to
> greatly reduce overhead and/or bugginess?

It's just there because it was simpler to implement.  I'm confident that we 
can come up with a more flexible format that won't have a detrimental effect 
on performance / stability.

> If the former, then there 
> seems enough interest in making the tracing more flexible to be worth
> changing; if the latter, then we should probably chose something and
> live with it, or perhaps a compromise (i.e., two record sizes).

Two record sizes might be an interesting compromise, but I suggest we go the 
whole hog and spec a design for fully variable data lengths with a fixed 
length header.

Cheers,
Mark

>  -George
>
> On 11/30/06, Mark Williamson <mark.williamson@cl.cam.ac.uk> wrote:
> > I guess one possibility would be to continue using unsigned longs but
> > store the machine word size and endianness in a header in the trace file.
> >  This gets us platform independence.
> >
> > This avoids adding extra overhead on the fast path, the extra processing
> > can happen offline (and probably not at all in the common case that
> > you're on the same endianness / word size as the trace was collected on).
> >
> > Another alternative would be to allow some combination of 32-bit or
> > (fewer) 64-bit words in the record.  This would let us keep the same
> > record size, but have a bit more flexibility.
> >
> > Going the whole hog, we could even make the trace data opaque to trace.c
> > - have a char[] for the data, and deal with the semantics in terms of
> > "longs" "u64" etc in macros in the traced code, and in xentrace_format.
> >
> > If we did this, the logical extension would be to have variable length
> > trace records with a fixed-size header giving the full length.  I think
> > this would be a good direction to go in, and would ensure that we
> > maximise use of the trace buffer space.  It shouldn't be that hard to
> > modify the system to do this - most of the work may even be in making it
> > nice to use!
> >
> > Cheers,
> > Mark
> >
> > On Thursday 30 November 2006 17:03, Keir Fraser wrote:
> > > On 30/11/06 16:58, "George Dunlap" <dunlapg@umich.edu> wrote:
> > > > Hmm... this has the unfortunate side-effect of doubling the size of
> > > > the trace, and effectively halving the effectiveness of the trace
> > > > buffer in avoiding drops.  My moderate-length traces are already in
> > > > the gigabyte range, and I occasionally lose trace records even with a
> > > > buffer size of 256.  It would be really nice if we could avoid that.
> > > >
> > > > I happen to be using the VMENTER/VMEXIT tracing, which could be
> > > > consolidated into one record if we went to a 64-bit trace.  Is anyone
> > > > else doing high-bandwidth tracing that this would affect in a
> > > > significantly negative way?
> > >
> > > As we move increasingly towards x86/64 this is an issue that will need
> > > to be addressed even if we leave the tracing fields as longs.
> > >
> > >  -- Keir
> > >
> > >
> > > _______________________________________________
> > > Xen-devel mailing list
> > > Xen-devel@lists.xensource.com
> > > http://lists.xensource.com/xen-devel
> >
> > --
> > Dave: Just a question. What use is a unicyle with no seat?  And no
> > pedals! Mark: To answer a question with a question: What use is a
> > skateboard? Dave: Skateboards have wheels.
> > Mark: My wheel has a wheel!

-- 
Dave: Just a question. What use is a unicyle with no seat?  And no pedals!
Mark: To answer a question with a question: What use is a skateboard?
Dave: Skateboards have wheels.
Mark: My wheel has a wheel!

LTTng Xen port

[Mathieu Desnoyers]

Hi,

My name is Mathieu Desnoyers, I am the current maintainer of the Linux Trace
Toolkit project, known as LTTng. This is a tracer for the 2.6 Linux kernels
oriented towards high performance and real-time applications.

I have read your tracing thread and I am surprised to see how much things
you would like in a tracer are already implemented and tested in LTTng. I am
currently porting my tracer to Xen, so I think it might be useful for you to
know what it provides. My goal is to do not duplicate the effort and save
everyone some time.

Here follows some key features of LTTng :

Architecture independant data types
Extensible event records
Self-describing traces
Variable size records
Fast (200 ns per event record)
Highly reentrant
Does not disable interrupts
Does not take lock on the critical path
Supports NMI tracing
Analysis/visualization tool (LTTV)

Looking at the integration of the existing LTTng implementation into Xen, I
came up with those two points for my Christmas whichlist :

Additionnal functionnalities that would be nice to have in Xen :

- RCU-style updates : would allow freeing the buffers without impact on tracing.
    * I guess I could currently use :
      for_each_domain( d )
        for_each_vcpu( d, v )
          vcpu_sleep_sync(v);
      I think it will have a huge impact on the system, but it would only be
      performed before trace buffers free.

- Polling for data in Xen from a dom0 process.
  Xentrace currently polls the hypervisor each 100ms to see if there is data
  that needs to be consumed. Instead of an active polling, it would be nice to
  use the dom0 OS capability to put a process to sleep while waiting for a
  resource. It would imply creating a module, loaded in dom0, that would wait
  for a specific virq coming from the Hypervisor to wake up such processes.
  We could think of exporting a complete poll() interface through sysfs or
  procfs that would be a directory filled with the resources exported from the
  Hypervisor to dom0 (which could include wait for resource freed, useful when
  shutting down a domU instead of busy looping). It would help dom0 to schedule
  other processes while a process is waiting for the Hypervisor.


You might also be interested in looking at :
- the website (http://ltt.polymtl.ca)
- LTTng Xen port design document (this one is different from the one posted by
  Jimi)
  (http://ltt.polymtl.ca/svn/ltt/branches/poly/doc/developer/lttng-xen.txt)
- OLS 2006 paper "The LTTng tracer : A Low Impact Performance and Behavior
  Monitor for GNU/Linux"
  (http://ltt.polymtl.ca/papers/desnoyers-ols2006.pdf)


Questions and constructive comments are welcome.

Mathieu


OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 

RE: LTTng Xen port

[Ian Pratt]

> I have read your tracing thread and I am surprised to see how much
things
> you would like in a tracer are already implemented and tested in
LTTng. I
> am currently porting my tracer to Xen, so I think it might be useful
for you
> to know what it provides. My goal is to do not duplicate the effort
and save
> everyone some time.

I like the work you've done with LTTng, but we have to be careful not to
go too overboard with how fancy we make the solution for Xen.  We don't
particularly need dynamic registering of trace types (though dynamically
turn off-and-on-able is good), and I'd like to keep as much complexity
compile time as possible. Having per CPU buffers using the TSC as the
timestamp is perfectly adequate (provided we drop in an appropriate
synchronization record whenever the Xen TSC/wall clock calibration code
runs on each CPU).

> - Polling for data in Xen from a dom0 process.
>   Xentrace currently polls the hypervisor each 100ms to see if there
is
> data
>   that needs to be consumed. Instead of an active polling, it would be
nice
> to
>   use the dom0 OS capability to put a process to sleep while waiting
for a
>   resource. It would imply creating a module, loaded in dom0, that
would
> wait
>   for a specific virq coming from the Hypervisor to wake up such
processes.
>   We could think of exporting a complete poll() interface through
sysfs or
>   procfs that would be a directory filled with the resources exported
from
> the
>   Hypervisor to dom0 (which could include wait for resource freed,
useful
> when
>   shutting down a domU instead of busy looping). It would help dom0 to
> schedule
>   other processes while a process is waiting for the Hypervisor.

I really thought we already had the functionality to enable the trac
ewriter to block on the trace buffer(s) becoming half full -- I thought
Rob Gardener fixed this ages ago. He certainly *promised* a patch to do
it :)

Thanks,
Ian

Re: LTTng Xen port

[Mathieu Desnoyers]

* Ian Pratt (m+Ian.Pratt@cl.cam.ac.uk) wrote:
> > I have read your tracing thread and I am surprised to see how much
> things
> > you would like in a tracer are already implemented and tested in
> LTTng. I
> > am currently porting my tracer to Xen, so I think it might be useful
> for you
> > to know what it provides. My goal is to do not duplicate the effort
> and save
> > everyone some time.
> 
> I like the work you've done with LTTng, but we have to be careful not to
> go too overboard with how fancy we make the solution for Xen.  We don't
> particularly need dynamic registering of trace types (though dynamically
> turn off-and-on-able is good), and I'd like to keep as much complexity
> compile time as possible. Having per CPU buffers using the TSC as the
> timestamp is perfectly adequate (provided we drop in an appropriate
> synchronization record whenever the Xen TSC/wall clock calibration code
> runs on each CPU).
> 

Hi Ian,

The good thing about being flexible is that we can easily trim down the
unneeded features like dynamic registering of trace types if you don't like
them (they are implemented in the ltt-facilities.c module which could be hacked
to take a statically known set of facilities). Some features like a small
control channel which records information about facilities, event types, arch
type sizes and endianness is still interesting even without dynamically loadable
facilities though, as we can expect the developers to extend the set of events
to add their own and it provides portability.

> > - Polling for data in Xen from a dom0 process.
> >   Xentrace currently polls the hypervisor each 100ms to see if there
> is
> > data
> >   that needs to be consumed. Instead of an active polling, it would be
> nice
> > to
> >   use the dom0 OS capability to put a process to sleep while waiting
> for a
> >   resource. It would imply creating a module, loaded in dom0, that
> would
> > wait
> >   for a specific virq coming from the Hypervisor to wake up such
> processes.
> >   We could think of exporting a complete poll() interface through
> sysfs or
> >   procfs that would be a directory filled with the resources exported
> from
> > the
> >   Hypervisor to dom0 (which could include wait for resource freed,
> useful
> > when
> >   shutting down a domU instead of busy looping). It would help dom0 to
> > schedule
> >   other processes while a process is waiting for the Hypervisor.
> 
> I really thought we already had the functionality to enable the trac
> ewriter to block on the trace buffer(s) becoming half full -- I thought
> Rob Gardener fixed this ages ago. He certainly *promised* a patch to do
> it :)
> 

As from the current mercurial tree :
http://lxr.xensource.com/lxr/source/tools/xentrace/xentrace.c

int monitor_tbufs(FILE *logfile)
310     /* now, scan buffers for events */
311     while ( !interrupted )
312     {
313         for ( i = 0; (i < num) && !interrupted; i++ )
314         {
315             while ( meta[i]->cons != meta[i]->prod )
316             {
317                 rmb(); /* read prod, then read item. */
318                 write_rec(i, data[i] + meta[i]->cons % size_in_recs, logfile);
319                 mb(); /* read item, then update cons. */
320                 meta[i]->cons++;
321             }
322         }
323 
324         nanosleep(&opts.poll_sleep, NULL);
325     }

So it seems like the implementation must be hiding either in someone's head or
in his mercurial tree. :)

I guess it would be easiter to implement if there was support for the dom0 OS
to block a process waiting for an Hypervisor resource. Or maybe is it already
implemented but eluded my attention ?

Regards,

Mathieu


OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 

Re: LTTng Xen port

[Tony Breeds]

On Tue, Dec 12, 2006 at 05:14:02PM -0500, Mathieu Desnoyers wrote:
 
> As from the current mercurial tree :
> http://lxr.xensource.com/lxr/source/tools/xentrace/xentrace.c
> 
> int monitor_tbufs(FILE *logfile)
> 310     /* now, scan buffers for events */
> 311     while ( !interrupted )
> 312     {
> 313         for ( i = 0; (i < num) && !interrupted; i++ )
> 314         {
> 315             while ( meta[i]->cons != meta[i]->prod )
> 316             {
> 317                 rmb(); /* read prod, then read item. */
> 318                 write_rec(i, data[i] + meta[i]->cons % size_in_recs, logfile);
> 319                 mb(); /* read item, then update cons. */
> 320                 meta[i]->cons++;
> 321             }
> 322         }
> 323 
> 324         nanosleep(&opts.poll_sleep, NULL);
> 325     }
> 
> So it seems like the implementation must be hiding either in someone's head or
> in his mercurial tree. :)

I suspect that xenbaked uses it,
http://lxr.xensource.com/lxr/source/tools/xenmon/xenbaked.c
---
262 int eventchn_init(void)
263 {
264   int rc;
265   
266   // to revert to old way:
267   if (0)
268     return -1;
269   
270   xce_handle = xc_evtchn_open();
271 
272   if (xce_handle < 0)
273     perror("Failed to open evtchn device");
274   
275   if ((rc = xc_evtchn_bind_virq(xce_handle, VIRQ_TBUF)) == -1)
276     perror("Failed to bind to domain exception virq port");
277   virq_port = rc;
278   
279   return xce_handle;
280 }
---

With appropriate triggering, in xen/common/trace.c

Yours Tony

   linux.conf.au       http://linux.conf.au/ || http://lca2007.linux.org.au/
   Jan 15-20 2007      The Australian Linux Technical Conference!

Re: LTTng Xen port

[Mark Williamson]

> I suspect that xenbaked uses it,
> http://lxr.xensource.com/lxr/source/tools/xenmon/xenbaked.c
> ---
> 262 int eventchn_init(void)
> 263 {
> 264   int rc;
> 265
> 266   // to revert to old way:
> 267   if (0)
> 268     return -1;
> 269
> 270   xce_handle = xc_evtchn_open();
> 271
> 272   if (xce_handle < 0)
> 273     perror("Failed to open evtchn device");
> 274
> 275   if ((rc = xc_evtchn_bind_virq(xce_handle, VIRQ_TBUF)) == -1)
> 276     perror("Failed to bind to domain exception virq port");
> 277   virq_port = rc;
> 278
> 279   return xce_handle;
> 280 }
> ---
>
> With appropriate triggering, in xen/common/trace.c

Ah yes.

It should be fairly trivial to copy this code into xentrace.c and then replace 
the nanosleep with a wait on that event.  This would probably be worthwhile 
as an interim solution until we figure out what next gen tracing ought to 
look like.

Cheers,
Mark

Re: LTTng Xen port

[M.A. Williamson]

> I have read your tracing thread and I am surprised to see how much things 
> you would like in a tracer are already implemented and tested in LTTng.

The Xen tracer was originally a fairly simple hack, although it has been 
updated a number of times to improve the scope of its functionality.

A more advanced tracing tool would be nice.

> I 
> am currently porting my tracer to Xen, so I think it might be useful for 
> you to know what it provides. My goal is to do not duplicate the effort 
> and save everyone some time.



>Here follows some key features of LTTng :
>
>Architecture independant data types

This should please those developing with cross compilers, so would be nice.

>Extensible event records

Definitely good.

>Self-describing traces
>Variable size records

Definitely good. Self-describing traces would be particularly useful - for 
our current xentrace, I had proposed adding a hypercall that would return 
event ID -> event description string mappings.

>Fast (200 ns per event record)
>Highly reentrant
>Does not disable interrupts
>Does not take lock on the critical path

Cool. We've been running with xentrace compiled by default for about a year 
now, although trace buffers are not usually allocated on production 
systems. It would be nice to keep the overhead low enough to continue to do 
this. Refinements like being able to enable specific event types might be 
nice, but aren't critical.

>Supports NMI tracing
>Analysis/visualization tool (LTTV)

Visualisation would be really useful - poring through trace files (even 
once formatted) is pretty hairy. Some kind of timeline visualisation tool 
(I think I've seen one for LTT at some point) would rock.

>- Polling for data in Xen from a dom0 process.
>  Xentrace currently polls the hypervisor each 100ms to see if there is 
> data
>  that needs to be consumed. Instead of an active polling, it would be 
> nice to
>  use the dom0 OS capability to put a process to sleep while waiting for a
>  resource. It would imply creating a module, loaded in dom0, that would 
> wait
>  for a specific virq coming from the Hypervisor to wake up such processes.
>  We could think of exporting a complete poll() interface through sysfs or
>  procfs that would be a directory filled with the resources exported 
> from the
>  Hypervisor to dom0 (which could include wait for resource freed, useful 
> when
>  shutting down a domU instead of busy looping). It would help dom0 to 
> schedule
>  other processes while a process is waiting for the Hypervisor.

We can actually do this already, it's just that xentrace doesn't implement 
it ;-) See xenbaked's code: it sleeps waiting for an event channel, which 
gets fired by Xen when the buffer has had a chance to fill up a bit. The 
nice bit is that the /dev/evtchn driver is compiled into Xen by default 
*anyhow*, so a new kernel module isn't required.

> You might also be interested in looking at : - the website 
> (http://ltt.polymtl.ca) - LTTng Xen port design document (this one is 
> different from the one posted by
>  Jimi)
>  (http://ltt.polymtl.ca/svn/ltt/branches/poly/doc/developer/lttng-xen.txt)
>- OLS 2006 paper "The LTTng tracer : A Low Impact Performance and Behavior
>  Monitor for GNU/Linux"
>  (http://ltt.polymtl.ca/papers/desnoyers-ols2006.pdf)

A few questions: How much commonality will this have with the Linux LLTng? 
Will there be direct code sharing, or a patchqueue, or simply code copying? 
Will existing userspace LTTng tools be applicable? Is there already a 
working visualiser? Is there a source repository online that we could take 
a look at?

It would be really great if you came up with something interesting on the 
tracing front: quite a lot of people seem to use it these days.

Cheers,
Mark

>Questions and constructive comments are welcome.
>
>Mathieu
>
>
> OpenPGP public key: http://krystal.dyndns.org:8080/key/compudj.gpg Key 
> fingerprint: 8CD5 52C3 8E3C 4140 715F BA06 3F25 A8FE 3BAE 9A68
>
>_______________________________________________
>Xen-devel mailing list
>Xen-devel@lists.xensource.com
>http://lists.xensource.com/xen-devel
>

Re: LTTng Xen port : finally in a repository near you

[Mathieu Desnoyers]

Hello,

I made a working version of the LTTng tracer for xen-unstable for x86.
Here is the pointer to my repository so you can try it out :

hg clone http://ltt.polymtl.ca/cgi-bin/hgweb.cgi xen-unstable-lttng.hg

Basic usage :

(see lttctl-xen -h)

lttctl-xen -c

(in a different console)
lttd-xen -t /tmp/xentrace1

(in the 1st console)
lttctl-xen -s

(tracing is active)

lttctl-xen -q
lttctl-xen -r

lttd-xen should automatically quit after writing the last buffers as
soon as lttctl-xen -r is issued.

Then, you must copy the XML facilities :

(see the http://ltt.polymtl.ca > QUICKSTART to see how to install the
ltt-control package which contains the XML facilities in your system)

lttctl-xen -e -t /tmp/xentrace1

View in the visualiser : (see the QUICKSTART to see how to install it)

lttv -m textDump -t /tmp/xentrace1

(not tested yet) : to visualize a dom0 trace with the xen hypervisor
information, one would have to collect the dom0 kernel trace and the Xen
trace and open them together with :
lttv -m textDump -t /tmp/xentrace1 -t /tmp/dom0trace

The current Linux kernel instrumentation is for 2.6.20. A backport might
be needed to 2.6.18 if there is no proper Xen support in 2.6.20 (I have
not followed the recent developments).


Currently broken/missing :

- Ressources are not freed when the trace channels are destroyed. So you
  basically have to reboot between taking different traces.
- My code in the hypervisor complains to the console that subbuffers
  have not been fully read when the trace channels are destroyed. The
  error printing is just done too fast : lttd-xen is still there and
  reading the buffers at that point. It will get fixed with proper
  ressource usage tracking of both Xen and lttd-xen (same as the first
  point above).
- x86_64 not tested, powerpc local.h and ltt.h missing (should be ripped
  from my Linux kernel LTTng).


Cheers,

Mathieu



* Mathieu Desnoyers (compudj@krystal.dyndns.org) wrote:
> Hi,
> 
> My name is Mathieu Desnoyers, I am the current maintainer of the Linux Trace
> Toolkit project, known as LTTng. This is a tracer for the 2.6 Linux kernels
> oriented towards high performance and real-time applications.
> 
> I have read your tracing thread and I am surprised to see how much things
> you would like in a tracer are already implemented and tested in LTTng. I am
> currently porting my tracer to Xen, so I think it might be useful for you to
> know what it provides. My goal is to do not duplicate the effort and save
> everyone some time.
> 
> Here follows some key features of LTTng :
> 
> Architecture independant data types
> Extensible event records
> Self-describing traces
> Variable size records
> Fast (200 ns per event record)
> Highly reentrant
> Does not disable interrupts
> Does not take lock on the critical path
> Supports NMI tracing
> Analysis/visualization tool (LTTV)
> 
> Looking at the integration of the existing LTTng implementation into Xen, I
> came up with those two points for my Christmas whichlist :
> 
> Additionnal functionnalities that would be nice to have in Xen :
> 
> - RCU-style updates : would allow freeing the buffers without impact on tracing.
>     * I guess I could currently use :
>       for_each_domain( d )
>         for_each_vcpu( d, v )
>           vcpu_sleep_sync(v);
>       I think it will have a huge impact on the system, but it would only be
>       performed before trace buffers free.
> 
> - Polling for data in Xen from a dom0 process.
>   Xentrace currently polls the hypervisor each 100ms to see if there is data
>   that needs to be consumed. Instead of an active polling, it would be nice to
>   use the dom0 OS capability to put a process to sleep while waiting for a
>   resource. It would imply creating a module, loaded in dom0, that would wait
>   for a specific virq coming from the Hypervisor to wake up such processes.
>   We could think of exporting a complete poll() interface through sysfs or
>   procfs that would be a directory filled with the resources exported from the
>   Hypervisor to dom0 (which could include wait for resource freed, useful when
>   shutting down a domU instead of busy looping). It would help dom0 to schedule
>   other processes while a process is waiting for the Hypervisor.
> 
> 
> You might also be interested in looking at :
> - the website (http://ltt.polymtl.ca)
> - LTTng Xen port design document (this one is different from the one posted by
>   Jimi)
>   (http://ltt.polymtl.ca/svn/ltt/branches/poly/doc/developer/lttng-xen.txt)
> - OLS 2006 paper "The LTTng tracer : A Low Impact Performance and Behavior
>   Monitor for GNU/Linux"
>   (http://ltt.polymtl.ca/papers/desnoyers-ols2006.pdf)
> 
> 
> Questions and constructive comments are welcome.
> 
> Mathieu
> 
> 
> OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
> Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
> 
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel
> 

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

Re: LTTng Xen port : finally in a repository near you

[INAKOSHI Hiroya]

Hi Mathieu,

I am interested in LTTng-xen because I thought that it would be nice if
I can get traces on both xen and guest linux at the same time.  I
reviewed LTTng-xen and found that

* LTTng and LTTng-xen have a quite similar structure,
* a trace buffer resides in a hypervisor for LTTng-xen,
* it is currently impossible to get traces from guest linux because
there is no LTTng for 2.6.18-xen kernel, as you mentioned.

I had coarsely ported LTTng to 2.6.18-xen, though it is only for
i386.  Now I can get traces on xen and guest linux simultaneously, even
though they put records in different trace buffers.  Then I thought that
it would be more useful if they put records in xen's trace buffer and I
can analyze events from xen and linux guests with a single lttd and
lttctl running on Domain-0.  Do you have an opinion about that?

Regards,
Hiroya


Mathieu Desnoyers wrote:
> Hello,
> 
> I made a working version of the LTTng tracer for xen-unstable for x86.
> Here is the pointer to my repository so you can try it out :
> 
> hg clone http://ltt.polymtl.ca/cgi-bin/hgweb.cgi xen-unstable-lttng.hg
> 
> Basic usage :
> 
> (see lttctl-xen -h)
> 
> lttctl-xen -c
> 
> (in a different console)
> lttd-xen -t /tmp/xentrace1
> 
> (in the 1st console)
> lttctl-xen -s
> 
> (tracing is active)
> 
> lttctl-xen -q
> lttctl-xen -r
> 
> lttd-xen should automatically quit after writing the last buffers as
> soon as lttctl-xen -r is issued.
> 
> Then, you must copy the XML facilities :
> 
> (see the http://ltt.polymtl.ca > QUICKSTART to see how to install the
> ltt-control package which contains the XML facilities in your system)
> 
> lttctl-xen -e -t /tmp/xentrace1
> 
> View in the visualiser : (see the QUICKSTART to see how to install it)
> 
> lttv -m textDump -t /tmp/xentrace1
> 
> (not tested yet) : to visualize a dom0 trace with the xen hypervisor
> information, one would have to collect the dom0 kernel trace and the Xen
> trace and open them together with :
> lttv -m textDump -t /tmp/xentrace1 -t /tmp/dom0trace
> 
> The current Linux kernel instrumentation is for 2.6.20. A backport might
> be needed to 2.6.18 if there is no proper Xen support in 2.6.20 (I have
> not followed the recent developments).
> 
> 
> Currently broken/missing :
> 
> - Ressources are not freed when the trace channels are destroyed. So you
>   basically have to reboot between taking different traces.
> - My code in the hypervisor complains to the console that subbuffers
>   have not been fully read when the trace channels are destroyed. The
>   error printing is just done too fast : lttd-xen is still there and
>   reading the buffers at that point. It will get fixed with proper
>   ressource usage tracking of both Xen and lttd-xen (same as the first
>   point above).
> - x86_64 not tested, powerpc local.h and ltt.h missing (should be ripped
>   from my Linux kernel LTTng).
> 
> 
> Cheers,
> 
> Mathieu
> 
> 
> 
> * Mathieu Desnoyers (compudj@krystal.dyndns.org) wrote:
>> Hi,
>>
>> My name is Mathieu Desnoyers, I am the current maintainer of the Linux Trace
>> Toolkit project, known as LTTng. This is a tracer for the 2.6 Linux kernels
>> oriented towards high performance and real-time applications.
>>
>> I have read your tracing thread and I am surprised to see how much things
>> you would like in a tracer are already implemented and tested in LTTng. I am
>> currently porting my tracer to Xen, so I think it might be useful for you to
>> know what it provides. My goal is to do not duplicate the effort and save
>> everyone some time.
>>
>> Here follows some key features of LTTng :
>>
>> Architecture independant data types
>> Extensible event records
>> Self-describing traces
>> Variable size records
>> Fast (200 ns per event record)
>> Highly reentrant
>> Does not disable interrupts
>> Does not take lock on the critical path
>> Supports NMI tracing
>> Analysis/visualization tool (LTTV)
>>
>> Looking at the integration of the existing LTTng implementation into Xen, I
>> came up with those two points for my Christmas whichlist :
>>
>> Additionnal functionnalities that would be nice to have in Xen :
>>
>> - RCU-style updates : would allow freeing the buffers without impact on tracing.
>>     * I guess I could currently use :
>>       for_each_domain( d )
>>         for_each_vcpu( d, v )
>>           vcpu_sleep_sync(v);
>>       I think it will have a huge impact on the system, but it would only be
>>       performed before trace buffers free.
>>
>> - Polling for data in Xen from a dom0 process.
>>   Xentrace currently polls the hypervisor each 100ms to see if there is data
>>   that needs to be consumed. Instead of an active polling, it would be nice to
>>   use the dom0 OS capability to put a process to sleep while waiting for a
>>   resource. It would imply creating a module, loaded in dom0, that would wait
>>   for a specific virq coming from the Hypervisor to wake up such processes.
>>   We could think of exporting a complete poll() interface through sysfs or
>>   procfs that would be a directory filled with the resources exported from the
>>   Hypervisor to dom0 (which could include wait for resource freed, useful when
>>   shutting down a domU instead of busy looping). It would help dom0 to schedule
>>   other processes while a process is waiting for the Hypervisor.
>>
>>
>> You might also be interested in looking at :
>> - the website (http://ltt.polymtl.ca)
>> - LTTng Xen port design document (this one is different from the one posted by
>>   Jimi)
>>   (http://ltt.polymtl.ca/svn/ltt/branches/poly/doc/developer/lttng-xen.txt)
>> - OLS 2006 paper "The LTTng tracer : A Low Impact Performance and Behavior
>>   Monitor for GNU/Linux"
>>   (http://ltt.polymtl.ca/papers/desnoyers-ols2006.pdf)
>>
>>
>> Questions and constructive comments are welcome.
>>
>> Mathieu
>>
>>
>> OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
>> Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
>>
>> _______________________________________________
>> Xen-devel mailing list
>> Xen-devel@lists.xensource.com
>> http://lists.xensource.com/xen-devel
>>
> 

Re: LTTng Xen port : finally in a repository near you

[Mark Williamson]

Mathieu,

Nice one, thank you for getting involved in this.  Are you planning to 
eventually submit this for inclusion in the Xen mainline?

Is there anything we (the community) can do to help?  E.g. patch review / 
advise / etc?

Cheers,
Mark

On Monday 25 June 2007, INAKOSHI Hiroya wrote:
> Hi Mathieu,
>
> I am interested in LTTng-xen because I thought that it would be nice if
> I can get traces on both xen and guest linux at the same time.  I
> reviewed LTTng-xen and found that
>
> * LTTng and LTTng-xen have a quite similar structure,
> * a trace buffer resides in a hypervisor for LTTng-xen,
> * it is currently impossible to get traces from guest linux because
> there is no LTTng for 2.6.18-xen kernel, as you mentioned.
>
> I had coarsely ported LTTng to 2.6.18-xen, though it is only for
> i386.  Now I can get traces on xen and guest linux simultaneously, even
> though they put records in different trace buffers.  Then I thought that
> it would be more useful if they put records in xen's trace buffer and I
> can analyze events from xen and linux guests with a single lttd and
> lttctl running on Domain-0.  Do you have an opinion about that?
>
> Regards,
> Hiroya
>
> Mathieu Desnoyers wrote:
> > Hello,
> >
> > I made a working version of the LTTng tracer for xen-unstable for x86.
> > Here is the pointer to my repository so you can try it out :
> >
> > hg clone http://ltt.polymtl.ca/cgi-bin/hgweb.cgi xen-unstable-lttng.hg
> >
> > Basic usage :
> >
> > (see lttctl-xen -h)
> >
> > lttctl-xen -c
> >
> > (in a different console)
> > lttd-xen -t /tmp/xentrace1
> >
> > (in the 1st console)
> > lttctl-xen -s
> >
> > (tracing is active)
> >
> > lttctl-xen -q
> > lttctl-xen -r
> >
> > lttd-xen should automatically quit after writing the last buffers as
> > soon as lttctl-xen -r is issued.
> >
> > Then, you must copy the XML facilities :
> >
> > (see the http://ltt.polymtl.ca > QUICKSTART to see how to install the
> > ltt-control package which contains the XML facilities in your system)
> >
> > lttctl-xen -e -t /tmp/xentrace1
> >
> > View in the visualiser : (see the QUICKSTART to see how to install it)
> >
> > lttv -m textDump -t /tmp/xentrace1
> >
> > (not tested yet) : to visualize a dom0 trace with the xen hypervisor
> > information, one would have to collect the dom0 kernel trace and the Xen
> > trace and open them together with :
> > lttv -m textDump -t /tmp/xentrace1 -t /tmp/dom0trace
> >
> > The current Linux kernel instrumentation is for 2.6.20. A backport might
> > be needed to 2.6.18 if there is no proper Xen support in 2.6.20 (I have
> > not followed the recent developments).
> >
> >
> > Currently broken/missing :
> >
> > - Ressources are not freed when the trace channels are destroyed. So you
> >   basically have to reboot between taking different traces.
> > - My code in the hypervisor complains to the console that subbuffers
> >   have not been fully read when the trace channels are destroyed. The
> >   error printing is just done too fast : lttd-xen is still there and
> >   reading the buffers at that point. It will get fixed with proper
> >   ressource usage tracking of both Xen and lttd-xen (same as the first
> >   point above).
> > - x86_64 not tested, powerpc local.h and ltt.h missing (should be ripped
> >   from my Linux kernel LTTng).
> >
> >
> > Cheers,
> >
> > Mathieu
> >
> > * Mathieu Desnoyers (compudj@krystal.dyndns.org) wrote:
> >> Hi,
> >>
> >> My name is Mathieu Desnoyers, I am the current maintainer of the Linux
> >> Trace Toolkit project, known as LTTng. This is a tracer for the 2.6
> >> Linux kernels oriented towards high performance and real-time
> >> applications.
> >>
> >> I have read your tracing thread and I am surprised to see how much
> >> things you would like in a tracer are already implemented and tested in
> >> LTTng. I am currently porting my tracer to Xen, so I think it might be
> >> useful for you to know what it provides. My goal is to do not duplicate
> >> the effort and save everyone some time.
> >>
> >> Here follows some key features of LTTng :
> >>
> >> Architecture independant data types
> >> Extensible event records
> >> Self-describing traces
> >> Variable size records
> >> Fast (200 ns per event record)
> >> Highly reentrant
> >> Does not disable interrupts
> >> Does not take lock on the critical path
> >> Supports NMI tracing
> >> Analysis/visualization tool (LTTV)
> >>
> >> Looking at the integration of the existing LTTng implementation into
> >> Xen, I came up with those two points for my Christmas whichlist :
> >>
> >> Additionnal functionnalities that would be nice to have in Xen :
> >>
> >> - RCU-style updates : would allow freeing the buffers without impact on
> >> tracing. * I guess I could currently use :
> >>       for_each_domain( d )
> >>         for_each_vcpu( d, v )
> >>           vcpu_sleep_sync(v);
> >>       I think it will have a huge impact on the system, but it would
> >> only be performed before trace buffers free.
> >>
> >> - Polling for data in Xen from a dom0 process.
> >>   Xentrace currently polls the hypervisor each 100ms to see if there is
> >> data that needs to be consumed. Instead of an active polling, it would
> >> be nice to use the dom0 OS capability to put a process to sleep while
> >> waiting for a resource. It would imply creating a module, loaded in
> >> dom0, that would wait for a specific virq coming from the Hypervisor to
> >> wake up such processes. We could think of exporting a complete poll()
> >> interface through sysfs or procfs that would be a directory filled with
> >> the resources exported from the Hypervisor to dom0 (which could include
> >> wait for resource freed, useful when shutting down a domU instead of
> >> busy looping). It would help dom0 to schedule other processes while a
> >> process is waiting for the Hypervisor.
> >>
> >>
> >> You might also be interested in looking at :
> >> - the website (http://ltt.polymtl.ca)
> >> - LTTng Xen port design document (this one is different from the one
> >> posted by Jimi)
> >>  
> >> (http://ltt.polymtl.ca/svn/ltt/branches/poly/doc/developer/lttng-xen.txt
> >>) - OLS 2006 paper "The LTTng tracer : A Low Impact Performance and
> >> Behavior Monitor for GNU/Linux"
> >>   (http://ltt.polymtl.ca/papers/desnoyers-ols2006.pdf)
> >>
> >>
> >> Questions and constructive comments are welcome.
> >>
> >> Mathieu
> >>
> >>
> >> OpenPGP public key:             
> >> http://krystal.dyndns.org:8080/key/compudj.gpg Key fingerprint:     8CD5
> >> 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68
> >>
> >> _______________________________________________
> >> Xen-devel mailing list
> >> Xen-devel@lists.xensource.com
> >> http://lists.xensource.com/xen-devel
>
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel



-- 
Dave: Just a question. What use is a unicyle with no seat?  And no pedals!
Mark: To answer a question with a question: What use is a skateboard?
Dave: Skateboards have wheels.
Mark: My wheel has a wheel!

Re: LTTng Xen port : finally in a repository near you

[Mathieu Desnoyers]

* Mark Williamson (mark.williamson@cl.cam.ac.uk) wrote:
> Mathieu,
> 
> Nice one, thank you for getting involved in this.  Are you planning to 
> eventually submit this for inclusion in the Xen mainline?
> 

I did the original port, but my current focus is more oriented towards
having LTTng included in Linux mainline. As you probably guess, it is
changing quite a bit in the process. I plan to first get the kernel
tracing polished, and afterward to port this "polished" tracer to
userspace tracing and hypervisor. Since a first port has been done, it
will not represent too much work to do it again.

> Is there anything we (the community) can do to help?  E.g. patch review / 
> advise / etc?
> 

You could go through a code review of the Xen port I have done. Not the
tracing infrastructure itself, since it is scheduled for major changes,
but mostly the low-level calls I did in lttd-xen, lttctl-xen and the Xen
ltt-tracer.



Note that you have to expect the current "Linux Kernel Markers" to be
ported to the Xen hypervisor eventually.

Some advice about how I currently (fail) to do the teardown of traces
correctly: I depend on synchronize_sched() under Linux to prodive a
race-less efficient teardown (using read-copy-update for my data
structures). It is a core concept I depend on to provide fast and
efficient tracing. Has RCU finally been implement in Xen since the last
time I checked?

Thanks,

Mathieu

> Cheers,
> Mark
> 
> On Monday 25 June 2007, INAKOSHI Hiroya wrote:
> > Hi Mathieu,
> >
> > I am interested in LTTng-xen because I thought that it would be nice if
> > I can get traces on both xen and guest linux at the same time.  I
> > reviewed LTTng-xen and found that
> >
> > * LTTng and LTTng-xen have a quite similar structure,
> > * a trace buffer resides in a hypervisor for LTTng-xen,
> > * it is currently impossible to get traces from guest linux because
> > there is no LTTng for 2.6.18-xen kernel, as you mentioned.
> >
> > I had coarsely ported LTTng to 2.6.18-xen, though it is only for
> > i386.  Now I can get traces on xen and guest linux simultaneously, even
> > though they put records in different trace buffers.  Then I thought that
> > it would be more useful if they put records in xen's trace buffer and I
> > can analyze events from xen and linux guests with a single lttd and
> > lttctl running on Domain-0.  Do you have an opinion about that?
> >
> > Regards,
> > Hiroya
> >
> > Mathieu Desnoyers wrote:
> > > Hello,
> > >
> > > I made a working version of the LTTng tracer for xen-unstable for x86.
> > > Here is the pointer to my repository so you can try it out :
> > >
> > > hg clone http://ltt.polymtl.ca/cgi-bin/hgweb.cgi xen-unstable-lttng.hg
> > >
> > > Basic usage :
> > >
> > > (see lttctl-xen -h)
> > >
> > > lttctl-xen -c
> > >
> > > (in a different console)
> > > lttd-xen -t /tmp/xentrace1
> > >
> > > (in the 1st console)
> > > lttctl-xen -s
> > >
> > > (tracing is active)
> > >
> > > lttctl-xen -q
> > > lttctl-xen -r
> > >
> > > lttd-xen should automatically quit after writing the last buffers as
> > > soon as lttctl-xen -r is issued.
> > >
> > > Then, you must copy the XML facilities :
> > >
> > > (see the http://ltt.polymtl.ca > QUICKSTART to see how to install the
> > > ltt-control package which contains the XML facilities in your system)
> > >
> > > lttctl-xen -e -t /tmp/xentrace1
> > >
> > > View in the visualiser : (see the QUICKSTART to see how to install it)
> > >
> > > lttv -m textDump -t /tmp/xentrace1
> > >
> > > (not tested yet) : to visualize a dom0 trace with the xen hypervisor
> > > information, one would have to collect the dom0 kernel trace and the Xen
> > > trace and open them together with :
> > > lttv -m textDump -t /tmp/xentrace1 -t /tmp/dom0trace
> > >
> > > The current Linux kernel instrumentation is for 2.6.20. A backport might
> > > be needed to 2.6.18 if there is no proper Xen support in 2.6.20 (I have
> > > not followed the recent developments).
> > >
> > >
> > > Currently broken/missing :
> > >
> > > - Ressources are not freed when the trace channels are destroyed. So you
> > >   basically have to reboot between taking different traces.
> > > - My code in the hypervisor complains to the console that subbuffers
> > >   have not been fully read when the trace channels are destroyed. The
> > >   error printing is just done too fast : lttd-xen is still there and
> > >   reading the buffers at that point. It will get fixed with proper
> > >   ressource usage tracking of both Xen and lttd-xen (same as the first
> > >   point above).
> > > - x86_64 not tested, powerpc local.h and ltt.h missing (should be ripped
> > >   from my Linux kernel LTTng).
> > >
> > >
> > > Cheers,
> > >
> > > Mathieu
> > >
> > > * Mathieu Desnoyers (compudj@krystal.dyndns.org) wrote:
> > >> Hi,
> > >>
> > >> My name is Mathieu Desnoyers, I am the current maintainer of the Linux
> > >> Trace Toolkit project, known as LTTng. This is a tracer for the 2.6
> > >> Linux kernels oriented towards high performance and real-time
> > >> applications.
> > >>
> > >> I have read your tracing thread and I am surprised to see how much
> > >> things you would like in a tracer are already implemented and tested in
> > >> LTTng. I am currently porting my tracer to Xen, so I think it might be
> > >> useful for you to know what it provides. My goal is to do not duplicate
> > >> the effort and save everyone some time.
> > >>
> > >> Here follows some key features of LTTng :
> > >>
> > >> Architecture independant data types
> > >> Extensible event records
> > >> Self-describing traces
> > >> Variable size records
> > >> Fast (200 ns per event record)
> > >> Highly reentrant
> > >> Does not disable interrupts
> > >> Does not take lock on the critical path
> > >> Supports NMI tracing
> > >> Analysis/visualization tool (LTTV)
> > >>
> > >> Looking at the integration of the existing LTTng implementation into
> > >> Xen, I came up with those two points for my Christmas whichlist :
> > >>
> > >> Additionnal functionnalities that would be nice to have in Xen :
> > >>
> > >> - RCU-style updates : would allow freeing the buffers without impact on
> > >> tracing. * I guess I could currently use :
> > >>       for_each_domain( d )
> > >>         for_each_vcpu( d, v )
> > >>           vcpu_sleep_sync(v);
> > >>       I think it will have a huge impact on the system, but it would
> > >> only be performed before trace buffers free.
> > >>
> > >> - Polling for data in Xen from a dom0 process.
> > >>   Xentrace currently polls the hypervisor each 100ms to see if there is
> > >> data that needs to be consumed. Instead of an active polling, it would
> > >> be nice to use the dom0 OS capability to put a process to sleep while
> > >> waiting for a resource. It would imply creating a module, loaded in
> > >> dom0, that would wait for a specific virq coming from the Hypervisor to
> > >> wake up such processes. We could think of exporting a complete poll()
> > >> interface through sysfs or procfs that would be a directory filled with
> > >> the resources exported from the Hypervisor to dom0 (which could include
> > >> wait for resource freed, useful when shutting down a domU instead of
> > >> busy looping). It would help dom0 to schedule other processes while a
> > >> process is waiting for the Hypervisor.
> > >>
> > >>
> > >> You might also be interested in looking at :
> > >> - the website (http://ltt.polymtl.ca)
> > >> - LTTng Xen port design document (this one is different from the one
> > >> posted by Jimi)
> > >>  
> > >> (http://ltt.polymtl.ca/svn/ltt/branches/poly/doc/developer/lttng-xen.txt
> > >>) - OLS 2006 paper "The LTTng tracer : A Low Impact Performance and
> > >> Behavior Monitor for GNU/Linux"
> > >>   (http://ltt.polymtl.ca/papers/desnoyers-ols2006.pdf)
> > >>
> > >>
> > >> Questions and constructive comments are welcome.
> > >>
> > >> Mathieu
> > >>
> > >>
> > >> OpenPGP public key:             
> > >> http://krystal.dyndns.org:8080/key/compudj.gpg Key fingerprint:     8CD5
> > >> 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68
> > >>
> > >> _______________________________________________
> > >> Xen-devel mailing list
> > >> Xen-devel@lists.xensource.com
> > >> http://lists.xensource.com/xen-devel
> >
> > _______________________________________________
> > Xen-devel mailing list
> > Xen-devel@lists.xensource.com
> > http://lists.xensource.com/xen-devel
> 
> 
> 
> -- 
> Dave: Just a question. What use is a unicyle with no seat?  And no pedals!
> Mark: To answer a question with a question: What use is a skateboard?
> Dave: Skateboards have wheels.
> Mark: My wheel has a wheel!
> 

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

Re: LTTng Xen port : finally in a repository near you

[Keir Fraser]

On 27/6/07 17:22, "Mathieu Desnoyers" <compudj@krystal.dyndns.org> wrote:

> Some advice about how I currently (fail) to do the teardown of traces
> correctly: I depend on synchronize_sched() under Linux to prodive a
> race-less efficient teardown (using read-copy-update for my data
> structures). It is a core concept I depend on to provide fast and
> efficient tracing. Has RCU finally been implement in Xen since the last
> time I checked?

Yes.

 K.

Re: LTTng Xen port : finally in a repository near you

[Mathieu Desnoyers]

* INAKOSHI Hiroya (inakoshi.hiroya@jp.fujitsu.com) wrote:
> Hi Mathieu,
> 
> I am interested in LTTng-xen because I thought that it would be nice if
> I can get traces on both xen and guest linux at the same time.  I
> reviewed LTTng-xen and found that
> 
> * LTTng and LTTng-xen have a quite similar structure,
> * a trace buffer resides in a hypervisor for LTTng-xen,
> * it is currently impossible to get traces from guest linux because
> there is no LTTng for 2.6.18-xen kernel, as you mentioned.
> 
> I had coarsely ported LTTng to 2.6.18-xen, though it is only for
> i386.  Now I can get traces on xen and guest linux simultaneously, even
> though they put records in different trace buffers.

Hi Ikanoski,

We did the same kind of coarse 2.6.18 port at our lab internally to get
traces from both Linux and Xen. The fact that the traces are recorded in
different buffers does not change anything to the fact that those trace
files can be copied in the same trace directory so they can be parsed
together by LTTV (traces coming from dom0, domUs and hypervisor). They
are synchronized by using the TSCs (hopefully, you will configure your
system to get a reliable TSC on AMD and older intels, see the
ltt-test-tsc kernel module in recent LTTng versions and ltt.polymtl.ca
website for info on that matter).


> Then I thought that
> it would be more useful if they put records in xen's trace buffer and I
> can analyze events

LTTV merges the information from all the valid trace files that appears
within the trace directory, so the analysis can be done on data coming
from userspace, kernels and hypervisor.

> from xen and linux guests with a single lttd and
> lttctl running on Domain-0.  Do you have an opinion about that?
> 

lttctl-xen and lttd-xen, although being quite similar to lttd and
lttctl, use hypercalls to get the data. The standard lttctl/lttd uses
debugfs files as a hook to the trace buffers.

As a distribution matter, I prefer to leave both separate for now,
because lttctl-xen and lttd-xen is highly tied to the Xen tree.

Also, merging the information within the buffers between Xen and Dom0 is
not such a great idea: The Hypervisor and dom0 can have a different
number of CPUs (Xen : real CPUs, dom0: vcpus). Since I use per-cpu
buffers, it does not fit.

Also, I don't want dom0 to overwrite data from the Xen buffers easily:
it's better if we keep some protection between dom0 and the Hypervisor.

Thanks for looking into this, don't hesitate to ask further questions,

Mathieu

> Regards,
> Hiroya
> 
> 
> Mathieu Desnoyers wrote:
> > Hello,
> > 
> > I made a working version of the LTTng tracer for xen-unstable for x86.
> > Here is the pointer to my repository so you can try it out :
> > 
> > hg clone http://ltt.polymtl.ca/cgi-bin/hgweb.cgi xen-unstable-lttng.hg
> > 
> > Basic usage :
> > 
> > (see lttctl-xen -h)
> > 
> > lttctl-xen -c
> > 
> > (in a different console)
> > lttd-xen -t /tmp/xentrace1
> > 
> > (in the 1st console)
> > lttctl-xen -s
> > 
> > (tracing is active)
> > 
> > lttctl-xen -q
> > lttctl-xen -r
> > 
> > lttd-xen should automatically quit after writing the last buffers as
> > soon as lttctl-xen -r is issued.
> > 
> > Then, you must copy the XML facilities :
> > 
> > (see the http://ltt.polymtl.ca > QUICKSTART to see how to install the
> > ltt-control package which contains the XML facilities in your system)
> > 
> > lttctl-xen -e -t /tmp/xentrace1
> > 
> > View in the visualiser : (see the QUICKSTART to see how to install it)
> > 
> > lttv -m textDump -t /tmp/xentrace1
> > 
> > (not tested yet) : to visualize a dom0 trace with the xen hypervisor
> > information, one would have to collect the dom0 kernel trace and the Xen
> > trace and open them together with :
> > lttv -m textDump -t /tmp/xentrace1 -t /tmp/dom0trace
> > 
> > The current Linux kernel instrumentation is for 2.6.20. A backport might
> > be needed to 2.6.18 if there is no proper Xen support in 2.6.20 (I have
> > not followed the recent developments).
> > 
> > 
> > Currently broken/missing :
> > 
> > - Ressources are not freed when the trace channels are destroyed. So you
> >   basically have to reboot between taking different traces.
> > - My code in the hypervisor complains to the console that subbuffers
> >   have not been fully read when the trace channels are destroyed. The
> >   error printing is just done too fast : lttd-xen is still there and
> >   reading the buffers at that point. It will get fixed with proper
> >   ressource usage tracking of both Xen and lttd-xen (same as the first
> >   point above).
> > - x86_64 not tested, powerpc local.h and ltt.h missing (should be ripped
> >   from my Linux kernel LTTng).
> > 
> > 
> > Cheers,
> > 
> > Mathieu
> > 
> > 
> > 
> > * Mathieu Desnoyers (compudj@krystal.dyndns.org) wrote:
> >> Hi,
> >>
> >> My name is Mathieu Desnoyers, I am the current maintainer of the Linux Trace
> >> Toolkit project, known as LTTng. This is a tracer for the 2.6 Linux kernels
> >> oriented towards high performance and real-time applications.
> >>
> >> I have read your tracing thread and I am surprised to see how much things
> >> you would like in a tracer are already implemented and tested in LTTng. I am
> >> currently porting my tracer to Xen, so I think it might be useful for you to
> >> know what it provides. My goal is to do not duplicate the effort and save
> >> everyone some time.
> >>
> >> Here follows some key features of LTTng :
> >>
> >> Architecture independant data types
> >> Extensible event records
> >> Self-describing traces
> >> Variable size records
> >> Fast (200 ns per event record)
> >> Highly reentrant
> >> Does not disable interrupts
> >> Does not take lock on the critical path
> >> Supports NMI tracing
> >> Analysis/visualization tool (LTTV)
> >>
> >> Looking at the integration of the existing LTTng implementation into Xen, I
> >> came up with those two points for my Christmas whichlist :
> >>
> >> Additionnal functionnalities that would be nice to have in Xen :
> >>
> >> - RCU-style updates : would allow freeing the buffers without impact on tracing.
> >>     * I guess I could currently use :
> >>       for_each_domain( d )
> >>         for_each_vcpu( d, v )
> >>           vcpu_sleep_sync(v);
> >>       I think it will have a huge impact on the system, but it would only be
> >>       performed before trace buffers free.
> >>
> >> - Polling for data in Xen from a dom0 process.
> >>   Xentrace currently polls the hypervisor each 100ms to see if there is data
> >>   that needs to be consumed. Instead of an active polling, it would be nice to
> >>   use the dom0 OS capability to put a process to sleep while waiting for a
> >>   resource. It would imply creating a module, loaded in dom0, that would wait
> >>   for a specific virq coming from the Hypervisor to wake up such processes.
> >>   We could think of exporting a complete poll() interface through sysfs or
> >>   procfs that would be a directory filled with the resources exported from the
> >>   Hypervisor to dom0 (which could include wait for resource freed, useful when
> >>   shutting down a domU instead of busy looping). It would help dom0 to schedule
> >>   other processes while a process is waiting for the Hypervisor.
> >>
> >>
> >> You might also be interested in looking at :
> >> - the website (http://ltt.polymtl.ca)
> >> - LTTng Xen port design document (this one is different from the one posted by
> >>   Jimi)
> >>   (http://ltt.polymtl.ca/svn/ltt/branches/poly/doc/developer/lttng-xen.txt)
> >> - OLS 2006 paper "The LTTng tracer : A Low Impact Performance and Behavior
> >>   Monitor for GNU/Linux"
> >>   (http://ltt.polymtl.ca/papers/desnoyers-ols2006.pdf)
> >>
> >>
> >> Questions and constructive comments are welcome.
> >>
> >> Mathieu
> >>
> >>
> >> OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
> >> Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
> >>
> >> _______________________________________________
> >> Xen-devel mailing list
> >> Xen-devel@lists.xensource.com
> >> http://lists.xensource.com/xen-devel
> >>
> > 
> 
> 
> 

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

Re: LTTng Xen port : finally in a repository near you

[INAKOSHI Hiroya]

Hi Mathieu,

thanks for your reply.  I can understand your opinion very well but a
concern is that cpu ids on a guest OS are different from those on Xen
because they are virtualized.  The number of vcpus in a guest OS is also
different from that of pcpus as you mentioned.  I wondered if the two
traces could be merged directly.  If you translate vcpu ids to pcpu ids
in writing records in the trace buffer in Xen, this concern is solved in
a natural way.

Mathieu Desnoyers wrote:
> * INAKOSHI Hiroya (inakoshi.hiroya@jp.fujitsu.com) wrote:
>> Hi Mathieu,
>>
>> I am interested in LTTng-xen because I thought that it would be nice if
>> I can get traces on both xen and guest linux at the same time.  I
>> reviewed LTTng-xen and found that
>>
>> * LTTng and LTTng-xen have a quite similar structure,
>> * a trace buffer resides in a hypervisor for LTTng-xen,
>> * it is currently impossible to get traces from guest linux because
>> there is no LTTng for 2.6.18-xen kernel, as you mentioned.
>>
>> I had coarsely ported LTTng to 2.6.18-xen, though it is only for
>> i386.  Now I can get traces on xen and guest linux simultaneously, even
>> though they put records in different trace buffers.
> 
> Hi Ikanoski,
> 
> We did the same kind of coarse 2.6.18 port at our lab internally to get
> traces from both Linux and Xen. The fact that the traces are recorded in
> different buffers does not change anything to the fact that those trace
> files can be copied in the same trace directory so they can be parsed
> together by LTTV (traces coming from dom0, domUs and hypervisor). They
> are synchronized by using the TSCs (hopefully, you will configure your
> system to get a reliable TSC on AMD and older intels, see the
> ltt-test-tsc kernel module in recent LTTng versions and ltt.polymtl.ca
> website for info on that matter).
> 
> 
>> Then I thought that
>> it would be more useful if they put records in xen's trace buffer and I
>> can analyze events
> 
> LTTV merges the information from all the valid trace files that appears
> within the trace directory, so the analysis can be done on data coming
> from userspace, kernels and hypervisor.
> 
>> from xen and linux guests with a single lttd and
>> lttctl running on Domain-0.  Do you have an opinion about that?
>>
> 
> lttctl-xen and lttd-xen, although being quite similar to lttd and
> lttctl, use hypercalls to get the data. The standard lttctl/lttd uses
> debugfs files as a hook to the trace buffers.
> 
> As a distribution matter, I prefer to leave both separate for now,
> because lttctl-xen and lttd-xen is highly tied to the Xen tree.
> 
> Also, merging the information within the buffers between Xen and Dom0 is
> not such a great idea: The Hypervisor and dom0 can have a different
> number of CPUs (Xen : real CPUs, dom0: vcpus). Since I use per-cpu
> buffers, it does not fit.
> 
> Also, I don't want dom0 to overwrite data from the Xen buffers easily:
> it's better if we keep some protection between dom0 and the Hypervisor.
> 
> Thanks for looking into this, don't hesitate to ask further questions,
> 
> Mathieu
> 
>> Regards,
>> Hiroya
>>
>>
>> Mathieu Desnoyers wrote:
>>> Hello,
>>>
>>> I made a working version of the LTTng tracer for xen-unstable for x86.
>>> Here is the pointer to my repository so you can try it out :
>>>
>>> hg clone http://ltt.polymtl.ca/cgi-bin/hgweb.cgi xen-unstable-lttng.hg
>>>
>>> Basic usage :
>>>
>>> (see lttctl-xen -h)
>>>
>>> lttctl-xen -c
>>>
>>> (in a different console)
>>> lttd-xen -t /tmp/xentrace1
>>>
>>> (in the 1st console)
>>> lttctl-xen -s
>>>
>>> (tracing is active)
>>>
>>> lttctl-xen -q
>>> lttctl-xen -r
>>>
>>> lttd-xen should automatically quit after writing the last buffers as
>>> soon as lttctl-xen -r is issued.
>>>
>>> Then, you must copy the XML facilities :
>>>
>>> (see the http://ltt.polymtl.ca > QUICKSTART to see how to install the
>>> ltt-control package which contains the XML facilities in your system)
>>>
>>> lttctl-xen -e -t /tmp/xentrace1
>>>
>>> View in the visualiser : (see the QUICKSTART to see how to install it)
>>>
>>> lttv -m textDump -t /tmp/xentrace1
>>>
>>> (not tested yet) : to visualize a dom0 trace with the xen hypervisor
>>> information, one would have to collect the dom0 kernel trace and the Xen
>>> trace and open them together with :
>>> lttv -m textDump -t /tmp/xentrace1 -t /tmp/dom0trace
>>>
>>> The current Linux kernel instrumentation is for 2.6.20. A backport might
>>> be needed to 2.6.18 if there is no proper Xen support in 2.6.20 (I have
>>> not followed the recent developments).
>>>
>>>
>>> Currently broken/missing :
>>>
>>> - Ressources are not freed when the trace channels are destroyed. So you
>>>   basically have to reboot between taking different traces.
>>> - My code in the hypervisor complains to the console that subbuffers
>>>   have not been fully read when the trace channels are destroyed. The
>>>   error printing is just done too fast : lttd-xen is still there and
>>>   reading the buffers at that point. It will get fixed with proper
>>>   ressource usage tracking of both Xen and lttd-xen (same as the first
>>>   point above).
>>> - x86_64 not tested, powerpc local.h and ltt.h missing (should be ripped
>>>   from my Linux kernel LTTng).
>>>
>>>
>>> Cheers,
>>>
>>> Mathieu
>>>
>>>
>>>
>>> * Mathieu Desnoyers (compudj@krystal.dyndns.org) wrote:
>>>> Hi,
>>>>
>>>> My name is Mathieu Desnoyers, I am the current maintainer of the Linux Trace
>>>> Toolkit project, known as LTTng. This is a tracer for the 2.6 Linux kernels
>>>> oriented towards high performance and real-time applications.
>>>>
>>>> I have read your tracing thread and I am surprised to see how much things
>>>> you would like in a tracer are already implemented and tested in LTTng. I am
>>>> currently porting my tracer to Xen, so I think it might be useful for you to
>>>> know what it provides. My goal is to do not duplicate the effort and save
>>>> everyone some time.
>>>>
>>>> Here follows some key features of LTTng :
>>>>
>>>> Architecture independant data types
>>>> Extensible event records
>>>> Self-describing traces
>>>> Variable size records
>>>> Fast (200 ns per event record)
>>>> Highly reentrant
>>>> Does not disable interrupts
>>>> Does not take lock on the critical path
>>>> Supports NMI tracing
>>>> Analysis/visualization tool (LTTV)
>>>>
>>>> Looking at the integration of the existing LTTng implementation into Xen, I
>>>> came up with those two points for my Christmas whichlist :
>>>>
>>>> Additionnal functionnalities that would be nice to have in Xen :
>>>>
>>>> - RCU-style updates : would allow freeing the buffers without impact on tracing.
>>>>     * I guess I could currently use :
>>>>       for_each_domain( d )
>>>>         for_each_vcpu( d, v )
>>>>           vcpu_sleep_sync(v);
>>>>       I think it will have a huge impact on the system, but it would only be
>>>>       performed before trace buffers free.
>>>>
>>>> - Polling for data in Xen from a dom0 process.
>>>>   Xentrace currently polls the hypervisor each 100ms to see if there is data
>>>>   that needs to be consumed. Instead of an active polling, it would be nice to
>>>>   use the dom0 OS capability to put a process to sleep while waiting for a
>>>>   resource. It would imply creating a module, loaded in dom0, that would wait
>>>>   for a specific virq coming from the Hypervisor to wake up such processes.
>>>>   We could think of exporting a complete poll() interface through sysfs or
>>>>   procfs that would be a directory filled with the resources exported from the
>>>>   Hypervisor to dom0 (which could include wait for resource freed, useful when
>>>>   shutting down a domU instead of busy looping). It would help dom0 to schedule
>>>>   other processes while a process is waiting for the Hypervisor.
>>>>
>>>>
>>>> You might also be interested in looking at :
>>>> - the website (http://ltt.polymtl.ca)
>>>> - LTTng Xen port design document (this one is different from the one posted by
>>>>   Jimi)
>>>>   (http://ltt.polymtl.ca/svn/ltt/branches/poly/doc/developer/lttng-xen.txt)
>>>> - OLS 2006 paper "The LTTng tracer : A Low Impact Performance and Behavior
>>>>   Monitor for GNU/Linux"
>>>>   (http://ltt.polymtl.ca/papers/desnoyers-ols2006.pdf)
>>>>
>>>>
>>>> Questions and constructive comments are welcome.
>>>>
>>>> Mathieu
>>>>
>>>>
>>>> OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
>>>> Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
>>>>
>>>> _______________________________________________
>>>> Xen-devel mailing list
>>>> Xen-devel@lists.xensource.com
>>>> http://lists.xensource.com/xen-devel
>>>>
>>
>>
> 

Re: LTTng Xen port : finally in a repository near you

[Mathieu Desnoyers]

* INAKOSHI Hiroya (inakoshi.hiroya@jp.fujitsu.com) wrote:
> Hi Mathieu,
> 
> thanks for your reply.  I can understand your opinion very well but a
> concern is that cpu ids on a guest OS are different from those on Xen
> because they are virtualized.  The number of vcpus in a guest OS is also
> different from that of pcpus as you mentioned.  I wondered if the two
> traces could be merged directly.  If you translate vcpu ids to pcpu ids
> in writing records in the trace buffer in Xen, this concern is solved in
> a natural way.
> 

When you are executing code in dom0 or domUs, how do you plan to get the
physical CPU number on which the tracing is done ?

> Mathieu Desnoyers wrote:
> > * INAKOSHI Hiroya (inakoshi.hiroya@jp.fujitsu.com) wrote:
> >> Hi Mathieu,
> >>
> >> I am interested in LTTng-xen because I thought that it would be nice if
> >> I can get traces on both xen and guest linux at the same time.  I
> >> reviewed LTTng-xen and found that
> >>
> >> * LTTng and LTTng-xen have a quite similar structure,
> >> * a trace buffer resides in a hypervisor for LTTng-xen,
> >> * it is currently impossible to get traces from guest linux because
> >> there is no LTTng for 2.6.18-xen kernel, as you mentioned.
> >>
> >> I had coarsely ported LTTng to 2.6.18-xen, though it is only for
> >> i386.  Now I can get traces on xen and guest linux simultaneously, even
> >> though they put records in different trace buffers.
> > 
> > Hi Ikanoski,
> > 
> > We did the same kind of coarse 2.6.18 port at our lab internally to get
> > traces from both Linux and Xen. The fact that the traces are recorded in
> > different buffers does not change anything to the fact that those trace
> > files can be copied in the same trace directory so they can be parsed
> > together by LTTV (traces coming from dom0, domUs and hypervisor). They
> > are synchronized by using the TSCs (hopefully, you will configure your
> > system to get a reliable TSC on AMD and older intels, see the
> > ltt-test-tsc kernel module in recent LTTng versions and ltt.polymtl.ca
> > website for info on that matter).
> > 
> > 
> >> Then I thought that
> >> it would be more useful if they put records in xen's trace buffer and I
> >> can analyze events
> > 
> > LTTV merges the information from all the valid trace files that appears
> > within the trace directory, so the analysis can be done on data coming
> > from userspace, kernels and hypervisor.
> > 
> >> from xen and linux guests with a single lttd and
> >> lttctl running on Domain-0.  Do you have an opinion about that?
> >>
> > 
> > lttctl-xen and lttd-xen, although being quite similar to lttd and
> > lttctl, use hypercalls to get the data. The standard lttctl/lttd uses
> > debugfs files as a hook to the trace buffers.
> > 
> > As a distribution matter, I prefer to leave both separate for now,
> > because lttctl-xen and lttd-xen is highly tied to the Xen tree.
> > 
> > Also, merging the information within the buffers between Xen and Dom0 is
> > not such a great idea: The Hypervisor and dom0 can have a different
> > number of CPUs (Xen : real CPUs, dom0: vcpus). Since I use per-cpu
> > buffers, it does not fit.
> > 
> > Also, I don't want dom0 to overwrite data from the Xen buffers easily:
> > it's better if we keep some protection between dom0 and the Hypervisor.
> > 
> > Thanks for looking into this, don't hesitate to ask further questions,
> > 
> > Mathieu
> > 
> >> Regards,
> >> Hiroya
> >>
> >>
> >> Mathieu Desnoyers wrote:
> >>> Hello,
> >>>
> >>> I made a working version of the LTTng tracer for xen-unstable for x86.
> >>> Here is the pointer to my repository so you can try it out :
> >>>
> >>> hg clone http://ltt.polymtl.ca/cgi-bin/hgweb.cgi xen-unstable-lttng.hg
> >>>
> >>> Basic usage :
> >>>
> >>> (see lttctl-xen -h)
> >>>
> >>> lttctl-xen -c
> >>>
> >>> (in a different console)
> >>> lttd-xen -t /tmp/xentrace1
> >>>
> >>> (in the 1st console)
> >>> lttctl-xen -s
> >>>
> >>> (tracing is active)
> >>>
> >>> lttctl-xen -q
> >>> lttctl-xen -r
> >>>
> >>> lttd-xen should automatically quit after writing the last buffers as
> >>> soon as lttctl-xen -r is issued.
> >>>
> >>> Then, you must copy the XML facilities :
> >>>
> >>> (see the http://ltt.polymtl.ca > QUICKSTART to see how to install the
> >>> ltt-control package which contains the XML facilities in your system)
> >>>
> >>> lttctl-xen -e -t /tmp/xentrace1
> >>>
> >>> View in the visualiser : (see the QUICKSTART to see how to install it)
> >>>
> >>> lttv -m textDump -t /tmp/xentrace1
> >>>
> >>> (not tested yet) : to visualize a dom0 trace with the xen hypervisor
> >>> information, one would have to collect the dom0 kernel trace and the Xen
> >>> trace and open them together with :
> >>> lttv -m textDump -t /tmp/xentrace1 -t /tmp/dom0trace
> >>>
> >>> The current Linux kernel instrumentation is for 2.6.20. A backport might
> >>> be needed to 2.6.18 if there is no proper Xen support in 2.6.20 (I have
> >>> not followed the recent developments).
> >>>
> >>>
> >>> Currently broken/missing :
> >>>
> >>> - Ressources are not freed when the trace channels are destroyed. So you
> >>>   basically have to reboot between taking different traces.
> >>> - My code in the hypervisor complains to the console that subbuffers
> >>>   have not been fully read when the trace channels are destroyed. The
> >>>   error printing is just done too fast : lttd-xen is still there and
> >>>   reading the buffers at that point. It will get fixed with proper
> >>>   ressource usage tracking of both Xen and lttd-xen (same as the first
> >>>   point above).
> >>> - x86_64 not tested, powerpc local.h and ltt.h missing (should be ripped
> >>>   from my Linux kernel LTTng).
> >>>
> >>>
> >>> Cheers,
> >>>
> >>> Mathieu
> >>>
> >>>
> >>>
> >>> * Mathieu Desnoyers (compudj@krystal.dyndns.org) wrote:
> >>>> Hi,
> >>>>
> >>>> My name is Mathieu Desnoyers, I am the current maintainer of the Linux Trace
> >>>> Toolkit project, known as LTTng. This is a tracer for the 2.6 Linux kernels
> >>>> oriented towards high performance and real-time applications.
> >>>>
> >>>> I have read your tracing thread and I am surprised to see how much things
> >>>> you would like in a tracer are already implemented and tested in LTTng. I am
> >>>> currently porting my tracer to Xen, so I think it might be useful for you to
> >>>> know what it provides. My goal is to do not duplicate the effort and save
> >>>> everyone some time.
> >>>>
> >>>> Here follows some key features of LTTng :
> >>>>
> >>>> Architecture independant data types
> >>>> Extensible event records
> >>>> Self-describing traces
> >>>> Variable size records
> >>>> Fast (200 ns per event record)
> >>>> Highly reentrant
> >>>> Does not disable interrupts
> >>>> Does not take lock on the critical path
> >>>> Supports NMI tracing
> >>>> Analysis/visualization tool (LTTV)
> >>>>
> >>>> Looking at the integration of the existing LTTng implementation into Xen, I
> >>>> came up with those two points for my Christmas whichlist :
> >>>>
> >>>> Additionnal functionnalities that would be nice to have in Xen :
> >>>>
> >>>> - RCU-style updates : would allow freeing the buffers without impact on tracing.
> >>>>     * I guess I could currently use :
> >>>>       for_each_domain( d )
> >>>>         for_each_vcpu( d, v )
> >>>>           vcpu_sleep_sync(v);
> >>>>       I think it will have a huge impact on the system, but it would only be
> >>>>       performed before trace buffers free.
> >>>>
> >>>> - Polling for data in Xen from a dom0 process.
> >>>>   Xentrace currently polls the hypervisor each 100ms to see if there is data
> >>>>   that needs to be consumed. Instead of an active polling, it would be nice to
> >>>>   use the dom0 OS capability to put a process to sleep while waiting for a
> >>>>   resource. It would imply creating a module, loaded in dom0, that would wait
> >>>>   for a specific virq coming from the Hypervisor to wake up such processes.
> >>>>   We could think of exporting a complete poll() interface through sysfs or
> >>>>   procfs that would be a directory filled with the resources exported from the
> >>>>   Hypervisor to dom0 (which could include wait for resource freed, useful when
> >>>>   shutting down a domU instead of busy looping). It would help dom0 to schedule
> >>>>   other processes while a process is waiting for the Hypervisor.
> >>>>
> >>>>
> >>>> You might also be interested in looking at :
> >>>> - the website (http://ltt.polymtl.ca)
> >>>> - LTTng Xen port design document (this one is different from the one posted by
> >>>>   Jimi)
> >>>>   (http://ltt.polymtl.ca/svn/ltt/branches/poly/doc/developer/lttng-xen.txt)
> >>>> - OLS 2006 paper "The LTTng tracer : A Low Impact Performance and Behavior
> >>>>   Monitor for GNU/Linux"
> >>>>   (http://ltt.polymtl.ca/papers/desnoyers-ols2006.pdf)
> >>>>
> >>>>
> >>>> Questions and constructive comments are welcome.
> >>>>
> >>>> Mathieu
> >>>>
> >>>>
> >>>> OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
> >>>> Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
> >>>>
> >>>> _______________________________________________
> >>>> Xen-devel mailing list
> >>>> Xen-devel@lists.xensource.com
> >>>> http://lists.xensource.com/xen-devel
> >>>>
> >>
> >>
> > 
> 
> 

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

Re: LTTng Xen port : finally in a repository near you

[INAKOSHI Hiroya]

Mathieu Desnoyers wrote:
> * INAKOSHI Hiroya (inakoshi.hiroya@jp.fujitsu.com) wrote:
>> Hi Mathieu,
>>
>> thanks for your reply.  I can understand your opinion very well but a
>> concern is that cpu ids on a guest OS are different from those on Xen
>> because they are virtualized.  The number of vcpus in a guest OS is also
>> different from that of pcpus as you mentioned.  I wondered if the two
>> traces could be merged directly.  If you translate vcpu ids to pcpu ids
>> in writing records in the trace buffer in Xen, this concern is solved in
>> a natural way.
>>
> 
> When you are executing code in dom0 or domUs, how do you plan to get the
> physical CPU number on which the tracing is done ?

I am considering the way where dom0 or domUs call hypercalls to write
records in the Xen's trace buffer.  In this setting, the vcpu info is
located in the xen kernel stack and the pcpu is the one performing the
hypercall.  So, I can resolve the mapping between vcpu id and pcpu id.

Regards,
Hiroya

> 
>> Mathieu Desnoyers wrote:
>>> * INAKOSHI Hiroya (inakoshi.hiroya@jp.fujitsu.com) wrote:
>>>> Hi Mathieu,
>>>>
>>>> I am interested in LTTng-xen because I thought that it would be nice if
>>>> I can get traces on both xen and guest linux at the same time.  I
>>>> reviewed LTTng-xen and found that
>>>>
>>>> * LTTng and LTTng-xen have a quite similar structure,
>>>> * a trace buffer resides in a hypervisor for LTTng-xen,
>>>> * it is currently impossible to get traces from guest linux because
>>>> there is no LTTng for 2.6.18-xen kernel, as you mentioned.
>>>>
>>>> I had coarsely ported LTTng to 2.6.18-xen, though it is only for
>>>> i386.  Now I can get traces on xen and guest linux simultaneously, even
>>>> though they put records in different trace buffers.
>>> Hi Ikanoski,
>>>
>>> We did the same kind of coarse 2.6.18 port at our lab internally to get
>>> traces from both Linux and Xen. The fact that the traces are recorded in
>>> different buffers does not change anything to the fact that those trace
>>> files can be copied in the same trace directory so they can be parsed
>>> together by LTTV (traces coming from dom0, domUs and hypervisor). They
>>> are synchronized by using the TSCs (hopefully, you will configure your
>>> system to get a reliable TSC on AMD and older intels, see the
>>> ltt-test-tsc kernel module in recent LTTng versions and ltt.polymtl.ca
>>> website for info on that matter).
>>>
>>>
>>>> Then I thought that
>>>> it would be more useful if they put records in xen's trace buffer and I
>>>> can analyze events
>>> LTTV merges the information from all the valid trace files that appears
>>> within the trace directory, so the analysis can be done on data coming
>>> from userspace, kernels and hypervisor.
>>>
>>>> from xen and linux guests with a single lttd and
>>>> lttctl running on Domain-0.  Do you have an opinion about that?
>>>>
>>> lttctl-xen and lttd-xen, although being quite similar to lttd and
>>> lttctl, use hypercalls to get the data. The standard lttctl/lttd uses
>>> debugfs files as a hook to the trace buffers.
>>>
>>> As a distribution matter, I prefer to leave both separate for now,
>>> because lttctl-xen and lttd-xen is highly tied to the Xen tree.
>>>
>>> Also, merging the information within the buffers between Xen and Dom0 is
>>> not such a great idea: The Hypervisor and dom0 can have a different
>>> number of CPUs (Xen : real CPUs, dom0: vcpus). Since I use per-cpu
>>> buffers, it does not fit.
>>>
>>> Also, I don't want dom0 to overwrite data from the Xen buffers easily:
>>> it's better if we keep some protection between dom0 and the Hypervisor.
>>>
>>> Thanks for looking into this, don't hesitate to ask further questions,
>>>
>>> Mathieu
>>>
>>>> Regards,
>>>> Hiroya
>>>>
>>>>
>>>> Mathieu Desnoyers wrote:
>>>>> Hello,
>>>>>
>>>>> I made a working version of the LTTng tracer for xen-unstable for x86.
>>>>> Here is the pointer to my repository so you can try it out :
>>>>>
>>>>> hg clone http://ltt.polymtl.ca/cgi-bin/hgweb.cgi xen-unstable-lttng.hg
>>>>>
>>>>> Basic usage :
>>>>>
>>>>> (see lttctl-xen -h)
>>>>>
>>>>> lttctl-xen -c
>>>>>
>>>>> (in a different console)
>>>>> lttd-xen -t /tmp/xentrace1
>>>>>
>>>>> (in the 1st console)
>>>>> lttctl-xen -s
>>>>>
>>>>> (tracing is active)
>>>>>
>>>>> lttctl-xen -q
>>>>> lttctl-xen -r
>>>>>
>>>>> lttd-xen should automatically quit after writing the last buffers as
>>>>> soon as lttctl-xen -r is issued.
>>>>>
>>>>> Then, you must copy the XML facilities :
>>>>>
>>>>> (see the http://ltt.polymtl.ca > QUICKSTART to see how to install the
>>>>> ltt-control package which contains the XML facilities in your system)
>>>>>
>>>>> lttctl-xen -e -t /tmp/xentrace1
>>>>>
>>>>> View in the visualiser : (see the QUICKSTART to see how to install it)
>>>>>
>>>>> lttv -m textDump -t /tmp/xentrace1
>>>>>
>>>>> (not tested yet) : to visualize a dom0 trace with the xen hypervisor
>>>>> information, one would have to collect the dom0 kernel trace and the Xen
>>>>> trace and open them together with :
>>>>> lttv -m textDump -t /tmp/xentrace1 -t /tmp/dom0trace
>>>>>
>>>>> The current Linux kernel instrumentation is for 2.6.20. A backport might
>>>>> be needed to 2.6.18 if there is no proper Xen support in 2.6.20 (I have
>>>>> not followed the recent developments).
>>>>>
>>>>>
>>>>> Currently broken/missing :
>>>>>
>>>>> - Ressources are not freed when the trace channels are destroyed. So you
>>>>>   basically have to reboot between taking different traces.
>>>>> - My code in the hypervisor complains to the console that subbuffers
>>>>>   have not been fully read when the trace channels are destroyed. The
>>>>>   error printing is just done too fast : lttd-xen is still there and
>>>>>   reading the buffers at that point. It will get fixed with proper
>>>>>   ressource usage tracking of both Xen and lttd-xen (same as the first
>>>>>   point above).
>>>>> - x86_64 not tested, powerpc local.h and ltt.h missing (should be ripped
>>>>>   from my Linux kernel LTTng).
>>>>>
>>>>>
>>>>> Cheers,
>>>>>
>>>>> Mathieu
>>>>>
>>>>>
>>>>>
>>>>> * Mathieu Desnoyers (compudj@krystal.dyndns.org) wrote:
>>>>>> Hi,
>>>>>>
>>>>>> My name is Mathieu Desnoyers, I am the current maintainer of the Linux Trace
>>>>>> Toolkit project, known as LTTng. This is a tracer for the 2.6 Linux kernels
>>>>>> oriented towards high performance and real-time applications.
>>>>>>
>>>>>> I have read your tracing thread and I am surprised to see how much things
>>>>>> you would like in a tracer are already implemented and tested in LTTng. I am
>>>>>> currently porting my tracer to Xen, so I think it might be useful for you to
>>>>>> know what it provides. My goal is to do not duplicate the effort and save
>>>>>> everyone some time.
>>>>>>
>>>>>> Here follows some key features of LTTng :
>>>>>>
>>>>>> Architecture independant data types
>>>>>> Extensible event records
>>>>>> Self-describing traces
>>>>>> Variable size records
>>>>>> Fast (200 ns per event record)
>>>>>> Highly reentrant
>>>>>> Does not disable interrupts
>>>>>> Does not take lock on the critical path
>>>>>> Supports NMI tracing
>>>>>> Analysis/visualization tool (LTTV)
>>>>>>
>>>>>> Looking at the integration of the existing LTTng implementation into Xen, I
>>>>>> came up with those two points for my Christmas whichlist :
>>>>>>
>>>>>> Additionnal functionnalities that would be nice to have in Xen :
>>>>>>
>>>>>> - RCU-style updates : would allow freeing the buffers without impact on tracing.
>>>>>>     * I guess I could currently use :
>>>>>>       for_each_domain( d )
>>>>>>         for_each_vcpu( d, v )
>>>>>>           vcpu_sleep_sync(v);
>>>>>>       I think it will have a huge impact on the system, but it would only be
>>>>>>       performed before trace buffers free.
>>>>>>
>>>>>> - Polling for data in Xen from a dom0 process.
>>>>>>   Xentrace currently polls the hypervisor each 100ms to see if there is data
>>>>>>   that needs to be consumed. Instead of an active polling, it would be nice to
>>>>>>   use the dom0 OS capability to put a process to sleep while waiting for a
>>>>>>   resource. It would imply creating a module, loaded in dom0, that would wait
>>>>>>   for a specific virq coming from the Hypervisor to wake up such processes.
>>>>>>   We could think of exporting a complete poll() interface through sysfs or
>>>>>>   procfs that would be a directory filled with the resources exported from the
>>>>>>   Hypervisor to dom0 (which could include wait for resource freed, useful when
>>>>>>   shutting down a domU instead of busy looping). It would help dom0 to schedule
>>>>>>   other processes while a process is waiting for the Hypervisor.
>>>>>>
>>>>>>
>>>>>> You might also be interested in looking at :
>>>>>> - the website (http://ltt.polymtl.ca)
>>>>>> - LTTng Xen port design document (this one is different from the one posted by
>>>>>>   Jimi)
>>>>>>   (http://ltt.polymtl.ca/svn/ltt/branches/poly/doc/developer/lttng-xen.txt)
>>>>>> - OLS 2006 paper "The LTTng tracer : A Low Impact Performance and Behavior
>>>>>>   Monitor for GNU/Linux"
>>>>>>   (http://ltt.polymtl.ca/papers/desnoyers-ols2006.pdf)
>>>>>>
>>>>>>
>>>>>> Questions and constructive comments are welcome.
>>>>>>
>>>>>> Mathieu
>>>>>>
>>>>>>
>>>>>> OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
>>>>>> Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
>>>>>>
>>>>>> _______________________________________________
>>>>>> Xen-devel mailing list
>>>>>> Xen-devel@lists.xensource.com
>>>>>> http://lists.xensource.com/xen-devel
>>>>>>
>>>>
>>
> 

Re: LTTng Xen port : finally in a repository near you

[Mathieu Desnoyers]

* INAKOSHI Hiroya (inakoshi.hiroya@jp.fujitsu.com) wrote:
> Mathieu Desnoyers wrote:
> > * INAKOSHI Hiroya (inakoshi.hiroya@jp.fujitsu.com) wrote:
> >> Hi Mathieu,
> >>
> >> thanks for your reply.  I can understand your opinion very well but a
> >> concern is that cpu ids on a guest OS are different from those on Xen
> >> because they are virtualized.  The number of vcpus in a guest OS is also
> >> different from that of pcpus as you mentioned.  I wondered if the two
> >> traces could be merged directly.  If you translate vcpu ids to pcpu ids
> >> in writing records in the trace buffer in Xen, this concern is solved in
> >> a natural way.
> >>
> > 
> > When you are executing code in dom0 or domUs, how do you plan to get the
> > physical CPU number on which the tracing is done ?
> 
> I am considering the way where dom0 or domUs call hypercalls to write
> records in the Xen's trace buffer.  In this setting, the vcpu info is
> located in the xen kernel stack and the pcpu is the one performing the
> hypercall.  So, I can resolve the mapping between vcpu id and pcpu id.
> 

The performance hit that goes with going through an hypercall for each
traced event would be too high. Typically, changing ring level involves
executing an interrupt routine, which takes a few thousands nanoseconds.
My tracing probes runs within the traced ring in about 270ns (as tested
on a Pentium 4 3GHz).

Mathieu

> Regards,
> Hiroya
> 
> > 
> >> Mathieu Desnoyers wrote:
> >>> * INAKOSHI Hiroya (inakoshi.hiroya@jp.fujitsu.com) wrote:
> >>>> Hi Mathieu,
> >>>>
> >>>> I am interested in LTTng-xen because I thought that it would be nice if
> >>>> I can get traces on both xen and guest linux at the same time.  I
> >>>> reviewed LTTng-xen and found that
> >>>>
> >>>> * LTTng and LTTng-xen have a quite similar structure,
> >>>> * a trace buffer resides in a hypervisor for LTTng-xen,
> >>>> * it is currently impossible to get traces from guest linux because
> >>>> there is no LTTng for 2.6.18-xen kernel, as you mentioned.
> >>>>
> >>>> I had coarsely ported LTTng to 2.6.18-xen, though it is only for
> >>>> i386.  Now I can get traces on xen and guest linux simultaneously, even
> >>>> though they put records in different trace buffers.
> >>> Hi Ikanoski,
> >>>
> >>> We did the same kind of coarse 2.6.18 port at our lab internally to get
> >>> traces from both Linux and Xen. The fact that the traces are recorded in
> >>> different buffers does not change anything to the fact that those trace
> >>> files can be copied in the same trace directory so they can be parsed
> >>> together by LTTV (traces coming from dom0, domUs and hypervisor). They
> >>> are synchronized by using the TSCs (hopefully, you will configure your
> >>> system to get a reliable TSC on AMD and older intels, see the
> >>> ltt-test-tsc kernel module in recent LTTng versions and ltt.polymtl.ca
> >>> website for info on that matter).
> >>>
> >>>
> >>>> Then I thought that
> >>>> it would be more useful if they put records in xen's trace buffer and I
> >>>> can analyze events
> >>> LTTV merges the information from all the valid trace files that appears
> >>> within the trace directory, so the analysis can be done on data coming
> >>> from userspace, kernels and hypervisor.
> >>>
> >>>> from xen and linux guests with a single lttd and
> >>>> lttctl running on Domain-0.  Do you have an opinion about that?
> >>>>
> >>> lttctl-xen and lttd-xen, although being quite similar to lttd and
> >>> lttctl, use hypercalls to get the data. The standard lttctl/lttd uses
> >>> debugfs files as a hook to the trace buffers.
> >>>
> >>> As a distribution matter, I prefer to leave both separate for now,
> >>> because lttctl-xen and lttd-xen is highly tied to the Xen tree.
> >>>
> >>> Also, merging the information within the buffers between Xen and Dom0 is
> >>> not such a great idea: The Hypervisor and dom0 can have a different
> >>> number of CPUs (Xen : real CPUs, dom0: vcpus). Since I use per-cpu
> >>> buffers, it does not fit.
> >>>
> >>> Also, I don't want dom0 to overwrite data from the Xen buffers easily:
> >>> it's better if we keep some protection between dom0 and the Hypervisor.
> >>>
> >>> Thanks for looking into this, don't hesitate to ask further questions,
> >>>
> >>> Mathieu
> >>>
> >>>> Regards,
> >>>> Hiroya
> >>>>
> >>>>
> >>>> Mathieu Desnoyers wrote:
> >>>>> Hello,
> >>>>>
> >>>>> I made a working version of the LTTng tracer for xen-unstable for x86.
> >>>>> Here is the pointer to my repository so you can try it out :
> >>>>>
> >>>>> hg clone http://ltt.polymtl.ca/cgi-bin/hgweb.cgi xen-unstable-lttng.hg
> >>>>>
> >>>>> Basic usage :
> >>>>>
> >>>>> (see lttctl-xen -h)
> >>>>>
> >>>>> lttctl-xen -c
> >>>>>
> >>>>> (in a different console)
> >>>>> lttd-xen -t /tmp/xentrace1
> >>>>>
> >>>>> (in the 1st console)
> >>>>> lttctl-xen -s
> >>>>>
> >>>>> (tracing is active)
> >>>>>
> >>>>> lttctl-xen -q
> >>>>> lttctl-xen -r
> >>>>>
> >>>>> lttd-xen should automatically quit after writing the last buffers as
> >>>>> soon as lttctl-xen -r is issued.
> >>>>>
> >>>>> Then, you must copy the XML facilities :
> >>>>>
> >>>>> (see the http://ltt.polymtl.ca > QUICKSTART to see how to install the
> >>>>> ltt-control package which contains the XML facilities in your system)
> >>>>>
> >>>>> lttctl-xen -e -t /tmp/xentrace1
> >>>>>
> >>>>> View in the visualiser : (see the QUICKSTART to see how to install it)
> >>>>>
> >>>>> lttv -m textDump -t /tmp/xentrace1
> >>>>>
> >>>>> (not tested yet) : to visualize a dom0 trace with the xen hypervisor
> >>>>> information, one would have to collect the dom0 kernel trace and the Xen
> >>>>> trace and open them together with :
> >>>>> lttv -m textDump -t /tmp/xentrace1 -t /tmp/dom0trace
> >>>>>
> >>>>> The current Linux kernel instrumentation is for 2.6.20. A backport might
> >>>>> be needed to 2.6.18 if there is no proper Xen support in 2.6.20 (I have
> >>>>> not followed the recent developments).
> >>>>>
> >>>>>
> >>>>> Currently broken/missing :
> >>>>>
> >>>>> - Ressources are not freed when the trace channels are destroyed. So you
> >>>>>   basically have to reboot between taking different traces.
> >>>>> - My code in the hypervisor complains to the console that subbuffers
> >>>>>   have not been fully read when the trace channels are destroyed. The
> >>>>>   error printing is just done too fast : lttd-xen is still there and
> >>>>>   reading the buffers at that point. It will get fixed with proper
> >>>>>   ressource usage tracking of both Xen and lttd-xen (same as the first
> >>>>>   point above).
> >>>>> - x86_64 not tested, powerpc local.h and ltt.h missing (should be ripped
> >>>>>   from my Linux kernel LTTng).
> >>>>>
> >>>>>
> >>>>> Cheers,
> >>>>>
> >>>>> Mathieu
> >>>>>
> >>>>>
> >>>>>
> >>>>> * Mathieu Desnoyers (compudj@krystal.dyndns.org) wrote:
> >>>>>> Hi,
> >>>>>>
> >>>>>> My name is Mathieu Desnoyers, I am the current maintainer of the Linux Trace
> >>>>>> Toolkit project, known as LTTng. This is a tracer for the 2.6 Linux kernels
> >>>>>> oriented towards high performance and real-time applications.
> >>>>>>
> >>>>>> I have read your tracing thread and I am surprised to see how much things
> >>>>>> you would like in a tracer are already implemented and tested in LTTng. I am
> >>>>>> currently porting my tracer to Xen, so I think it might be useful for you to
> >>>>>> know what it provides. My goal is to do not duplicate the effort and save
> >>>>>> everyone some time.
> >>>>>>
> >>>>>> Here follows some key features of LTTng :
> >>>>>>
> >>>>>> Architecture independant data types
> >>>>>> Extensible event records
> >>>>>> Self-describing traces
> >>>>>> Variable size records
> >>>>>> Fast (200 ns per event record)
> >>>>>> Highly reentrant
> >>>>>> Does not disable interrupts
> >>>>>> Does not take lock on the critical path
> >>>>>> Supports NMI tracing
> >>>>>> Analysis/visualization tool (LTTV)
> >>>>>>
> >>>>>> Looking at the integration of the existing LTTng implementation into Xen, I
> >>>>>> came up with those two points for my Christmas whichlist :
> >>>>>>
> >>>>>> Additionnal functionnalities that would be nice to have in Xen :
> >>>>>>
> >>>>>> - RCU-style updates : would allow freeing the buffers without impact on tracing.
> >>>>>>     * I guess I could currently use :
> >>>>>>       for_each_domain( d )
> >>>>>>         for_each_vcpu( d, v )
> >>>>>>           vcpu_sleep_sync(v);
> >>>>>>       I think it will have a huge impact on the system, but it would only be
> >>>>>>       performed before trace buffers free.
> >>>>>>
> >>>>>> - Polling for data in Xen from a dom0 process.
> >>>>>>   Xentrace currently polls the hypervisor each 100ms to see if there is data
> >>>>>>   that needs to be consumed. Instead of an active polling, it would be nice to
> >>>>>>   use the dom0 OS capability to put a process to sleep while waiting for a
> >>>>>>   resource. It would imply creating a module, loaded in dom0, that would wait
> >>>>>>   for a specific virq coming from the Hypervisor to wake up such processes.
> >>>>>>   We could think of exporting a complete poll() interface through sysfs or
> >>>>>>   procfs that would be a directory filled with the resources exported from the
> >>>>>>   Hypervisor to dom0 (which could include wait for resource freed, useful when
> >>>>>>   shutting down a domU instead of busy looping). It would help dom0 to schedule
> >>>>>>   other processes while a process is waiting for the Hypervisor.
> >>>>>>
> >>>>>>
> >>>>>> You might also be interested in looking at :
> >>>>>> - the website (http://ltt.polymtl.ca)
> >>>>>> - LTTng Xen port design document (this one is different from the one posted by
> >>>>>>   Jimi)
> >>>>>>   (http://ltt.polymtl.ca/svn/ltt/branches/poly/doc/developer/lttng-xen.txt)
> >>>>>> - OLS 2006 paper "The LTTng tracer : A Low Impact Performance and Behavior
> >>>>>>   Monitor for GNU/Linux"
> >>>>>>   (http://ltt.polymtl.ca/papers/desnoyers-ols2006.pdf)
> >>>>>>
> >>>>>>
> >>>>>> Questions and constructive comments are welcome.
> >>>>>>
> >>>>>> Mathieu
> >>>>>>
> >>>>>>
> >>>>>> OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
> >>>>>> Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
> >>>>>>
> >>>>>> _______________________________________________
> >>>>>> Xen-devel mailing list
> >>>>>> Xen-devel@lists.xensource.com
> >>>>>> http://lists.xensource.com/xen-devel
> >>>>>>
> >>>>
> >>
> > 
> 
> 

-- 
Mathieu Desnoyers
Computer Engineering Ph.D. Student, Ecole Polytechnique de Montreal
OpenPGP key fingerprint: 8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68

Re: LTTng Xen port : finally in a repository near you

[INAKOSHI Hiroya]

Hi Mathieu,

I see, thanks.  I'm looking forward LTTng be captured in linux kernel
first, and then, in xen.

Regards,
Hiroya

Mathieu Desnoyers wrote:
> * INAKOSHI Hiroya (inakoshi.hiroya@jp.fujitsu.com) wrote:
>> Mathieu Desnoyers wrote:
>>> * INAKOSHI Hiroya (inakoshi.hiroya@jp.fujitsu.com) wrote:
>>>> Hi Mathieu,
>>>>
>>>> thanks for your reply.  I can understand your opinion very well but a
>>>> concern is that cpu ids on a guest OS are different from those on Xen
>>>> because they are virtualized.  The number of vcpus in a guest OS is also
>>>> different from that of pcpus as you mentioned.  I wondered if the two
>>>> traces could be merged directly.  If you translate vcpu ids to pcpu ids
>>>> in writing records in the trace buffer in Xen, this concern is solved in
>>>> a natural way.
>>>>
>>> When you are executing code in dom0 or domUs, how do you plan to get the
>>> physical CPU number on which the tracing is done ?
>> I am considering the way where dom0 or domUs call hypercalls to write
>> records in the Xen's trace buffer.  In this setting, the vcpu info is
>> located in the xen kernel stack and the pcpu is the one performing the
>> hypercall.  So, I can resolve the mapping between vcpu id and pcpu id.
>>
> 
> The performance hit that goes with going through an hypercall for each
> traced event would be too high. Typically, changing ring level involves
> executing an interrupt routine, which takes a few thousands nanoseconds.
> My tracing probes runs within the traced ring in about 270ns (as tested
> on a Pentium 4 3GHz).
> 
> Mathieu
> 
>> Regards,
>> Hiroya
>>
>>>> Mathieu Desnoyers wrote:
>>>>> * INAKOSHI Hiroya (inakoshi.hiroya@jp.fujitsu.com) wrote:
>>>>>> Hi Mathieu,
>>>>>>
>>>>>> I am interested in LTTng-xen because I thought that it would be nice if
>>>>>> I can get traces on both xen and guest linux at the same time.  I
>>>>>> reviewed LTTng-xen and found that
>>>>>>
>>>>>> * LTTng and LTTng-xen have a quite similar structure,
>>>>>> * a trace buffer resides in a hypervisor for LTTng-xen,
>>>>>> * it is currently impossible to get traces from guest linux because
>>>>>> there is no LTTng for 2.6.18-xen kernel, as you mentioned.
>>>>>>
>>>>>> I had coarsely ported LTTng to 2.6.18-xen, though it is only for
>>>>>> i386.  Now I can get traces on xen and guest linux simultaneously, even
>>>>>> though they put records in different trace buffers.
>>>>> Hi Ikanoski,
>>>>>
>>>>> We did the same kind of coarse 2.6.18 port at our lab internally to get
>>>>> traces from both Linux and Xen. The fact that the traces are recorded in
>>>>> different buffers does not change anything to the fact that those trace
>>>>> files can be copied in the same trace directory so they can be parsed
>>>>> together by LTTV (traces coming from dom0, domUs and hypervisor). They
>>>>> are synchronized by using the TSCs (hopefully, you will configure your
>>>>> system to get a reliable TSC on AMD and older intels, see the
>>>>> ltt-test-tsc kernel module in recent LTTng versions and ltt.polymtl.ca
>>>>> website for info on that matter).
>>>>>
>>>>>
>>>>>> Then I thought that
>>>>>> it would be more useful if they put records in xen's trace buffer and I
>>>>>> can analyze events
>>>>> LTTV merges the information from all the valid trace files that appears
>>>>> within the trace directory, so the analysis can be done on data coming
>>>>> from userspace, kernels and hypervisor.
>>>>>
>>>>>> from xen and linux guests with a single lttd and
>>>>>> lttctl running on Domain-0.  Do you have an opinion about that?
>>>>>>
>>>>> lttctl-xen and lttd-xen, although being quite similar to lttd and
>>>>> lttctl, use hypercalls to get the data. The standard lttctl/lttd uses
>>>>> debugfs files as a hook to the trace buffers.
>>>>>
>>>>> As a distribution matter, I prefer to leave both separate for now,
>>>>> because lttctl-xen and lttd-xen is highly tied to the Xen tree.
>>>>>
>>>>> Also, merging the information within the buffers between Xen and Dom0 is
>>>>> not such a great idea: The Hypervisor and dom0 can have a different
>>>>> number of CPUs (Xen : real CPUs, dom0: vcpus). Since I use per-cpu
>>>>> buffers, it does not fit.
>>>>>
>>>>> Also, I don't want dom0 to overwrite data from the Xen buffers easily:
>>>>> it's better if we keep some protection between dom0 and the Hypervisor.
>>>>>
>>>>> Thanks for looking into this, don't hesitate to ask further questions,
>>>>>
>>>>> Mathieu
>>>>>
>>>>>> Regards,
>>>>>> Hiroya
>>>>>>
>>>>>>
>>>>>> Mathieu Desnoyers wrote:
>>>>>>> Hello,
>>>>>>>
>>>>>>> I made a working version of the LTTng tracer for xen-unstable for x86.
>>>>>>> Here is the pointer to my repository so you can try it out :
>>>>>>>
>>>>>>> hg clone http://ltt.polymtl.ca/cgi-bin/hgweb.cgi xen-unstable-lttng.hg
>>>>>>>
>>>>>>> Basic usage :
>>>>>>>
>>>>>>> (see lttctl-xen -h)
>>>>>>>
>>>>>>> lttctl-xen -c
>>>>>>>
>>>>>>> (in a different console)
>>>>>>> lttd-xen -t /tmp/xentrace1
>>>>>>>
>>>>>>> (in the 1st console)
>>>>>>> lttctl-xen -s
>>>>>>>
>>>>>>> (tracing is active)
>>>>>>>
>>>>>>> lttctl-xen -q
>>>>>>> lttctl-xen -r
>>>>>>>
>>>>>>> lttd-xen should automatically quit after writing the last buffers as
>>>>>>> soon as lttctl-xen -r is issued.
>>>>>>>
>>>>>>> Then, you must copy the XML facilities :
>>>>>>>
>>>>>>> (see the http://ltt.polymtl.ca > QUICKSTART to see how to install the
>>>>>>> ltt-control package which contains the XML facilities in your system)
>>>>>>>
>>>>>>> lttctl-xen -e -t /tmp/xentrace1
>>>>>>>
>>>>>>> View in the visualiser : (see the QUICKSTART to see how to install it)
>>>>>>>
>>>>>>> lttv -m textDump -t /tmp/xentrace1
>>>>>>>
>>>>>>> (not tested yet) : to visualize a dom0 trace with the xen hypervisor
>>>>>>> information, one would have to collect the dom0 kernel trace and the Xen
>>>>>>> trace and open them together with :
>>>>>>> lttv -m textDump -t /tmp/xentrace1 -t /tmp/dom0trace
>>>>>>>
>>>>>>> The current Linux kernel instrumentation is for 2.6.20. A backport might
>>>>>>> be needed to 2.6.18 if there is no proper Xen support in 2.6.20 (I have
>>>>>>> not followed the recent developments).
>>>>>>>
>>>>>>>
>>>>>>> Currently broken/missing :
>>>>>>>
>>>>>>> - Ressources are not freed when the trace channels are destroyed. So you
>>>>>>>   basically have to reboot between taking different traces.
>>>>>>> - My code in the hypervisor complains to the console that subbuffers
>>>>>>>   have not been fully read when the trace channels are destroyed. The
>>>>>>>   error printing is just done too fast : lttd-xen is still there and
>>>>>>>   reading the buffers at that point. It will get fixed with proper
>>>>>>>   ressource usage tracking of both Xen and lttd-xen (same as the first
>>>>>>>   point above).
>>>>>>> - x86_64 not tested, powerpc local.h and ltt.h missing (should be ripped
>>>>>>>   from my Linux kernel LTTng).
>>>>>>>
>>>>>>>
>>>>>>> Cheers,
>>>>>>>
>>>>>>> Mathieu
>>>>>>>
>>>>>>>
>>>>>>>
>>>>>>> * Mathieu Desnoyers (compudj@krystal.dyndns.org) wrote:
>>>>>>>> Hi,
>>>>>>>>
>>>>>>>> My name is Mathieu Desnoyers, I am the current maintainer of the Linux Trace
>>>>>>>> Toolkit project, known as LTTng. This is a tracer for the 2.6 Linux kernels
>>>>>>>> oriented towards high performance and real-time applications.
>>>>>>>>
>>>>>>>> I have read your tracing thread and I am surprised to see how much things
>>>>>>>> you would like in a tracer are already implemented and tested in LTTng. I am
>>>>>>>> currently porting my tracer to Xen, so I think it might be useful for you to
>>>>>>>> know what it provides. My goal is to do not duplicate the effort and save
>>>>>>>> everyone some time.
>>>>>>>>
>>>>>>>> Here follows some key features of LTTng :
>>>>>>>>
>>>>>>>> Architecture independant data types
>>>>>>>> Extensible event records
>>>>>>>> Self-describing traces
>>>>>>>> Variable size records
>>>>>>>> Fast (200 ns per event record)
>>>>>>>> Highly reentrant
>>>>>>>> Does not disable interrupts
>>>>>>>> Does not take lock on the critical path
>>>>>>>> Supports NMI tracing
>>>>>>>> Analysis/visualization tool (LTTV)
>>>>>>>>
>>>>>>>> Looking at the integration of the existing LTTng implementation into Xen, I
>>>>>>>> came up with those two points for my Christmas whichlist :
>>>>>>>>
>>>>>>>> Additionnal functionnalities that would be nice to have in Xen :
>>>>>>>>
>>>>>>>> - RCU-style updates : would allow freeing the buffers without impact on tracing.
>>>>>>>>     * I guess I could currently use :
>>>>>>>>       for_each_domain( d )
>>>>>>>>         for_each_vcpu( d, v )
>>>>>>>>           vcpu_sleep_sync(v);
>>>>>>>>       I think it will have a huge impact on the system, but it would only be
>>>>>>>>       performed before trace buffers free.
>>>>>>>>
>>>>>>>> - Polling for data in Xen from a dom0 process.
>>>>>>>>   Xentrace currently polls the hypervisor each 100ms to see if there is data
>>>>>>>>   that needs to be consumed. Instead of an active polling, it would be nice to
>>>>>>>>   use the dom0 OS capability to put a process to sleep while waiting for a
>>>>>>>>   resource. It would imply creating a module, loaded in dom0, that would wait
>>>>>>>>   for a specific virq coming from the Hypervisor to wake up such processes.
>>>>>>>>   We could think of exporting a complete poll() interface through sysfs or
>>>>>>>>   procfs that would be a directory filled with the resources exported from the
>>>>>>>>   Hypervisor to dom0 (which could include wait for resource freed, useful when
>>>>>>>>   shutting down a domU instead of busy looping). It would help dom0 to schedule
>>>>>>>>   other processes while a process is waiting for the Hypervisor.
>>>>>>>>
>>>>>>>>
>>>>>>>> You might also be interested in looking at :
>>>>>>>> - the website (http://ltt.polymtl.ca)
>>>>>>>> - LTTng Xen port design document (this one is different from the one posted by
>>>>>>>>   Jimi)
>>>>>>>>   (http://ltt.polymtl.ca/svn/ltt/branches/poly/doc/developer/lttng-xen.txt)
>>>>>>>> - OLS 2006 paper "The LTTng tracer : A Low Impact Performance and Behavior
>>>>>>>>   Monitor for GNU/Linux"
>>>>>>>>   (http://ltt.polymtl.ca/papers/desnoyers-ols2006.pdf)
>>>>>>>>
>>>>>>>>
>>>>>>>> Questions and constructive comments are welcome.
>>>>>>>>
>>>>>>>> Mathieu
>>>>>>>>
>>>>>>>>
>>>>>>>> OpenPGP public key:              http://krystal.dyndns.org:8080/key/compudj.gpg
>>>>>>>> Key fingerprint:     8CD5 52C3 8E3C 4140 715F  BA06 3F25 A8FE 3BAE 9A68 
>>>>>>>>
>>>>>>>> _______________________________________________
>>>>>>>> Xen-devel mailing list
>>>>>>>> Xen-devel@lists.xensource.com
>>>>>>>> http://lists.xensource.com/xen-devel
>>>>>>>>
>>
> 

[RFC][PATCH] 2/3] [TOOLS][XENTRACE] Update tools to write data to disk in a known endian'ness.

[Tony Breeds]

Use network byte order when writing data to disk, making the data portable to
any machine.

Signed-off-by: Tony Breeds <tony@bakeyournoodle.com>
---

 tools/xentrace/xentrace.c      |   47 ++++++++++++++++++++++++++++++++++-------
 tools/xentrace/xentrace_format |    7 +++---
 2 files changed, 44 insertions(+), 10 deletions(-)

Index: xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace.c
===================================================================
--- xen-unstable.hg-mainline.xentrace.orig/tools/xentrace/xentrace.c
+++ xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace.c
@@ -28,6 +28,20 @@
 
 #include <xenctrl.h>
 
+#include <arpa/inet.h> /* hton*(), ntoh*() */
+#include <endian.h>
+
+/* There is no 64-bit htonll, so create one */
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+#define        htonll(x)   ( (((uint64_t)htonl(x)) << 32) + htonl(x >> 32) )
+#define        ntohll(x)   ( (((uint64_t)ntohl(x)) << 32) + ntohl(x >> 32) )
+#else
+#define        htonll(x)   (x)
+#define        ntohll(x)   (x)
+#endif
+
+#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
+
 #define PERROR(_m, _a...)                                       \
 do {                                                            \
     int __saved_errno = errno;                                  \
@@ -90,12 +104,30 @@ struct timespec millis_to_timespec(unsig
  * Outputs the trace record to a filestream, prepending the CPU ID of the
  * source trace buffer.
  */
-void write_rec(unsigned int cpu, struct t_rec *rec, FILE *out)
+void write_rec(uint32_t cpu, struct t_rec *rec, FILE *out)
 {
     size_t written = 0;
-    written += fwrite(&cpu, sizeof(cpu), 1, out);
-    written += fwrite(rec, sizeof(*rec), 1, out);
-    if ( written != 2 )
+    int i;
+    /* Place network byte order representation in temp vars, rather than
+     * write back into kernel/xen memory */
+    uint64_t tmp64;
+    uint32_t tmp32;
+
+    tmp32 = htonl(cpu);
+    written += fwrite(&tmp32, sizeof(tmp32), 1, out);
+
+    tmp64 = htonll(rec->cycles);
+    written += fwrite(&tmp64, sizeof(tmp64), 1, out);
+
+    tmp32 = htonl(rec->event);
+    written += fwrite(&tmp32, sizeof(tmp32), 1, out);
+
+    for ( i=0; i<ARRAY_SIZE(rec->data); i++ ) {
+        tmp64 = htonl(rec->data[i]);
+        written += fwrite(&tmp64, sizeof(tmp64), 1, out);
+    }
+
+    if ( written != 8 )
     {
         PERROR("Failed to write trace record");
         exit(EXIT_FAILURE);
@@ -147,6 +179,7 @@ struct t_buf *map_tbufs(unsigned long tb
         exit(EXIT_FAILURE);
     }
 
+    /* On PPC (At least) the DOMID arg is ignored in dom0 */
     tbufs_mapped = xc_map_foreign_range(xc_handle, DOMID_XEN,
                                         size * num, PROT_READ | PROT_WRITE,
                                         tbufs_mfn);
@@ -253,7 +286,7 @@ struct t_rec **init_rec_ptrs(struct t_bu
 /**
  * get_num_cpus - get the number of logical CPUs
  */
-unsigned int get_num_cpus(void)
+uint32_t get_num_cpus(void)
 {
     xc_physinfo_t physinfo;
     int xc_handle = xc_interface_open();
@@ -282,14 +315,14 @@ unsigned int get_num_cpus(void)
  */
 int monitor_tbufs(FILE *logfile)
 {
-    int i;
+    uint32_t i;
 
     void *tbufs_mapped;          /* pointer to where the tbufs are mapped    */
     struct t_buf **meta;         /* pointers to the trace buffer metadata    */
     struct t_rec **data;         /* pointers to the trace buffer data areas
                                   * where they are mapped into user space.   */
     unsigned long tbufs_mfn;     /* mfn of the tbufs                         */
-    unsigned int  num;           /* number of trace buffers / logical CPUS   */
+    uint32_t num;                /* number of trace buffers / logical CPUS   */
     unsigned long size;          /* size of a single trace buffer            */
 
     int size_in_recs;
Index: xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace_format
===================================================================
--- xen-unstable.hg-mainline.xentrace.orig/tools/xentrace/xentrace_format
+++ xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace_format
@@ -84,10 +84,11 @@ interrupted = 0
 defs = read_defs(arg[0])
 
 # structure of trace record + prepended CPU id (as output by xentrace):
-# CPU(I) TSC(Q) EVENT(L) D1(L) D2(L) D3(L) D4(L) D5(L)
+# CPU(L) TSC(Q) EVENT(L) D1(Q) D2(Q) D3(Q) D4(Q) D5(Q)
 # read CPU id separately to avoid structure packing problems on 64-bit arch.
-CPUREC = "I"
-TRCREC = "QLLLLLL"
+# Force network byte order.
+CPUREC = "!L"
+TRCREC = "!QLQQQQQ"
 
 last_tsc = [0]

Re: [RFC][PATCH] 2/3] [TOOLS][XENTRACE] Update tools to write data to disk in a known endian'ness.

[Mark Williamson]

> Use network byte order when writing data to disk, making the data portable
> to any machine.

Thanks for splitting this from the formatting changes - it's a big help.

No objections to this here, seems like a nice improvement.  I did wonder if 
perhaps some sort of trace file header would be useful, giving metadata.  For 
instance, a record of what dom0 / Xen builds were being used, and type of 
machine might be useful.  A future extensible format would be a plus too, I 
guess.  None of this is strictly necessary, it just might help with 
organising trace files.

Cheers,
Mark

> Signed-off-by: Tony Breeds <tony@bakeyournoodle.com>
> ---
>
>  tools/xentrace/xentrace.c      |   47
> ++++++++++++++++++++++++++++++++++------- tools/xentrace/xentrace_format | 
>   7 +++---
>  2 files changed, 44 insertions(+), 10 deletions(-)
>
> Index: xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace.c
> ===================================================================
> --- xen-unstable.hg-mainline.xentrace.orig/tools/xentrace/xentrace.c
> +++ xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace.c
> @@ -28,6 +28,20 @@
>
>  #include <xenctrl.h>
>
> +#include <arpa/inet.h> /* hton*(), ntoh*() */
> +#include <endian.h>
> +
> +/* There is no 64-bit htonll, so create one */
> +#if __BYTE_ORDER == __LITTLE_ENDIAN
> +#define        htonll(x)   ( (((uint64_t)htonl(x)) << 32) + htonl(x >> 32)
> ) +#define        ntohll(x)   ( (((uint64_t)ntohl(x)) << 32) + ntohl(x >>
> 32) ) +#else
> +#define        htonll(x)   (x)
> +#define        ntohll(x)   (x)
> +#endif
> +
> +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
> +
>  #define PERROR(_m, _a...)                                       \
>  do {                                                            \
>      int __saved_errno = errno;                                  \
> @@ -90,12 +104,30 @@ struct timespec millis_to_timespec(unsig
>   * Outputs the trace record to a filestream, prepending the CPU ID of the
>   * source trace buffer.
>   */
> -void write_rec(unsigned int cpu, struct t_rec *rec, FILE *out)
> +void write_rec(uint32_t cpu, struct t_rec *rec, FILE *out)
>  {
>      size_t written = 0;
> -    written += fwrite(&cpu, sizeof(cpu), 1, out);
> -    written += fwrite(rec, sizeof(*rec), 1, out);
> -    if ( written != 2 )
> +    int i;
> +    /* Place network byte order representation in temp vars, rather than
> +     * write back into kernel/xen memory */
> +    uint64_t tmp64;
> +    uint32_t tmp32;
> +
> +    tmp32 = htonl(cpu);
> +    written += fwrite(&tmp32, sizeof(tmp32), 1, out);
> +
> +    tmp64 = htonll(rec->cycles);
> +    written += fwrite(&tmp64, sizeof(tmp64), 1, out);
> +
> +    tmp32 = htonl(rec->event);
> +    written += fwrite(&tmp32, sizeof(tmp32), 1, out);
> +
> +    for ( i=0; i<ARRAY_SIZE(rec->data); i++ ) {
> +        tmp64 = htonl(rec->data[i]);
> +        written += fwrite(&tmp64, sizeof(tmp64), 1, out);
> +    }
> +
> +    if ( written != 8 )
>      {
>          PERROR("Failed to write trace record");
>          exit(EXIT_FAILURE);
> @@ -147,6 +179,7 @@ struct t_buf *map_tbufs(unsigned long tb
>          exit(EXIT_FAILURE);
>      }
>
> +    /* On PPC (At least) the DOMID arg is ignored in dom0 */
>      tbufs_mapped = xc_map_foreign_range(xc_handle, DOMID_XEN,
>                                          size * num, PROT_READ |
> PROT_WRITE, tbufs_mfn);
> @@ -253,7 +286,7 @@ struct t_rec **init_rec_ptrs(struct t_bu
>  /**
>   * get_num_cpus - get the number of logical CPUs
>   */
> -unsigned int get_num_cpus(void)
> +uint32_t get_num_cpus(void)
>  {
>      xc_physinfo_t physinfo;
>      int xc_handle = xc_interface_open();
> @@ -282,14 +315,14 @@ unsigned int get_num_cpus(void)
>   */
>  int monitor_tbufs(FILE *logfile)
>  {
> -    int i;
> +    uint32_t i;
>
>      void *tbufs_mapped;          /* pointer to where the tbufs are mapped 
>   */ struct t_buf **meta;         /* pointers to the trace buffer metadata 
>   */ struct t_rec **data;         /* pointers to the trace buffer data
> areas * where they are mapped into user space.   */ unsigned long
> tbufs_mfn;     /* mfn of the tbufs                         */ -    unsigned
> int  num;           /* number of trace buffers / logical CPUS   */ +   
> uint32_t num;                /* number of trace buffers / logical CPUS   */
> unsigned long size;          /* size of a single trace buffer            */
>
>      int size_in_recs;
> Index: xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace_format
> ===================================================================
> --- xen-unstable.hg-mainline.xentrace.orig/tools/xentrace/xentrace_format
> +++ xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace_format
> @@ -84,10 +84,11 @@ interrupted = 0
>  defs = read_defs(arg[0])
>
>  # structure of trace record + prepended CPU id (as output by xentrace):
> -# CPU(I) TSC(Q) EVENT(L) D1(L) D2(L) D3(L) D4(L) D5(L)
> +# CPU(L) TSC(Q) EVENT(L) D1(Q) D2(Q) D3(Q) D4(Q) D5(Q)
>  # read CPU id separately to avoid structure packing problems on 64-bit
> arch. -CPUREC = "I"
> -TRCREC = "QLLLLLL"
> +# Force network byte order.
> +CPUREC = "!L"
> +TRCREC = "!QLQQQQQ"
>
>  last_tsc = [0]
>
>
>
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel

-- 
Dave: Just a question. What use is a unicyle with no seat?  And no pedals!
Mark: To answer a question with a question: What use is a skateboard?
Dave: Skateboards have wheels.
Mark: My wheel has a wheel!

Re: [RFC][PATCH] 2/3] [TOOLS][XENTRACE] Update tools to write data to disk in a known endian'ness.

[Tony Breeds]

On Thu, Nov 30, 2006 at 01:12:43PM +0000, Mark Williamson wrote:
 
> No objections to this here, seems like a nice improvement.  

That's what we like to hear ;P

>                                                             I did wonder if 
> perhaps some sort of trace file header would be useful, giving metadata.  For 
> instance, a record of what dom0 / Xen builds were being used, and type of 
> machine might be useful. 

I did think about doing something similar.  I though I could steal the
high 16bits of the first CPU number in the file as a xentrace format
version number.  Sure this would limit us to 64k CPUs but we can deal
with that limit later ;P 

>                           A future extensible format would be a plus too, I 
> guess.  None of this is strictly necessary, it just might help with 
> organising trace files.

You spoke in more detail about that in another email so I'll keep the
discussion there.

Yours Tony

   linux.conf.au       http://linux.conf.au/ || http://lca2007.linux.org.au/
   Jan 15-20 2007      The Australian Linux Technical Conference!

Re: [RFC][PATCH] 2/3] [TOOLS][XENTRACE] Update tools to write data to disk in a known endian'ness.

[Mark Williamson]

> >                                                             I did wonder
> > if perhaps some sort of trace file header would be useful, giving
> > metadata.  For instance, a record of what dom0 / Xen builds were being
> > used, and type of machine might be useful.
>
> I did think about doing something similar.  I though I could steal the
> high 16bits of the first CPU number in the file as a xentrace format
> version number.  Sure this would limit us to 64k CPUs but we can deal
> with that limit later ;P

Since traces are typically not going to be longlived but used for 
problem-specific debugging work, I think we can probably just make a clean 
break and define a new format.

Is anybody is really relying on the trace format being stable?  Please speak 
up now :-)

Cheers,
Mark


> >                           A future extensible format would be a plus too,
> > I guess.  None of this is strictly necessary, it just might help with
> > organising trace files.
>
> You spoke in more detail about that in another email so I'll keep the
> discussion there.
>
> Yours Tony
>
>    linux.conf.au       http://linux.conf.au/ ||
> http://lca2007.linux.org.au/ Jan 15-20 2007      The Australian Linux
> Technical Conference!

-- 
Dave: Just a question. What use is a unicyle with no seat?  And no pedals!
Mark: To answer a question with a question: What use is a skateboard?
Dave: Skateboards have wheels.
Mark: My wheel has a wheel!

Re: [RFC][PATCH] 2/3] [TOOLS][XENTRACE] Update tools to write data to disk in a known endian'ness.

[George Dunlap]

Do you have an idea how much extra cpu this takes for high-bandwidth tracing?

I'm using xentrace for performance analysis, and it's already got a
measurable impact due to the disk bandwidth and copying.  I'd like to
keep that as minimal as possible.

 -George

On 11/30/06, Tony Breeds <tony@bakeyournoodle.com> wrote:
> Use network byte order when writing data to disk, making the data portable to
> any machine.
>
> Signed-off-by: Tony Breeds <tony@bakeyournoodle.com>
> ---
>
>  tools/xentrace/xentrace.c      |   47 ++++++++++++++++++++++++++++++++++-------
>  tools/xentrace/xentrace_format |    7 +++---
>  2 files changed, 44 insertions(+), 10 deletions(-)
>
> Index: xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace.c
> ===================================================================
> --- xen-unstable.hg-mainline.xentrace.orig/tools/xentrace/xentrace.c
> +++ xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace.c
> @@ -28,6 +28,20 @@
>
>  #include <xenctrl.h>
>
> +#include <arpa/inet.h> /* hton*(), ntoh*() */
> +#include <endian.h>
> +
> +/* There is no 64-bit htonll, so create one */
> +#if __BYTE_ORDER == __LITTLE_ENDIAN
> +#define        htonll(x)   ( (((uint64_t)htonl(x)) << 32) + htonl(x >> 32) )
> +#define        ntohll(x)   ( (((uint64_t)ntohl(x)) << 32) + ntohl(x >> 32) )
> +#else
> +#define        htonll(x)   (x)
> +#define        ntohll(x)   (x)
> +#endif
> +
> +#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
> +
>  #define PERROR(_m, _a...)                                       \
>  do {                                                            \
>      int __saved_errno = errno;                                  \
> @@ -90,12 +104,30 @@ struct timespec millis_to_timespec(unsig
>   * Outputs the trace record to a filestream, prepending the CPU ID of the
>   * source trace buffer.
>   */
> -void write_rec(unsigned int cpu, struct t_rec *rec, FILE *out)
> +void write_rec(uint32_t cpu, struct t_rec *rec, FILE *out)
>  {
>      size_t written = 0;
> -    written += fwrite(&cpu, sizeof(cpu), 1, out);
> -    written += fwrite(rec, sizeof(*rec), 1, out);
> -    if ( written != 2 )
> +    int i;
> +    /* Place network byte order representation in temp vars, rather than
> +     * write back into kernel/xen memory */
> +    uint64_t tmp64;
> +    uint32_t tmp32;
> +
> +    tmp32 = htonl(cpu);
> +    written += fwrite(&tmp32, sizeof(tmp32), 1, out);
> +
> +    tmp64 = htonll(rec->cycles);
> +    written += fwrite(&tmp64, sizeof(tmp64), 1, out);
> +
> +    tmp32 = htonl(rec->event);
> +    written += fwrite(&tmp32, sizeof(tmp32), 1, out);
> +
> +    for ( i=0; i<ARRAY_SIZE(rec->data); i++ ) {
> +        tmp64 = htonl(rec->data[i]);
> +        written += fwrite(&tmp64, sizeof(tmp64), 1, out);
> +    }
> +
> +    if ( written != 8 )
>      {
>          PERROR("Failed to write trace record");
>          exit(EXIT_FAILURE);
> @@ -147,6 +179,7 @@ struct t_buf *map_tbufs(unsigned long tb
>          exit(EXIT_FAILURE);
>      }
>
> +    /* On PPC (At least) the DOMID arg is ignored in dom0 */
>      tbufs_mapped = xc_map_foreign_range(xc_handle, DOMID_XEN,
>                                          size * num, PROT_READ | PROT_WRITE,
>                                          tbufs_mfn);
> @@ -253,7 +286,7 @@ struct t_rec **init_rec_ptrs(struct t_bu
>  /**
>   * get_num_cpus - get the number of logical CPUs
>   */
> -unsigned int get_num_cpus(void)
> +uint32_t get_num_cpus(void)
>  {
>      xc_physinfo_t physinfo;
>      int xc_handle = xc_interface_open();
> @@ -282,14 +315,14 @@ unsigned int get_num_cpus(void)
>   */
>  int monitor_tbufs(FILE *logfile)
>  {
> -    int i;
> +    uint32_t i;
>
>      void *tbufs_mapped;          /* pointer to where the tbufs are mapped    */
>      struct t_buf **meta;         /* pointers to the trace buffer metadata    */
>      struct t_rec **data;         /* pointers to the trace buffer data areas
>                                    * where they are mapped into user space.   */
>      unsigned long tbufs_mfn;     /* mfn of the tbufs                         */
> -    unsigned int  num;           /* number of trace buffers / logical CPUS   */
> +    uint32_t num;                /* number of trace buffers / logical CPUS   */
>      unsigned long size;          /* size of a single trace buffer            */
>
>      int size_in_recs;
> Index: xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace_format
> ===================================================================
> --- xen-unstable.hg-mainline.xentrace.orig/tools/xentrace/xentrace_format
> +++ xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace_format
> @@ -84,10 +84,11 @@ interrupted = 0
>  defs = read_defs(arg[0])
>
>  # structure of trace record + prepended CPU id (as output by xentrace):
> -# CPU(I) TSC(Q) EVENT(L) D1(L) D2(L) D3(L) D4(L) D5(L)
> +# CPU(L) TSC(Q) EVENT(L) D1(Q) D2(Q) D3(Q) D4(Q) D5(Q)
>  # read CPU id separately to avoid structure packing problems on 64-bit arch.
> -CPUREC = "I"
> -TRCREC = "QLLLLLL"
> +# Force network byte order.
> +CPUREC = "!L"
> +TRCREC = "!QLQQQQQ"
>
>  last_tsc = [0]
>
>
>
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel
>

Re: [RFC][PATCH] 2/3] [TOOLS][XENTRACE] Update tools to write data to disk in a known endian'ness.

[Tony Breeds]

On Thu, Nov 30, 2006 at 11:27:27AM -0500, George Dunlap  wrote:
> Do you have an idea how much extra cpu this takes for high-bandwidth 
> tracing?

I don't imagine that hton*() are that expensive, and fwrite already
buffers intreranll for 128k writes so with these patches you won't be
touching the disk anymore often.
 
> I'm using xentrace for performance analysis, and it's already got a
> measurable impact due to the disk bandwidth and copying.  I'd like to
> keep that as minimal as possible.

To be honnest I'm not doing any tracing, this patchset is the start
required to use xentrace on PPC.

Yours Tony

   linux.conf.au       http://linux.conf.au/ || http://lca2007.linux.org.au/
   Jan 15-20 2007      The Australian Linux Technical Conference!

[RFC][PATCH] 3/3] [TOOLS][XENTRACE] Various tidyups to xentrace tools.

[Tony Breeds]

 - use err/errx/warn, instead of PERROR, perror and fprintf
 - Match () place ment consistent in this files
 - Use consistent tabs and whitespacing.

Signed-off-by: Tony Breeds <tony@bakeyournoodle.com>
---

 tools/xentrace/xentrace.c      |  137 ++++++++++++++---------------------------
 tools/xentrace/xentrace_format |   51 +++++++--------
 2 files changed, 73 insertions(+), 115 deletions(-)

Index: xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace.c
===================================================================
--- xen-unstable.hg-mainline.xentrace.orig/tools/xentrace/xentrace.c
+++ xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace.c
@@ -22,6 +22,7 @@
 #include <signal.h>
 #include <inttypes.h>
 #include <string.h>
+#include <err.h>
 
 #include <xen/xen.h>
 #include <xen/trace.h>
@@ -42,16 +43,6 @@
 
 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
 
-#define PERROR(_m, _a...)                                       \
-do {                                                            \
-    int __saved_errno = errno;                                  \
-    fprintf(stderr, "ERROR: " _m " (%d = %s)\n" , ## _a ,       \
-            __saved_errno, strerror(__saved_errno));            \
-    errno = __saved_errno;                                      \
-} while (0)
-
-extern FILE *stderr;
-
 /***** Compile time configuration of defaults ********************************/
 
 /* when we've got more records than this waiting, we log it to the output */
@@ -88,7 +79,7 @@ void close_handler(int signal)
 struct timespec millis_to_timespec(unsigned long millis)
 {
     struct timespec spec;
-    
+
     spec.tv_sec = millis / 1000;
     spec.tv_nsec = (millis % 1000) * 1000;
 
@@ -128,10 +119,7 @@ void write_rec(uint32_t cpu, struct t_re
     }
 
     if ( written != 8 )
-    {
-        PERROR("Failed to write trace record");
-        exit(EXIT_FAILURE);
-    }
+        err(EXIT_FAILURE, "Failed to write trace record");
 }
 
 static void get_tbufs(unsigned long *mfn, unsigned long *size)
@@ -139,21 +127,16 @@ static void get_tbufs(unsigned long *mfn
     int xc_handle = xc_interface_open();
     int ret;
 
-    if ( xc_handle < 0 ) 
-    {
-        exit(EXIT_FAILURE);
-    }
+    if ( xc_handle < 0 )
+        errx(EXIT_FAILURE, "Unable to open the xc interface");
 
-    if(!opts.tbuf_size)
-      opts.tbuf_size = DEFAULT_TBUF_SIZE;
+    if ( !opts.tbuf_size )
+        opts.tbuf_size = DEFAULT_TBUF_SIZE;
 
     ret = xc_tbuf_enable(xc_handle, opts.tbuf_size, mfn, size);
 
     if ( ret != 0 )
-    {
-        perror("Couldn't enable trace buffers");
-        exit(1);
-    }
+        err(EXIT_FAILURE, "Couldn't enable trace buffers");
 
     xc_interface_close(xc_handle);
 }
@@ -174,10 +157,8 @@ struct t_buf *map_tbufs(unsigned long tb
 
     xc_handle = xc_interface_open();
 
-    if ( xc_handle < 0 ) 
-    {
-        exit(EXIT_FAILURE);
-    }
+    if ( xc_handle < 0 )
+        errx(EXIT_FAILURE, "Unable to open the xc interface");
 
     /* On PPC (At least) the DOMID arg is ignored in dom0 */
     tbufs_mapped = xc_map_foreign_range(xc_handle, DOMID_XEN,
@@ -186,18 +167,15 @@ struct t_buf *map_tbufs(unsigned long tb
 
     xc_interface_close(xc_handle);
 
-    if ( tbufs_mapped == 0 ) 
-    {
-        PERROR("Failed to mmap trace buffers");
-        exit(EXIT_FAILURE);
-    }
+    if ( tbufs_mapped == 0 )
+        err(EXIT_FAILURE, "Failed to mmap trace buffers");
 
     return tbufs_mapped;
 }
 
 /**
  * set_mask - set the cpu/event mask in HV
- * @mask:           the new mask 
+ * @mask:           the new mask
  * @type:           the new mask type,0-event mask, 1-cpu mask
  *
  */
@@ -206,21 +184,19 @@ void set_mask(uint32_t mask, int type)
     int ret = 0;
     int xc_handle = xc_interface_open(); /* for accessing control interface */
 
-    if (type == 1) {
+    if ( type == 1 ) {
         ret = xc_tbuf_set_cpu_mask(xc_handle, mask);
-        fprintf(stderr, "change cpumask to 0x%x\n", mask);
-    } else if (type == 0) {
+        warnx("change cpumask to 0x%x\n", mask);
+    } else if ( type == 0 ) {
         ret = xc_tbuf_set_evt_mask(xc_handle, mask);
-        fprintf(stderr, "change evtmask to 0x%x\n", mask);
+        warnx("change evtmask to 0x%x\n", mask);
     }
 
     xc_interface_close(xc_handle);
 
     if ( ret != 0 )
-    {
-        PERROR("Failure to get trace buffer pointer from Xen and set the new mask");
-        exit(EXIT_FAILURE);
-    }
+        err(EXIT_FAILURE, "Failure to get trace buffer pointer from "
+	                  "Xen and set the new mask");
 }
 
 /**
@@ -240,11 +216,8 @@ struct t_buf **init_bufs_ptrs(void *bufs
 
     user_ptrs = (struct t_buf **)calloc(num, sizeof(struct t_buf *));
     if ( user_ptrs == NULL )
-    {
-        PERROR( "Failed to allocate memory for buffer pointers\n");
-        exit(EXIT_FAILURE);
-    }
-    
+        err(EXIT_FAILURE, "Failed to allocate memory for buffer pointers");
+
     /* initialise pointers to the trace buffers - given the size of a trace
      * buffer and the value of bufs_maped, we can easily calculate these */
     for ( i = 0; i<num; i++ )
@@ -269,13 +242,10 @@ struct t_rec **init_rec_ptrs(struct t_bu
 {
     int i;
     struct t_rec **data;
-    
+
     data = calloc(num, sizeof(struct t_rec *));
     if ( data == NULL )
-    {
-        PERROR("Failed to allocate memory for data pointers\n");
-        exit(EXIT_FAILURE);
-    }
+        err(EXIT_FAILURE, "Failed to allocate memory for data pointers");
 
     for ( i = 0; i < num; i++ )
         data[i] = (struct t_rec *)(meta[i] + 1);
@@ -291,14 +261,11 @@ uint32_t get_num_cpus(void)
     xc_physinfo_t physinfo;
     int xc_handle = xc_interface_open();
     int ret;
-    
+
     ret = xc_physinfo(xc_handle, &physinfo);
-    
+
     if ( ret != 0 )
-    {
-        PERROR("Failure to get logical CPU count from Xen");
-        exit(EXIT_FAILURE);
-    }
+        errx(EXIT_FAILURE, "Failure to get logical CPU count from Xen");
 
     xc_interface_close(xc_handle);
 
@@ -377,17 +344,17 @@ int parse_evtmask(char *arg, struct argp
     char *inval;
 
     /* search filtering class */
-    if (strcmp(arg, "gen") == 0){ 
+    if ( strcmp(arg, "gen") == 0 )
         setup->evt_mask |= TRC_GEN;
-    } else if(strcmp(arg, "sched") == 0){ 
+    else if ( strcmp(arg, "sched") == 0 )
         setup->evt_mask |= TRC_SCHED;
-    } else if(strcmp(arg, "dom0op") == 0){ 
+    else if ( strcmp(arg, "dom0op") == 0 )
         setup->evt_mask |= TRC_DOM0OP;
-    } else if(strcmp(arg, "vmx") == 0){ 
+    else if ( strcmp(arg, "vmx") == 0 )
         setup->evt_mask |= TRC_VMX;
-    } else if(strcmp(arg, "all") == 0){ 
+    else if ( strcmp(arg, "all") == 0 )
         setup->evt_mask |= TRC_ALL;
-    } else {
+    else {
         setup->evt_mask = strtol(arg, &inval, 0);
         if ( inval == arg )
             argp_usage(state);
@@ -430,13 +397,13 @@ error_t cmd_parser(int key, char *arg, s
             argp_usage(state);
     }
     break;
-    
+
     case 'e': /* set new event mask for filtering*/
     {
         parse_evtmask(arg, state);
     }
     break;
-    
+
     case 'S': /* set tbuf size (given in pages) */
     {
         char *inval;
@@ -454,7 +421,7 @@ error_t cmd_parser(int key, char *arg, s
             argp_usage(state);
     }
     break;
-        
+
     default:
         return ARGP_ERR_UNKNOWN;
     }
@@ -473,16 +440,16 @@ const struct argp_option cmd_opts[] =
       "(default " xstr(NEW_DATA_THRESH) ")." },
 
     { .name = "poll-sleep", .key='s', .arg="p",
-      .doc = 
+      .doc =
       "Set sleep time, p, in milliseconds between polling the trace buffer "
       "for new data (default " xstr(POLL_SLEEP_MILLIS) ")." },
 
     { .name = "cpu-mask", .key='c', .arg="c",
-      .doc = 
+      .doc =
       "set cpu-mask " },
 
     { .name = "evt-mask", .key='e', .arg="e",
-      .doc = 
+      .doc =
       "set evt-mask " },
 
     { .name = "trace-buf-size", .key='S', .arg="N",
@@ -504,7 +471,7 @@ const struct argp parser_def =
     "\v"
     "This tool is used to capture trace buffer data from Xen.  The data is "
     "output in a binary format, in the following order:\n\n"
-    "  CPU(uint) TSC(uint64_t) EVENT(uint32_t) D1 D2 D3 D4 D5 "
+    "  CPU(uint32_t) TSC(uint64_t) EVENT(uint32_t) D1 D2 D3 D4 D5 "
     "(all uint32_t)\n\n"
     "The output should be parsed using the tool xentrace_format, which can "
     "produce human-readable output in ASCII format."
@@ -513,8 +480,8 @@ const struct argp parser_def =
 
 const char *argp_program_version     = "xentrace v1.1";
 const char *argp_program_bug_address = "<mark.a.williamson@intel.com>";
-        
-    
+
+
 int main(int argc, char **argv)
 {
     int outfd = 1, ret;
@@ -529,31 +496,23 @@ int main(int argc, char **argv)
 
     argp_parse(&parser_def, argc, argv, 0, 0, &opts);
 
-    if (opts.evt_mask != 0) { 
+    if ( opts.evt_mask != 0 )
         set_mask(opts.evt_mask, 0);
-    }
 
-    if (opts.cpu_mask != 0) {
+    if ( opts.cpu_mask != 0 )
         set_mask(opts.cpu_mask, 1);
-    }
 
     if ( opts.outfile )
         outfd = open(opts.outfile, O_WRONLY | O_CREAT | O_LARGEFILE, 0644);
 
-    if(outfd < 0)
-    {
-        perror("Could not open output file");
-        exit(EXIT_FAILURE);
-    }        
+    if ( outfd < 0 )
+        err(EXIT_FAILURE, "Could not open output file");
 
-    if(isatty(outfd))
-    {
-        fprintf(stderr, "Cannot output to a TTY, specify a log file.\n");
-        exit(EXIT_FAILURE);
-    }
+    if ( isatty(outfd) )
+        errx(EXIT_FAILURE, "Cannot output to a TTY, specify a log file.\n");
 
     logfile = fdopen(outfd, "w");
-    
+
     /* ensure that if we get a signal, we'll do cleanup, then exit */
     act.sa_handler = close_handler;
     act.sa_flags = 0;
Index: xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace_format
===================================================================
--- xen-unstable.hg-mainline.xentrace.orig/tools/xentrace/xentrace_format
+++ xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace_format
@@ -24,7 +24,7 @@ def usage():
           Which correspond to the CPU number, event ID, timestamp counter and
           the 5 data fields from the trace record.  There should be one such
           rule for each type of event.
-          
+
           Depending on your system and the volume of trace buffer data,
           this script may not be able to keep up with the output of xentrace
           if it is piped directly.  In these circumstances you should have
@@ -34,7 +34,7 @@ def usage():
 
 def read_defs(defs_file):
     defs = {}
-    
+
     fd = open(defs_file)
 
     reg = re.compile('(\S+)\s+(\S.*)')
@@ -43,14 +43,14 @@ def read_defs(defs_file):
         line = fd.readline()
         if not line:
             break
-	
-	if line[0] == '#' or line[0] == '\n':
-	    continue
-        
+
+        if line[0] == '#' or line[0] == '\n':
+            continue
+
         m = reg.match(line)
 
         if not m: print >> sys.stderr, "Bad format file" ; sys.exit(1)
-        
+
         defs[str(eval(m.group(1)))] = m.group(2)
 
     return defs
@@ -70,7 +70,7 @@ try:
     opts, arg = getopt.getopt(sys.argv[1:], "c:" )
 
     for opt in opts:
-	if opt[0] == '-c' : mhz = int(opt[1])
+        if opt[0] == '-c' : mhz = int(opt[1])
 
 except getopt.GetoptError:
     usage()
@@ -96,7 +96,7 @@ i=0
 
 while not interrupted:
     try:
-	i=i+1
+        i=i+1
         line = sys.stdin.read(struct.calcsize(CPUREC))
         if not line:
             break
@@ -108,19 +108,20 @@ while not interrupted:
 
         (tsc, event, d1, d2, d3, d4, d5) = struct.unpack(TRCREC, line)
 
-	#tsc = (tscH<<32) | tscL
+        #tsc = (tscH<<32) | tscL
 
-	#print i, tsc
+        #print i, tsc
 
         if cpu >= len(last_tsc):
             last_tsc += [0] * (cpu - len(last_tsc) + 1)
-	elif tsc < last_tsc[cpu]:
-	    print "TSC stepped backward cpu %d !  %d %d" % (cpu,tsc,last_tsc[cpu])
+        elif tsc < last_tsc[cpu]:
+            print "TSC stepped backward cpu %d !  %d %d" % (cpu, tsc,
+                                                            last_tsc[cpu])
 
-	last_tsc[cpu] = tsc
+        last_tsc[cpu] = tsc
 
-	if mhz:
-	    tsc = tsc / (mhz*1000000.0)
+        if mhz:
+            tsc = tsc / (mhz*1000000.0)
 
         args = {'cpu'   : cpu,
                 'tsc'   : tsc,
@@ -131,15 +132,13 @@ while not interrupted:
                 '4'     : d4,
                 '5'     : d5    }
 
-	try:
-
-	    if defs.has_key(str(event)): 
-		print defs[str(event)] % args
-	    else: 
-		if defs.has_key(str(0)): print defs[str(0)] % args
-	except TypeError:
-	    print defs[str(event)]
-	    print args
-	    
+        try:
+            if defs.has_key(str(event)):
+                print defs[str(event)] % args
+            else:
+                if defs.has_key(str(0)): print defs[str(0)] % args
+        except TypeError:
+            print defs[str(event)]
+            print args
 
     except IOError, struct.error: sys.exit()

Re: [RFC][PATCH] 3/3] [TOOLS][XENTRACE] Various tidyups to xentrace tools.

[Mark Williamson]

>  - use err/errx/warn, instead of PERROR, perror and fprintf

Good stuff!  I like this.

>  - Match () place ment consistent in this files

Thanks for doing this.  While you're at it, would you like to fix the bracing?  
Some of it is in K&R style:

if ( aoeuaoeu ) {

Whereas other Xen code uses:

if ( aoeuaoeu )
{

Braces always starting on a new line, for all types of block.

>  - Use consistent tabs and whitespacing.

Thanks for doing this.

Cheers,
Mark

>
> Signed-off-by: Tony Breeds <tony@bakeyournoodle.com>
> ---
>
>  tools/xentrace/xentrace.c      |  137
> ++++++++++++++--------------------------- tools/xentrace/xentrace_format | 
>  51 +++++++--------
>  2 files changed, 73 insertions(+), 115 deletions(-)
>
> Index: xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace.c
> ===================================================================
> --- xen-unstable.hg-mainline.xentrace.orig/tools/xentrace/xentrace.c
> +++ xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace.c
> @@ -22,6 +22,7 @@
>  #include <signal.h>
>  #include <inttypes.h>
>  #include <string.h>
> +#include <err.h>
>
>  #include <xen/xen.h>
>  #include <xen/trace.h>
> @@ -42,16 +43,6 @@
>
>  #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
>
> -#define PERROR(_m, _a...)                                       \
> -do {                                                            \
> -    int __saved_errno = errno;                                  \
> -    fprintf(stderr, "ERROR: " _m " (%d = %s)\n" , ## _a ,       \
> -            __saved_errno, strerror(__saved_errno));            \
> -    errno = __saved_errno;                                      \
> -} while (0)
> -
> -extern FILE *stderr;
> -
>  /***** Compile time configuration of defaults
> ********************************/
>
>  /* when we've got more records than this waiting, we log it to the output
> */ @@ -88,7 +79,7 @@ void close_handler(int signal)
>  struct timespec millis_to_timespec(unsigned long millis)
>  {
>      struct timespec spec;
> -
> +
>      spec.tv_sec = millis / 1000;
>      spec.tv_nsec = (millis % 1000) * 1000;
>
> @@ -128,10 +119,7 @@ void write_rec(uint32_t cpu, struct t_re
>      }
>
>      if ( written != 8 )
> -    {
> -        PERROR("Failed to write trace record");
> -        exit(EXIT_FAILURE);
> -    }
> +        err(EXIT_FAILURE, "Failed to write trace record");
>  }
>
>  static void get_tbufs(unsigned long *mfn, unsigned long *size)
> @@ -139,21 +127,16 @@ static void get_tbufs(unsigned long *mfn
>      int xc_handle = xc_interface_open();
>      int ret;
>
> -    if ( xc_handle < 0 )
> -    {
> -        exit(EXIT_FAILURE);
> -    }
> +    if ( xc_handle < 0 )
> +        errx(EXIT_FAILURE, "Unable to open the xc interface");
>
> -    if(!opts.tbuf_size)
> -      opts.tbuf_size = DEFAULT_TBUF_SIZE;
> +    if ( !opts.tbuf_size )
> +        opts.tbuf_size = DEFAULT_TBUF_SIZE;
>
>      ret = xc_tbuf_enable(xc_handle, opts.tbuf_size, mfn, size);
>
>      if ( ret != 0 )
> -    {
> -        perror("Couldn't enable trace buffers");
> -        exit(1);
> -    }
> +        err(EXIT_FAILURE, "Couldn't enable trace buffers");
>
>      xc_interface_close(xc_handle);
>  }
> @@ -174,10 +157,8 @@ struct t_buf *map_tbufs(unsigned long tb
>
>      xc_handle = xc_interface_open();
>
> -    if ( xc_handle < 0 )
> -    {
> -        exit(EXIT_FAILURE);
> -    }
> +    if ( xc_handle < 0 )
> +        errx(EXIT_FAILURE, "Unable to open the xc interface");
>
>      /* On PPC (At least) the DOMID arg is ignored in dom0 */
>      tbufs_mapped = xc_map_foreign_range(xc_handle, DOMID_XEN,
> @@ -186,18 +167,15 @@ struct t_buf *map_tbufs(unsigned long tb
>
>      xc_interface_close(xc_handle);
>
> -    if ( tbufs_mapped == 0 )
> -    {
> -        PERROR("Failed to mmap trace buffers");
> -        exit(EXIT_FAILURE);
> -    }
> +    if ( tbufs_mapped == 0 )
> +        err(EXIT_FAILURE, "Failed to mmap trace buffers");
>
>      return tbufs_mapped;
>  }
>
>  /**
>   * set_mask - set the cpu/event mask in HV
> - * @mask:           the new mask
> + * @mask:           the new mask
>   * @type:           the new mask type,0-event mask, 1-cpu mask
>   *
>   */
> @@ -206,21 +184,19 @@ void set_mask(uint32_t mask, int type)
>      int ret = 0;
>      int xc_handle = xc_interface_open(); /* for accessing control
> interface */
>
> -    if (type == 1) {
> +    if ( type == 1 ) {
>          ret = xc_tbuf_set_cpu_mask(xc_handle, mask);
> -        fprintf(stderr, "change cpumask to 0x%x\n", mask);
> -    } else if (type == 0) {
> +        warnx("change cpumask to 0x%x\n", mask);
> +    } else if ( type == 0 ) {
>          ret = xc_tbuf_set_evt_mask(xc_handle, mask);
> -        fprintf(stderr, "change evtmask to 0x%x\n", mask);
> +        warnx("change evtmask to 0x%x\n", mask);
>      }
>
>      xc_interface_close(xc_handle);
>
>      if ( ret != 0 )
> -    {
> -        PERROR("Failure to get trace buffer pointer from Xen and set the
> new mask"); -        exit(EXIT_FAILURE);
> -    }
> +        err(EXIT_FAILURE, "Failure to get trace buffer pointer from "
> +	                  "Xen and set the new mask");
>  }
>
>  /**
> @@ -240,11 +216,8 @@ struct t_buf **init_bufs_ptrs(void *bufs
>
>      user_ptrs = (struct t_buf **)calloc(num, sizeof(struct t_buf *));
>      if ( user_ptrs == NULL )
> -    {
> -        PERROR( "Failed to allocate memory for buffer pointers\n");
> -        exit(EXIT_FAILURE);
> -    }
> -
> +        err(EXIT_FAILURE, "Failed to allocate memory for buffer
> pointers"); +
>      /* initialise pointers to the trace buffers - given the size of a
> trace * buffer and the value of bufs_maped, we can easily calculate these
> */ for ( i = 0; i<num; i++ )
> @@ -269,13 +242,10 @@ struct t_rec **init_rec_ptrs(struct t_bu
>  {
>      int i;
>      struct t_rec **data;
> -
> +
>      data = calloc(num, sizeof(struct t_rec *));
>      if ( data == NULL )
> -    {
> -        PERROR("Failed to allocate memory for data pointers\n");
> -        exit(EXIT_FAILURE);
> -    }
> +        err(EXIT_FAILURE, "Failed to allocate memory for data pointers");
>
>      for ( i = 0; i < num; i++ )
>          data[i] = (struct t_rec *)(meta[i] + 1);
> @@ -291,14 +261,11 @@ uint32_t get_num_cpus(void)
>      xc_physinfo_t physinfo;
>      int xc_handle = xc_interface_open();
>      int ret;
> -
> +
>      ret = xc_physinfo(xc_handle, &physinfo);
> -
> +
>      if ( ret != 0 )
> -    {
> -        PERROR("Failure to get logical CPU count from Xen");
> -        exit(EXIT_FAILURE);
> -    }
> +        errx(EXIT_FAILURE, "Failure to get logical CPU count from Xen");
>
>      xc_interface_close(xc_handle);
>
> @@ -377,17 +344,17 @@ int parse_evtmask(char *arg, struct argp
>      char *inval;
>
>      /* search filtering class */
> -    if (strcmp(arg, "gen") == 0){
> +    if ( strcmp(arg, "gen") == 0 )
>          setup->evt_mask |= TRC_GEN;
> -    } else if(strcmp(arg, "sched") == 0){
> +    else if ( strcmp(arg, "sched") == 0 )
>          setup->evt_mask |= TRC_SCHED;
> -    } else if(strcmp(arg, "dom0op") == 0){
> +    else if ( strcmp(arg, "dom0op") == 0 )
>          setup->evt_mask |= TRC_DOM0OP;
> -    } else if(strcmp(arg, "vmx") == 0){
> +    else if ( strcmp(arg, "vmx") == 0 )
>          setup->evt_mask |= TRC_VMX;
> -    } else if(strcmp(arg, "all") == 0){
> +    else if ( strcmp(arg, "all") == 0 )
>          setup->evt_mask |= TRC_ALL;
> -    } else {
> +    else {
>          setup->evt_mask = strtol(arg, &inval, 0);
>          if ( inval == arg )
>              argp_usage(state);
> @@ -430,13 +397,13 @@ error_t cmd_parser(int key, char *arg, s
>              argp_usage(state);
>      }
>      break;
> -
> +
>      case 'e': /* set new event mask for filtering*/
>      {
>          parse_evtmask(arg, state);
>      }
>      break;
> -
> +
>      case 'S': /* set tbuf size (given in pages) */
>      {
>          char *inval;
> @@ -454,7 +421,7 @@ error_t cmd_parser(int key, char *arg, s
>              argp_usage(state);
>      }
>      break;
> -
> +
>      default:
>          return ARGP_ERR_UNKNOWN;
>      }
> @@ -473,16 +440,16 @@ const struct argp_option cmd_opts[] =
>        "(default " xstr(NEW_DATA_THRESH) ")." },
>
>      { .name = "poll-sleep", .key='s', .arg="p",
> -      .doc =
> +      .doc =
>        "Set sleep time, p, in milliseconds between polling the trace buffer
> " "for new data (default " xstr(POLL_SLEEP_MILLIS) ")." },
>
>      { .name = "cpu-mask", .key='c', .arg="c",
> -      .doc =
> +      .doc =
>        "set cpu-mask " },
>
>      { .name = "evt-mask", .key='e', .arg="e",
> -      .doc =
> +      .doc =
>        "set evt-mask " },
>
>      { .name = "trace-buf-size", .key='S', .arg="N",
> @@ -504,7 +471,7 @@ const struct argp parser_def =
>      "\v"
>      "This tool is used to capture trace buffer data from Xen.  The data is
> " "output in a binary format, in the following order:\n\n"
> -    "  CPU(uint) TSC(uint64_t) EVENT(uint32_t) D1 D2 D3 D4 D5 "
> +    "  CPU(uint32_t) TSC(uint64_t) EVENT(uint32_t) D1 D2 D3 D4 D5 "
>      "(all uint32_t)\n\n"
>      "The output should be parsed using the tool xentrace_format, which can
> " "produce human-readable output in ASCII format."
> @@ -513,8 +480,8 @@ const struct argp parser_def =
>
>  const char *argp_program_version     = "xentrace v1.1";
>  const char *argp_program_bug_address = "<mark.a.williamson@intel.com>";
> -
> -
> +
> +
>  int main(int argc, char **argv)
>  {
>      int outfd = 1, ret;
> @@ -529,31 +496,23 @@ int main(int argc, char **argv)
>
>      argp_parse(&parser_def, argc, argv, 0, 0, &opts);
>
> -    if (opts.evt_mask != 0) {
> +    if ( opts.evt_mask != 0 )
>          set_mask(opts.evt_mask, 0);
> -    }
>
> -    if (opts.cpu_mask != 0) {
> +    if ( opts.cpu_mask != 0 )
>          set_mask(opts.cpu_mask, 1);
> -    }
>
>      if ( opts.outfile )
>          outfd = open(opts.outfile, O_WRONLY | O_CREAT | O_LARGEFILE,
> 0644);
>
> -    if(outfd < 0)
> -    {
> -        perror("Could not open output file");
> -        exit(EXIT_FAILURE);
> -    }
> +    if ( outfd < 0 )
> +        err(EXIT_FAILURE, "Could not open output file");
>
> -    if(isatty(outfd))
> -    {
> -        fprintf(stderr, "Cannot output to a TTY, specify a log file.\n");
> -        exit(EXIT_FAILURE);
> -    }
> +    if ( isatty(outfd) )
> +        errx(EXIT_FAILURE, "Cannot output to a TTY, specify a log
> file.\n");
>
>      logfile = fdopen(outfd, "w");
> -
> +
>      /* ensure that if we get a signal, we'll do cleanup, then exit */
>      act.sa_handler = close_handler;
>      act.sa_flags = 0;
> Index: xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace_format
> ===================================================================
> --- xen-unstable.hg-mainline.xentrace.orig/tools/xentrace/xentrace_format
> +++ xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace_format
> @@ -24,7 +24,7 @@ def usage():
>            Which correspond to the CPU number, event ID, timestamp counter
> and the 5 data fields from the trace record.  There should be one such rule
> for each type of event.
> -
> +
>            Depending on your system and the volume of trace buffer data,
>            this script may not be able to keep up with the output of
> xentrace if it is piped directly.  In these circumstances you should have
> @@ -34,7 +34,7 @@ def usage():
>
>  def read_defs(defs_file):
>      defs = {}
> -
> +
>      fd = open(defs_file)
>
>      reg = re.compile('(\S+)\s+(\S.*)')
> @@ -43,14 +43,14 @@ def read_defs(defs_file):
>          line = fd.readline()
>          if not line:
>              break
> -
> -	if line[0] == '#' or line[0] == '\n':
> -	    continue
> -
> +
> +        if line[0] == '#' or line[0] == '\n':
> +            continue
> +
>          m = reg.match(line)
>
>          if not m: print >> sys.stderr, "Bad format file" ; sys.exit(1)
> -
> +
>          defs[str(eval(m.group(1)))] = m.group(2)
>
>      return defs
> @@ -70,7 +70,7 @@ try:
>      opts, arg = getopt.getopt(sys.argv[1:], "c:" )
>
>      for opt in opts:
> -	if opt[0] == '-c' : mhz = int(opt[1])
> +        if opt[0] == '-c' : mhz = int(opt[1])
>
>  except getopt.GetoptError:
>      usage()
> @@ -96,7 +96,7 @@ i=0
>
>  while not interrupted:
>      try:
> -	i=i+1
> +        i=i+1
>          line = sys.stdin.read(struct.calcsize(CPUREC))
>          if not line:
>              break
> @@ -108,19 +108,20 @@ while not interrupted:
>
>          (tsc, event, d1, d2, d3, d4, d5) = struct.unpack(TRCREC, line)
>
> -	#tsc = (tscH<<32) | tscL
> +        #tsc = (tscH<<32) | tscL
>
> -	#print i, tsc
> +        #print i, tsc
>
>          if cpu >= len(last_tsc):
>              last_tsc += [0] * (cpu - len(last_tsc) + 1)
> -	elif tsc < last_tsc[cpu]:
> -	    print "TSC stepped backward cpu %d !  %d %d" %
> (cpu,tsc,last_tsc[cpu]) +        elif tsc < last_tsc[cpu]:
> +            print "TSC stepped backward cpu %d !  %d %d" % (cpu, tsc,
> +                                                            last_tsc[cpu])
>
> -	last_tsc[cpu] = tsc
> +        last_tsc[cpu] = tsc
>
> -	if mhz:
> -	    tsc = tsc / (mhz*1000000.0)
> +        if mhz:
> +            tsc = tsc / (mhz*1000000.0)
>
>          args = {'cpu'   : cpu,
>                  'tsc'   : tsc,
> @@ -131,15 +132,13 @@ while not interrupted:
>                  '4'     : d4,
>                  '5'     : d5    }
>
> -	try:
> -
> -	    if defs.has_key(str(event)):
> -		print defs[str(event)] % args
> -	    else:
> -		if defs.has_key(str(0)): print defs[str(0)] % args
> -	except TypeError:
> -	    print defs[str(event)]
> -	    print args
> -
> +        try:
> +            if defs.has_key(str(event)):
> +                print defs[str(event)] % args
> +            else:
> +                if defs.has_key(str(0)): print defs[str(0)] % args
> +        except TypeError:
> +            print defs[str(event)]
> +            print args
>
>      except IOError, struct.error: sys.exit()
>
>
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel

-- 
Dave: Just a question. What use is a unicyle with no seat?  And no pedals!
Mark: To answer a question with a question: What use is a skateboard?
Dave: Skateboards have wheels.
Mark: My wheel has a wheel!

Re: [RFC][PATCH] 3/3] [TOOLS][XENTRACE] Various tidyups to xentrace tools.

[Tony Breeds]

On Thu, Nov 30, 2006 at 12:57:53PM +0000, Mark Williamson wrote:

> Thanks for doing this.  While you're at it, would you like to fix the bracing?  
> Some of it is in K&R style:

Sure.  The version below fixes the couple of places that needed it, it
needs [2/3] to apply cleanly.

I'll post a complete series tomorrow, pending a decision on the 32/64bit
discussion.

---
From: Tony Breeds <tony@bakeyournoodle.com>
Subject: [RFC][PATCH] 3/3] [TOOLS][XENTRACE] Various tidyups to xentrace tools.

 - use err/errx/warn, instead of PERROR, perror and fprintf
 - Make () and {} placement consistent in this file, and with the xen
   codeing style.
 - Use consistent tabs and whitespacing.

Signed-off-by: Tony Breeds <tony@bakeyournoodle.com>
---

 tools/xentrace/xentrace.c      |  142 +++++++++++++++--------------------------
 tools/xentrace/xentrace_format |   51 +++++++-------
 2 files changed, 78 insertions(+), 115 deletions(-)

Index: xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace.c
===================================================================
--- xen-unstable.hg-mainline.xentrace.orig/tools/xentrace/xentrace.c
+++ xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace.c
@@ -22,6 +22,7 @@
 #include <signal.h>
 #include <inttypes.h>
 #include <string.h>
+#include <err.h>
 
 #include <xen/xen.h>
 #include <xen/trace.h>
@@ -42,16 +43,6 @@
 
 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
 
-#define PERROR(_m, _a...)                                       \
-do {                                                            \
-    int __saved_errno = errno;                                  \
-    fprintf(stderr, "ERROR: " _m " (%d = %s)\n" , ## _a ,       \
-            __saved_errno, strerror(__saved_errno));            \
-    errno = __saved_errno;                                      \
-} while (0)
-
-extern FILE *stderr;
-
 /***** Compile time configuration of defaults ********************************/
 
 /* when we've got more records than this waiting, we log it to the output */
@@ -88,7 +79,7 @@ void close_handler(int signal)
 struct timespec millis_to_timespec(unsigned long millis)
 {
     struct timespec spec;
-    
+
     spec.tv_sec = millis / 1000;
     spec.tv_nsec = (millis % 1000) * 1000;
 
@@ -122,16 +113,14 @@ void write_rec(uint32_t cpu, struct t_re
     tmp32 = htonl(rec->event);
     written += fwrite(&tmp32, sizeof(tmp32), 1, out);
 
-    for ( i=0; i<ARRAY_SIZE(rec->data); i++ ) {
+    for ( i=0; i<ARRAY_SIZE(rec->data); i++ )
+    {
         tmp64 = htonl(rec->data[i]);
         written += fwrite(&tmp64, sizeof(tmp64), 1, out);
     }
 
     if ( written != 8 )
-    {
-        PERROR("Failed to write trace record");
-        exit(EXIT_FAILURE);
-    }
+        err(EXIT_FAILURE, "Failed to write trace record");
 }
 
 static void get_tbufs(unsigned long *mfn, unsigned long *size)
@@ -139,21 +128,16 @@ static void get_tbufs(unsigned long *mfn
     int xc_handle = xc_interface_open();
     int ret;
 
-    if ( xc_handle < 0 ) 
-    {
-        exit(EXIT_FAILURE);
-    }
+    if ( xc_handle < 0 )
+        errx(EXIT_FAILURE, "Unable to open the xc interface");
 
-    if(!opts.tbuf_size)
-      opts.tbuf_size = DEFAULT_TBUF_SIZE;
+    if ( !opts.tbuf_size )
+        opts.tbuf_size = DEFAULT_TBUF_SIZE;
 
     ret = xc_tbuf_enable(xc_handle, opts.tbuf_size, mfn, size);
 
     if ( ret != 0 )
-    {
-        perror("Couldn't enable trace buffers");
-        exit(1);
-    }
+        err(EXIT_FAILURE, "Couldn't enable trace buffers");
 
     xc_interface_close(xc_handle);
 }
@@ -174,10 +158,8 @@ struct t_buf *map_tbufs(unsigned long tb
 
     xc_handle = xc_interface_open();
 
-    if ( xc_handle < 0 ) 
-    {
-        exit(EXIT_FAILURE);
-    }
+    if ( xc_handle < 0 )
+        errx(EXIT_FAILURE, "Unable to open the xc interface");
 
     /* On PPC (At least) the DOMID arg is ignored in dom0 */
     tbufs_mapped = xc_map_foreign_range(xc_handle, DOMID_XEN,
@@ -186,18 +168,15 @@ struct t_buf *map_tbufs(unsigned long tb
 
     xc_interface_close(xc_handle);
 
-    if ( tbufs_mapped == 0 ) 
-    {
-        PERROR("Failed to mmap trace buffers");
-        exit(EXIT_FAILURE);
-    }
+    if ( tbufs_mapped == 0 )
+        err(EXIT_FAILURE, "Failed to mmap trace buffers");
 
     return tbufs_mapped;
 }
 
 /**
  * set_mask - set the cpu/event mask in HV
- * @mask:           the new mask 
+ * @mask:           the new mask
  * @type:           the new mask type,0-event mask, 1-cpu mask
  *
  */
@@ -206,21 +185,22 @@ void set_mask(uint32_t mask, int type)
     int ret = 0;
     int xc_handle = xc_interface_open(); /* for accessing control interface */
 
-    if (type == 1) {
+    if ( type == 1 )
+    {
         ret = xc_tbuf_set_cpu_mask(xc_handle, mask);
-        fprintf(stderr, "change cpumask to 0x%x\n", mask);
-    } else if (type == 0) {
+        warnx("change cpumask to 0x%x\n", mask);
+    }
+    else if ( type == 0 )
+    {
         ret = xc_tbuf_set_evt_mask(xc_handle, mask);
-        fprintf(stderr, "change evtmask to 0x%x\n", mask);
+        warnx("change evtmask to 0x%x\n", mask);
     }
 
     xc_interface_close(xc_handle);
 
     if ( ret != 0 )
-    {
-        PERROR("Failure to get trace buffer pointer from Xen and set the new mask");
-        exit(EXIT_FAILURE);
-    }
+        err(EXIT_FAILURE, "Failure to get trace buffer pointer from "
+	                  "Xen and set the new mask");
 }
 
 /**
@@ -240,11 +220,8 @@ struct t_buf **init_bufs_ptrs(void *bufs
 
     user_ptrs = (struct t_buf **)calloc(num, sizeof(struct t_buf *));
     if ( user_ptrs == NULL )
-    {
-        PERROR( "Failed to allocate memory for buffer pointers\n");
-        exit(EXIT_FAILURE);
-    }
-    
+        err(EXIT_FAILURE, "Failed to allocate memory for buffer pointers");
+
     /* initialise pointers to the trace buffers - given the size of a trace
      * buffer and the value of bufs_maped, we can easily calculate these */
     for ( i = 0; i<num; i++ )
@@ -269,13 +246,10 @@ struct t_rec **init_rec_ptrs(struct t_bu
 {
     int i;
     struct t_rec **data;
-    
+
     data = calloc(num, sizeof(struct t_rec *));
     if ( data == NULL )
-    {
-        PERROR("Failed to allocate memory for data pointers\n");
-        exit(EXIT_FAILURE);
-    }
+        err(EXIT_FAILURE, "Failed to allocate memory for data pointers");
 
     for ( i = 0; i < num; i++ )
         data[i] = (struct t_rec *)(meta[i] + 1);
@@ -291,14 +265,11 @@ uint32_t get_num_cpus(void)
     xc_physinfo_t physinfo;
     int xc_handle = xc_interface_open();
     int ret;
-    
+
     ret = xc_physinfo(xc_handle, &physinfo);
-    
+
     if ( ret != 0 )
-    {
-        PERROR("Failure to get logical CPU count from Xen");
-        exit(EXIT_FAILURE);
-    }
+        errx(EXIT_FAILURE, "Failure to get logical CPU count from Xen");
 
     xc_interface_close(xc_handle);
 
@@ -377,17 +348,18 @@ int parse_evtmask(char *arg, struct argp
     char *inval;
 
     /* search filtering class */
-    if (strcmp(arg, "gen") == 0){ 
+    if ( strcmp(arg, "gen") == 0 )
         setup->evt_mask |= TRC_GEN;
-    } else if(strcmp(arg, "sched") == 0){ 
+    else if ( strcmp(arg, "sched") == 0 )
         setup->evt_mask |= TRC_SCHED;
-    } else if(strcmp(arg, "dom0op") == 0){ 
+    else if ( strcmp(arg, "dom0op") == 0 )
         setup->evt_mask |= TRC_DOM0OP;
-    } else if(strcmp(arg, "vmx") == 0){ 
+    else if ( strcmp(arg, "vmx") == 0 )
         setup->evt_mask |= TRC_VMX;
-    } else if(strcmp(arg, "all") == 0){ 
+    else if ( strcmp(arg, "all") == 0 )
         setup->evt_mask |= TRC_ALL;
-    } else {
+    else
+    {
         setup->evt_mask = strtol(arg, &inval, 0);
         if ( inval == arg )
             argp_usage(state);
@@ -430,13 +402,13 @@ error_t cmd_parser(int key, char *arg, s
             argp_usage(state);
     }
     break;
-    
+
     case 'e': /* set new event mask for filtering*/
     {
         parse_evtmask(arg, state);
     }
     break;
-    
+
     case 'S': /* set tbuf size (given in pages) */
     {
         char *inval;
@@ -454,7 +426,7 @@ error_t cmd_parser(int key, char *arg, s
             argp_usage(state);
     }
     break;
-        
+
     default:
         return ARGP_ERR_UNKNOWN;
     }
@@ -473,16 +445,16 @@ const struct argp_option cmd_opts[] =
       "(default " xstr(NEW_DATA_THRESH) ")." },
 
     { .name = "poll-sleep", .key='s', .arg="p",
-      .doc = 
+      .doc =
       "Set sleep time, p, in milliseconds between polling the trace buffer "
       "for new data (default " xstr(POLL_SLEEP_MILLIS) ")." },
 
     { .name = "cpu-mask", .key='c', .arg="c",
-      .doc = 
+      .doc =
       "set cpu-mask " },
 
     { .name = "evt-mask", .key='e', .arg="e",
-      .doc = 
+      .doc =
       "set evt-mask " },
 
     { .name = "trace-buf-size", .key='S', .arg="N",
@@ -514,8 +486,8 @@ const struct argp parser_def =
 
 const char *argp_program_version     = "xentrace v1.1";
 const char *argp_program_bug_address = "<mark.a.williamson@intel.com>";
-        
-    
+
+
 int main(int argc, char **argv)
 {
     int outfd = 1, ret;
@@ -530,31 +502,23 @@ int main(int argc, char **argv)
 
     argp_parse(&parser_def, argc, argv, 0, 0, &opts);
 
-    if (opts.evt_mask != 0) { 
+    if ( opts.evt_mask != 0 )
         set_mask(opts.evt_mask, 0);
-    }
 
-    if (opts.cpu_mask != 0) {
+    if ( opts.cpu_mask != 0 )
         set_mask(opts.cpu_mask, 1);
-    }
 
     if ( opts.outfile )
         outfd = open(opts.outfile, O_WRONLY | O_CREAT | O_LARGEFILE, 0644);
 
-    if(outfd < 0)
-    {
-        perror("Could not open output file");
-        exit(EXIT_FAILURE);
-    }        
+    if ( outfd < 0 )
+        err(EXIT_FAILURE, "Could not open output file");
 
-    if(isatty(outfd))
-    {
-        fprintf(stderr, "Cannot output to a TTY, specify a log file.\n");
-        exit(EXIT_FAILURE);
-    }
+    if ( isatty(outfd) )
+        errx(EXIT_FAILURE, "Cannot output to a TTY, specify a log file.\n");
 
     logfile = fdopen(outfd, "w");
-    
+
     /* ensure that if we get a signal, we'll do cleanup, then exit */
     act.sa_handler = close_handler;
     act.sa_flags = 0;
Index: xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace_format
===================================================================
--- xen-unstable.hg-mainline.xentrace.orig/tools/xentrace/xentrace_format
+++ xen-unstable.hg-mainline.xentrace/tools/xentrace/xentrace_format
@@ -24,7 +24,7 @@ def usage():
           Which correspond to the CPU number, event ID, timestamp counter and
           the 5 data fields from the trace record.  There should be one such
           rule for each type of event.
-          
+
           Depending on your system and the volume of trace buffer data,
           this script may not be able to keep up with the output of xentrace
           if it is piped directly.  In these circumstances you should have
@@ -34,7 +34,7 @@ def usage():
 
 def read_defs(defs_file):
     defs = {}
-    
+
     fd = open(defs_file)
 
     reg = re.compile('(\S+)\s+(\S.*)')
@@ -43,14 +43,14 @@ def read_defs(defs_file):
         line = fd.readline()
         if not line:
             break
-	
-	if line[0] == '#' or line[0] == '\n':
-	    continue
-        
+
+        if line[0] == '#' or line[0] == '\n':
+            continue
+
         m = reg.match(line)
 
         if not m: print >> sys.stderr, "Bad format file" ; sys.exit(1)
-        
+
         defs[str(eval(m.group(1)))] = m.group(2)
 
     return defs
@@ -70,7 +70,7 @@ try:
     opts, arg = getopt.getopt(sys.argv[1:], "c:" )
 
     for opt in opts:
-	if opt[0] == '-c' : mhz = int(opt[1])
+        if opt[0] == '-c' : mhz = int(opt[1])
 
 except getopt.GetoptError:
     usage()
@@ -96,7 +96,7 @@ i=0
 
 while not interrupted:
     try:
-	i=i+1
+        i=i+1
         line = sys.stdin.read(struct.calcsize(CPUREC))
         if not line:
             break
@@ -108,19 +108,20 @@ while not interrupted:
 
         (tsc, event, d1, d2, d3, d4, d5) = struct.unpack(TRCREC, line)
 
-	#tsc = (tscH<<32) | tscL
+        #tsc = (tscH<<32) | tscL
 
-	#print i, tsc
+        #print i, tsc
 
         if cpu >= len(last_tsc):
             last_tsc += [0] * (cpu - len(last_tsc) + 1)
-	elif tsc < last_tsc[cpu]:
-	    print "TSC stepped backward cpu %d !  %d %d" % (cpu,tsc,last_tsc[cpu])
+        elif tsc < last_tsc[cpu]:
+            print "TSC stepped backward cpu %d !  %d %d" % (cpu, tsc,
+                                                            last_tsc[cpu])
 
-	last_tsc[cpu] = tsc
+        last_tsc[cpu] = tsc
 
-	if mhz:
-	    tsc = tsc / (mhz*1000000.0)
+        if mhz:
+            tsc = tsc / (mhz*1000000.0)
 
         args = {'cpu'   : cpu,
                 'tsc'   : tsc,
@@ -131,15 +132,13 @@ while not interrupted:
                 '4'     : d4,
                 '5'     : d5    }
 
-	try:
-
-	    if defs.has_key(str(event)): 
-		print defs[str(event)] % args
-	    else: 
-		if defs.has_key(str(0)): print defs[str(0)] % args
-	except TypeError:
-	    print defs[str(event)]
-	    print args
-	    
+        try:
+            if defs.has_key(str(event)):
+                print defs[str(event)] % args
+            else:
+                if defs.has_key(str(0)): print defs[str(0)] % args
+        except TypeError:
+            print defs[str(event)]
+            print args
 
     except IOError, struct.error: sys.exit()

Yours Tony

   linux.conf.au       http://linux.conf.au/ || http://lca2007.linux.org.au/
   Jan 15-20 2007      The Australian Linux Technical Conference!