Re: [Qemu-devel] [RFC PATCH v3 1/3] Converting tracetool.sh to tracetool.py

[Harsh Bora <harsh@linux.vnet.ibm.com>]

On 01/11/2012 04:21 AM, Lluís Vilanova wrote:
> Harsh Prateek Bora writes:
>
>> Signed-off-by: Harsh Prateek Bora<harsh@linux.vnet.ibm.com>
>> ---
>>   Makefile.objs        |    6 +-
>>   Makefile.target      |   10 +-
>>   configure            |    7 +-
>>   scripts/tracetool    |  643 --------------------------------------------------
>>   scripts/tracetool.py |  585 +++++++++++++++++++++++++++++++++++++++++++++
>>   5 files changed, 597 insertions(+), 654 deletions(-)
>>   delete mode 100755 scripts/tracetool
>>   create mode 100755 scripts/tracetool.py
>
> [...]
>> diff --git a/scripts/tracetool.py b/scripts/tracetool.py
>> new file mode 100755
>> index 0000000..6874f66
>> --- /dev/null
>> +++ b/scripts/tracetool.py
>> @@ -0,0 +1,585 @@
>> +#!/usr/bin/env python
>> +# Python based tracetool script (Code generator for trace events)
>> +#
>> +# Copyright IBM, Corp. 2011
>> +#
>> +# This work is licensed under the terms of the GNU GPL, version 2.  See
>> +# the COPYING file in the top-level directory.
>> +#
>> +#
>> +
>> +import sys
>> +import getopt
>> +
>> +def usage():
>> +    print "Tracetool: Generate tracing code for trace events file on stdin"
>> +    print "Usage:"
>> +    print sys.argv[0], " --backend=[nop|simple|stderr|dtrace|ust] [-h|-c|-d|--stap]"
>> +    print '''
>> +Backends:
>> +  --nop     Tracing disabled
>> +  --simple  Simple built-in backend
>> +  --stderr  Stderr built-in backend
>> +  --dtrace  DTrace/SystemTAP backend
>> +  --ust     LTTng User Space Tracing backend
>> +
>> +Output formats:
>> +  -h     Generate .h file
>> +  -c     Generate .c file
>> +  -d     Generate .d file (DTrace only)
>> +  --stap Generate .stp file (DTrace with SystemTAP only)
>> +
>> +Options:
>> +  --binary       [path]    Full path to QEMU binary
>> +  --target-arch  [arch]    QEMU emulator target arch
>> +  --target-type  [type]    QEMU emulator target type ('system' or 'user')
>> +  --probe-prefix [prefix]  Prefix for dtrace probe names
>> +                           (default: qemu-targettype-targetarch)
>> +'''
>> +    sys.exit(1)
>> +
>> +def get_name(line, sep='('):
>> +    head, sep, tail = line.partition(sep)
>> +    return head
>> +
>> +def get_args(line, sep1='(', sep2=')'):
>> +    head, sep1, tail = line.partition(sep1)
>> +    args, sep2, fmt_str = tail.partition(sep2)
>> +    return args
>> +
>> +def get_argnames(line, sep=','):
>> +    nfields = 0
>> +    str = []
>> +    args = get_args(line)
>> +    for field in args.split():
>> +      nfields = nfields + 1
>> +      # Drop pointer star
>> +      type, ptr, tail = field.partition('*')
>> +      if type != field:
>> +        field = tail
>> +
>> +      name, sep, tail = field.partition(',')
>> +
>> +      if name == field:
>> +        continue
>> +      str.append(name)
>> +      str.append(", ")
>> +
>> +    if nfields>  1:
>> +      str.append(name)
>> +      return ''.join(str)
>> +    else:
>> +      return ''
>> +
>> +def get_argc(line):
>> +    argc = 0
>> +    argnames = get_argnames(line)
>> +    if argnames:
>> +      for name in argnames.split(','):
>> +        argc = argc + 1
>> +    return argc
>> +
>> +def get_fmt(line, sep=')'):
>> +    event, sep, fmt = line.partition(sep)
>> +    return fmt
>> +
>> +def calc_sizeofargs(line):
>> +    args = get_args(line)
>> +    argc = get_argc(line)
>> +    strtype = ('const char*', 'char*', 'const char *', 'char *')
>> +    str = []
>> +    newstr = ""
>> +    if argc>  0:
>> +      str = args.split(',')
>> +      for elem in str:
>> +        if elem.lstrip().startswith(strtype): #strings
>> +          type, sep, var = elem.rpartition('*')
>> +          newstr = newstr+"4 + strlen("+var.lstrip()+") + "
>> +        #elif '*' in elem:
>> +        #  newstr = newstr + "4 + " # pointer vars
>> +        else:
>> +          #type, sep, var = elem.rpartition(' ')
>> +          #newstr = newstr+"sizeof("+type.lstrip()+") + "
>> +          newstr = newstr + '8 + '
>> +    newstr = newstr + '0' # for last +
>> +    return newstr
>> +
>> +
>> +def trace_h_begin():
>> +    print '''#ifndef TRACE_H
>> +#define TRACE_H
>> +
>> +/* This file is autogenerated by tracetool, do not edit. */
>> +
>> +#include "qemu-common.h"'''
>> +    return
>> +
>> +def trace_h_end():
>> +    print '#endif /* TRACE_H */'
>> +    return
>> +
>> +def trace_c_begin():
>> +    print '/* This file is autogenerated by tracetool, do not edit. */'
>> +    return
>> +
>> +def trace_c_end():
>> +    # nop, required for trace_gen
>> +    return
>> +
>> +def nop_h(events):
>> +    print
>> +    for event in events:
>> +        print '''static inline void trace_%(name)s(%(args)s)
>> +{
>> +}
>> +''' % {
>> +    'name': event.name,
>> +    'args': event.args
>> +}
>> +    return
>> +
>> +def nop_c(events):
>> +    # nop, reqd for converters
>> +    return
>> +
>> +
>> +def simple_h(events):
>> +    print '#include "trace/simple.h"'
>> +    print
>> +    for event in events:
>> +        print 'void trace_%(name)s(%(args)s);' % {
>> +    'name': event.name,
>> +    'args': event.args
>> +}
>> +    print
>> +    print '#define NR_TRACE_EVENTS %d' % (event.num + 1)
>> +    print 'extern TraceEvent trace_list[NR_TRACE_EVENTS];'
>> +
>> +    return
>> +
>> +def is_string(arg):
>> +    strtype = ('const char*', 'char*', 'const char *', 'char *')
>> +    if arg.lstrip().startswith(strtype):
>> +        return True
>> +    else:
>> +        return False
>> +
>> +def simple_c(events):
>> +    rec_off = 0
>> +    print '#include "trace.h"'
>> +    print '#include "trace/simple.h"'
>> +    print
>> +    print 'TraceEvent trace_list[] = {'
>> +    print
>> +    eventlist = list(events)
>> +    for event in eventlist:
>> +        print '{.tp_name = "%(name)s", .state=0},' % {
>> +    'name': event.name
>> +}
>> +        print
>> +    print '};'
>> +    print
>> +    for event in eventlist:
>> +        argc = event.argc
>> +        print '''void trace_%(name)s(%(args)s)
>> +{
>> +    unsigned int tbuf_idx, rec_off;
>> +    uint64_t var64 __attribute__ ((unused));
>> +    uint64_t pvar64 __attribute__ ((unused));
>> +    uint32_t slen __attribute__ ((unused));
>> +
>> +    if (!trace_list[%(event_id)s].state) {
>> +        return;
>> +    }
>> +''' % {
>> +    'name': event.name,
>> +    'args': event.args,
>> +    'event_id': event.num,
>> +}
>> +        print '''
>> +    tbuf_idx = trace_alloc_record(%(event_id)s, %(sizestr)s);
>> +    rec_off = (tbuf_idx + ST_V2_REC_HDR_LEN) %% TRACE_BUF_LEN; /* seek record header */
>> +''' % {'event_id': event.num, 'sizestr': event.sizestr,}
>> +
>> +        if argc>  0:
>> +            str = event.arglist
>> +            for elem in str:
>> +                if is_string(elem): # if string
>> +                    type, sep, var = elem.rpartition('*')
>> +                    print '''
>> +    slen = strlen(%(var)s);
>> +    write_to_buffer(rec_off, (uint8_t*)&slen, sizeof(slen));
>> +    rec_off += sizeof(slen);''' % {
>> +    'var': var.lstrip()
>> +}
>> +                    print '''
>> +    write_to_buffer(rec_off, (uint8_t*)%(var)s, slen);
>> +    rec_off += slen;''' % {
>> +    'var': var.lstrip()
>> +}
>> +                elif '*' in elem: # pointer var (not string)
>> +                    type, sep, var = elem.rpartition('*')
>> +                    print '''
>> +    pvar64 = (uint64_t)(uint64_t*)%(var)s;
>> +    write_to_buffer(rec_off, (uint8_t*)&pvar64, sizeof(uint64_t));
>> +    rec_off += sizeof(uint64_t);''' % {
>> +    'var': var.lstrip()
>> +}
>> +                else: # primitive data type
>> +                    type, sep, var = elem.rpartition(' ')
>> +                    print '''
>> +    var64 = (uint64_t)%(var)s;
>> +    write_to_buffer(rec_off, (uint8_t*)&var64, sizeof(uint64_t));
>> +    rec_off += sizeof(uint64_t);''' % {
>> +    'var': var.lstrip()
>> +}
>> +        print '''
>> +    trace_mark_record_complete(tbuf_idx);'''
>> +        print '}'
>> +        print
>> +
>> +    return
>> +
>> +def stderr_h(events):
>> +    print '''#include<stdio.h>
>> +#include "trace/stderr.h"
>> +
>> +extern TraceEvent trace_list[];'''
>> +    for event in events:
>> +        argnames = event.argnames
>> +        if event.argc>  0:
>> +            argnames = ', ' + event.argnames
>> +        else:
>> +            argnames = ''
>> +        print '''
>> +static inline void trace_%(name)s(%(args)s)
>> +{
>> +    if (trace_list[%(event_num)s].state != 0) {
>> +        fprintf(stderr, "%(name)s " %(fmt)s "\\n" %(argnames)s);
>> +    }
>> +}''' % {
>> +    'name': event.name,
>> +    'args': event.args,
>> +    'event_num': event.num,
>> +    'fmt': event.fmt.rstrip('\n'),
>> +    'argnames': argnames
>> +}
>> +    print
>> +    print '#define NR_TRACE_EVENTS %d' % (event.num + 1)
>> +
>> +def stderr_c(events):
>> +    print '''#include "trace.h"
>> +
>> +TraceEvent trace_list[] = {
>> +'''
>> +    for event in events:
>> +        print '{.tp_name = "%(name)s", .state=0},' % {
>> +    'name': event.name
>> +}
>> +        print
>> +    print '};'
>> +
>> +def ust_h(events):
>> +    print '''#include<ust/tracepoint.h>
>> +#undef mutex_lock
>> +#undef mutex_unlock
>> +#undef inline
>> +#undef wmb'''
>> +
>> +    for event in events:
>> +        if event.argc>  0:
>> +            print '''
>> +DECLARE_TRACE(ust_%(name)s, TP_PROTO(%(args)s), TP_ARGS(%(argnames)s));
>> +#define trace_%(name)s trace_ust_%(name)s''' % {
>> +    'name': event.name,
>> +    'args': event.args,
>> +    'argnames': event.argnames
>> +}
>> +        else:
>> +            print '''
>> +_DECLARE_TRACEPOINT_NOARGS(ust_%(name)s);
>> +#define trace_%(name)s trace_ust_%(name)s''' % {
>> +    'name': event.name,
>> +}
>> +    print
>> +    return
>> +
>> +def ust_c(events):
>> +    print '''#include<ust/marker.h>
>> +#undef mutex_lock
>> +#undef mutex_unlock
>> +#undef inline
>> +#undef wmb
>> +#include "trace.h"'''
>> +    eventlist = list(events)
>> +    for event in eventlist:
>> +        argnames = event.argnames
>> +        if event.argc>  0:
>> +            argnames = ', ' + event.argnames
>> +            print '''
>> +DEFINE_TRACE(ust_%(name)s);
>> +
>> +static void ust_%(name)s_probe(%(args)s)
>> +{
>> +    trace_mark(ust, %(name)s, %(fmt)s%(argnames)s);
>> +}''' % {
>> +    'name': event.name,
>> +    'args': event.args,
>> +    'fmt': event.fmt.rstrip('\n'),
>> +    'argnames': argnames
>> +}
>> +        else:
>> +            print '''
>> +DEFINE_TRACE(ust_%(name)s);
>> +
>> +static void ust_%(name)s_probe(%(args)s)
>> +{
>> +    trace_mark(ust, %(name)s, UST_MARKER_NOARGS);
>> +}''' % {
>> +    'name': event.name,
>> +    'args': event.args,
>> +}
>> +
>> +    # register probes
>> +    print '''
>> +static void __attribute__((constructor)) trace_init(void)
>> +{'''
>> +    for event in eventlist:
>> +        print '    register_trace_ust_%(name)s(ust_%(name)s_probe);' % {
>> +    'name': event.name
>> +}
>> +    print '}'
>> +
>> +    return
>> +
>> +def dtrace_h(events):
>> +    print '#include "trace-dtrace.h"'
>> +    print
>> +    for event in events:
>> +        print '''static inline void trace_%(name)s(%(args)s) {
>> +    if (QEMU_%(uppername)s_ENABLED()) {
>> +        QEMU_%(uppername)s(%(argnames)s);
>> +    }
>> +}
>> +''' % {
>> +    'name': event.name,
>> +    'args': event.args,
>> +    'uppername': event.name.upper(),
>> +    'argnames': event.argnames,
>> +}
>> +
>> +def dtrace_c(events):
>> +    return # No need for function definitions in dtrace backend
>> +
>> +def dtrace_d(events):
>> +    print 'provider qemu {'
>> +    for event in events:
>> +        args = event.args
>> +
>> +        # DTrace provider syntax expects foo() for empty
>> +        # params, not foo(void)
>> +        if args == 'void':
>> +            args = ''
>> +
>> +        # Define prototype for probe arguments
>> +        print '''
>> +        probe %(name)s(%(args)s);''' % {
>> +        'name': event.name,
>> +        'args': args
>> +}
>> +    print
>> +    print '};'
>> +    return
>> +
>> +def dtrace_stp(events):
>> +    for event in events:
>> +        # Define prototype for probe arguments
>> +        print '''
>> +probe %(probeprefix)s.%(name)s = process("%(binary)s").mark("%(name)s")
>> +{''' % {
>> +    'probeprefix': probeprefix,
>> +    'name': event.name,
>> +    'binary': binary
>> +}
>> +        i = 1
>> +        if event.argc>  0:
>> +            for arg in event.argnames.split(','):
>> +                # 'limit' is a reserved keyword
>> +                if arg == 'limit':
>> +                    arg = '_limit'
>> +                print '  %s = $arg%d;' % (arg.lstrip(), i)
>> +                i += 1
>> +        print '}'
>> +    print
>> +    return
>> +
>> +def trace_stap_begin():
>> +    global probeprefix
>> +    if backend != "dtrace":
>> +        print 'SystemTAP tapset generator not applicable to %s backend' % backend
>> +        sys.exit(1)
>> +    if binary == "":
>> +        print '--binary is required for SystemTAP tapset generator'
>> +        sys.exit(1)
>> +    if ((probeprefix == "") and (targettype == "")):
>> +        print '--target-type is required for SystemTAP tapset generator'
>> +        sys.exit(1)
>> +    if ((probeprefix == "") and (targetarch == "")):
>> +        print '--target-arch is required for SystemTAP tapset generator'
>> +        sys.exit(1)
>> +    if probeprefix == "":
>> +        probeprefix = 'qemu.' + targettype + '.' + targetarch
>> +    print '/* This file is autogenerated by tracetool, do not edit. */'
>> +    return
>> +
>> +def trace_stap_end():
>> +    return #nop, reqd for trace_gen
>> +
>> +def trace_d_begin():
>> +    if backend != 'dtrace':
>> +        print 'DTrace probe generator not applicable to %s backend' % backend
>> +        sys.exit(1)
>> +    print '/* This file is autogenerated by tracetool, do not edit. */'
>> +
>> +def trace_d_end():
>> +    return #nop, reqd for trace_gen
>> +
>> +
>> +# Registry of backends and their converter functions
>> +converters = {
>> +    'simple': {
>> +        'h': simple_h,
>> +	'c': simple_c,
>> +    },
>> +
>> +    'nop': {
>> +        'h': nop_h,
>> +        'c': nop_c,
>> +    },
>> +
>> +    'stderr': {
>> +        'h': stderr_h,
>> +        'c': stderr_c,
>> +    },
>> +
>> +    'dtrace': {
>> +        'h': dtrace_h,
>> +        'c': dtrace_c,
>> +        'd': dtrace_d,
>> +        'stap': dtrace_stp
>> +    },
>> +
>> +    'ust': {
>> +        'h': ust_h,
>> +        'c': ust_c,
>> +    },
>> +
>> +}
>> +
>> +# Trace file header and footer code generators
>> +trace_gen = {
>> +    'h': {
>> +        'begin': trace_h_begin,
>> +        'end': trace_h_end,
>> +    },
>> +    'c': {
>> +        'begin': trace_c_begin,
>> +        'end': trace_c_end,
>> +    },
>> +    'd': {
>> +        'begin': trace_d_begin,
>> +        'end': trace_d_end,
>> +    },
>> +    'stap': {
>> +        'begin': trace_stap_begin,
>> +        'end': trace_stap_end,
>> +    },
>> +}
>> +
>> +# A trace event
>> +class Event(object):
>> +    def __init__(self, num, line):
>> +        self.num = num
>> +        self.args = get_args(line)
>> +        self.arglist = self.args.split(',')
>> +        self.name = get_name(line)
>> +        self.argc = get_argc(line)
>> +        self.argnames = get_argnames(line)
>> +        self.sizestr = calc_sizeofargs(line)
>> +        self.fmt = get_fmt(line)
>
> This is not not extracting the event properties (e.g., disable). A set of
> strings should suffice.
>

Ok, I had a misunderstanding about 'disable' being removed completely 
since all events were enabled by default. I will update the patch as 
required. Are there any other properties except 'disable' ?

> Arguments could be converted to an Arguments class (or similar, like
> ArgumentList) and derive the rest from there using methods (e.g., names, types,
> sizestr, arglist, etc.).
>
> If you want, I can send a patch on top of this one with that.

Any improvements are always welcome.

- Harsh

>
>
>> +
>> +# Generator that yields Event objects given a trace-events file object
>> +def read_events(fobj):
>> +    event_num = 0
>> +    for line in fobj:
>> +        if not line.strip():
>> +            continue
>> +        if line.lstrip().startswith('#'):
>> +	    continue
>> +	yield Event(event_num, line)
>> +	event_num += 1
>> +
>> +backend = ""
>> +output = ""
>> +binary = ""
>> +targettype = ""
>> +targetarch = ""
>> +probeprefix = ""
>> +
>> +def main():
>> +    global backend, output, binary, targettype, targetarch, probeprefix
>> +    supported_backends = ["simple", "nop", "stderr", "dtrace", "ust"]
>> +    short_options = "hcd"
>> +    long_options = ["stap", "backend=", "binary=", "target-arch=", "target-type=", "probe-prefix=", "list-backends", "check-backend"]
>> +    try:
>> +        opts, args = getopt.getopt(sys.argv[1:], short_options, long_options)
>> +    except getopt.GetoptError, err:
>> +        # print help information and exit:
>> +        print str(err) # will print something like "option -a not recognized"
>> +        usage()
>> +        sys.exit(2)
>> +    for opt, arg in opts:
>> +        if opt == '-h':
>> +            output = 'h'
>> +        elif opt == '-c':
>> +            output = 'c'
>> +        elif opt == '-d':
>> +            output = 'd'
>> +        elif opt == '--stap':
>> +            output = 'stap'
>> +        elif opt == '--backend':
>> +            backend = arg
>> +        elif opt == '--binary':
>> +            binary = arg
>> +        elif opt == '--target-arch':
>> +            targetarch = arg
>> +        elif opt == '--target-type':
>> +            targettype = arg
>> +        elif opt == '--probe-prefix':
>> +            probeprefix = arg
>> +        elif opt == '--list-backends':
>> +            print 'simple, nop, stderr, dtrace'
>> +            sys.exit(0)
>> +        elif opt == "--check-backend":
>> +            if any(backend in s for s in supported_backends):
>> +                sys.exit(0)
>> +            else:
>> +                sys.exit(1)
>> +        else:
>> +            #assert False, "unhandled option"
>> +            print "unhandled option: ", opt
>> +            usage()
>> +
>> +    if backend == "" or output == "":
>> +        usage()
>> +        sys.exit(0)
>> +
>> +    events = read_events(sys.stdin)
>> +    trace_gen[output]['begin']()
>> +    converters[backend][output](events)
>
> This should use the "disable" property to establish whether to use output or
> "nop".
>
>> +    trace_gen[output]['end']()
>> +    return
>> +
>> +if __name__ == "__main__":
>> +    main()
>> +
>> --
>> 1.7.1.1
>
> Lluis
>
>