[PATCH 3/4] trace: improve usefulness of hypercall trace record

[David Vrabel <david.vrabel@citrix.com>]

From: David Vrabel <david.vrabel@citrix.com>

Trace hypercalls using a more useful trace record format.

The EIP field is removed (it was always somewhere in the hypercall
page) and include selected hypercall arguments (e.g., the number of
calls in a multicall, and the number of PTE updates in an mmu_update
etc.).  12 bits in the first extra word are used to indicate which
arguments are present in the record and what size they are (32 or
64-bit).

This is an incompatible record format so a new event ID is used so
tools can distinguish between the two formats.

Signed-off-by: David Vrabel <david.vrabel@citrix.com>
---
 tools/xentrace/formats         |    1 +
 tools/xentrace/xentrace_format |    6 +++
 xen/arch/x86/trace.c           |   35 ++++++++-----------
 xen/common/trace.c             |   70 ++++++++++++++++++++++++++++++++++++++++
 xen/include/public/trace.h     |   30 +++++++++++++++++
 xen/include/xen/trace.h        |    2 +
 6 files changed, 124 insertions(+), 20 deletions(-)

diff --git a/tools/xentrace/formats b/tools/xentrace/formats
index 71220c0..fa935c8 100644
--- a/tools/xentrace/formats
+++ b/tools/xentrace/formats
@@ -104,6 +104,7 @@
 0x0020110b  CPU%(cpu)d  %(tsc)d (+%(reltsc)8d)  ptwr_emulation      [ addr = 0x%(4)08x%(3)08x, rip = 0x%(6)08x%(5)08x, npte = 0x%(2)08x%(1)08x ]
 0x0020100c  CPU%(cpu)d  %(tsc)d (+%(reltsc)8d)  ptwr_emulation_pae  [ addr = 0x%(3)08x, eip = 0x%(4)08x, npte = 0x%(2)08x%(1)08x ]
 0x0020110c  CPU%(cpu)d  %(tsc)d (+%(reltsc)8d)  ptwr_emulation_pae  [ addr = 0x%(4)08x%(3)08x, rip = 0x%(6)08x%(5)08x, npte = 0x%(2)08x%(1)08x ]
+0x0020100d  CPU%(cpu)d  %(tsc)d (+%(reltsc)8d)  hypercall  [ op = 0x%(1)08x ]
 
 0x0040f001  CPU%(cpu)d  %(tsc)d (+%(reltsc)8d)  shadow_not_shadow                 [ gl1e = 0x%(2)08x%(1)08x, va = 0x%(3)08x, flags = 0x%(4)08x ]
 0x0040f101  CPU%(cpu)d  %(tsc)d (+%(reltsc)8d)  shadow_not_shadow                 [ gl1e = 0x%(2)08x%(1)08x, va = 0x%(4)08x%(3)08x, flags = 0x%(5)08x ]
diff --git a/tools/xentrace/xentrace_format b/tools/xentrace/xentrace_format
index e13b05b..bdcab09 100644
--- a/tools/xentrace/xentrace_format
+++ b/tools/xentrace/xentrace_format
@@ -111,6 +111,8 @@ D7REC  = "IIIIIII"
 last_tsc = [0]
 
 TRC_TRACE_IRQ = 0x1f004
+TRC_PV_HYPERCALL_V2 = 0x20100d
+
 NR_VECTORS = 256
 irq_measure = [{'count':0, 'tot_cycles':0, 'max_cycles':0}] * NR_VECTORS
 
@@ -197,6 +199,10 @@ while not interrupted:
             d3 = irq_measure[d1]['tot_cycles']
             d4 = irq_measure[d1]['max_cycles']
 
+        if event == TRC_PV_HYPERCALL_V2:
+            # Mask off the argument present bits.
+            d1 &= 0x000fffff
+
         #tsc = (tscH<<32) | tscL
 
         #print i, tsc
diff --git a/xen/arch/x86/trace.c b/xen/arch/x86/trace.c
index da4e974..9c43f17 100644
--- a/xen/arch/x86/trace.c
+++ b/xen/arch/x86/trace.c
@@ -9,33 +9,28 @@
 void __trace_hypercall_entry(void)
 {
     struct cpu_user_regs *regs = guest_cpu_user_regs();
+    unsigned long args[6];
 
     if ( is_pv_32on64_vcpu(current) )
     {
-        struct {
-            u32 eip,eax;
-        } __attribute__((packed)) d;
-            
-        d.eip = regs->eip;
-        d.eax = regs->eax;
-
-        __trace_var(TRC_PV_HYPERCALL, 1, sizeof(d), &d);
+        args[0] = regs->ebx;
+        args[1] = regs->ecx;
+        args[2] = regs->edx;
+        args[3] = regs->esi;
+        args[4] = regs->edi;
+        args[5] = regs->ebp;
     }
     else
     {
-        struct {
-            unsigned long eip;
-            u32 eax;
-        } __attribute__((packed)) d;
-        u32 event;
-
-        event = TRC_PV_HYPERCALL;
-        event |= TRC_64_FLAG;
-        d.eip = regs->eip;
-        d.eax = regs->eax;
-
-        __trace_var(event, 1/*tsc*/, sizeof(d), &d);
+        args[0] = regs->rdi;
+        args[1] = regs->rsi;
+        args[2] = regs->rdx;
+        args[3] = regs->r10;
+        args[4] = regs->r8;
+        args[5] = regs->r9;
     }
+
+    __trace_hypercall(regs->eax, args);
 }
 
 void __trace_pv_trap(int trapnr, unsigned long eip,
diff --git a/xen/common/trace.c b/xen/common/trace.c
index cacaeb2..059020d 100644
--- a/xen/common/trace.c
+++ b/xen/common/trace.c
@@ -816,6 +816,76 @@ unlock:
         tasklet_schedule(&trace_notify_dom0_tasklet);
 }
 
+struct trace_hypercall {
+    struct trace_arg {
+        uint8_t words;
+        uint8_t idx;
+    } args[6];
+};
+
+#define ARG32(i) { .words = 1, .idx = (i) }
+#define ARG64(i) { .words = 2, .idx = (i) }
+
+/*
+ * Table of hypercall arguments to be added to the PV_HYPERCALL_V2
+ * trace record.
+ *
+ * Guest pointers are usually omitted.
+ */
+static struct trace_hypercall trace_hypercall_table[] = {
+    [__HYPERVISOR_mmu_update]     = {{ ARG32(1) }}, /* count */
+    [__HYPERVISOR_multicall]      = {{ ARG32(1) }}, /* count */
+    [__HYPERVISOR_grant_table_op] = {{ ARG32(0), ARG32(2) }}, /* cmd, count */
+    [__HYPERVISOR_vcpu_op]        = {{ ARG32(0), ARG32(1) }}, /* cmd, vcpuid */
+    [__HYPERVISOR_mmuext_op]      = {{ ARG32(1) }}, /* count */
+    [__HYPERVISOR_sched_op]       = {{ ARG32(0) }}, /* cmd */
+};
+
+#undef ARG32
+#undef ARG64
+
+void __trace_hypercall(unsigned long op, const unsigned long *args)
+{
+    struct {
+        uint32_t op;
+        uint32_t args[6];
+    } __attribute__((packed)) d;
+    uint32_t *a = d.args;
+
+    /*
+     * This shouldn't happen as @op should be small enough but just in
+     * case, warn if the argument bits in the trace record would
+     * clobber the hypercall op.
+     */
+    WARN_ON(op & TRC_PV_HYPERCALL_V2_ARG_MASK);
+
+    d.op = op;
+
+    if ( op < ARRAY_SIZE(trace_hypercall_table) )
+    {
+        struct trace_arg *i;
+
+        for ( i = trace_hypercall_table[op].args; i->words; i++ )
+        {
+            switch (i->words)
+            {
+            case 1:
+                *a++ = args[i->idx];
+                d.op |= TRC_PV_HYPERCALL_V2_ARG_32(i->idx);
+                break;
+            case 2:
+                *a++ = args[i->idx];
+                *a++ = args[i->idx] >> 32;
+                d.op |= TRC_PV_HYPERCALL_V2_ARG_64(i->idx);
+                break;
+            }
+        }
+    }
+
+    __trace_var(TRC_PV_HYPERCALL_V2, 1,
+                sizeof(uint32_t) * (1 + (a - d.args)), &d);
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/xen/include/public/trace.h b/xen/include/public/trace.h
index 1f154bb..ef43b23 100644
--- a/xen/include/public/trace.h
+++ b/xen/include/public/trace.h
@@ -107,6 +107,36 @@
 #define TRC_PV_GDT_LDT_MAPPING_FAULT (TRC_PV_ENTRY + 10)
 #define TRC_PV_PTWR_EMULATION        (TRC_PV_ENTRY + 11)
 #define TRC_PV_PTWR_EMULATION_PAE    (TRC_PV_ENTRY + 12)
+#define TRC_PV_HYPERCALL_V2          (TRC_PV_ENTRY + 13)
+
+/*
+ * TRC_PV_HYPERCALL_V2 format
+ *
+ * Only some of the hypercall argument are recorded. Bit fields A0 to
+ * A5 in the first extra word are set if the argument is present and
+ * the arguments themselves are packed sequentially in the following
+ * words.
+ *
+ * The TRC_64_FLAG bit is not set for these events (even if there are
+ * 64-bit arguments in the record).
+ *
+ * Word
+ * 0    bit 31 30|29 28|27 26|25 24|23 22|21 20|19 ... 0
+ *          A5   |A4   |A3   |A2   |A1   |A0   |Hypercall op
+ * 1    First 32 bit (or low word of first 64 bit) arg in record
+ * 2    Second 32 bit (or high word of first 64 bit) arg in record
+ * ...
+ *
+ * A0-A5 bitfield values:
+ *
+ *   00b  Argument not present
+ *   01b  32-bit argument present
+ *   10b  64-bit argument present
+ *   11b  Reserved
+ */
+#define TRC_PV_HYPERCALL_V2_ARG_32(i) (0x1 << (20 + 2*(i)))
+#define TRC_PV_HYPERCALL_V2_ARG_64(i) (0x2 << (20 + 2*(i)))
+#define TRC_PV_HYPERCALL_V2_ARG_MASK  (0xfff00000)
 
 #define TRC_SHADOW_NOT_SHADOW                 (TRC_SHADOW +  1)
 #define TRC_SHADOW_FAST_PROPAGATE             (TRC_SHADOW +  2)
diff --git a/xen/include/xen/trace.h b/xen/include/xen/trace.h
index b32f6c5..f601aeb 100644
--- a/xen/include/xen/trace.h
+++ b/xen/include/xen/trace.h
@@ -44,6 +44,8 @@ static inline void trace_var(u32 event, int cycles, int extra,
         __trace_var(event, cycles, extra, extra_data);
 }
 
+void __trace_hypercall(unsigned long call, const unsigned long *args);
+
 /* Convenience macros for calling the trace function. */
 #define TRACE_0D(_e)                            \
     do {                                        \
-- 
1.7.2.5