diff -r e3b13e1ecf6c xen/arch/x86/hvm/hpet.c
--- a/xen/arch/x86/hvm/hpet.c	Thu May 15 15:10:05 2008 +0100
+++ b/xen/arch/x86/hvm/hpet.c	Wed May 21 11:37:31 2008 -0600
@@ -29,9 +29,9 @@
 #define S_TO_NS  1000000000ULL           /* 1s  = 10^9  ns */
 #define S_TO_FS  1000000000000000ULL     /* 1s  = 10^15 fs */
 
-/* Frequency_of_TSC / frequency_of_HPET = 32 */
-#define TSC_PER_HPET_TICK 32
-#define guest_time_hpet(v) (hvm_get_guest_time(v) / TSC_PER_HPET_TICK)
+/* Frequency_of_Xen_systeme_time / frequency_of_HPET = 16 */
+#define STIME_PER_HPET_TICK 16
+#define guest_time_hpet(v) (hvm_get_guest_time_mono(v) / STIME_PER_HPET_TICK)
 
 #define HPET_ID         0x000
 #define HPET_PERIOD     0x004
@@ -192,7 +192,7 @@ static void hpet_stop_timer(HPETState *h
 
 /* the number of HPET tick that stands for
  * 1/(2^10) second, namely, 0.9765625 milliseconds */
-#define  HPET_TINY_TIME_SPAN  ((h->tsc_freq >> 10) / TSC_PER_HPET_TICK)
+#define  HPET_TINY_TIME_SPAN  ((h->stime_freq >> 10) / STIME_PER_HPET_TICK)
 
 static void hpet_set_timer(HPETState *h, unsigned int tn)
 {
@@ -558,17 +558,17 @@ void hpet_init(struct vcpu *v)
     spin_lock_init(&h->lock);
 
     h->vcpu = v;
-    h->tsc_freq = ticks_per_sec(v);
+    h->stime_freq = S_TO_NS;
 
-    h->hpet_to_ns_scale = ((S_TO_NS * TSC_PER_HPET_TICK) << 10) / h->tsc_freq;
+    h->hpet_to_ns_scale = ((S_TO_NS * STIME_PER_HPET_TICK) << 10) / h->stime_freq;
     h->hpet_to_ns_limit = ~0ULL / h->hpet_to_ns_scale;
 
     /* 64-bit main counter; 3 timers supported; LegacyReplacementRoute. */
     h->hpet.capability = 0x8086A201ULL;
 
     /* This is the number of femptoseconds per HPET tick. */
-    /* Here we define HPET's frequency to be 1/32 of the TSC's */
-    h->hpet.capability |= ((S_TO_FS*TSC_PER_HPET_TICK/h->tsc_freq) << 32);
+    /* Here we define HPET's frequency to be 1/16 of Xen system time */
+    h->hpet.capability |= ((S_TO_FS*STIME_PER_HPET_TICK/h->stime_freq) << 32);
 
     for ( i = 0; i < HPET_TIMER_NUM; i++ )
     {
diff -r e3b13e1ecf6c xen/arch/x86/hvm/hvm.c
--- a/xen/arch/x86/hvm/hvm.c	Thu May 15 15:10:05 2008 +0100
+++ b/xen/arch/x86/hvm/hvm.c	Wed May 21 11:37:31 2008 -0600
@@ -152,6 +152,49 @@ u64 hvm_get_guest_tsc(struct vcpu *v)
     return host_tsc + v->arch.hvm_vcpu.cache_tsc_offset;
 }
 
+/*
+ * Guest time is Xen system time (measured in nsec) plus an offset.
+ * It is guaranteed to be monotonic
+ */
+void hvm_init_guest_time(struct hvm_domain *hd)
+{
+    struct pl_time *pl = &hd->pl_time;
+
+    spin_lock_init(&pl->pl_time_lock);
+    pl->stime_offset = -(u64)get_s_time_mono();
+    pl->last_guest_time = 0ULL;
+}
+
+void hvm_set_guest_time(struct vcpu *v, u64 guest_time)
+{
+    struct pl_time *pl = &v->domain->arch.hvm_domain.pl_time;
+    u64 now;
+
+    spin_lock(&pl->pl_time_lock);
+    now = (u64)get_s_time_mono() + pl->stime_offset;
+    if (now <= guest_time) {
+         pl->stime_offset = guest_time - now;
+         pl->last_guest_time = guest_time - now;
+    }
+    /* ignore attempts to set guest time backwards */
+    spin_unlock(&pl->pl_time_lock);
+}
+
+u64 hvm_get_guest_time_mono(struct vcpu *v)
+{
+    struct pl_time *pl = &v->domain->arch.hvm_domain.pl_time;
+    u64 now;
+
+    spin_lock(&pl->pl_time_lock);
+    now = get_s_time_mono() + pl->stime_offset;
+    if (now > pl->last_guest_time)
+        pl->last_guest_time = now;
+    else
+        now = pl->last_guest_time;
+    spin_unlock(&pl->pl_time_lock);
+    return now;
+}
+
 void hvm_migrate_timers(struct vcpu *v)
 {
     rtc_migrate_timers(v);
@@ -295,6 +338,8 @@ int hvm_domain_initialise(struct domain 
     spin_lock_init(&d->arch.hvm_domain.pbuf_lock);
     spin_lock_init(&d->arch.hvm_domain.irq_lock);
     spin_lock_init(&d->arch.hvm_domain.uc_lock);
+
+    hvm_init_guest_time(&d->arch.hvm_domain);
 
     d->arch.hvm_domain.params[HVM_PARAM_HPET_ENABLED] = 1;
 
@@ -661,7 +706,7 @@ int hvm_vcpu_initialise(struct vcpu *v)
         hpet_init(v);
  
         /* Init guest TSC to start from zero. */
-        hvm_set_guest_time(v, 0);
+        hvm_set_guest_tsc(v, 0);
 
         /* Can start up without SIPI-SIPI or setvcpucontext domctl. */
         v->is_initialised = 1;
@@ -1632,7 +1677,7 @@ int hvm_msr_read_intercept(struct cpu_us
     switch ( ecx )
     {
     case MSR_IA32_TSC:
-        msr_content = hvm_get_guest_time(v);
+        msr_content = hvm_get_guest_tsc(v);
         break;
 
     case MSR_IA32_APICBASE:
@@ -1725,7 +1770,7 @@ int hvm_msr_write_intercept(struct cpu_u
     switch ( ecx )
     {
      case MSR_IA32_TSC:
-        hvm_set_guest_time(v, msr_content);
+        hvm_set_guest_tsc(v, msr_content);
         pt_reset(v);
         break;
 
diff -r e3b13e1ecf6c xen/arch/x86/hvm/i8254.c
--- a/xen/arch/x86/hvm/i8254.c	Thu May 15 15:10:05 2008 +0100
+++ b/xen/arch/x86/hvm/i8254.c	Wed May 21 11:37:31 2008 -0600
@@ -31,6 +31,7 @@
 #include <xen/lib.h>
 #include <xen/errno.h>
 #include <xen/sched.h>
+#include <asm/time.h>
 #include <asm/hvm/hvm.h>
 #include <asm/hvm/io.h>
 #include <asm/hvm/support.h>
@@ -86,8 +87,8 @@ static int pit_get_count(PITState *pit, 
 
     ASSERT(spin_is_locked(&pit->lock));
 
-    d = muldiv64(hvm_get_guest_time(v) - pit->count_load_time[channel],
-                 PIT_FREQ, ticks_per_sec(v));
+    d = muldiv64(hvm_get_guest_time_mono(v) - pit->count_load_time[channel],
+                 PIT_FREQ, s_ticks_per_sec(v));
 
     switch ( c->mode )
     {
@@ -117,8 +118,8 @@ static int pit_get_out(PITState *pit, in
 
     ASSERT(spin_is_locked(&pit->lock));
 
-    d = muldiv64(hvm_get_guest_time(v) - pit->count_load_time[channel], 
-                 PIT_FREQ, ticks_per_sec(v));
+    d = muldiv64(hvm_get_guest_time_mono(v) - pit->count_load_time[channel], 
+                 PIT_FREQ, s_ticks_per_sec(v));
 
     switch ( s->mode )
     {
@@ -164,7 +165,7 @@ static void pit_set_gate(PITState *pit, 
     case 3:
         /* Restart counting on rising edge. */
         if ( s->gate < val )
-            pit->count_load_time[channel] = hvm_get_guest_time(v);
+            pit->count_load_time[channel] = hvm_get_guest_time_mono(v);
         break;
     }
 
@@ -180,7 +181,7 @@ static void pit_time_fired(struct vcpu *
 static void pit_time_fired(struct vcpu *v, void *priv)
 {
     uint64_t *count_load_time = priv;
-    *count_load_time = hvm_get_guest_time(v);
+    *count_load_time = hvm_get_guest_time_mono(v);
 }
 
 static void pit_load_count(PITState *pit, int channel, int val)
@@ -195,11 +196,11 @@ static void pit_load_count(PITState *pit
         val = 0x10000;
 
     if ( v == NULL )
-        rdtscll(pit->count_load_time[channel]);
+        pit->count_load_time[channel] = 0;
     else
-        pit->count_load_time[channel] = hvm_get_guest_time(v);
+        pit->count_load_time[channel] = hvm_get_guest_time_mono(v);
     s->count = val;
-    period = DIV_ROUND((val * 1000000000ULL), PIT_FREQ);
+    period = DIV_ROUND((val * s_ticks_per_sec(v)), PIT_FREQ);
 
     if ( (v == NULL) || !is_hvm_vcpu(v) || (channel != 0) )
         return;
@@ -435,7 +436,7 @@ static int pit_load(struct domain *d, hv
      * time jitter here, but the wall-clock will have jumped massively, so 
      * we hope the guest can handle it.
      */
-    pit->pt0.last_plt_gtime = hvm_get_guest_time(d->vcpu[0]);
+    pit->pt0.last_plt_gtime = hvm_get_guest_time_mono(d->vcpu[0]);
     for ( i = 0; i < 3; i++ )
         pit_load_count(pit, i, pit->hw.channels[i].count);
 
diff -r e3b13e1ecf6c xen/arch/x86/hvm/pmtimer.c
--- a/xen/arch/x86/hvm/pmtimer.c	Thu May 15 15:10:05 2008 +0100
+++ b/xen/arch/x86/hvm/pmtimer.c	Wed May 21 11:37:31 2008 -0600
@@ -71,7 +71,7 @@ static void pmt_update_time(PMTState *s)
     ASSERT(spin_is_locked(&s->lock));
 
     /* Update the timer */
-    curr_gtime = hvm_get_guest_time(s->vcpu);
+    curr_gtime = hvm_get_guest_time_mono(s->vcpu);
     s->pm.tmr_val += ((curr_gtime - s->last_gtime) * s->scale) >> 32;
     s->pm.tmr_val &= TMR_VAL_MASK;
     s->last_gtime = curr_gtime;
@@ -238,7 +238,7 @@ static int pmtimer_load(struct domain *d
     }
 
     /* Calculate future counter values from now. */
-    s->last_gtime = hvm_get_guest_time(s->vcpu);
+    s->last_gtime = hvm_get_guest_time_mono(s->vcpu);
 
     /* Set the SCI state from the registers */ 
     pmt_update_sci(s);
@@ -257,7 +257,7 @@ void pmtimer_init(struct vcpu *v)
 
     spin_lock_init(&s->lock);
 
-    s->scale = ((uint64_t)FREQUENCE_PMTIMER << 32) / ticks_per_sec(v);
+    s->scale = ((uint64_t)FREQUENCE_PMTIMER << 32) / s_ticks_per_sec(v);
     s->vcpu = v;
 
     /* Intercept port I/O (need two handlers because PM1a_CNT is between
diff -r e3b13e1ecf6c xen/arch/x86/hvm/svm/svm.c
--- a/xen/arch/x86/hvm/svm/svm.c	Thu May 15 15:10:05 2008 +0100
+++ b/xen/arch/x86/hvm/svm/svm.c	Wed May 21 11:37:31 2008 -0600
@@ -299,7 +299,7 @@ static void svm_save_cpu_state(struct vc
     data->msr_efer         = v->arch.hvm_vcpu.guest_efer;
     data->msr_flags        = -1ULL;
 
-    data->tsc = hvm_get_guest_time(v);
+    data->tsc = hvm_get_guest_tsc(v);
 }
 
 
@@ -315,7 +315,7 @@ static void svm_load_cpu_state(struct vc
     v->arch.hvm_vcpu.guest_efer = data->msr_efer;
     svm_update_guest_efer(v);
 
-    hvm_set_guest_time(v, data->tsc);
+    hvm_set_guest_tsc(v, data->tsc);
 }
 
 static void svm_save_vmcb_ctxt(struct vcpu *v, struct hvm_hw_cpu *ctxt)
diff -r e3b13e1ecf6c xen/arch/x86/hvm/vlapic.c
--- a/xen/arch/x86/hvm/vlapic.c	Thu May 15 15:10:05 2008 +0100
+++ b/xen/arch/x86/hvm/vlapic.c	Wed May 21 11:37:31 2008 -0600
@@ -535,8 +535,7 @@ static uint32_t vlapic_get_tmcct(struct 
     uint32_t tmcct, tmict = vlapic_get_reg(vlapic, APIC_TMICT);
     uint64_t counter_passed;
 
-    counter_passed = ((hvm_get_guest_time(v) - vlapic->timer_last_update)
-                      * 1000000000ULL / ticks_per_sec(v)
+    counter_passed = ((hvm_get_guest_time_mono(v) - vlapic->timer_last_update)
                       / APIC_BUS_CYCLE_NS / vlapic->hw.timer_divisor);
     tmcct = tmict - counter_passed;
 
@@ -638,7 +637,7 @@ static int vlapic_read(
 
 void vlapic_pt_cb(struct vcpu *v, void *data)
 {
-    *(s_time_t *)data = hvm_get_guest_time(v);
+    *(s_time_t *)data = hvm_get_guest_time_mono(v);
 }
 
 static int vlapic_write(struct vcpu *v, unsigned long address,
diff -r e3b13e1ecf6c xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c	Thu May 15 15:10:05 2008 +0100
+++ b/xen/arch/x86/hvm/vmx/vmx.c	Wed May 21 11:37:31 2008 -0600
@@ -607,7 +607,7 @@ static void vmx_save_cpu_state(struct vc
     data->msr_syscall_mask = guest_state->msrs[VMX_INDEX_MSR_SYSCALL_MASK];
 #endif
 
-    data->tsc = hvm_get_guest_time(v);
+    data->tsc = hvm_get_guest_tsc(v);
 }
 
 static void vmx_load_cpu_state(struct vcpu *v, struct hvm_hw_cpu *data)
@@ -625,7 +625,7 @@ static void vmx_load_cpu_state(struct vc
     v->arch.hvm_vmx.shadow_gs = data->shadow_gs;
 #endif
 
-    hvm_set_guest_time(v, data->tsc);
+    hvm_set_guest_tsc(v, data->tsc);
 }
 
 
diff -r e3b13e1ecf6c xen/arch/x86/hvm/vpt.c
--- a/xen/arch/x86/hvm/vpt.c	Thu May 15 15:10:05 2008 +0100
+++ b/xen/arch/x86/hvm/vpt.c	Wed May 21 11:37:31 2008 -0600
@@ -108,7 +108,7 @@ static void pt_freeze_time(struct vcpu *
     if ( !mode_is(v->domain, delay_for_missed_ticks) )
         return;
 
-    v->arch.hvm_vcpu.guest_time = hvm_get_guest_time(v);
+    v->arch.hvm_vcpu.guest_time = hvm_get_guest_time_mono(v);
 }
 
 static void pt_thaw_time(struct vcpu *v)
@@ -267,7 +267,7 @@ void pt_intr_post(struct vcpu *v, struct
         if ( mode_is(v->domain, one_missed_tick_pending) ||
              mode_is(v->domain, no_missed_ticks_pending) )
         {
-            pt->last_plt_gtime = hvm_get_guest_time(v);
+            pt->last_plt_gtime = hvm_get_guest_time_mono(v);
             pt->pending_intr_nr = 0; /* 'collapse' all missed ticks */
         }
         else
@@ -278,7 +278,7 @@ void pt_intr_post(struct vcpu *v, struct
     }
 
     if ( mode_is(v->domain, delay_for_missed_ticks) &&
-         (hvm_get_guest_time(v) < pt->last_plt_gtime) )
+         (hvm_get_guest_time_mono(v) < pt->last_plt_gtime) )
         hvm_set_guest_time(v, pt->last_plt_gtime);
 
     cb = pt->cb;
@@ -300,7 +300,7 @@ void pt_reset(struct vcpu *v)
     list_for_each_entry ( pt, head, list )
     {
         pt->pending_intr_nr = 0;
-        pt->last_plt_gtime = hvm_get_guest_time(pt->vcpu);
+        pt->last_plt_gtime = hvm_get_guest_time_mono(pt->vcpu);
         pt->scheduled = NOW() + pt->period;
         set_timer(&pt->timer, pt->scheduled);
     }
@@ -346,9 +346,9 @@ void create_periodic_time(
 
     pt->period = period;
     pt->vcpu = v;
-    pt->last_plt_gtime = hvm_get_guest_time(pt->vcpu);
+    pt->last_plt_gtime = hvm_get_guest_time_mono(pt->vcpu);
     pt->irq = irq;
-    pt->period_cycles = (u64)period * cpu_khz / 1000000L;
+    pt->period_cycles = (u64)period;
     pt->one_shot = one_shot;
     pt->scheduled = NOW() + period;
     /*
diff -r e3b13e1ecf6c xen/arch/x86/time.c
--- a/xen/arch/x86/time.c	Thu May 15 15:10:05 2008 +0100
+++ b/xen/arch/x86/time.c	Wed May 21 11:37:31 2008 -0600
@@ -535,6 +535,8 @@ static s_time_t read_platform_stime(void
     return stime;
 }
 
+extern void get_s_time_mono_init(void);
+
 static void platform_time_calibration(void)
 {
     u64 count;
@@ -593,6 +595,7 @@ static void init_platform_timer(void)
     plt_overflow(NULL);
 
     platform_timer_stamp = plt_stamp64;
+    get_s_time_mono_init();
 
     printk("Platform timer is %s %s\n",
            freq_string(pts->frequency), pts->name);
@@ -728,6 +731,29 @@ s_time_t get_s_time(void)
     rdtscll(tsc);
     delta = tsc - t->local_tsc_stamp;
     now = t->stime_local_stamp + scale_delta(delta, &t->tsc_scale);
+
+    return now;
+}
+
+spinlock_t get_s_time_mono_lock;
+
+void get_s_time_mono_init(void)
+{
+    spin_lock_init(&get_s_time_mono_lock);
+}
+
+s_time_t get_s_time_mono(void)
+{
+    s_time_t now;
+    static s_time_t last = (s_time_t)0;
+
+    spin_lock(&get_s_time_mono_lock);
+    now = get_s_time();
+    if (now > last)
+        last = now;
+    else
+        now = last;
+    spin_unlock(&get_s_time_mono_lock);
 
     return now;
 }
diff -r e3b13e1ecf6c xen/include/asm-x86/hvm/hvm.h
--- a/xen/include/asm-x86/hvm/hvm.h	Thu May 15 15:10:05 2008 +0100
+++ b/xen/include/asm-x86/hvm/hvm.h	Wed May 21 11:37:31 2008 -0600
@@ -146,8 +146,9 @@ void hvm_send_assist_req(struct vcpu *v)
 
 void hvm_set_guest_tsc(struct vcpu *v, u64 guest_tsc);
 u64 hvm_get_guest_tsc(struct vcpu *v);
-#define hvm_set_guest_time(vcpu, gtime) hvm_set_guest_tsc(vcpu, gtime)
-#define hvm_get_guest_time(vcpu)        hvm_get_guest_tsc(vcpu)
+void hvm_set_guest_time(struct vcpu *v, u64 guest_time);
+u64 hvm_get_guest_time(struct vcpu *v);
+u64 hvm_get_guest_time_mono(struct vcpu *v);
 
 #define hvm_paging_enabled(v) \
     (!!((v)->arch.hvm_vcpu.guest_cr[0] & X86_CR0_PG))
diff -r e3b13e1ecf6c xen/include/asm-x86/hvm/vmx/vmx.h
--- a/xen/include/asm-x86/hvm/vmx/vmx.h	Thu May 15 15:10:05 2008 +0100
+++ b/xen/include/asm-x86/hvm/vmx/vmx.h	Wed May 21 11:37:31 2008 -0600
@@ -49,7 +49,6 @@ void vmx_asm_do_vmentry(void);
 void vmx_asm_do_vmentry(void);
 void vmx_intr_assist(void);
 void vmx_do_resume(struct vcpu *);
-void set_guest_time(struct vcpu *v, u64 gtime);
 void vmx_vlapic_msr_changed(struct vcpu *v);
 void vmx_realmode(struct cpu_user_regs *regs);
 
diff -r e3b13e1ecf6c xen/include/asm-x86/hvm/vpt.h
--- a/xen/include/asm-x86/hvm/vpt.h	Thu May 15 15:10:05 2008 +0100
+++ b/xen/include/asm-x86/hvm/vpt.h	Wed May 21 11:37:31 2008 -0600
@@ -57,7 +57,7 @@ typedef struct HPETState {
 typedef struct HPETState {
     struct hpet_registers hpet;
     struct vcpu *vcpu;
-    uint64_t tsc_freq;
+    uint64_t stime_freq;
     uint64_t hpet_to_ns_scale; /* hpet ticks to ns (multiplied by 2^10) */
     uint64_t hpet_to_ns_limit; /* max hpet ticks convertable to ns      */
     uint64_t mc_offset;
@@ -137,6 +137,9 @@ struct pl_time {    /* platform time */
     struct RTCState  vrtc;
     struct HPETState vhpet;
     struct PMTState  vpmt;
+    int64_t stime_offset;       /* guest_time = Xen sys time + stime_offset */
+    uint64_t last_guest_time;   /* ensure monotonicity */
+    spinlock_t pl_time_lock;
 };
 
 #define ticks_per_sec(v) (v->domain->arch.hvm_domain.tsc_frequency)
diff -r e3b13e1ecf6c xen/include/asm-x86/time.h
--- a/xen/include/asm-x86/time.h	Thu May 15 15:10:05 2008 +0100
+++ b/xen/include/asm-x86/time.h	Wed May 21 11:37:31 2008 -0600
@@ -34,4 +34,6 @@ struct tm;
 struct tm;
 struct tm wallclock_time(void);
 
+#define s_ticks_per_sec(v)	1000000000ULL
+
 #endif /* __X86_TIME_H__ */
diff -r e3b13e1ecf6c xen/include/xen/time.h
--- a/xen/include/xen/time.h	Thu May 15 15:10:05 2008 +0100
+++ b/xen/include/xen/time.h	Wed May 21 11:37:31 2008 -0600
@@ -24,6 +24,8 @@ struct domain;
  * System Time
  * 64 bit value containing the nanoseconds elapsed since boot time.
  * This value is adjusted by frequency drift.
+ * mono version guarantees that consecutive reads are non-decreasing,
+ * (e.g. when first read is on a different processor than second read)
  * NOW() returns the current time.
  * The other macros are for convenience to approximate short intervals
  * of real time into system time 
@@ -32,6 +34,7 @@ typedef s64 s_time_t;
 typedef s64 s_time_t;
 
 s_time_t get_s_time(void);
+s_time_t get_s_time_mono(void);
 unsigned long get_localtime(struct domain *d);
 
 struct tm {
