[PATCH kvm-unit-tests] x86:kvmclock-test: add testcases for x-cpu clock warp

[PATCH kvm-unit-tests] x86:kvmclock-test: add testcases for x-cpu clock warp

[Roman Kagan]

Add testcases for clock monotonicity across vCPUs.  In a sense this is a
reduced adaptation of Ingo Molnar's time-wrap-test.

These testcases are extremely unlikely to pass on SMP VMs, so they are
reported as XFAILs.  Their purpose is to demonstrate that it's
impossible to achieve clock monotonicity across vCPUs (unless you can
completely exclude preemption of vCPUs by the hypervisor, of course).

Note that even in the (faked) case of PVCLOCK_TSC_STABLE_BIT unset, when
pvclock_clocksource_read() tries to monotonize the clock using basically
the same technique as the testcase uses, it still fails the global
monotonicity test.

Signed-off-by: Roman Kagan <rkagan@virtuozzo.com>
---
the patch applies on top of my earlier patch "x86/kvmclock_test: rework
to avoid spinlocks"

 x86/kvmclock_test.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)

diff --git a/x86/kvmclock_test.c b/x86/kvmclock_test.c
index fc3cb2e..0af30c8 100644
--- a/x86/kvmclock_test.c
+++ b/x86/kvmclock_test.c
@@ -15,6 +15,8 @@ struct warp_test_info {
 	long long worst;
 };
 struct warp_test_info wti[MAX_CPU];
+struct warp_test_info xwti[MAX_CPU];
+atomic64_t last_value = ATOMIC64_INIT(0);
 
 struct perf_test_info {
 	unsigned long long cycles;
@@ -80,6 +82,38 @@ static void warp_test_cpu(void *data)
 	atomic_dec(&cpus_left);
 }
 
+static void xwarp_test_cpu(void *data)
+{
+	struct warp_test_info *ti = data;
+	unsigned long long t = kvm_clock_read();
+	unsigned long long end = t + DURATION * NSEC_PER_SEC;
+	ti->warps = 0;
+	ti->stalls = 0;
+	ti->worst = 0;
+
+	do {
+		long long delta;
+		u64 last;
+
+		t = kvm_clock_read();
+		last = atomic64_read(&last_value);
+		delta = t - last;
+
+		while (t > last)
+			last = atomic64_cmpxchg(&last_value, last, t);
+
+		if (delta < 0) {
+			ti->warps++;
+			if (delta < ti->worst)
+				ti->worst = delta;
+		}
+		if (delta == 0)
+			ti->stalls++;
+	} while (t < end);
+
+	atomic_dec(&cpus_left);
+}
+
 static void perf_test_cpu(void *data)
 {
 	struct perf_test_info *ti = data;
@@ -121,6 +155,32 @@ static void warp_test(int ncpus, bool stable)
 	       warps == 0, stable ? "" : "out", warps, worst, stalls);
 }
 
+static void xwarp_test(int ncpus, bool stable)
+{
+	int i;
+	unsigned long long warps = 0, stalls = 0;
+	long long worst = 0;
+
+	pvclock_set_flags(stable ? PVCLOCK_RAW_CYCLE_BIT : 0);
+
+	atomic_set(&cpus_left, ncpus);
+	for (i = ncpus - 1; i >= 0; i--)
+		on_cpu_async(i, xwarp_test_cpu, &xwti[i]);
+	while (atomic_read(&cpus_left));
+
+	for (i = 0; i < ncpus; i++) {
+		warps += xwti[i].warps;
+		stalls += xwti[i].stalls;
+		if (xwti[i].worst < worst)
+			worst = xwti[i].worst;
+	}
+
+	report_xfail("with%s TSC_STABLE: x-cpu warps:"
+		     " %llu (worst %lld), stalls: %llu",
+		     ncpus > 1, warps == 0, stable ? "" : "out",
+		     warps, worst, stalls);
+}
+
 static void perf_test(int ncpus, bool stable)
 {
 	int i;
@@ -169,6 +229,8 @@ int main(int ac, char **av)
 
 	warp_test(ncpus, true);
 	warp_test(ncpus, false);
+	xwarp_test(ncpus, true);
+	xwarp_test(ncpus, false);
 	perf_test(ncpus, true);
 	perf_test(ncpus, false);
 
-- 
2.5.5

Re: [PATCH kvm-unit-tests] x86:kvmclock-test: add testcases for x-cpu clock warp

[Marcelo Tosatti]

On Wed, Jun 15, 2016 at 09:41:50PM +0300, Roman Kagan wrote:
> Add testcases for clock monotonicity across vCPUs.  In a sense this is a
> reduced adaptation of Ingo Molnar's time-wrap-test.
> 
> These testcases are extremely unlikely to pass on SMP VMs, so they are
> reported as XFAILs.  Their purpose is to demonstrate that it's
> impossible to achieve clock monotonicity across vCPUs (unless you can
> completely exclude preemption of vCPUs by the hypervisor, of course).

Don't think this is true: to confirm, run time-warp-test.c on a
multi-guest vcpu with SMP, and check that TSC monotonicity does not fail
across vCPUs.

This means that TSCs are monotonic, which means that any paravirt clock
based on TSC should be able to be monotonic as well.

Please submit Hyper-V patchset with the rdmsr(REF_CLOCK) reading Hyper-V
tsc page, i'll submit code to fix kvmclock update monotonicity
on top of it.

> 
> Note that even in the (faked) case of PVCLOCK_TSC_STABLE_BIT unset, when
> pvclock_clocksource_read() tries to monotonize the clock using basically
> the same technique as the testcase uses, it still fails the global
> monotonicity test.
> 
> Signed-off-by: Roman Kagan <rkagan@virtuozzo.com>
> ---
> the patch applies on top of my earlier patch "x86/kvmclock_test: rework
> to avoid spinlocks"
> 
>  x86/kvmclock_test.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 62 insertions(+)
> 
> diff --git a/x86/kvmclock_test.c b/x86/kvmclock_test.c
> index fc3cb2e..0af30c8 100644
> --- a/x86/kvmclock_test.c
> +++ b/x86/kvmclock_test.c
> @@ -15,6 +15,8 @@ struct warp_test_info {
>  	long long worst;
>  };
>  struct warp_test_info wti[MAX_CPU];
> +struct warp_test_info xwti[MAX_CPU];
> +atomic64_t last_value = ATOMIC64_INIT(0);
>  
>  struct perf_test_info {
>  	unsigned long long cycles;
> @@ -80,6 +82,38 @@ static void warp_test_cpu(void *data)
>  	atomic_dec(&cpus_left);
>  }
>  
> +static void xwarp_test_cpu(void *data)
> +{
> +	struct warp_test_info *ti = data;
> +	unsigned long long t = kvm_clock_read();
> +	unsigned long long end = t + DURATION * NSEC_PER_SEC;
> +	ti->warps = 0;
> +	ti->stalls = 0;
> +	ti->worst = 0;
> +
> +	do {
> +		long long delta;
> +		u64 last;
> +
> +		t = kvm_clock_read();
> +		last = atomic64_read(&last_value);
> +		delta = t - last;
> +
> +		while (t > last)
> +			last = atomic64_cmpxchg(&last_value, last, t);
> +
> +		if (delta < 0) {
> +			ti->warps++;
> +			if (delta < ti->worst)
> +				ti->worst = delta;
> +		}
> +		if (delta == 0)
> +			ti->stalls++;
> +	} while (t < end);
> +
> +	atomic_dec(&cpus_left);
> +}
> +
>  static void perf_test_cpu(void *data)
>  {
>  	struct perf_test_info *ti = data;
> @@ -121,6 +155,32 @@ static void warp_test(int ncpus, bool stable)
>  	       warps == 0, stable ? "" : "out", warps, worst, stalls);
>  }
>  
> +static void xwarp_test(int ncpus, bool stable)
> +{
> +	int i;
> +	unsigned long long warps = 0, stalls = 0;
> +	long long worst = 0;
> +
> +	pvclock_set_flags(stable ? PVCLOCK_RAW_CYCLE_BIT : 0);
> +
> +	atomic_set(&cpus_left, ncpus);
> +	for (i = ncpus - 1; i >= 0; i--)
> +		on_cpu_async(i, xwarp_test_cpu, &xwti[i]);
> +	while (atomic_read(&cpus_left));
> +
> +	for (i = 0; i < ncpus; i++) {
> +		warps += xwti[i].warps;
> +		stalls += xwti[i].stalls;
> +		if (xwti[i].worst < worst)
> +			worst = xwti[i].worst;
> +	}
> +
> +	report_xfail("with%s TSC_STABLE: x-cpu warps:"
> +		     " %llu (worst %lld), stalls: %llu",
> +		     ncpus > 1, warps == 0, stable ? "" : "out",
> +		     warps, worst, stalls);
> +}
> +
>  static void perf_test(int ncpus, bool stable)
>  {
>  	int i;
> @@ -169,6 +229,8 @@ int main(int ac, char **av)
>  
>  	warp_test(ncpus, true);
>  	warp_test(ncpus, false);
> +	xwarp_test(ncpus, true);
> +	xwarp_test(ncpus, false);
>  	perf_test(ncpus, true);
>  	perf_test(ncpus, false);
>  
> -- 
> 2.5.5

Re: [PATCH kvm-unit-tests] x86:kvmclock-test: add testcases for x-cpu clock warp

[Roman Kagan]

On Wed, Jun 15, 2016 at 09:41:50PM +0300, Roman Kagan wrote:
> Add testcases for clock monotonicity across vCPUs.  In a sense this is a
> reduced adaptation of Ingo Molnar's time-wrap-test.
> 
> These testcases are extremely unlikely to pass on SMP VMs, so they are
> reported as XFAILs.  Their purpose is to demonstrate that it's
> impossible to achieve clock monotonicity across vCPUs (unless you can
> completely exclude preemption of vCPUs by the hypervisor, of course).
> 
> Note that even in the (faked) case of PVCLOCK_TSC_STABLE_BIT unset, when
> pvclock_clocksource_read() tries to monotonize the clock using basically
> the same technique as the testcase uses, it still fails the global
> monotonicity test.

I've finally realized the obvious "how I screwed it all up": the
standard pattern is to read the clock and compare to other vCPUs /
update the shared value in a single critical section.  So please
disregard my patches to the tests, per-cpu counters are of little
interest here.

Sorry for the noise,
Roman.

Re: [PATCH kvm-unit-tests] x86:kvmclock-test: add testcases for x-cpu clock warp

[Paolo Bonzini]

On 20/06/2016 18:55, Roman Kagan wrote:
> > Note that even in the (faked) case of PVCLOCK_TSC_STABLE_BIT unset, when
> > pvclock_clocksource_read() tries to monotonize the clock using basically
> > the same technique as the testcase uses, it still fails the global
> > monotonicity test.
>
> I've finally realized the obvious "how I screwed it all up": the
> standard pattern is to read the clock and compare to other vCPUs /
> update the shared value in a single critical section.  So please
> disregard my patches to the tests, per-cpu counters are of little
> interest here.

Actually aren't per-cpu counters used by the vDSO?

Paolo

Re: [PATCH kvm-unit-tests] x86:kvmclock-test: add testcases for x-cpu clock warp

[Roman Kagan]

On Mon, Jun 27, 2016 at 01:26:54PM +0200, Paolo Bonzini wrote:
> 
> 
> On 20/06/2016 18:55, Roman Kagan wrote:
> > > Note that even in the (faked) case of PVCLOCK_TSC_STABLE_BIT unset, when
> > > pvclock_clocksource_read() tries to monotonize the clock using basically
> > > the same technique as the testcase uses, it still fails the global
> > > monotonicity test.
> >
> > I've finally realized the obvious "how I screwed it all up": the
> > standard pattern is to read the clock and compare to other vCPUs /
> > update the shared value in a single critical section.  So please
> > disregard my patches to the tests, per-cpu counters are of little
> > interest here.
> 
> Actually aren't per-cpu counters used by the vDSO?

Yes they are, but the vDSO code explicitly protects against warps as
this is considered normal situation.

Roman.