[PATCH v7 1/3] KVM: make halt_poll_ns per-vCPU

[PATCH v7 1/3] KVM: make halt_poll_ns per-vCPU

[Wanpeng Li]

Change halt_poll_ns into per-VCPU variable, seeded from module parameter,
to allow greater flexibility.

Signed-off-by: Wanpeng Li <wanpeng.li@hotmail.com>
---
 include/linux/kvm_host.h | 1 +
 virt/kvm/kvm_main.c      | 5 +++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 81089cf..1bef9e2 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -242,6 +242,7 @@ struct kvm_vcpu {
 	int sigset_active;
 	sigset_t sigset;
 	struct kvm_vcpu_stat stat;
+	unsigned int halt_poll_ns;
 
 #ifdef CONFIG_HAS_IOMEM
 	int mmio_needed;
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index d8db2f8f..c06e57c 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -217,6 +217,7 @@ int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
 	vcpu->kvm = kvm;
 	vcpu->vcpu_id = id;
 	vcpu->pid = NULL;
+	vcpu->halt_poll_ns = 0;
 	init_waitqueue_head(&vcpu->wq);
 	kvm_async_pf_vcpu_init(vcpu);
 
@@ -1930,8 +1931,8 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
 	bool waited = false;
 
 	start = cur = ktime_get();
-	if (halt_poll_ns) {
-		ktime_t stop = ktime_add_ns(ktime_get(), halt_poll_ns);
+	if (vcpu->halt_poll_ns) {
+		ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
 
 		do {
 			/*
-- 
1.9.1

[PATCH v7 2/3] KVM: dynamic halt-polling

[Wanpeng Li]

There is a downside of always-poll since poll is still happened for idle 
vCPUs which can waste cpu usage. This patchset add the ability to adjust 
halt_poll_ns dynamically, to grow halt_poll_ns when shot halt is detected,  
and to shrink halt_poll_ns when long halt is detected.

There are two new kernel parameters for changing the halt_poll_ns:
halt_poll_ns_grow and halt_poll_ns_shrink. 

                        no-poll      always-poll    dynamic-poll
-----------------------------------------------------------------------
Idle (nohz) vCPU %c0     0.15%        0.3%            0.2%  
Idle (250HZ) vCPU %c0    1.1%         4.6%~14%        1.2%
TCP_RR latency           34us         27us            26.7us

"Idle (X) vCPU %c0" is the percent of time the physical cpu spent in
c0 over 60 seconds (each vCPU is pinned to a pCPU). (nohz) means the
guest was tickless. (250HZ) means the guest was ticking at 250HZ.

The big win is with ticking operating systems. Running the linux guest
with nohz=off (and HZ=250), we save 3.4%~12.8% CPUs/second and get close 
to no-polling overhead levels by using the dynamic-poll. The savings
should be even higher for higher frequency ticks.

Suggested-by: David Matlack <dmatlack@google.com>
Signed-off-by: Wanpeng Li <wanpeng.li@hotmail.com>
---
 virt/kvm/kvm_main.c | 63 +++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 59 insertions(+), 4 deletions(-)

diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index c06e57c..d5e07e9 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -66,9 +66,18 @@
 MODULE_AUTHOR("Qumranet");
 MODULE_LICENSE("GPL");
 
-static unsigned int halt_poll_ns;
+/* halt polling only reduces halt latency by 5-7 us, 500us is enough */
+static unsigned int halt_poll_ns = 500000;
 module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR);
 
+/* Default doubles per-vcpu halt_poll_ns. */
+static unsigned int halt_poll_ns_grow = 2;
+module_param(halt_poll_ns_grow, int, S_IRUGO);
+
+/* Default resets per-vcpu halt_poll_ns . */
+static unsigned int halt_poll_ns_shrink;
+module_param(halt_poll_ns_shrink, int, S_IRUGO);
+
 /*
  * Ordering of locks:
  *
@@ -1907,6 +1916,31 @@ void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn)
 }
 EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);
 
+static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
+{
+	int val = vcpu->halt_poll_ns;
+
+	/* 10us base */
+	if (val == 0 && halt_poll_ns_grow)
+		val = 10000;
+	else
+		val *= halt_poll_ns_grow;
+
+	vcpu->halt_poll_ns = val;
+}
+
+static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
+{
+	int val = vcpu->halt_poll_ns;
+
+	if (halt_poll_ns_shrink == 0)
+		val = 0;
+	else
+		val /= halt_poll_ns_shrink;
+
+	vcpu->halt_poll_ns = val;
+}
+
 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
 {
 	if (kvm_arch_vcpu_runnable(vcpu)) {
@@ -1928,7 +1962,8 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
 {
 	ktime_t start, cur;
 	DEFINE_WAIT(wait);
-	bool waited = false;
+	bool polled = false, waited = false;
+	u64 poll_ns = 0, wait_ns = 0, block_ns = 0;
 
 	start = cur = ktime_get();
 	if (vcpu->halt_poll_ns) {
@@ -1940,11 +1975,16 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
 			 * arrives.
 			 */
 			if (kvm_vcpu_check_block(vcpu) < 0) {
+				polled = true;
 				++vcpu->stat.halt_successful_poll;
-				goto out;
+				break;
 			}
 			cur = ktime_get();
 		} while (single_task_running() && ktime_before(cur, stop));
+
+		poll_ns = ktime_to_ns(cur) - ktime_to_ns(start);
+		if (polled)
+			goto out;
 	}
 
 	for (;;) {
@@ -1959,9 +1999,24 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
 
 	finish_wait(&vcpu->wq, &wait);
 	cur = ktime_get();
+	wait_ns = ktime_to_ns(cur) - ktime_to_ns(start);
 
 out:
-	trace_kvm_vcpu_wakeup(ktime_to_ns(cur) - ktime_to_ns(start), waited);
+	block_ns = poll_ns + wait_ns;
+
+	if (halt_poll_ns) {
+		if (block_ns <= vcpu->halt_poll_ns)
+			;
+		/* we had a long block, shrink polling */
+		else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns)
+			shrink_halt_poll_ns(vcpu);
+		/* we had a short halt and our poll time is too small */
+		else if (vcpu->halt_poll_ns < halt_poll_ns &&
+			block_ns < halt_poll_ns)
+			grow_halt_poll_ns(vcpu);
+	}
+
+	trace_kvm_vcpu_wakeup(block_ns, waited);
 }
 EXPORT_SYMBOL_GPL(kvm_vcpu_block);
 
-- 
1.9.1

[PATCH v7 3/3] KVM: trace kvm_halt_poll_ns grow/shrink

[Wanpeng Li]

Tracepoint for dynamic halt_pool_ns, fired on every potential change.

Signed-off-by: Wanpeng Li <wanpeng.li@hotmail.com>
---
 include/trace/events/kvm.h | 30 ++++++++++++++++++++++++++++++
 virt/kvm/kvm_main.c        |  8 ++++++--
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/include/trace/events/kvm.h b/include/trace/events/kvm.h
index a44062d..0d1bc47 100644
--- a/include/trace/events/kvm.h
+++ b/include/trace/events/kvm.h
@@ -356,6 +356,36 @@ TRACE_EVENT(
 		  __entry->address)
 );
 
+TRACE_EVENT(kvm_halt_poll_ns,
+	TP_PROTO(bool grow, unsigned int vcpu_id, int new, int old),
+	TP_ARGS(grow, vcpu_id, new, old),
+
+	TP_STRUCT__entry(
+		__field(bool, grow)
+		__field(unsigned int, vcpu_id)
+		__field(int, new)
+		__field(int, old)
+	),
+
+	TP_fast_assign(
+		__entry->grow           = grow;
+		__entry->vcpu_id        = vcpu_id;
+		__entry->new            = new;
+		__entry->old            = old;
+	),
+
+	TP_printk("vcpu %u: halt_poll_ns %d (%s %d)",
+			__entry->vcpu_id,
+			__entry->new,
+			__entry->grow ? "grow" : "shrink",
+			__entry->old)
+);
+
+#define trace_kvm_halt_poll_ns_grow(vcpu_id, new, old) \
+	trace_kvm_halt_poll_ns(true, vcpu_id, new, old)
+#define trace_kvm_halt_poll_ns_shrink(vcpu_id, new, old) \
+	trace_kvm_halt_poll_ns(false, vcpu_id, new, old)
+
 #endif
 
 #endif /* _TRACE_KVM_MAIN_H */
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index d5e07e9..44a1301 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -1918,8 +1918,9 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_mark_page_dirty);
 
 static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
 {
-	int val = vcpu->halt_poll_ns;
+	int old, val;
 
+	old = val = vcpu->halt_poll_ns;
 	/* 10us base */
 	if (val == 0 && halt_poll_ns_grow)
 		val = 10000;
@@ -1927,18 +1928,21 @@ static void grow_halt_poll_ns(struct kvm_vcpu *vcpu)
 		val *= halt_poll_ns_grow;
 
 	vcpu->halt_poll_ns = val;
+	trace_kvm_halt_poll_ns_grow(vcpu->vcpu_id, val, old);
 }
 
 static void shrink_halt_poll_ns(struct kvm_vcpu *vcpu)
 {
-	int val = vcpu->halt_poll_ns;
+	int old, val;
 
+	old = val = vcpu->halt_poll_ns;
 	if (halt_poll_ns_shrink == 0)
 		val = 0;
 	else
 		val /= halt_poll_ns_shrink;
 
 	vcpu->halt_poll_ns = val;
+	trace_kvm_halt_poll_ns_shrink(vcpu->vcpu_id, val, old);
 }
 
 static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
-- 
1.9.1

Re: [PATCH v7 2/3] KVM: dynamic halt-polling

[Wanpeng Li]

Hi Paolo,
On 9/3/15 10:07 PM, Wanpeng Li wrote:
> [...]
>   static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
>   {
>   	if (kvm_arch_vcpu_runnable(vcpu)) {
> @@ -1928,7 +1962,8 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
>   {
>   	ktime_t start, cur;
>   	DEFINE_WAIT(wait);
> -	bool waited = false;
> +	bool polled = false, waited = false;
> +	u64 poll_ns = 0, wait_ns = 0, block_ns = 0;
>   
>   	start = cur = ktime_get();
>   	if (vcpu->halt_poll_ns) {
> @@ -1940,11 +1975,16 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
>   			 * arrives.
>   			 */
>   			if (kvm_vcpu_check_block(vcpu) < 0) {
> +				polled = true;
>   				++vcpu->stat.halt_successful_poll;
> -				goto out;
> +				break;
>   			}
>   			cur = ktime_get();
>   		} while (single_task_running() && ktime_before(cur, stop));
> +
> +		poll_ns = ktime_to_ns(cur) - ktime_to_ns(start);
> +		if (polled)
> +			goto out;
>

Please move poll_ns caculation under if() when you applied, as I 
explained in reply to v6.

Regards,
Wanpeng Li

Re: [PATCH v7 2/3] KVM: dynamic halt-polling

[Paolo Bonzini]

On 05/09/2015 00:38, Wanpeng Li wrote:
>>
>> @@ -1940,11 +1975,16 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
>>                * arrives.
>>                */
>>               if (kvm_vcpu_check_block(vcpu) < 0) {
>> +                polled = true;
>>                   ++vcpu->stat.halt_successful_poll;
>> -                goto out;
>> +                break;
>>               }
>>               cur = ktime_get();
>>           } while (single_task_running() && ktime_before(cur, stop));
>> +
>> +        poll_ns = ktime_to_ns(cur) - ktime_to_ns(start);
>> +        if (polled)
>> +            goto out;
>>
> 
> Please move poll_ns caculation under if() when you applied, as I
> explained in reply to v6.

You can do much more than just that, the patch reduces to this:

@@ -1929,6 +1963,7 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
 	ktime_t start, cur;
 	DEFINE_WAIT(wait);
 	bool waited = false;
+	u64 block_ns;
 
 	start = cur = ktime_get();
 	if (vcpu->halt_poll_ns) {
@@ -1961,7 +1996,21 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
 	cur = ktime_get();
 
 out:
-	trace_kvm_vcpu_wakeup(ktime_to_ns(cur) - ktime_to_ns(start), waited);
+	block_ns = ktime_to_ns(cur) - ktime_to_ns(start);
+
+	if (halt_poll_ns) {
+		if (block_ns <= vcpu->halt_poll_ns)
+			;
+		/* we had a long block, shrink polling */
+		else if (vcpu->halt_poll_ns && block_ns > halt_poll_ns)
+			shrink_halt_poll_ns(vcpu);
+		/* we had a short halt and our poll time is too small */
+		else if (vcpu->halt_poll_ns < halt_poll_ns &&
+			block_ns < halt_poll_ns)
+			grow_halt_poll_ns(vcpu);
+	}
+
+	trace_kvm_vcpu_wakeup(block_ns, waited);
 }
 EXPORT_SYMBOL_GPL(kvm_vcpu_block);
 

Paolo

Re: [PATCH v7 1/3] KVM: make halt_poll_ns per-vCPU

[Paolo Bonzini]

On 03/09/2015 16:07, Wanpeng Li wrote:
> Change halt_poll_ns into per-VCPU variable, seeded from module parameter,
> to allow greater flexibility.
> 
> Signed-off-by: Wanpeng Li <wanpeng.li@hotmail.com>
> ---
>  include/linux/kvm_host.h | 1 +
>  virt/kvm/kvm_main.c      | 5 +++--
>  2 files changed, 4 insertions(+), 2 deletions(-)
> 
> diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
> index 81089cf..1bef9e2 100644
> --- a/include/linux/kvm_host.h
> +++ b/include/linux/kvm_host.h
> @@ -242,6 +242,7 @@ struct kvm_vcpu {
>  	int sigset_active;
>  	sigset_t sigset;
>  	struct kvm_vcpu_stat stat;
> +	unsigned int halt_poll_ns;
>  
>  #ifdef CONFIG_HAS_IOMEM
>  	int mmio_needed;
> diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
> index d8db2f8f..c06e57c 100644
> --- a/virt/kvm/kvm_main.c
> +++ b/virt/kvm/kvm_main.c
> @@ -217,6 +217,7 @@ int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id)
>  	vcpu->kvm = kvm;
>  	vcpu->vcpu_id = id;
>  	vcpu->pid = NULL;
> +	vcpu->halt_poll_ns = 0;
>  	init_waitqueue_head(&vcpu->wq);
>  	kvm_async_pf_vcpu_init(vcpu);
>  
> @@ -1930,8 +1931,8 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
>  	bool waited = false;
>  
>  	start = cur = ktime_get();
> -	if (halt_poll_ns) {
> -		ktime_t stop = ktime_add_ns(ktime_get(), halt_poll_ns);
> +	if (vcpu->halt_poll_ns) {
> +		ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns);
>  
>  		do {
>  			/*
> 

Applied, thanks.

Paolo

Re: [PATCH v7 2/3] KVM: dynamic halt-polling

[Wanpeng Li]

On 9/6/15 10:32 PM, Paolo Bonzini wrote:
>
> On 05/09/2015 00:38, Wanpeng Li wrote:
>>> @@ -1940,11 +1975,16 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)
>>>                 * arrives.
>>>                 */
>>>                if (kvm_vcpu_check_block(vcpu) < 0) {
>>> +                polled = true;
>>>                    ++vcpu->stat.halt_successful_poll;
>>> -                goto out;
>>> +                break;
>>>                }
>>>                cur = ktime_get();
>>>            } while (single_task_running() && ktime_before(cur, stop));
>>> +
>>> +        poll_ns = ktime_to_ns(cur) - ktime_to_ns(start);
>>> +        if (polled)
>>> +            goto out;
>>>
>> Please move poll_ns caculation under if() when you applied, as I
>> explained in reply to v6.
> You can do much more than just that, the patch reduces to this:
>

Cool, many thanks for your help, Paolo! :)

Regards,
Wanpeng Li