[PATCH 0/3] KVM: x86: legacy NMI watchdog fixes

[PATCH 0/3] KVM: x86: legacy NMI watchdog fixes

[Radim Krčmář]

Until v2.6.37, Linux used NMI watchdog that utilized IO-APIC and LVT0.
This series fixes some problems with APICv, restore, and concurrency
while keeping the monster asleep.


Radim Krčmář (3):
  KVM: x86: keep track of LVT0 changes under APICv
  KVM: x86: properly restore LVT0
  KVM: x86: make vapics_in_nmi_mode atomic

 arch/x86/include/asm/kvm_host.h |  2 +-
 arch/x86/kvm/i8254.c            |  2 +-
 arch/x86/kvm/lapic.c            | 17 ++++++++++-------
 arch/x86/kvm/lapic.h            |  1 +
 4 files changed, 13 insertions(+), 9 deletions(-)

-- 
2.4.5

[PATCH 1/3] KVM: x86: keep track of LVT0 changes under APICv

[Radim Krčmář]

Memory-mapped LVT0 register already contains the new value when APICv
traps so we can't directly detect a change.
Memorize a bit we are interested in to enable legacy NMI watchdog.

Suggested-by: Yoshida Nobuo <yoshida.nb@ncos.nec.co.jp>
Signed-off-by: Radim Krčmář <rkrcmar@redhat.com>
---
 arch/x86/kvm/lapic.c | 14 ++++++++------
 arch/x86/kvm/lapic.h |  1 +
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 36e9de1b4127..f49c7cca1de6 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -1257,16 +1257,17 @@ static void start_apic_timer(struct kvm_lapic *apic)
 
 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
 {
-	int nmi_wd_enabled = apic_lvt_nmi_mode(kvm_apic_get_reg(apic, APIC_LVT0));
+	bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val);
 
-	if (apic_lvt_nmi_mode(lvt0_val)) {
-		if (!nmi_wd_enabled) {
+	if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) {
+		apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode;
+		if (lvt0_in_nmi_mode) {
 			apic_debug("Receive NMI setting on APIC_LVT0 "
 				   "for cpu %d\n", apic->vcpu->vcpu_id);
 			apic->vcpu->kvm->arch.vapics_in_nmi_mode++;
-		}
-	} else if (nmi_wd_enabled)
-		apic->vcpu->kvm->arch.vapics_in_nmi_mode--;
+		} else
+			apic->vcpu->kvm->arch.vapics_in_nmi_mode--;
+	}
 }
 
 static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
@@ -1597,6 +1598,7 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
 	if (!(vcpu->kvm->arch.disabled_quirks & KVM_QUIRK_LINT0_REENABLED))
 		apic_set_reg(apic, APIC_LVT0,
 			     SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
+	apic_manage_nmi_watchdog(apic, kvm_apic_get_reg(apic, APIC_LVT0));
 
 	apic_set_reg(apic, APIC_DFR, 0xffffffffU);
 	apic_set_spiv(apic, 0xff);
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index f2f4e10ab772..71952748222a 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -26,6 +26,7 @@ struct kvm_lapic {
 	struct kvm_vcpu *vcpu;
 	bool sw_enabled;
 	bool irr_pending;
+	bool lvt0_in_nmi_mode;
 	/* Number of bits set in ISR. */
 	s16 isr_count;
 	/* The highest vector set in ISR; if -1 - invalid, must scan ISR. */
-- 
2.4.5

[PATCH 2/3] KVM: x86: properly restore LVT0

[Radim Krčmář]

Legacy NMI watchdog didn't work after migration/resume, because
vapics_in_nmi_mode was left at 0.

Signed-off-by: Radim Krčmář <rkrcmar@redhat.com>
---
 arch/x86/kvm/lapic.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index f49c7cca1de6..8dc32b5a4e0d 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -1824,6 +1824,7 @@ void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu,
 	apic_update_ppr(apic);
 	hrtimer_cancel(&apic->lapic_timer.timer);
 	apic_update_lvtt(apic);
+	apic_manage_nmi_watchdog(apic, kvm_apic_get_reg(apic, APIC_LVT0));
 	update_divide_count(apic);
 	start_apic_timer(apic);
 	apic->irr_pending = true;
-- 
2.4.5

Re: [PATCH 2/3] KVM: x86: properly restore LVT0

[Paolo Bonzini]

On 30/06/2015 22:19, Radim Krčmář wrote:
> Legacy NMI watchdog didn't work after migration/resume, because
> vapics_in_nmi_mode was left at 0.
> 
> Signed-off-by: Radim Krčmář <rkrcmar@redhat.com>
> ---
>  arch/x86/kvm/lapic.c | 1 +
>  1 file changed, 1 insertion(+)
> 
> diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
> index f49c7cca1de6..8dc32b5a4e0d 100644
> --- a/arch/x86/kvm/lapic.c
> +++ b/arch/x86/kvm/lapic.c
> @@ -1824,6 +1824,7 @@ void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu,
>  	apic_update_ppr(apic);
>  	hrtimer_cancel(&apic->lapic_timer.timer);
>  	apic_update_lvtt(apic);
> +	apic_manage_nmi_watchdog(apic, kvm_apic_get_reg(apic, APIC_LVT0));
>  	update_divide_count(apic);
>  	start_apic_timer(apic);
>  	apic->irr_pending = true;
> 

Applied already, with Cc: stable, as it is not related to APICv.

Paolo

[PATCH 3/3] KVM: x86: make vapics_in_nmi_mode atomic

[Radim Krčmář]

Writes were a bit racy, but hard to turn into a bug at the same time.
(Particularly because modern Linux doesn't use this feature anymore.)

Signed-off-by: Radim Krčmář <rkrcmar@redhat.com>
---
 arch/x86/include/asm/kvm_host.h | 2 +-
 arch/x86/kvm/i8254.c            | 2 +-
 arch/x86/kvm/lapic.c            | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index c7fa57b529d2..2a7f5d782c33 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -607,7 +607,7 @@ struct kvm_arch {
 	struct kvm_pic *vpic;
 	struct kvm_ioapic *vioapic;
 	struct kvm_pit *vpit;
-	int vapics_in_nmi_mode;
+	atomic_t vapics_in_nmi_mode;
 	struct mutex apic_map_lock;
 	struct kvm_apic_map *apic_map;
 
diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c
index 4dce6f8b6129..f90952f64e79 100644
--- a/arch/x86/kvm/i8254.c
+++ b/arch/x86/kvm/i8254.c
@@ -305,7 +305,7 @@ static void pit_do_work(struct kthread_work *work)
 		 * LVT0 to NMI delivery. Other PIC interrupts are just sent to
 		 * VCPU0, and only if its LVT0 is in EXTINT mode.
 		 */
-		if (kvm->arch.vapics_in_nmi_mode > 0)
+		if (atomic_read(&kvm->arch.vapics_in_nmi_mode) > 0)
 			kvm_for_each_vcpu(i, vcpu, kvm)
 				kvm_apic_nmi_wd_deliver(vcpu);
 	}
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 8dc32b5a4e0d..954e98a8c2e3 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -1264,9 +1264,9 @@ static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
 		if (lvt0_in_nmi_mode) {
 			apic_debug("Receive NMI setting on APIC_LVT0 "
 				   "for cpu %d\n", apic->vcpu->vcpu_id);
-			apic->vcpu->kvm->arch.vapics_in_nmi_mode++;
+			atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
 		} else
-			apic->vcpu->kvm->arch.vapics_in_nmi_mode--;
+			atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
 	}
 }
 
-- 
2.4.5

Re: [PATCH 3/3] KVM: x86: make vapics_in_nmi_mode atomic

[Paolo Bonzini]

On 30/06/2015 22:19, Radim Krčmář wrote:
> Writes were a bit racy, but hard to turn into a bug at the same time.
> (Particularly because modern Linux doesn't use this feature anymore.)

I suspect patch 2 makes this race much easier to trigger, so it deserves
Cc: stable@ as well.

Paolo

> Signed-off-by: Radim Krčmář <rkrcmar@redhat.com>
> ---
>  arch/x86/include/asm/kvm_host.h | 2 +-
>  arch/x86/kvm/i8254.c            | 2 +-
>  arch/x86/kvm/lapic.c            | 4 ++--
>  3 files changed, 4 insertions(+), 4 deletions(-)
> 
> diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
> index c7fa57b529d2..2a7f5d782c33 100644
> --- a/arch/x86/include/asm/kvm_host.h
> +++ b/arch/x86/include/asm/kvm_host.h
> @@ -607,7 +607,7 @@ struct kvm_arch {
>  	struct kvm_pic *vpic;
>  	struct kvm_ioapic *vioapic;
>  	struct kvm_pit *vpit;
> -	int vapics_in_nmi_mode;
> +	atomic_t vapics_in_nmi_mode;
>  	struct mutex apic_map_lock;
>  	struct kvm_apic_map *apic_map;
>  
> diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c
> index 4dce6f8b6129..f90952f64e79 100644
> --- a/arch/x86/kvm/i8254.c
> +++ b/arch/x86/kvm/i8254.c
> @@ -305,7 +305,7 @@ static void pit_do_work(struct kthread_work *work)
>  		 * LVT0 to NMI delivery. Other PIC interrupts are just sent to
>  		 * VCPU0, and only if its LVT0 is in EXTINT mode.
>  		 */
> -		if (kvm->arch.vapics_in_nmi_mode > 0)
> +		if (atomic_read(&kvm->arch.vapics_in_nmi_mode) > 0)
>  			kvm_for_each_vcpu(i, vcpu, kvm)
>  				kvm_apic_nmi_wd_deliver(vcpu);
>  	}
> diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
> index 8dc32b5a4e0d..954e98a8c2e3 100644
> --- a/arch/x86/kvm/lapic.c
> +++ b/arch/x86/kvm/lapic.c
> @@ -1264,9 +1264,9 @@ static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
>  		if (lvt0_in_nmi_mode) {
>  			apic_debug("Receive NMI setting on APIC_LVT0 "
>  				   "for cpu %d\n", apic->vcpu->vcpu_id);
> -			apic->vcpu->kvm->arch.vapics_in_nmi_mode++;
> +			atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
>  		} else
> -			apic->vcpu->kvm->arch.vapics_in_nmi_mode--;
> +			atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
>  	}
>  }
>  
> 

Re: [PATCH 0/3] KVM: x86: legacy NMI watchdog fixes

[Paolo Bonzini]

On 30/06/2015 22:19, Radim Krčmář wrote:
> Until v2.6.37, Linux used NMI watchdog that utilized IO-APIC and LVT0.
> This series fixes some problems with APICv, restore, and concurrency
> while keeping the monster asleep.

Queued for 4.2.

Paolo