Re: [Qemu-devel] E5-2620v2 - emulation stop error

[Andrey Korolyov <andrey@xdel.ru>]

On Tue, Mar 31, 2015 at 7:45 PM, Radim Krčmář <rkrcmar@redhat.com> wrote:
> 2015-03-31 17:56+0300, Andrey Korolyov:
>> > Chasing the culprit this way could take a long time, so a new tracepoint
>> > that shows if 0xef is set on entry would let us guess the bug faster ...
>> >
>> > Please provide a failing trace with the following patch:
>>
>> Thanks, please see below:
>>
>> http://xdel.ru/downloads/kvm-e5v2-issue/new-tracepoint-fail-with-apicv.dat.gz
>
>  qemu-system-x86-4022  [006]  255.915978:
>   kvm_entry:            vcpu 0
>   kvm_emulate_insn:     f0000:d275: ea 7a d2 00 f0
>   kvm_emulate_insn:     f0000:d27a: 2e 0f 01 1e f0 6c
>   kvm_emulate_insn:     f0000:d280: 31 c0
>   kvm_emulate_insn:     f0000:d282: 8e e0
>   kvm_emulate_insn:     f0000:d284: 8e e8
>   kvm_emulate_insn:     f0000:d286: 8e c0
>   kvm_emulate_insn:     f0000:d288: 8e d8
>   kvm_emulate_insn:     f0000:d28a: 8e d0
>   kvm_entry:            vcpu 0
>   kvm_0xef:             irr clear, isr clear, vmcs 0x0
>   kvm_exit:             reason EPT_VIOLATION rip 0x8dd0 info 184 0
>   kvm_page_fault:       address f8dd0 error_code 184
>   kvm_entry:            vcpu 0
>   kvm_0xef:             irr clear, isr clear, vmcs 0x0
>   kvm_exit:             reason EPT_VIOLATION rip 0x76d6 info 184 0
>   kvm_page_fault:       address f76d6 error_code 184
>   kvm_entry:            vcpu 0
>   kvm_0xef:             irr clear, isr clear, vmcs 0x0
>   kvm_exit:             reason EXCEPTION_NMI rip 0xd331 info 0 80000b0d
>   kvm_userspace_exit:   reason KVM_EXIT_INTERNAL_ERROR (17)
>
> Ok, nothing obvious here either ... I've desperately added all
> information I know about.  Please run it again, thanks.
>
> (The patch has to be applied instead of the previous one.)
> ---
> diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h
> index 7c7bc8bef21f..f986636ad9d0 100644
> --- a/arch/x86/kvm/trace.h
> +++ b/arch/x86/kvm/trace.h
> @@ -742,6 +742,41 @@ TRACE_EVENT(kvm_emulate_insn,
>  #define trace_kvm_emulate_insn_start(vcpu) trace_kvm_emulate_insn(vcpu, 0)
>  #define trace_kvm_emulate_insn_failed(vcpu) trace_kvm_emulate_insn(vcpu, 1)
>
> +TRACE_EVENT(kvm_0xef,
> +       TP_PROTO(bool irr, bool isr, u32 info, bool on, bool pir, u16 status),
> +       TP_ARGS(irr, isr, info, on, pir, status),
> +
> +       TP_STRUCT__entry(
> +               __field(bool,  irr )
> +               __field(bool,  isr )
> +               __field(u32,   info)
> +               __field(bool,  on  )
> +               __field(bool,  pir )
> +               __field(u8,    rvi )
> +               __field(u8,    svi )
> +               ),
> +
> +       TP_fast_assign(
> +               __entry->irr  = irr;
> +               __entry->isr  = isr;
> +               __entry->info = info;
> +               __entry->on   = on;
> +               __entry->pir  = pir;
> +               __entry->rvi  = status & 0xff;
> +               __entry->svi  = status >> 8;
> +               ),
> +
> +       TP_printk("irr %s, isr %s, info 0x%x, on %s, pir %s, rvi 0x%x, svi 0x%x",
> +                 __entry->irr ? "set  " : "clear",
> +                 __entry->isr ? "set  " : "clear",
> +                 __entry->info,
> +                 __entry->on  ? "set  " : "clear",
> +                 __entry->pir ? "set  " : "clear",
> +                 __entry->rvi,
> +                 __entry->svi
> +                )
> +       );
> +
>  TRACE_EVENT(
>         vcpu_match_mmio,
>         TP_PROTO(gva_t gva, gpa_t gpa, bool write, bool gpa_match),
> diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
> index eee63dc33d89..b461edc93d53 100644
> --- a/arch/x86/kvm/vmx.c
> +++ b/arch/x86/kvm/vmx.c
> @@ -5047,6 +5047,25 @@ static int handle_machine_check(struct kvm_vcpu *vcpu)
>         return 1;
>  }
>
> +#define VEC_POS(v) ((v) & (32 - 1))
> +#define REG_POS(v) (((v) >> 5) << 4)
> +static inline int apic_test_vector(int vec, void *bitmap)
> +{
> +       return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
> +}
> +
> +static inline void random_trace(struct kvm_vcpu *vcpu)
> +{
> +       struct vcpu_vmx *vmx = to_vmx(vcpu);
> +
> +       trace_kvm_0xef(apic_test_vector(0xef, vcpu->arch.apic->regs + APIC_IRR),
> +                      apic_test_vector(0xef, vcpu->arch.apic->regs + APIC_ISR),
> +                      vmcs_read32(VM_ENTRY_INTR_INFO_FIELD),
> +                      test_bit(POSTED_INTR_ON, (unsigned long *)&vmx->pi_desc.control),
> +                      test_bit(0xef, (unsigned long *)vmx->pi_desc.pir),
> +                      vmcs_read16(GUEST_INTR_STATUS));
> +}
> +
>  static int handle_exception(struct kvm_vcpu *vcpu)
>  {
>         struct vcpu_vmx *vmx = to_vmx(vcpu);
> @@ -5077,6 +5096,8 @@ static int handle_exception(struct kvm_vcpu *vcpu)
>                 return 1;
>         }
>
> +       random_trace(vcpu);
> +
>         error_code = 0;
>         if (intr_info & INTR_INFO_DELIVER_CODE_MASK)
>                 error_code = vmcs_read32(VM_EXIT_INTR_ERROR_CODE);
> @@ -8143,6 +8164,8 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
>         if (vmx->emulation_required)
>                 return;
>
> +       random_trace(vcpu);
> +
>         if (vmx->ple_window_dirty) {
>                 vmx->ple_window_dirty = false;
>                 vmcs_write32(PLE_WINDOW, vmx->ple_window);
> @@ -8312,6 +8335,8 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
>         vmx->exit_reason = vmcs_read32(VM_EXIT_REASON);
>         trace_kvm_exit(vmx->exit_reason, vcpu, KVM_ISA_VMX);
>
> +       random_trace(vcpu);
> +
>         /*
>          * the KVM_REQ_EVENT optimization bit is only on for one entry, and if
>          * we did not inject a still-pending event to L1 now because of
> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index 32bf19ef3115..a45fa01bd354 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -7881,3 +7881,4 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_nested_intercepts);
>  EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_write_tsc_offset);
>  EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ple_window);
>  EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_pml_full);
> +EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_0xef);

http://xdel.ru/downloads/kvm-e5v2-issue/another-tracepoint-fail-with-apicv.dat.gz

Something a bit more interesting, but the mess is happening just
*after* NMI firing.