Re: [PATCH] KVM: x86: ioapic: Optimize EOI handling to reduce unnecessary VM exits

[Sean Christopherson <seanjc@google.com>]

On Fri, Dec 27, 2024, wzj wrote:
> On December 18, 2024, Sean Christopherson wrote:
> > On Thu, Nov 21, 2024, weizijie wrote:
> We make the code on the common path as simple as possible.
> > > +		clear_bit(vector, vcpu->arch.ioapic_pending_vectors);
> > > +		clear_bit(vector, vcpu->arch.ioapic_handled_vectors);
> > > +		kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu);
> > > +	}
> > >   	return 1;
> > >   }
> KVM: x86: ioapic: Optimize EOI handling to reduce
>  unnecessary VM exits
> 
> Address performance issues caused by a vector being reused by a
> non-IOAPIC source.
> 
> commit 0fc5a36dd6b3
> ("KVM: x86: ioapic: Fix level-triggered EOI and IOAPIC reconfigure race")
> addressed the issues related to EOI and IOAPIC reconfiguration races.
> However, it has introduced some performance concerns:
> 
> Configuring IOAPIC interrupts while an interrupt request (IRQ) is
> already in service can unintentionally trigger a VM exit for other
> interrupts that normally do not require one, due to the settings of
> `ioapic_handled_vectors`. If the IOAPIC is not reconfigured during
> runtime, this issue persists, continuing to adversely affect
> performance.
> 
> Simple Fix Proposal:
> A straightforward solution is to record highest in-service IRQ that
> is pending at the time of the last scan. Then, upon the next guest
> exit, do a full KVM_REQ_SCAN_IOAPIC. This ensures that a re-scan of
> the ioapic occurs only when the recorded vector is EOI'd, and
> subsequently, the extra bit in the eoi_exit_bitmap are cleared,
> avoiding unnecessary VM exits.
> 
> Co-developed-by: xuyun <xuyun_xy.xy@linux.alibaba.com>
> Signed-off-by: xuyun <xuyun_xy.xy@linux.alibaba.com>
> Signed-off-by: weizijie <zijie.wei@linux.alibaba.com>
> ---

This is whitespace damaged beyond what I am willing to massage and apply, and
honestly even beyond what I am willing to review.  Please fix your setup and
resend.

> v1 -> v2
> 
> * Move my SoB to the end and add Co-developed-by for Xuyun
> 
> * Use a u8 type to record a pending IRQ during the ioapic scan process
> 
> * Made the same changes for the split IRQ chip mode
> 
> arch/x86/include/asm/kvm_host.h |  1 +
>  arch/x86/kvm/ioapic.c           | 10 ++++++++--
>  arch/x86/kvm/irq_comm.c         |  9 +++++++--
>  arch/x86/kvm/vmx/vmx.c          |  9 +++++++++
>  4 files changed, 25 insertions(+), 4 deletions(-)
> 
> diff --git a/arch/x86/include/asm/kvm_host.h
> b/arch/x86/include/asm/kvm_host.h
> index e159e44a6a1b..f84a4881afa4 100644
> --- a/arch/x86/include/asm/kvm_host.h
> +++ b/arch/x86/include/asm/kvm_host.h
> @@ -1041,6 +1041,7 @@ struct kvm_vcpu_arch {
>  #if IS_ENABLED(CONFIG_HYPERV)
>         hpa_t hv_root_tdp;
>  #endif
> +       u8 last_pending_vector;
>  };
> 
>  struct kvm_lpage_info {
> diff --git a/arch/x86/kvm/ioapic.c b/arch/x86/kvm/ioapic.c
> index 995eb5054360..40252a800897 100644
> --- a/arch/x86/kvm/ioapic.c
> +++ b/arch/x86/kvm/ioapic.c
> @@ -297,10 +297,16 @@ void kvm_ioapic_scan_entry(struct kvm_vcpu *vcpu,
> ulong *ioapic_handled_vectors)
>                         u16 dm =
> kvm_lapic_irq_dest_mode(!!e->fields.dest_mode);
> 
>                         if (kvm_apic_match_dest(vcpu, NULL,
> APIC_DEST_NOSHORT,
> - e->fields.dest_id, dm) ||
> -                           kvm_apic_pending_eoi(vcpu, e->fields.vector))
> + e->fields.dest_id, dm))
>                                 __set_bit(e->fields.vector,
>                                           ioapic_handled_vectors);
> +                       else if (kvm_apic_pending_eoi(vcpu,
> e->fields.vector)) {
> +                               __set_bit(e->fields.vector,
> +                                         ioapic_handled_vectors);
> +                               vcpu->arch.last_pending_vector =
> e->fields.vector >
> + vcpu->arch.last_pending_vector ? e->fields.vector :
> + vcpu->arch.last_pending_vector;
> +                       }
>                 }
>         }
>         spin_unlock(&ioapic->lock);
> diff --git a/arch/x86/kvm/irq_comm.c b/arch/x86/kvm/irq_comm.c
> index 8136695f7b96..1d23c52576e1 100644
> --- a/arch/x86/kvm/irq_comm.c
> +++ b/arch/x86/kvm/irq_comm.c
> @@ -426,9 +426,14 @@ void kvm_scan_ioapic_routes(struct kvm_vcpu *vcpu,
> 
>                         if (irq.trig_mode &&
>                             (kvm_apic_match_dest(vcpu, NULL,
> APIC_DEST_NOSHORT,
> -                                                irq.dest_id, irq.dest_mode)
> ||
> -                            kvm_apic_pending_eoi(vcpu, irq.vector)))
> +                                                irq.dest_id,
> irq.dest_mode)))
>                                 __set_bit(irq.vector,
> ioapic_handled_vectors);
> +                       else if (kvm_apic_pending_eoi(vcpu, irq.vector)) {
> +                               __set_bit(irq.vector,
> ioapic_handled_vectors);
> +                               vcpu->arch.last_pending_vector = irq.vector
> >
> + vcpu->arch.last_pending_vector ? irq.vector :
> + vcpu->arch.last_pending_vector;
> +                       }
>                 }
>         }
>         srcu_read_unlock(&kvm->irq_srcu, idx);
> diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
> index 893366e53732..cd0db1496ce7 100644
> --- a/arch/x86/kvm/vmx/vmx.c
> +++ b/arch/x86/kvm/vmx/vmx.c
> @@ -5702,6 +5702,15 @@ static int handle_apic_eoi_induced(struct kvm_vcpu
> *vcpu)
> 
>         /* EOI-induced VM exit is trap-like and thus no need to adjust IP */
>         kvm_apic_set_eoi_accelerated(vcpu, vector);
> +
> +       /* When there are instances where ioapic_handled_vectors is
> +        * set due to pending interrupts, clean up the record and do
> +        * a full KVM_REQ_SCAN_IOAPIC.
> +        */
> +       if (vcpu->arch.last_pending_vector == vector) {
> +               vcpu->arch.last_pending_vector = 0;
> +               kvm_make_request(KVM_REQ_SCAN_IOAPIC, vcpu);
> +       }
>         return 1;
>  }
> 
> > > -- 
> > > 2.43.5
> > >