Re: [PATCH v11 3/3] x86, apicv: add virtual interrupt delivery support

[Marcelo Tosatti <mtosatti@redhat.com>]

On Wed, Jan 16, 2013 at 06:21:12PM +0800, Yang Zhang wrote:
> From: Yang Zhang <yang.z.zhang@Intel.com>
> 
> Virtual interrupt delivery avoids KVM to inject vAPIC interrupts
> manually, which is fully taken care of by the hardware. This needs
> some special awareness into existing interrupr injection path:
> 
> - for pending interrupt, instead of direct injection, we may need
>   update architecture specific indicators before resuming to guest.
> 
> - A pending interrupt, which is masked by ISR, should be also
>   considered in above update action, since hardware will decide
>   when to inject it at right time. Current has_interrupt and
>   get_interrupt only returns a valid vector from injection p.o.v.
> 
> Signed-off-by: Kevin Tian <kevin.tian@intel.com>
> Signed-off-by: Yang Zhang <yang.z.zhang@Intel.com>
> ---
>  arch/ia64/kvm/lapic.h           |    6 ++
>  arch/x86/include/asm/kvm_host.h |    7 ++
>  arch/x86/include/asm/vmx.h      |   11 +++
>  arch/x86/kvm/irq.c              |   56 +++++++++++-
>  arch/x86/kvm/lapic.c            |   69 +++++++++------
>  arch/x86/kvm/lapic.h            |   23 +++++
>  arch/x86/kvm/svm.c              |   25 +++++
>  arch/x86/kvm/vmx.c              |  184 +++++++++++++++++++++++++++++++++++++--
>  arch/x86/kvm/x86.c              |   11 ++-
>  arch/x86/kvm/x86.h              |    2 +
>  include/linux/kvm_host.h        |    3 +
>  virt/kvm/ioapic.c               |   38 ++++++++
>  virt/kvm/ioapic.h               |    1 +
>  virt/kvm/irq_comm.c             |   25 +++++
>  virt/kvm/kvm_main.c             |    5 +
>  15 files changed, 421 insertions(+), 45 deletions(-)
> 

> diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
> index 35aa8e6..d90bfa8 100644
> --- a/arch/x86/include/asm/kvm_host.h
> +++ b/arch/x86/include/asm/kvm_host.h
> @@ -697,6 +697,12 @@ struct kvm_x86_ops {
>  	void (*enable_nmi_window)(struct kvm_vcpu *vcpu);
>  	void (*enable_irq_window)(struct kvm_vcpu *vcpu);
>  	void (*update_cr8_intercept)(struct kvm_vcpu *vcpu, int tpr, int irr);
> +	int (*has_virtual_interrupt_delivery)(void);
> +	void (*update_apic_irq)(struct kvm_vcpu *vcpu, int max_irr);

Call it hwapic_irr_update().

> +	void (*update_eoi_exitmap)(struct kvm_vcpu *vcpu);
> +	void (*set_svi)(int isr);

SVI is VMX-specific. Call it hwapic_isr_update().

> diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
> index 526955d..8b9752a 100644
> --- a/arch/x86/kvm/vmx.c
> +++ b/arch/x86/kvm/vmx.c
> @@ -84,8 +84,8 @@ module_param(vmm_exclusive, bool, S_IRUGO);
>  static bool __read_mostly fasteoi = 1;
>  module_param(fasteoi, bool, S_IRUGO);
>  
> -static bool __read_mostly enable_apicv_reg = 1;
> -module_param(enable_apicv_reg, bool, S_IRUGO);
> +static bool __read_mostly enable_apicv_reg_vid = 1;
> +module_param(enable_apicv_reg_vid, bool, S_IRUGO);
>  
>  /*
>   * If nested=1, nested virtualization is supported, i.e., guests may use

> @@ -2783,8 +2793,14 @@ static __init int hardware_setup(void)
>  	if (!cpu_has_vmx_ple())
>  		ple_gap = 0;
>  
> -	if (!cpu_has_vmx_apic_register_virt())
> -		enable_apicv_reg = 0;
> +	if (!cpu_has_vmx_apic_register_virt() ||
> +				!cpu_has_vmx_virtual_intr_delivery())
> +		enable_apicv_reg_vid = 0;
> +
> +	if (enable_apicv_reg_vid)
> +		kvm_x86_ops->update_cr8_intercept = NULL;
> +	else
> +		kvm_x86_ops->update_apic_irq = NULL;

Please be consistent regarding ->update_apic_irq: if HW does not support 
acceleration, or HW acceleration has been disabled, it is NULL (SVM
should be NULL, then).

If so, then please delete enable_apic_reg_vid check in vmx_set_svi.

> +static void vmx_set_svi(int isr)
> +{
> +	u16 status;
> +	u8 old;
> +
> +	if (!enable_apicv_reg_vid)
> +		return;

See above.

> +static void set_eoi_exitmap_one(struct kvm_vcpu *vcpu,
> +				u32 vector)
> +{
> +	struct vcpu_vmx *vmx = to_vmx(vcpu);
> +
> +	if (WARN_ONCE((vector > 255),
> +		"KVM VMX: vector (%d) out of range\n", vector))
> +		return;

Please remove the WARN_ON, inject triple fault if this happens
(and add a tracepoint).

> +
> +	__set_bit(vector, (unsigned long *)vmx->eoi_exit_bitmap);
> +}
> +
> +static void vmx_check_ioapic_entry(struct kvm_vcpu *vcpu,
> +				   struct kvm_lapic_irq *irq)
> +{
> +	struct kvm_lapic **dst;
> +	struct kvm_apic_map *map;
> +	unsigned long bitmap = 1;
> +	int i;
> +
> +	rcu_read_lock();
> +	map = rcu_dereference(vcpu->kvm->arch.apic_map);
> +
> +	if (unlikely(!map)) {
> +		set_eoi_exitmap_one(vcpu, irq->vector);
> +		goto out;
> +	}
> +
> +	if (irq->dest_mode == 0) { /* physical mode */
> +		if (irq->delivery_mode == APIC_DM_LOWEST ||
> +				irq->dest_id == 0xff) {
> +			set_eoi_exitmap_one(vcpu, irq->vector);
> +			goto out;
> +		}
> +		dst = &map->phys_map[irq->dest_id & 0xff];
> +	} else {
> +		u32 mda = irq->dest_id << (32 - map->ldr_bits);
> +
> +		dst = map->logical_map[apic_cluster_id(map, mda)];
> +
> +		bitmap = apic_logical_id(map, mda);
> +	}
> +
> +	for_each_set_bit(i, &bitmap, 16) {
> +		if (!dst[i])
> +			continue;
> +		if (dst[i]->vcpu == vcpu) {
> +			set_eoi_exitmap_one(vcpu, irq->vector);
> +			break;
> +		}
> +	}
> +
> +out:
> +	rcu_read_unlock();
> +}

All this can be moved to generic code, then

hwapic_vector_intercept_on_eoi() becomes a callback to kvm_x86_ops.

> index e224f7a..e8e7b6a 100644
> --- a/arch/x86/kvm/x86.h
> +++ b/arch/x86/kvm/x86.h
> @@ -122,6 +122,8 @@ int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt,
>  	gva_t addr, void *val, unsigned int bytes,
>  	struct x86_exception *exception);
>  
> +void set_eoi_exitmap(struct kvm_vcpu *vcpu);
> +

This should be in virt/kvm/ioapic.h.

> @@ -82,5 +82,6 @@ int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src,
>  		struct kvm_lapic_irq *irq);
>  int kvm_get_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state);
>  int kvm_set_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state);
> +void ioapic_update_eoi_exitmap(struct kvm *kvm);

prefix with kvm_