Re: KVM: protect assigned dev workqueue, int handler and irq acker

["Yang, Sheng" <sheng.yang@intel.com>]

On Friday 27 February 2009 07:50:54 Marcelo Tosatti wrote:
> Can someone with HW test this please?

Good catch! The patch works fine on my side.

Can it be a per-device lock? One big lock for all assigned device seems 
restrict scalability. 

> -----
>
> kvm_assigned_dev_ack_irq is vulnerable to a race condition with the
> interrupt handler function. It does:
>
>         if (dev->host_irq_disabled) {
>                 enable_irq(dev->host_irq);
>                 dev->host_irq_disabled = false;
> 	}
>
> If an interrupt triggers before the host->dev_irq_disabled assignment,
> it will disable the interrupt and set dev->host_irq_disabled to true.
>
> On return to kvm_assigned_dev_ack_irq, dev->host_irq_disabled is set to
> false, and the next kvm_assigned_dev_ack_irq call will fail to reenable
> it.
>
> Other than that, having the interrupt handler and work handlers run in
> parallel sounds like asking for trouble (could not spot any obvious
> problem, but better not have to, its fragile).

Well, my original purpose is a FIFO between interrupt handler and work(for 
MSI-X), but seems too complex... And I also don't see any problem for now...

-- 
regards
Yang, Sheng

> diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
> index 3832243..faaf386 100644
> --- a/include/linux/kvm_host.h
> +++ b/include/linux/kvm_host.h
> @@ -152,6 +152,7 @@ struct kvm {
>  	unsigned long mmu_notifier_seq;
>  	long mmu_notifier_count;
>  #endif
> +	spinlock_t assigned_dev_lock;
>  };
>
>  /* The guest did something we don't support. */
> diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
> index 4d2be16..2bbf074 100644
> --- a/virt/kvm/kvm_main.c
> +++ b/virt/kvm/kvm_main.c
> @@ -41,6 +41,7 @@
>  #include <linux/pagemap.h>
>  #include <linux/mman.h>
>  #include <linux/swap.h>
> +#include <linux/spinlock.h>
>
>  #include <asm/processor.h>
>  #include <asm/io.h>
> @@ -132,6 +133,7 @@ static void
> kvm_assigned_dev_interrupt_work_handler(struct work_struct *work) *
> finer-grained lock, update this
>  	 */
>  	mutex_lock(&kvm->lock);
> +	spin_lock_irq(&kvm->assigned_dev_lock);
>  	if (assigned_dev->irq_requested_type & KVM_ASSIGNED_DEV_MSIX) {
>  		struct kvm_guest_msix_entry *guest_entries =
>  			assigned_dev->guest_msix_entries;
> @@ -158,18 +160,21 @@ static void
> kvm_assigned_dev_interrupt_work_handler(struct work_struct *work) }
>  	}
>
> +	spin_unlock_irq(&kvm->assigned_dev_lock);
>  	mutex_unlock(&assigned_dev->kvm->lock);
>  }
>
>  static irqreturn_t kvm_assigned_dev_intr(int irq, void *dev_id)
>  {
> +	unsigned long flags;
>  	struct kvm_assigned_dev_kernel *assigned_dev =
>  		(struct kvm_assigned_dev_kernel *) dev_id;
>
> +	spin_lock_irqsave(&assigned_dev->kvm->assigned_dev_lock, flags);
>  	if (assigned_dev->irq_requested_type == KVM_ASSIGNED_DEV_MSIX) {
>  		int index = find_index_from_host_irq(assigned_dev, irq);
>  		if (index < 0)
> -			return IRQ_HANDLED;
> +			goto out;
>  		assigned_dev->guest_msix_entries[index].flags |=
>  			KVM_ASSIGNED_MSIX_PENDING;
>  	}
> @@ -179,6 +184,8 @@ static irqreturn_t kvm_assigned_dev_intr(int irq, void
> *dev_id) disable_irq_nosync(irq);
>  	assigned_dev->host_irq_disabled = true;
>
> +out:
> +	spin_unlock_irqrestore(&assigned_dev->kvm->assigned_dev_lock, flags);
>  	return IRQ_HANDLED;
>  }
>
> @@ -186,6 +193,7 @@ static irqreturn_t kvm_assigned_dev_intr(int irq, void
> *dev_id) static void kvm_assigned_dev_ack_irq(struct kvm_irq_ack_notifier
> *kian) {
>  	struct kvm_assigned_dev_kernel *dev;
> +	unsigned long flags;
>
>  	if (kian->gsi == -1)
>  		return;
> @@ -198,10 +206,12 @@ static void kvm_assigned_dev_ack_irq(struct
> kvm_irq_ack_notifier *kian) /* The guest irq may be shared so this ack may
> be
>  	 * from another device.
>  	 */
> +	spin_lock_irqsave(&dev->kvm->assigned_dev_lock, flags);
>  	if (dev->host_irq_disabled) {
>  		enable_irq(dev->host_irq);
>  		dev->host_irq_disabled = false;
>  	}
> +	spin_unlock_irqrestore(&dev->kvm->assigned_dev_lock, flags);
>  }
>
>  /* The function implicit hold kvm->lock mutex due to cancel_work_sync() */
> @@ -955,6 +965,7 @@ static struct kvm *kvm_create_vm(void)
>  	kvm->mm = current->mm;
>  	atomic_inc(&kvm->mm->mm_count);
>  	spin_lock_init(&kvm->mmu_lock);
> +	spin_lock_init(&kvm->assigned_dev_lock);
>  	kvm_io_bus_init(&kvm->pio_bus);
>  	mutex_init(&kvm->lock);
>  	kvm_io_bus_init(&kvm->mmio_bus);
>
>
> ----- End forwarded message -----