[PATCH 6/6] Add MSI support to XEN

[PATCH 6/6] Add MSI support to XEN

[Shan, Haitao]

[-- Attachment #1: Type: text/plain, Size: 222 bytes --]

This patch add MSI support to domain0/domain U.
 
Signed-off-by: Jiang Yunhong <yunhong.jiang@intel.com>
Signed-off-by: Shan Haitao     <haitao.shan@intel.co 
<<msi_kernel.patch>> m>
 

Best Regards
Haitao Shan


[-- Attachment #2: msi_kernel.patch --]
[-- Type: application/octet-stream, Size: 34067 bytes --]

diff -r f3d354a4b403 drivers/pci/Kconfig
--- a/drivers/pci/Kconfig	Fri Apr 25 00:01:08 2008 +0800
+++ b/drivers/pci/Kconfig	Wed Apr 30 19:34:58 2008 +0800
@@ -5,7 +5,6 @@
 	bool "Message Signaled Interrupts (MSI and MSI-X)"
 	depends on PCI
 	depends on (X86_LOCAL_APIC && X86_IO_APIC) || IA64
-	depends on !XEN
 	help
 	   This allows device drivers to enable MSI (Message Signaled
 	   Interrupts).  Message Signaled Interrupts enable a device to
diff -r f3d354a4b403 drivers/pci/msi-xen.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/drivers/pci/msi-xen.c	Wed Apr 30 19:34:58 2008 +0800
@@ -0,0 +1,710 @@
+/*
+ * File:	msi.c
+ * Purpose:	PCI Message Signaled Interrupt (MSI)
+ *
+ * Copyright (C) 2003-2004 Intel
+ * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
+ */
+
+#include <linux/mm.h>
+#include <linux/irq.h>
+#include <linux/interrupt.h>
+#include <linux/init.h>
+#include <linux/ioport.h>
+#include <linux/smp_lock.h>
+#include <linux/pci.h>
+#include <linux/proc_fs.h>
+
+#include <asm/errno.h>
+#include <asm/io.h>
+#include <asm/smp.h>
+
+#include "pci.h"
+#include "msi.h"
+
+static int pci_msi_enable = 1;
+
+static struct msi_ops *msi_ops;
+
+int msi_register(struct msi_ops *ops)
+{
+	msi_ops = ops;
+	return 0;
+}
+
+static struct list_head msi_dev_head;
+static int msi_dev_head_inited = 0;
+DEFINE_SPINLOCK(msi_dev_lock);
+
+struct msi_dev_list {
+	struct pci_dev *dev;
+	struct list_head list;
+	spinlock_t pirq_list_lock;
+	struct list_head pirq_list_head;
+};
+
+struct msi_pirq_entry {
+	struct list_head list;
+	int pirq;
+	int entry_nr;
+};
+
+static struct msi_dev_list *get_msi_dev_pirq_list(struct pci_dev *dev)
+{
+	struct msi_dev_list *msi_dev_list, *ret = NULL;
+	unsigned long flags;
+
+	if (!msi_dev_head_inited) {
+		INIT_LIST_HEAD(&msi_dev_head);
+		msi_dev_head_inited = 1;
+	}
+
+	spin_lock_irqsave(&msi_dev_lock, flags);
+
+	list_for_each_entry(msi_dev_list, &msi_dev_head, list)
+		if ( msi_dev_list->dev == dev )
+			ret = msi_dev_list;
+
+	if ( ret ) {
+		spin_unlock_irqrestore(&msi_dev_lock, flags);
+		return ret;
+	}
+
+	/* Has not allocate msi_dev until now. */
+	ret = kmalloc(sizeof(struct msi_dev_list), GFP_ATOMIC);
+
+	/* Failed to allocate msi_dev structure */
+	if ( !ret ) {
+		spin_unlock_irqrestore(&msi_dev_lock, flags);
+		return NULL;
+	}
+
+	list_add_tail(&ret->list, &msi_dev_head);
+	spin_unlock_irqrestore(&msi_dev_lock, flags);
+	spin_lock_init(&ret->pirq_list_lock);
+	INIT_LIST_HEAD(&ret->pirq_list_head);
+	return ret;
+}
+
+static int attach_pirq_entry(int pirq, int entry_nr,
+                             struct msi_dev_list *msi_dev_entry)
+{
+	struct msi_pirq_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
+	unsigned long flags;
+
+	if (!entry)
+		return -ENOMEM;
+	entry->pirq = pirq;
+	entry->entry_nr = entry_nr;
+	spin_lock_irqsave(&msi_dev_entry->pirq_list_lock, flags);
+	list_add_tail(&entry->list, &msi_dev_entry->pirq_list_head);
+	spin_unlock_irqrestore(&msi_dev_entry->pirq_list_lock, flags);
+	return 0;
+}
+
+/*
+ * pciback will provide device's owner
+ */
+int (*get_owner)(struct pci_dev *dev);
+
+int register_msi_get_owner(int (*func)(struct pci_dev *dev))
+{
+	if (get_owner) {
+		printk(KERN_WARNING "register msi_get_owner again\n");
+		return -EEXIST;
+	}
+	get_owner = func;
+	return 0;
+}
+
+int unregister_msi_get_owner(int (*func)(struct pci_dev *dev))
+{
+	if (get_owner == func)
+		get_owner = NULL;
+	return 0;
+}
+
+static int msi_get_dev_owner(struct pci_dev *dev)
+{
+	int owner = DOMID_SELF;
+
+	BUG_ON(!is_initial_xendomain());
+	if (get_owner && (owner = get_owner(dev)) >=0 ) {
+		printk(KERN_INFO "get owner for dev %x get %x \n",
+				    dev->devfn, owner);
+		return owner;
+	}
+	else
+		return DOMID_SELF;
+}
+
+static int msi_unmap_pirq(struct pci_dev *dev, int pirq)
+{
+	struct physdev_unmap_pirq unmap;
+	int rc;
+	domid_t domid = DOMID_SELF;
+
+	domid = msi_get_dev_owner(dev);
+	unmap.domid = domid;
+	unmap.pirq = pirq;
+
+	if ((rc = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap)))
+		printk(KERN_WARNING "unmap irq %x failed\n", pirq);
+
+	if (rc < 0)
+		return rc;
+    return 0;
+}
+
+/*
+ * Protected by msi_lock
+ */
+static int msi_map_pirq_to_vector(struct pci_dev *dev, int pirq,
+                                  int entry_nr, int msi)
+{
+	struct physdev_map_pirq map_irq;
+	int rc;
+	domid_t domid = DOMID_SELF;
+
+	domid = msi_get_dev_owner(dev);
+
+	map_irq.domid = domid;
+	map_irq.type = MAP_PIRQ_TYPE_MSI;
+	map_irq.index = -1;
+	map_irq.pirq = pirq;
+    map_irq.msi_info.bus = dev->bus->number;
+    map_irq.msi_info.devfn = dev->devfn;
+	map_irq.msi_info.entry_nr = entry_nr;
+    map_irq.msi_info.msi = msi;
+
+	if ((rc = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_irq)))
+		printk(KERN_WARNING "map irq failed\n");
+
+	if (rc < 0)
+		return rc;
+
+	return map_irq.pirq;
+}
+
+static int msi_map_vector(struct pci_dev *dev, int entry_nr, int msi)
+{
+	return msi_map_pirq_to_vector(dev, -1, entry_nr, msi);
+}
+
+static int msi_init(void)
+{
+	static int status = 0;
+
+	if (pci_msi_quirk) {
+		pci_msi_enable = 0;
+		printk(KERN_WARNING "PCI: MSI quirk detected. MSI disabled.\n");
+		status = -EINVAL;
+	}
+
+	return status;
+}
+
+void pci_scan_msi_device(struct pci_dev *dev) { }
+
+void disable_msi_mode(struct pci_dev *dev, int pos, int type)
+{
+	u16 control;
+
+	pci_read_config_word(dev, msi_control_reg(pos), &control);
+	if (type == PCI_CAP_ID_MSI) {
+		/* Set enabled bits to single MSI & enable MSI_enable bit */
+		msi_disable(control);
+		pci_write_config_word(dev, msi_control_reg(pos), control);
+		dev->msi_enabled = 0;
+	} else {
+		msix_disable(control);
+		pci_write_config_word(dev, msi_control_reg(pos), control);
+		dev->msix_enabled = 0;
+	}
+    	if (pci_find_capability(dev, PCI_CAP_ID_EXP)) {
+		/* PCI Express Endpoint device detected */
+		pci_intx(dev, 1);  /* enable intx */
+	}
+}
+
+static void enable_msi_mode(struct pci_dev *dev, int pos, int type)
+{
+	u16 control;
+
+	pci_read_config_word(dev, msi_control_reg(pos), &control);
+	if (type == PCI_CAP_ID_MSI) {
+		/* Set enabled bits to single MSI & enable MSI_enable bit */
+		msi_enable(control, 1);
+		pci_write_config_word(dev, msi_control_reg(pos), control);
+		dev->msi_enabled = 1;
+	} else {
+		msix_enable(control);
+		pci_write_config_word(dev, msi_control_reg(pos), control);
+		dev->msix_enabled = 1;
+	}
+    	if (pci_find_capability(dev, PCI_CAP_ID_EXP)) {
+		/* PCI Express Endpoint device detected */
+		pci_intx(dev, 0);  /* disable intx */
+	}
+}
+
+#ifdef CONFIG_PM
+int pci_save_msi_state(struct pci_dev *dev)
+{
+	int pos;
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
+	if (pos <= 0 || dev->no_msi)
+		return 0;
+
+	if (!dev->msi_enabled)
+		return 0;
+
+	/* Restore dev->irq to its default pin-assertion vector */
+	msi_unmap_pirq(dev, dev->irq);
+	/* Disable MSI mode */
+	disable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
+	/* Set the flags for use of restore */
+	dev->msi_enabled = 1;
+	return 0;
+}
+
+void pci_restore_msi_state(struct pci_dev *dev)
+{
+	int pos, pirq;
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
+	if (pos <= 0)
+		return;
+
+	if (!dev->msi_enabled)
+		return;
+
+	pirq = msi_map_pirq_to_vector(dev, dev->irq, 0, 1);
+	if (pirq < 0)
+		return;
+	enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
+}
+
+int pci_save_msix_state(struct pci_dev *dev)
+{
+	int pos;
+	unsigned long flags;
+	struct msi_dev_list *msi_dev_entry;
+	struct msi_pirq_entry *pirq_entry, *tmp;
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
+	if (pos <= 0 || dev->no_msi)
+		return 0;
+
+	/* save the capability */
+	if (!dev->msix_enabled)
+		return 0;
+
+	msi_dev_entry = get_msi_dev_pirq_list(dev);
+
+	spin_lock_irqsave(&msi_dev_entry->pirq_list_lock, flags);
+	if (!list_empty_careful(&msi_dev_entry->pirq_list_head))
+		list_for_each_entry_safe(pirq_entry, tmp,
+		                         &msi_dev_entry->pirq_list_head, list)
+			msi_unmap_pirq(dev, pirq_entry->pirq);
+	spin_unlock_irqrestore(&msi_dev_entry->pirq_list_lock, flags);
+
+	disable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
+	/* Set the flags for use of restore */
+	dev->msix_enabled = 1;
+
+	return 0;
+}
+
+void pci_restore_msix_state(struct pci_dev *dev)
+{
+	int pos;
+	unsigned long flags;
+	struct msi_dev_list *msi_dev_entry;
+	struct msi_pirq_entry *pirq_entry, *tmp;
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
+	if (pos <= 0)
+		return;
+
+	if (!dev->msix_enabled)
+		return;
+
+	msi_dev_entry = get_msi_dev_pirq_list(dev);
+
+	spin_lock_irqsave(&msi_dev_entry->pirq_list_lock, flags);
+	list_for_each_entry_safe(pirq_entry, tmp,
+							 &msi_dev_entry->pirq_list_head, list)
+		msi_map_pirq_to_vector(dev, pirq_entry->pirq, pirq_entry->entry_nr, 0);
+	spin_unlock_irqrestore(&msi_dev_entry->pirq_list_lock, flags);
+
+	enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
+}
+#endif
+
+/**
+ * msi_capability_init - configure device's MSI capability structure
+ * @dev: pointer to the pci_dev data structure of MSI device function
+ *
+ * Setup the MSI capability structure of device function with a single
+ * MSI vector, regardless of device function is capable of handling
+ * multiple messages. A return of zero indicates the successful setup
+ * of an entry zero with the new MSI vector or non-zero for otherwise.
+ **/
+static int msi_capability_init(struct pci_dev *dev)
+{
+	int pos, pirq;
+	u16 control;
+
+   	pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
+	pci_read_config_word(dev, msi_control_reg(pos), &control);
+
+	pirq = msi_map_vector(dev, 0, 1);
+	if (pirq < 0)
+		return -EBUSY;
+
+	dev->irq = pirq;
+	/* Set MSI enabled bits	 */
+	enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
+	dev->msi_enabled = 1;
+
+	return 0;
+}
+
+/**
+ * msix_capability_init - configure device's MSI-X capability
+ * @dev: pointer to the pci_dev data structure of MSI-X device function
+ * @entries: pointer to an array of struct msix_entry entries
+ * @nvec: number of @entries
+ *
+ * Setup the MSI-X capability structure of device function with a
+ * single MSI-X vector. A return of zero indicates the successful setup of
+ * requested MSI-X entries with allocated vectors or non-zero for otherwise.
+ **/
+static int msix_capability_init(struct pci_dev *dev,
+				struct msix_entry *entries, int nvec)
+{
+	int pirq, i, pos;
+	struct msi_dev_list *msi_dev_entry = get_msi_dev_pirq_list(dev);
+	struct msi_pirq_entry *pirq_entry, *tmp;
+	unsigned long flags;
+
+	if (!msi_dev_entry)
+		return -ENOMEM;
+
+	spin_lock_irqsave(&msi_dev_entry->pirq_list_lock, flags);
+	if (!list_empty_careful(&msi_dev_entry->pirq_list_head))
+	{
+		printk(KERN_WARNING "msix pirqs for dev %02x:%02x:%01x are not freed \
+		       before acquire again.\n", dev->bus->number, PCI_SLOT(dev->devfn),
+			   PCI_FUNC(dev->devfn));
+		list_for_each_entry_safe(pirq_entry, tmp,
+		                         &msi_dev_entry->pirq_list_head, list) {
+			msi_unmap_pirq(dev, pirq_entry->pirq);
+			list_del(&pirq_entry->list);
+			kfree(pirq_entry);
+		}
+	}
+	spin_unlock_irqrestore(&msi_dev_entry->pirq_list_lock, flags);
+
+	/* MSI-X Table Initialization */
+	for (i = 0; i < nvec; i++) {
+		pirq = msi_map_vector(dev, entries[i].entry, 0);
+		if (pirq < 0)
+			break;
+		attach_pirq_entry(pirq, entries[i].entry, msi_dev_entry);
+		dev->irq = pirq;
+		(entries + i)->vector = pirq;
+	}
+	if (i != nvec) {
+		msi_unmap_pirq(dev, dev->irq);
+		(entries + i)->vector = 0;
+		return -EBUSY;
+	}
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
+	enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
+	dev->msix_enabled = 1;
+
+	return 0;
+}
+
+/**
+ * pci_enable_msi - configure device's MSI capability structure
+ * @dev: pointer to the pci_dev data structure of MSI device function
+ *
+ * Setup the MSI capability structure of device function with
+ * a single MSI vector upon its software driver call to request for
+ * MSI mode enabled on its hardware device function. A return of zero
+ * indicates the successful setup of an entry zero with the new MSI
+ * vector or non-zero for otherwise.
+ **/
+extern int pci_frontend_enable_msi(struct pci_dev *dev);
+int pci_enable_msi(struct pci_dev* dev)
+{
+	struct pci_bus *bus;
+	int pos, temp, status = -EINVAL;
+
+	if (!pci_msi_enable || !dev)
+ 		return status;
+
+	if (dev->no_msi)
+		return status;
+
+	for (bus = dev->bus; bus; bus = bus->parent)
+		if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI)
+			return -EINVAL;
+
+	status = msi_init();
+	if (status < 0)
+		return status;
+
+#ifdef CONFIG_XEN_PCIDEV_FRONTEND
+	if (!is_initial_xendomain())
+	{
+		int ret;
+
+		temp = dev->irq;
+		ret = pci_frontend_enable_msi(dev);
+		if (ret)
+			return ret;
+
+		dev->irq_old = temp;
+
+		return ret;
+	}
+#endif
+
+	temp = dev->irq;
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
+	if (!pos)
+		return -EINVAL;
+
+	/* Check whether driver already requested for MSI-X vectors */
+	if (dev->msix_enabled) {
+		printk(KERN_INFO "PCI: %s: Can't enable MSI.  "
+			   "Device already has MSI-X vectors assigned\n",
+			   pci_name(dev));
+		dev->irq = temp;
+		return -EINVAL;
+	}
+
+	status = msi_capability_init(dev);
+	if ( !status )
+		dev->irq_old = temp;
+    else
+		dev->irq = temp;
+
+	return status;
+}
+
+extern void pci_frontend_disable_msi(struct pci_dev* dev);
+void pci_disable_msi(struct pci_dev* dev)
+{
+	int pos;
+	int pirq;
+
+	if (!pci_msi_enable)
+		return;
+	if (!dev)
+		return;
+
+#ifdef CONFIG_XEN_PCIDEV_FRONTEND
+	if (!is_initial_xendomain()) {
+		pci_frontend_disable_msi(dev);
+		dev->irq = dev->irq_old;
+		return;
+	}
+#endif
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
+	if (!pos)
+		return;
+
+	pirq = dev->irq;
+	/* Restore dev->irq to its default pin-assertion vector */
+	dev->irq = dev->irq_old;
+	msi_unmap_pirq(dev, pirq);
+
+	/* Disable MSI mode */
+	disable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
+}
+
+/**
+ * pci_enable_msix - configure device's MSI-X capability structure
+ * @dev: pointer to the pci_dev data structure of MSI-X device function
+ * @entries: pointer to an array of MSI-X entries
+ * @nvec: number of MSI-X vectors requested for allocation by device driver
+ *
+ * Setup the MSI-X capability structure of device function with the number
+ * of requested vectors upon its software driver call to request for
+ * MSI-X mode enabled on its hardware device function. A return of zero
+ * indicates the successful configuration of MSI-X capability structure
+ * with new allocated MSI-X vectors. A return of < 0 indicates a failure.
+ * Or a return of > 0 indicates that driver request is exceeding the number
+ * of vectors available. Driver should use the returned value to re-send
+ * its request.
+ **/
+extern int pci_frontend_enable_msix(struct pci_dev *dev,
+		struct msix_entry *entries, int nvec);
+int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec)
+{
+	struct pci_bus *bus;
+	int status, pos, nr_entries;
+	int i, j, temp;
+	u16 control;
+
+	if (!pci_msi_enable || !dev || !entries)
+ 		return -EINVAL;
+
+	if (dev->no_msi)
+		return -EINVAL;
+
+	for (bus = dev->bus; bus; bus = bus->parent)
+		if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI)
+			return -EINVAL;
+
+#ifdef CONFIG_XEN_PCIDEV_FRONTEND
+	if (!is_initial_xendomain()) {
+		int ret;
+
+		ret = pci_frontend_enable_msix(dev, entries, nvec);
+		if (ret) {
+			printk("get %x from pci_frontend_enable_msix\n", ret);
+			return ret;
+		}
+
+        return 0;
+	}
+#endif
+
+	status = msi_init();
+	if (status < 0)
+		return status;
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
+	if (!pos)
+ 		return -EINVAL;
+
+	pci_read_config_word(dev, msi_control_reg(pos), &control);
+	nr_entries = multi_msix_capable(control);
+	if (nvec > nr_entries)
+		return -EINVAL;
+
+	/* Check for any invalid entries */
+	for (i = 0; i < nvec; i++) {
+		if (entries[i].entry >= nr_entries)
+			return -EINVAL;		/* invalid entry */
+		for (j = i + 1; j < nvec; j++) {
+			if (entries[i].entry == entries[j].entry)
+				return -EINVAL;	/* duplicate entry */
+		}
+	}
+
+	temp = dev->irq;
+	/* Check whether driver already requested for MSI vector */
+	if (dev->msi_enabled) {
+		printk(KERN_INFO "PCI: %s: Can't enable MSI-X.  "
+		       "Device already has an MSI vector assigned\n",
+		       pci_name(dev));
+		dev->irq = temp;
+		return -EINVAL;
+	}
+
+	status = msix_capability_init(dev, entries, nvec);
+
+	if ( !status )
+		dev->irq_old = temp;
+	else
+		dev->irq = temp;
+
+	return status;
+}
+
+extern void pci_frontend_disable_msix(struct pci_dev* dev);
+void pci_disable_msix(struct pci_dev* dev)
+{
+	int pos;
+	u16 control;
+
+
+	if (!pci_msi_enable)
+		return;
+	if (!dev)
+		return;
+
+#ifdef CONFIG_XEN_PCIDEV_FRONTEND
+	if (!is_initial_xendomain()) {
+		pci_frontend_disable_msix(dev);
+		dev->irq = dev->irq_old;
+		return;
+	}
+#endif
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
+	if (!pos)
+		return;
+
+	pci_read_config_word(dev, msi_control_reg(pos), &control);
+	if (!(control & PCI_MSIX_FLAGS_ENABLE))
+		return;
+
+	msi_remove_pci_irq_vectors(dev);
+
+	/* Disable MSI mode */
+	disable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
+}
+
+/**
+ * msi_remove_pci_irq_vectors - reclaim MSI(X) vectors to unused state
+ * @dev: pointer to the pci_dev data structure of MSI(X) device function
+ *
+ * Being called during hotplug remove, from which the device function
+ * is hot-removed. All previous assigned MSI/MSI-X vectors, if
+ * allocated for this device function, are reclaimed to unused state,
+ * which may be used later on.
+ **/
+void msi_remove_pci_irq_vectors(struct pci_dev* dev)
+{
+	unsigned long flags;
+	struct msi_dev_list *msi_dev_entry;
+	struct msi_pirq_entry *pirq_entry, *tmp;
+
+	if (!pci_msi_enable || !dev)
+ 		return;
+
+	msi_dev_entry = get_msi_dev_pirq_list(dev);
+
+	spin_lock_irqsave(&msi_dev_entry->pirq_list_lock, flags);
+	if (!list_empty_careful(&msi_dev_entry->pirq_list_head))
+	{
+		printk(KERN_WARNING "msix pirqs for dev %02x:%02x:%01x are not freed \
+		       before acquire again.\n", dev->bus->number, PCI_SLOT(dev->devfn),
+			   PCI_FUNC(dev->devfn));
+		list_for_each_entry_safe(pirq_entry, tmp,
+		                         &msi_dev_entry->pirq_list_head, list) {
+			msi_unmap_pirq(dev, pirq_entry->pirq);
+			list_del(&pirq_entry->list);
+			kfree(pirq_entry);
+		}
+	}
+	spin_unlock_irqrestore(&msi_dev_entry->pirq_list_lock, flags);
+	dev->irq = dev->irq_old;
+}
+
+void pci_no_msi(void)
+{
+	pci_msi_enable = 0;
+}
+
+EXPORT_SYMBOL(pci_enable_msi);
+EXPORT_SYMBOL(pci_disable_msi);
+EXPORT_SYMBOL(pci_enable_msix);
+EXPORT_SYMBOL(pci_disable_msix);
+#ifdef CONFIG_XEN
+EXPORT_SYMBOL(register_msi_get_owner);
+EXPORT_SYMBOL(unregister_msi_get_owner);
+#endif
+
diff -r f3d354a4b403 drivers/pci/msi.c
--- a/drivers/pci/msi.c	Fri Apr 25 00:01:08 2008 +0800
+++ b/drivers/pci/msi.c	Wed Apr 30 19:34:58 2008 +0800
@@ -1022,21 +1022,13 @@
 	}
 }
 
-static int msi_free_vector(struct pci_dev* dev, int vector, int reassign)
+static int msi_free_vector(struct pci_dev* dev, int pirq)
 {
-	struct msi_desc *entry;
-	int head, entry_nr, type;
-	void __iomem *base;
-	unsigned long flags;
+	struct msi_dev_list *msi_dev_entry = get_msi_dev_pirq_list(dev);
 
-	msi_ops->teardown(vector);
+	if (!list_empty_careful(msi_dev_entry->pirq_list_head))
+		return 0;
 
-	spin_lock_irqsave(&msi_lock, flags);
-	entry = msi_desc[vector];
-	if (!entry || entry->dev != dev) {
-		spin_unlock_irqrestore(&msi_lock, flags);
-		return -EINVAL;
-	}
 	type = entry->msi_attrib.type;
 	entry_nr = entry->msi_attrib.entry_nr;
 	head = entry->link.head;
diff -r f3d354a4b403 drivers/pci/msi.h
--- a/drivers/pci/msi.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/drivers/pci/msi.h	Wed Apr 30 19:34:58 2008 +0800
@@ -84,6 +84,11 @@
 extern void (*interrupt[NR_IRQS])(void);
 extern int pci_vector_resources(int last, int nr_released);
 
+#ifdef CONFIG_XEN
+extern int unregister_msi_get_owner(int (*func)(struct pci_dev *dev));
+extern int register_msi_get_owner(int (*func)(struct pci_dev *dev));
+#endif
+
 /*
  * MSI-X Address Register
  */
diff -r f3d354a4b403 drivers/xen/pciback/Makefile
--- a/drivers/xen/pciback/Makefile	Fri Apr 25 00:01:08 2008 +0800
+++ b/drivers/xen/pciback/Makefile	Wed Apr 30 19:34:58 2008 +0800
@@ -6,6 +6,7 @@
 	     conf_space_capability_vpd.o \
 	     conf_space_capability_pm.o \
              conf_space_quirks.o
+pciback-$(CONFIG_PCI_MSI) += conf_space_capability_msi.o
 pciback-$(CONFIG_XEN_PCIDEV_BACKEND_VPCI) += vpci.o
 pciback-$(CONFIG_XEN_PCIDEV_BACKEND_SLOT) += slot.o
 pciback-$(CONFIG_XEN_PCIDEV_BACKEND_PASS) += passthrough.o
diff -r f3d354a4b403 drivers/xen/pciback/conf_space_capability_msi.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/drivers/xen/pciback/conf_space_capability_msi.c	Wed Apr 30 19:34:58 2008 +0800
@@ -0,0 +1,60 @@
+/*
+ * PCI Backend -- Configuration overlay for MSI capability
+ */
+#include <linux/pci.h>
+#include "conf_space.h"
+#include "conf_space_capability.h"
+#include <xen/interface/io/pciif.h>
+#include "pciback.h"
+
+int pciback_enable_msi(struct pciback_device *pdev,
+		struct pci_dev *dev, struct xen_pci_op *op)
+{
+	int otherend = pdev->xdev->otherend_id;
+	int irq;
+	int status;
+
+	status = pci_enable_msi(dev);
+
+	if (status) {
+		printk("error enable msi for guest %x status %x\n", otherend, status);
+		op->value = 0;
+		return XEN_PCI_ERR_op_failed;
+	}
+
+	op->value = dev->irq;
+	return 0;
+}
+
+int pciback_disable_msi(struct pciback_device *pdev,
+		struct pci_dev *dev, struct xen_pci_op *op)
+{
+	int old_irq = dev->irq;
+
+	pci_disable_msi(dev);
+
+	op->value = dev->irq;
+	return 0;
+}
+
+int pciback_enable_msix(struct pciback_device *pdev,
+		struct pci_dev *dev, struct xen_pci_op *op)
+{
+	int otherend = pdev->xdev->otherend_id, result, i;
+
+	result = pci_enable_msix(dev, op->msix_entries, op->value);
+
+	op->value = result;
+	return result;
+}
+
+int pciback_disable_msix(struct pciback_device *pdev,
+		struct pci_dev *dev, struct xen_pci_op *op)
+{
+
+	pci_disable_msix(dev);
+
+	op->value = dev->irq;
+	return 0;
+}
+
diff -r f3d354a4b403 drivers/xen/pciback/pci_stub.c
--- a/drivers/xen/pciback/pci_stub.c	Fri Apr 25 00:01:08 2008 +0800
+++ b/drivers/xen/pciback/pci_stub.c	Wed Apr 30 19:34:58 2008 +0800
@@ -805,6 +805,23 @@
 
 DRIVER_ATTR(permissive, S_IRUSR | S_IWUSR, permissive_show, permissive_add);
 
+#ifdef CONFIG_PCI_MSI
+
+int pciback_get_owner(struct pci_dev *dev)
+{
+	struct pcistub_device *psdev;
+
+	psdev = pcistub_device_find(pci_domain_nr(dev->bus), dev->bus->number,
+			PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn));
+	/* XXX will other domain has pciback support ??? */
+	if (!psdev || !psdev->pdev) {
+		printk(KERN_WARNING "no ownder\n");
+		return -1;
+	}
+	return psdev->pdev->xdev->otherend_id;
+}
+#endif
+
 static void pcistub_exit(void)
 {
 	driver_remove_file(&pciback_pci_driver.driver, &driver_attr_new_slot);
@@ -815,6 +832,9 @@
 	driver_remove_file(&pciback_pci_driver.driver, &driver_attr_permissive);
 
 	pci_unregister_driver(&pciback_pci_driver);
+#ifdef CONFIG_PCI_MSI
+	unregister_msi_get_owner(pciback_get_owner);
+#endif
 }
 
 static int __init pcistub_init(void)
@@ -872,6 +892,10 @@
 		err = driver_create_file(&pciback_pci_driver.driver,
 					 &driver_attr_permissive);
 
+#ifdef CONFIG_PCI_MSI
+	if (!err)
+		err = register_msi_get_owner(pciback_get_owner);
+#endif
 	if (err)
 		pcistub_exit();
 
diff -r f3d354a4b403 drivers/xen/pciback/pciback.h
--- a/drivers/xen/pciback/pciback.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/drivers/xen/pciback/pciback.h	Wed Apr 30 19:34:58 2008 +0800
@@ -93,5 +93,19 @@
 int pciback_xenbus_register(void);
 void pciback_xenbus_unregister(void);
 
+#ifdef CONFIG_PCI_MSI
+int pciback_enable_msi(struct pciback_device *pdev,
+                       struct pci_dev *dev, struct xen_pci_op *op);
+
+int pciback_disable_msi(struct pciback_device *pdev,
+                         struct pci_dev *dev, struct xen_pci_op *op);
+
+
+int pciback_enable_msix(struct pciback_device *pdev,
+                        struct pci_dev *dev, struct xen_pci_op *op);
+
+int pciback_disable_msix(struct pciback_device *pdev,
+                        struct pci_dev *dev, struct xen_pci_op *op);
+#endif
 extern int verbose_request;
 #endif
diff -r f3d354a4b403 drivers/xen/pciback/pciback_ops.c
--- a/drivers/xen/pciback/pciback_ops.c	Fri Apr 25 00:01:08 2008 +0800
+++ b/drivers/xen/pciback/pciback_ops.c	Wed Apr 30 19:34:58 2008 +0800
@@ -61,15 +61,37 @@
 
 	if (dev == NULL)
 		op->err = XEN_PCI_ERR_dev_not_found;
-	else if (op->cmd == XEN_PCI_OP_conf_read)
-		op->err = pciback_config_read(dev, op->offset, op->size,
-					      &op->value);
-	else if (op->cmd == XEN_PCI_OP_conf_write)
-		op->err = pciback_config_write(dev, op->offset, op->size,
-					       op->value);
 	else
-		op->err = XEN_PCI_ERR_not_implemented;
-
+	{
+		switch (op->cmd)
+		{
+			case XEN_PCI_OP_conf_read:
+				op->err = pciback_config_read(dev,
+					  op->offset, op->size, &op->value);
+				break;
+			case XEN_PCI_OP_conf_write:
+				op->err = pciback_config_write(dev,
+					  op->offset, op->size,	op->value);
+				break;
+#ifdef CONFIG_PCI_MSI
+			case XEN_PCI_OP_enable_msi:
+				op->err = pciback_enable_msi(pdev, dev, op);
+				break;
+			case XEN_PCI_OP_disable_msi:
+				op->err = pciback_disable_msi(pdev, dev, op);
+				break;
+			case XEN_PCI_OP_enable_msix:
+				op->err = pciback_enable_msix(pdev, dev, op);
+				break;
+			case XEN_PCI_OP_disable_msix:
+				op->err = pciback_disable_msix(pdev, dev, op);
+				break;
+#endif
+			default:
+				op->err = XEN_PCI_ERR_not_implemented;
+				break;
+		}
+	}
 	/* Tell the driver domain that we're done. */ 
 	wmb();
 	clear_bit(_XEN_PCIF_active, (unsigned long *)&pdev->sh_info->flags);
diff -r f3d354a4b403 drivers/xen/pcifront/pci_op.c
--- a/drivers/xen/pcifront/pci_op.c	Fri Apr 25 00:01:08 2008 +0800
+++ b/drivers/xen/pcifront/pci_op.c	Wed Apr 30 19:34:58 2008 +0800
@@ -277,6 +277,122 @@
 	.write = pcifront_bus_write,
 };
 
+#ifdef CONFIG_PCI_MSI
+int pci_frontend_enable_msix(struct pci_dev *dev,
+		struct msix_entry *entries,
+		int nvec)
+{
+	int err;
+	int i;
+	struct xen_pci_op op = {
+		.cmd    = XEN_PCI_OP_enable_msix,
+		.domain = pci_domain_nr(dev->bus),
+		.bus = dev->bus->number,
+		.devfn = dev->devfn,
+		.value = nvec,
+	};
+	struct pcifront_sd *sd = dev->bus->sysdata;
+	struct pcifront_device *pdev = pcifront_get_pdev(sd);
+
+	if (nvec > SH_INFO_MAX_VEC) {
+		printk("too much vector for pci frontend%x\n", nvec);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < nvec; i++) {
+		op.msix_entries[i].entry = entries[i].entry;
+		op.msix_entries[i].vector = entries[i].vector;
+	}
+
+	err = do_pci_op(pdev, &op);
+
+	if (!err) {
+		if (!op.value) {
+			/* we get the result */
+			for ( i = 0; i < nvec; i++)
+				entries[i].vector = op.msix_entries[i].vector;
+			return 0;
+		}
+		else {
+            printk("enable msix get value %x\n", op.value);
+			return op.value;
+		}
+	}
+	else {
+        printk("enable msix get err %x\n", err);
+		return err;
+	}
+}
+
+void pci_frontend_disable_msix(struct pci_dev* dev)
+{
+	int err;
+	struct xen_pci_op op = {
+		.cmd    = XEN_PCI_OP_disable_msix,
+		.domain = pci_domain_nr(dev->bus),
+		.bus = dev->bus->number,
+		.devfn = dev->devfn,
+	};
+	struct pcifront_sd *sd = dev->bus->sysdata;
+	struct pcifront_device *pdev = pcifront_get_pdev(sd);
+
+	err = do_pci_op(pdev, &op);
+
+	/* What should do for error ? */
+	if (err)
+		printk("pci_disable_msix get err %x\n", err);
+}
+
+int pci_frontend_enable_msi(struct pci_dev *dev)
+{
+	int err;
+	struct xen_pci_op op = {
+		.cmd    = XEN_PCI_OP_enable_msi,
+		.domain = pci_domain_nr(dev->bus),
+		.bus = dev->bus->number,
+		.devfn = dev->devfn,
+	};
+	struct pcifront_sd *sd = dev->bus->sysdata;
+	struct pcifront_device *pdev = pcifront_get_pdev(sd);
+
+	err = do_pci_op(pdev, &op);
+	if (likely(!err)) {
+		dev->irq = op.value;
+	}
+	else {
+		printk("pci frontend enable msi failed for dev %x:%x \n",
+				op.bus, op.devfn);
+		err = -EINVAL;
+	}
+	return err;
+}
+
+void pci_frontend_disable_msi(struct pci_dev* dev)
+{
+	int err;
+	struct xen_pci_op op = {
+		.cmd    = XEN_PCI_OP_disable_msi,
+		.domain = pci_domain_nr(dev->bus),
+		.bus = dev->bus->number,
+		.devfn = dev->devfn,
+	};
+	struct pcifront_sd *sd = dev->bus->sysdata;
+	struct pcifront_device *pdev = pcifront_get_pdev(sd);
+
+	err = do_pci_op(pdev, &op);
+	if (err == XEN_PCI_ERR_dev_not_found) {
+		/* XXX No response from backend, what shall we do? */
+		printk("get no response from backend for disable MSI\n");
+		return;
+	}
+	if (likely(!err))
+		dev->irq = op.value;
+	else
+		/* how can pciback notify us fail? */
+		printk("get fake response frombackend \n");
+}
+#endif /* CONFIG_PCI_MSI */
+
 /* Claim resources for the PCI frontend as-is, backend won't allow changes */
 static void pcifront_claim_resource(struct pci_dev *dev, void *data)
 {
diff -r f3d354a4b403 include/asm-i386/io_apic.h
--- a/include/asm-i386/io_apic.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/include/asm-i386/io_apic.h	Wed Apr 30 19:34:58 2008 +0800
@@ -12,7 +12,7 @@
 
 #ifdef CONFIG_X86_IO_APIC
 
-#ifdef CONFIG_PCI_MSI
+#if defined(CONFIG_PCI_MSI) && !defined(CONFIG_XEN)
 static inline int use_pci_vector(void)	{return 1;}
 static inline void disable_edge_ioapic_vector(unsigned int vector) { }
 static inline void mask_and_ack_level_ioapic_vector(unsigned int vector) { }
diff -r f3d354a4b403 include/asm-x86_64/io_apic.h
--- a/include/asm-x86_64/io_apic.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/include/asm-x86_64/io_apic.h	Wed Apr 30 19:34:58 2008 +0800
@@ -12,7 +12,7 @@
 
 #ifdef CONFIG_X86_IO_APIC
 
-#ifdef CONFIG_PCI_MSI
+#if defined(CONFIG_PCI_MSI) && !defined(CONFIG_XEN)
 static inline int use_pci_vector(void)	{return 1;}
 static inline void disable_edge_ioapic_vector(unsigned int vector) { }
 static inline void mask_and_ack_level_ioapic_vector(unsigned int vector) { }
diff -r f3d354a4b403 include/asm-x86_64/msi.h
--- a/include/asm-x86_64/msi.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/include/asm-x86_64/msi.h	Wed Apr 30 19:34:58 2008 +0800
@@ -7,14 +7,21 @@
 #define ASM_MSI_H
 
 #include <asm/desc.h>
+#ifndef CONFIG_XEN
 #include <asm/mach_apic.h>
+#endif
 #include <asm/smp.h>
 
+#ifndef CONFIG_XEN
 #define LAST_DEVICE_VECTOR	(FIRST_SYSTEM_VECTOR - 1)
+#else
+#define LAST_DYNAMIC_VECTOR 0xdf
+#define LAST_DEVICE_VECTOR	(LAST_DYNAMIC_VECTOR)
+#endif
+
 #define MSI_TARGET_CPU_SHIFT	12
 
 extern struct msi_ops msi_apic_ops;
-
 static inline int msi_arch_init(void)
 {
 	msi_register(&msi_apic_ops);
diff -r f3d354a4b403 include/linux/pci.h
--- a/include/linux/pci.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/include/linux/pci.h	Wed Apr 30 19:34:58 2008 +0800
@@ -152,6 +152,9 @@
 	 * directly, use the values stored here. They might be different!
 	 */
 	unsigned int	irq;
+#ifdef CONFIG_XEN
+	unsigned int    irq_old;
+#endif
 	struct resource resource[DEVICE_COUNT_RESOURCE]; /* I/O and memory regions + expansion ROMs */
 
 	/* These fields are used by common fixups */
diff -r f3d354a4b403 include/xen/evtchn.h
--- a/include/xen/evtchn.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/include/xen/evtchn.h	Wed Apr 30 19:34:58 2008 +0800
@@ -136,4 +136,18 @@
 void notify_remote_via_irq(int irq);
 int irq_to_evtchn_port(int irq);
 
+#define PIRQ_SET_MAPPING 0x0
+#define PIRQ_CLEAR_MAPPING 0x1
+#define PIRQ_GET_MAPPING 0x3
+int pirq_mapstatus(int pirq, int action);
+int set_pirq_hw_action(int pirq, int (*action)(int pirq, int action));
+int clear_pirq_hw_action(int pirq);
+
+#define PIRQ_STARTUP 1
+#define PIRQ_SHUTDOWN 2
+#define PIRQ_ENABLE 3
+#define PIRQ_DISABLE 4
+#define PIRQ_END 5
+#define PIRQ_ACK 6
+
 #endif /* __ASM_EVTCHN_H__ */
diff -r f3d354a4b403 include/xen/interface/io/pciif.h
--- a/include/xen/interface/io/pciif.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/include/xen/interface/io/pciif.h	Wed Apr 30 19:34:58 2008 +0800
@@ -34,6 +34,10 @@
 /* xen_pci_op commands */
 #define XEN_PCI_OP_conf_read    (0)
 #define XEN_PCI_OP_conf_write   (1)
+#define XEN_PCI_OP_enable_msi   (2)
+#define XEN_PCI_OP_enable_msix  (3)
+#define XEN_PCI_OP_disable_msi  (4)
+#define XEN_PCI_OP_disable_msix (5)
 
 /* xen_pci_op error numbers */
 #define XEN_PCI_ERR_success          (0)
@@ -43,6 +47,12 @@
 #define XEN_PCI_ERR_not_implemented (-4)
 /* XEN_PCI_ERR_op_failed - backend failed to complete the operation */
 #define XEN_PCI_ERR_op_failed       (-5)
+
+/*
+ * it should be PAGE_SIZE-sizeof(struct xen_pci_op))/sizeof(struct msix_entry))
+ * Should not exceed 128
+ */
+#define SH_INFO_MAX_VEC     128
 
 struct xen_pci_op {
     /* IN: what action to perform: XEN_PCI_OP_* */
@@ -62,6 +72,11 @@
 
     /* IN/OUT: Contains the result after a READ or the value to WRITE */
     uint32_t value;
+    /* IN: Contains extra infor for this operation */
+    uint32_t info;
+    /*IN:  param for msi-x */
+    struct msix_entry msix_entries[SH_INFO_MAX_VEC];
+
 };
 
 struct xen_pci_sharedinfo {
diff -r f3d354a4b403 include/xen/interface/physdev.h
--- a/include/xen/interface/physdev.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/include/xen/interface/physdev.h	Wed Apr 30 19:34:58 2008 +0800
@@ -122,6 +122,38 @@
 typedef struct physdev_irq physdev_irq_t;
 DEFINE_XEN_GUEST_HANDLE(physdev_irq_t);
 
+#define MAP_PIRQ_TYPE_MSI               0x0
+#define MAP_PIRQ_TYPE_GSI               0x1
+#define MAP_PIRQ_TYPE_UNKNOWN           0x2
+
+#define PHYSDEVOP_map_pirq               13
+struct physdev_map_pirq {
+    domid_t domid;
+    /* IN */
+    int type;
+    /* IN */
+    int index;
+    /* IN or OUT */
+    int pirq;
+    /* msi info passed to VMM */
+    struct {
+        int bus, devfn, entry_nr;
+		int msi;  /* 0 - MSIX  1 - MSI */
+    } msi_info;
+};
+typedef struct physdev_map_pirq physdev_map_pirq_t;
+DEFINE_XEN_GUEST_HANDLE(physdev_map_pirq_t);
+
+#define PHYSDEVOP_unmap_pirq             14
+struct physdev_unmap_pirq {
+    domid_t domid;
+    /* IN */
+    int pirq;
+};
+
+typedef struct physdev_unmap_pirq physdev_unmap_pirq_t;
+DEFINE_XEN_GUEST_HANDLE(physdev_unmap_pirq_t);
+
 /*
  * Argument to physdev_op_compat() hypercall. Superceded by new physdev_op()
  * hypercall since 0x00030202.

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [PATCH 6/6] Add MSI support to XEN

[Jan Beulich]

>>> "Shan, Haitao" <haitao.shan@intel.com> 30.04.08 09:25 >>>
>This patch add MSI support to domain0/domain U.
> 
>Signed-off-by: Jiang Yunhong <yunhong.jiang@intel.com>
>Signed-off-by: Shan Haitao     <haitao.shan@intel.co 
><<msi_kernel.patch>> m>

The change to drivers/pci/msi.c looks bogus, not only because the
patch adds a drivers/pci/msi-xen.c.

Jan

RE: [PATCH 6/6] Add MSI support to XEN

[Shan, Haitao]

[-- Attachment #1: Type: text/plain, Size: 872 bytes --]

Hi, Jan,
I do not want to touch drivers/pci/msi.c. Sorry, this part of code is
left in the patch duo to my mistake. The attached is the updated patch.
The reason that I add a new file is that I did not find a clean way to
use the original msi.c. There is code there managing vectors, which is
definitly not needed in dom0. Also I do not want to insert too many
"ifdef"s in that file.
Do I understand your comments correctly?

Best Regards
Haitao Shan

Jan Beulich wrote:
>>>> "Shan, Haitao" <haitao.shan@intel.com> 30.04.08 09:25 >>>
>> This patch add MSI support to domain0/domain U.
>> 
>> Signed-off-by: Jiang Yunhong <yunhong.jiang@intel.com>
>> Signed-off-by: Shan Haitao     <haitao.shan@intel.co
>> <<msi_kernel.patch>> m>
> 
> The change to drivers/pci/msi.c looks bogus, not only because the
> patch adds a drivers/pci/msi-xen.c.
> 
> Jan

[-- Attachment #2: msi_kernel.patch --]
[-- Type: application/octet-stream, Size: 33205 bytes --]

diff -r f3d354a4b403 drivers/pci/Kconfig
--- a/drivers/pci/Kconfig	Fri Apr 25 00:01:08 2008 +0800
+++ b/drivers/pci/Kconfig	Wed Apr 30 19:34:58 2008 +0800
@@ -5,7 +5,6 @@
 	bool "Message Signaled Interrupts (MSI and MSI-X)"
 	depends on PCI
 	depends on (X86_LOCAL_APIC && X86_IO_APIC) || IA64
-	depends on !XEN
 	help
 	   This allows device drivers to enable MSI (Message Signaled
 	   Interrupts).  Message Signaled Interrupts enable a device to
diff -r f3d354a4b403 drivers/pci/msi-xen.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/drivers/pci/msi-xen.c	Wed Apr 30 19:34:58 2008 +0800
@@ -0,0 +1,710 @@
+/*
+ * File:	msi.c
+ * Purpose:	PCI Message Signaled Interrupt (MSI)
+ *
+ * Copyright (C) 2003-2004 Intel
+ * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
+ */
+
+#include <linux/mm.h>
+#include <linux/irq.h>
+#include <linux/interrupt.h>
+#include <linux/init.h>
+#include <linux/ioport.h>
+#include <linux/smp_lock.h>
+#include <linux/pci.h>
+#include <linux/proc_fs.h>
+
+#include <asm/errno.h>
+#include <asm/io.h>
+#include <asm/smp.h>
+
+#include "pci.h"
+#include "msi.h"
+
+static int pci_msi_enable = 1;
+
+static struct msi_ops *msi_ops;
+
+int msi_register(struct msi_ops *ops)
+{
+	msi_ops = ops;
+	return 0;
+}
+
+static struct list_head msi_dev_head;
+static int msi_dev_head_inited = 0;
+DEFINE_SPINLOCK(msi_dev_lock);
+
+struct msi_dev_list {
+	struct pci_dev *dev;
+	struct list_head list;
+	spinlock_t pirq_list_lock;
+	struct list_head pirq_list_head;
+};
+
+struct msi_pirq_entry {
+	struct list_head list;
+	int pirq;
+	int entry_nr;
+};
+
+static struct msi_dev_list *get_msi_dev_pirq_list(struct pci_dev *dev)
+{
+	struct msi_dev_list *msi_dev_list, *ret = NULL;
+	unsigned long flags;
+
+	if (!msi_dev_head_inited) {
+		INIT_LIST_HEAD(&msi_dev_head);
+		msi_dev_head_inited = 1;
+	}
+
+	spin_lock_irqsave(&msi_dev_lock, flags);
+
+	list_for_each_entry(msi_dev_list, &msi_dev_head, list)
+		if ( msi_dev_list->dev == dev )
+			ret = msi_dev_list;
+
+	if ( ret ) {
+		spin_unlock_irqrestore(&msi_dev_lock, flags);
+		return ret;
+	}
+
+	/* Has not allocate msi_dev until now. */
+	ret = kmalloc(sizeof(struct msi_dev_list), GFP_ATOMIC);
+
+	/* Failed to allocate msi_dev structure */
+	if ( !ret ) {
+		spin_unlock_irqrestore(&msi_dev_lock, flags);
+		return NULL;
+	}
+
+	list_add_tail(&ret->list, &msi_dev_head);
+	spin_unlock_irqrestore(&msi_dev_lock, flags);
+	spin_lock_init(&ret->pirq_list_lock);
+	INIT_LIST_HEAD(&ret->pirq_list_head);
+	return ret;
+}
+
+static int attach_pirq_entry(int pirq, int entry_nr,
+                             struct msi_dev_list *msi_dev_entry)
+{
+	struct msi_pirq_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
+	unsigned long flags;
+
+	if (!entry)
+		return -ENOMEM;
+	entry->pirq = pirq;
+	entry->entry_nr = entry_nr;
+	spin_lock_irqsave(&msi_dev_entry->pirq_list_lock, flags);
+	list_add_tail(&entry->list, &msi_dev_entry->pirq_list_head);
+	spin_unlock_irqrestore(&msi_dev_entry->pirq_list_lock, flags);
+	return 0;
+}
+
+/*
+ * pciback will provide device's owner
+ */
+int (*get_owner)(struct pci_dev *dev);
+
+int register_msi_get_owner(int (*func)(struct pci_dev *dev))
+{
+	if (get_owner) {
+		printk(KERN_WARNING "register msi_get_owner again\n");
+		return -EEXIST;
+	}
+	get_owner = func;
+	return 0;
+}
+
+int unregister_msi_get_owner(int (*func)(struct pci_dev *dev))
+{
+	if (get_owner == func)
+		get_owner = NULL;
+	return 0;
+}
+
+static int msi_get_dev_owner(struct pci_dev *dev)
+{
+	int owner = DOMID_SELF;
+
+	BUG_ON(!is_initial_xendomain());
+	if (get_owner && (owner = get_owner(dev)) >=0 ) {
+		printk(KERN_INFO "get owner for dev %x get %x \n",
+				    dev->devfn, owner);
+		return owner;
+	}
+	else
+		return DOMID_SELF;
+}
+
+static int msi_unmap_pirq(struct pci_dev *dev, int pirq)
+{
+	struct physdev_unmap_pirq unmap;
+	int rc;
+	domid_t domid = DOMID_SELF;
+
+	domid = msi_get_dev_owner(dev);
+	unmap.domid = domid;
+	unmap.pirq = pirq;
+
+	if ((rc = HYPERVISOR_physdev_op(PHYSDEVOP_unmap_pirq, &unmap)))
+		printk(KERN_WARNING "unmap irq %x failed\n", pirq);
+
+	if (rc < 0)
+		return rc;
+    return 0;
+}
+
+/*
+ * Protected by msi_lock
+ */
+static int msi_map_pirq_to_vector(struct pci_dev *dev, int pirq,
+                                  int entry_nr, int msi)
+{
+	struct physdev_map_pirq map_irq;
+	int rc;
+	domid_t domid = DOMID_SELF;
+
+	domid = msi_get_dev_owner(dev);
+
+	map_irq.domid = domid;
+	map_irq.type = MAP_PIRQ_TYPE_MSI;
+	map_irq.index = -1;
+	map_irq.pirq = pirq;
+    map_irq.msi_info.bus = dev->bus->number;
+    map_irq.msi_info.devfn = dev->devfn;
+	map_irq.msi_info.entry_nr = entry_nr;
+    map_irq.msi_info.msi = msi;
+
+	if ((rc = HYPERVISOR_physdev_op(PHYSDEVOP_map_pirq, &map_irq)))
+		printk(KERN_WARNING "map irq failed\n");
+
+	if (rc < 0)
+		return rc;
+
+	return map_irq.pirq;
+}
+
+static int msi_map_vector(struct pci_dev *dev, int entry_nr, int msi)
+{
+	return msi_map_pirq_to_vector(dev, -1, entry_nr, msi);
+}
+
+static int msi_init(void)
+{
+	static int status = 0;
+
+	if (pci_msi_quirk) {
+		pci_msi_enable = 0;
+		printk(KERN_WARNING "PCI: MSI quirk detected. MSI disabled.\n");
+		status = -EINVAL;
+	}
+
+	return status;
+}
+
+void pci_scan_msi_device(struct pci_dev *dev) { }
+
+void disable_msi_mode(struct pci_dev *dev, int pos, int type)
+{
+	u16 control;
+
+	pci_read_config_word(dev, msi_control_reg(pos), &control);
+	if (type == PCI_CAP_ID_MSI) {
+		/* Set enabled bits to single MSI & enable MSI_enable bit */
+		msi_disable(control);
+		pci_write_config_word(dev, msi_control_reg(pos), control);
+		dev->msi_enabled = 0;
+	} else {
+		msix_disable(control);
+		pci_write_config_word(dev, msi_control_reg(pos), control);
+		dev->msix_enabled = 0;
+	}
+    	if (pci_find_capability(dev, PCI_CAP_ID_EXP)) {
+		/* PCI Express Endpoint device detected */
+		pci_intx(dev, 1);  /* enable intx */
+	}
+}
+
+static void enable_msi_mode(struct pci_dev *dev, int pos, int type)
+{
+	u16 control;
+
+	pci_read_config_word(dev, msi_control_reg(pos), &control);
+	if (type == PCI_CAP_ID_MSI) {
+		/* Set enabled bits to single MSI & enable MSI_enable bit */
+		msi_enable(control, 1);
+		pci_write_config_word(dev, msi_control_reg(pos), control);
+		dev->msi_enabled = 1;
+	} else {
+		msix_enable(control);
+		pci_write_config_word(dev, msi_control_reg(pos), control);
+		dev->msix_enabled = 1;
+	}
+    	if (pci_find_capability(dev, PCI_CAP_ID_EXP)) {
+		/* PCI Express Endpoint device detected */
+		pci_intx(dev, 0);  /* disable intx */
+	}
+}
+
+#ifdef CONFIG_PM
+int pci_save_msi_state(struct pci_dev *dev)
+{
+	int pos;
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
+	if (pos <= 0 || dev->no_msi)
+		return 0;
+
+	if (!dev->msi_enabled)
+		return 0;
+
+	/* Restore dev->irq to its default pin-assertion vector */
+	msi_unmap_pirq(dev, dev->irq);
+	/* Disable MSI mode */
+	disable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
+	/* Set the flags for use of restore */
+	dev->msi_enabled = 1;
+	return 0;
+}
+
+void pci_restore_msi_state(struct pci_dev *dev)
+{
+	int pos, pirq;
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
+	if (pos <= 0)
+		return;
+
+	if (!dev->msi_enabled)
+		return;
+
+	pirq = msi_map_pirq_to_vector(dev, dev->irq, 0, 1);
+	if (pirq < 0)
+		return;
+	enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
+}
+
+int pci_save_msix_state(struct pci_dev *dev)
+{
+	int pos;
+	unsigned long flags;
+	struct msi_dev_list *msi_dev_entry;
+	struct msi_pirq_entry *pirq_entry, *tmp;
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
+	if (pos <= 0 || dev->no_msi)
+		return 0;
+
+	/* save the capability */
+	if (!dev->msix_enabled)
+		return 0;
+
+	msi_dev_entry = get_msi_dev_pirq_list(dev);
+
+	spin_lock_irqsave(&msi_dev_entry->pirq_list_lock, flags);
+	if (!list_empty_careful(&msi_dev_entry->pirq_list_head))
+		list_for_each_entry_safe(pirq_entry, tmp,
+		                         &msi_dev_entry->pirq_list_head, list)
+			msi_unmap_pirq(dev, pirq_entry->pirq);
+	spin_unlock_irqrestore(&msi_dev_entry->pirq_list_lock, flags);
+
+	disable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
+	/* Set the flags for use of restore */
+	dev->msix_enabled = 1;
+
+	return 0;
+}
+
+void pci_restore_msix_state(struct pci_dev *dev)
+{
+	int pos;
+	unsigned long flags;
+	struct msi_dev_list *msi_dev_entry;
+	struct msi_pirq_entry *pirq_entry, *tmp;
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
+	if (pos <= 0)
+		return;
+
+	if (!dev->msix_enabled)
+		return;
+
+	msi_dev_entry = get_msi_dev_pirq_list(dev);
+
+	spin_lock_irqsave(&msi_dev_entry->pirq_list_lock, flags);
+	list_for_each_entry_safe(pirq_entry, tmp,
+							 &msi_dev_entry->pirq_list_head, list)
+		msi_map_pirq_to_vector(dev, pirq_entry->pirq, pirq_entry->entry_nr, 0);
+	spin_unlock_irqrestore(&msi_dev_entry->pirq_list_lock, flags);
+
+	enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
+}
+#endif
+
+/**
+ * msi_capability_init - configure device's MSI capability structure
+ * @dev: pointer to the pci_dev data structure of MSI device function
+ *
+ * Setup the MSI capability structure of device function with a single
+ * MSI vector, regardless of device function is capable of handling
+ * multiple messages. A return of zero indicates the successful setup
+ * of an entry zero with the new MSI vector or non-zero for otherwise.
+ **/
+static int msi_capability_init(struct pci_dev *dev)
+{
+	int pos, pirq;
+	u16 control;
+
+   	pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
+	pci_read_config_word(dev, msi_control_reg(pos), &control);
+
+	pirq = msi_map_vector(dev, 0, 1);
+	if (pirq < 0)
+		return -EBUSY;
+
+	dev->irq = pirq;
+	/* Set MSI enabled bits	 */
+	enable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
+	dev->msi_enabled = 1;
+
+	return 0;
+}
+
+/**
+ * msix_capability_init - configure device's MSI-X capability
+ * @dev: pointer to the pci_dev data structure of MSI-X device function
+ * @entries: pointer to an array of struct msix_entry entries
+ * @nvec: number of @entries
+ *
+ * Setup the MSI-X capability structure of device function with a
+ * single MSI-X vector. A return of zero indicates the successful setup of
+ * requested MSI-X entries with allocated vectors or non-zero for otherwise.
+ **/
+static int msix_capability_init(struct pci_dev *dev,
+				struct msix_entry *entries, int nvec)
+{
+	int pirq, i, pos;
+	struct msi_dev_list *msi_dev_entry = get_msi_dev_pirq_list(dev);
+	struct msi_pirq_entry *pirq_entry, *tmp;
+	unsigned long flags;
+
+	if (!msi_dev_entry)
+		return -ENOMEM;
+
+	spin_lock_irqsave(&msi_dev_entry->pirq_list_lock, flags);
+	if (!list_empty_careful(&msi_dev_entry->pirq_list_head))
+	{
+		printk(KERN_WARNING "msix pirqs for dev %02x:%02x:%01x are not freed \
+		       before acquire again.\n", dev->bus->number, PCI_SLOT(dev->devfn),
+			   PCI_FUNC(dev->devfn));
+		list_for_each_entry_safe(pirq_entry, tmp,
+		                         &msi_dev_entry->pirq_list_head, list) {
+			msi_unmap_pirq(dev, pirq_entry->pirq);
+			list_del(&pirq_entry->list);
+			kfree(pirq_entry);
+		}
+	}
+	spin_unlock_irqrestore(&msi_dev_entry->pirq_list_lock, flags);
+
+	/* MSI-X Table Initialization */
+	for (i = 0; i < nvec; i++) {
+		pirq = msi_map_vector(dev, entries[i].entry, 0);
+		if (pirq < 0)
+			break;
+		attach_pirq_entry(pirq, entries[i].entry, msi_dev_entry);
+		dev->irq = pirq;
+		(entries + i)->vector = pirq;
+	}
+	if (i != nvec) {
+		msi_unmap_pirq(dev, dev->irq);
+		(entries + i)->vector = 0;
+		return -EBUSY;
+	}
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
+	enable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
+	dev->msix_enabled = 1;
+
+	return 0;
+}
+
+/**
+ * pci_enable_msi - configure device's MSI capability structure
+ * @dev: pointer to the pci_dev data structure of MSI device function
+ *
+ * Setup the MSI capability structure of device function with
+ * a single MSI vector upon its software driver call to request for
+ * MSI mode enabled on its hardware device function. A return of zero
+ * indicates the successful setup of an entry zero with the new MSI
+ * vector or non-zero for otherwise.
+ **/
+extern int pci_frontend_enable_msi(struct pci_dev *dev);
+int pci_enable_msi(struct pci_dev* dev)
+{
+	struct pci_bus *bus;
+	int pos, temp, status = -EINVAL;
+
+	if (!pci_msi_enable || !dev)
+ 		return status;
+
+	if (dev->no_msi)
+		return status;
+
+	for (bus = dev->bus; bus; bus = bus->parent)
+		if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI)
+			return -EINVAL;
+
+	status = msi_init();
+	if (status < 0)
+		return status;
+
+#ifdef CONFIG_XEN_PCIDEV_FRONTEND
+	if (!is_initial_xendomain())
+	{
+		int ret;
+
+		temp = dev->irq;
+		ret = pci_frontend_enable_msi(dev);
+		if (ret)
+			return ret;
+
+		dev->irq_old = temp;
+
+		return ret;
+	}
+#endif
+
+	temp = dev->irq;
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
+	if (!pos)
+		return -EINVAL;
+
+	/* Check whether driver already requested for MSI-X vectors */
+	if (dev->msix_enabled) {
+		printk(KERN_INFO "PCI: %s: Can't enable MSI.  "
+			   "Device already has MSI-X vectors assigned\n",
+			   pci_name(dev));
+		dev->irq = temp;
+		return -EINVAL;
+	}
+
+	status = msi_capability_init(dev);
+	if ( !status )
+		dev->irq_old = temp;
+    else
+		dev->irq = temp;
+
+	return status;
+}
+
+extern void pci_frontend_disable_msi(struct pci_dev* dev);
+void pci_disable_msi(struct pci_dev* dev)
+{
+	int pos;
+	int pirq;
+
+	if (!pci_msi_enable)
+		return;
+	if (!dev)
+		return;
+
+#ifdef CONFIG_XEN_PCIDEV_FRONTEND
+	if (!is_initial_xendomain()) {
+		pci_frontend_disable_msi(dev);
+		dev->irq = dev->irq_old;
+		return;
+	}
+#endif
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSI);
+	if (!pos)
+		return;
+
+	pirq = dev->irq;
+	/* Restore dev->irq to its default pin-assertion vector */
+	dev->irq = dev->irq_old;
+	msi_unmap_pirq(dev, pirq);
+
+	/* Disable MSI mode */
+	disable_msi_mode(dev, pos, PCI_CAP_ID_MSI);
+}
+
+/**
+ * pci_enable_msix - configure device's MSI-X capability structure
+ * @dev: pointer to the pci_dev data structure of MSI-X device function
+ * @entries: pointer to an array of MSI-X entries
+ * @nvec: number of MSI-X vectors requested for allocation by device driver
+ *
+ * Setup the MSI-X capability structure of device function with the number
+ * of requested vectors upon its software driver call to request for
+ * MSI-X mode enabled on its hardware device function. A return of zero
+ * indicates the successful configuration of MSI-X capability structure
+ * with new allocated MSI-X vectors. A return of < 0 indicates a failure.
+ * Or a return of > 0 indicates that driver request is exceeding the number
+ * of vectors available. Driver should use the returned value to re-send
+ * its request.
+ **/
+extern int pci_frontend_enable_msix(struct pci_dev *dev,
+		struct msix_entry *entries, int nvec);
+int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec)
+{
+	struct pci_bus *bus;
+	int status, pos, nr_entries;
+	int i, j, temp;
+	u16 control;
+
+	if (!pci_msi_enable || !dev || !entries)
+ 		return -EINVAL;
+
+	if (dev->no_msi)
+		return -EINVAL;
+
+	for (bus = dev->bus; bus; bus = bus->parent)
+		if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI)
+			return -EINVAL;
+
+#ifdef CONFIG_XEN_PCIDEV_FRONTEND
+	if (!is_initial_xendomain()) {
+		int ret;
+
+		ret = pci_frontend_enable_msix(dev, entries, nvec);
+		if (ret) {
+			printk("get %x from pci_frontend_enable_msix\n", ret);
+			return ret;
+		}
+
+        return 0;
+	}
+#endif
+
+	status = msi_init();
+	if (status < 0)
+		return status;
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
+	if (!pos)
+ 		return -EINVAL;
+
+	pci_read_config_word(dev, msi_control_reg(pos), &control);
+	nr_entries = multi_msix_capable(control);
+	if (nvec > nr_entries)
+		return -EINVAL;
+
+	/* Check for any invalid entries */
+	for (i = 0; i < nvec; i++) {
+		if (entries[i].entry >= nr_entries)
+			return -EINVAL;		/* invalid entry */
+		for (j = i + 1; j < nvec; j++) {
+			if (entries[i].entry == entries[j].entry)
+				return -EINVAL;	/* duplicate entry */
+		}
+	}
+
+	temp = dev->irq;
+	/* Check whether driver already requested for MSI vector */
+	if (dev->msi_enabled) {
+		printk(KERN_INFO "PCI: %s: Can't enable MSI-X.  "
+		       "Device already has an MSI vector assigned\n",
+		       pci_name(dev));
+		dev->irq = temp;
+		return -EINVAL;
+	}
+
+	status = msix_capability_init(dev, entries, nvec);
+
+	if ( !status )
+		dev->irq_old = temp;
+	else
+		dev->irq = temp;
+
+	return status;
+}
+
+extern void pci_frontend_disable_msix(struct pci_dev* dev);
+void pci_disable_msix(struct pci_dev* dev)
+{
+	int pos;
+	u16 control;
+
+
+	if (!pci_msi_enable)
+		return;
+	if (!dev)
+		return;
+
+#ifdef CONFIG_XEN_PCIDEV_FRONTEND
+	if (!is_initial_xendomain()) {
+		pci_frontend_disable_msix(dev);
+		dev->irq = dev->irq_old;
+		return;
+	}
+#endif
+
+	pos = pci_find_capability(dev, PCI_CAP_ID_MSIX);
+	if (!pos)
+		return;
+
+	pci_read_config_word(dev, msi_control_reg(pos), &control);
+	if (!(control & PCI_MSIX_FLAGS_ENABLE))
+		return;
+
+	msi_remove_pci_irq_vectors(dev);
+
+	/* Disable MSI mode */
+	disable_msi_mode(dev, pos, PCI_CAP_ID_MSIX);
+}
+
+/**
+ * msi_remove_pci_irq_vectors - reclaim MSI(X) vectors to unused state
+ * @dev: pointer to the pci_dev data structure of MSI(X) device function
+ *
+ * Being called during hotplug remove, from which the device function
+ * is hot-removed. All previous assigned MSI/MSI-X vectors, if
+ * allocated for this device function, are reclaimed to unused state,
+ * which may be used later on.
+ **/
+void msi_remove_pci_irq_vectors(struct pci_dev* dev)
+{
+	unsigned long flags;
+	struct msi_dev_list *msi_dev_entry;
+	struct msi_pirq_entry *pirq_entry, *tmp;
+
+	if (!pci_msi_enable || !dev)
+ 		return;
+
+	msi_dev_entry = get_msi_dev_pirq_list(dev);
+
+	spin_lock_irqsave(&msi_dev_entry->pirq_list_lock, flags);
+	if (!list_empty_careful(&msi_dev_entry->pirq_list_head))
+	{
+		printk(KERN_WARNING "msix pirqs for dev %02x:%02x:%01x are not freed \
+		       before acquire again.\n", dev->bus->number, PCI_SLOT(dev->devfn),
+			   PCI_FUNC(dev->devfn));
+		list_for_each_entry_safe(pirq_entry, tmp,
+		                         &msi_dev_entry->pirq_list_head, list) {
+			msi_unmap_pirq(dev, pirq_entry->pirq);
+			list_del(&pirq_entry->list);
+			kfree(pirq_entry);
+		}
+	}
+	spin_unlock_irqrestore(&msi_dev_entry->pirq_list_lock, flags);
+	dev->irq = dev->irq_old;
+}
+
+void pci_no_msi(void)
+{
+	pci_msi_enable = 0;
+}
+
+EXPORT_SYMBOL(pci_enable_msi);
+EXPORT_SYMBOL(pci_disable_msi);
+EXPORT_SYMBOL(pci_enable_msix);
+EXPORT_SYMBOL(pci_disable_msix);
+#ifdef CONFIG_XEN
+EXPORT_SYMBOL(register_msi_get_owner);
+EXPORT_SYMBOL(unregister_msi_get_owner);
+#endif
+
diff -r f3d354a4b403 drivers/pci/msi.h
--- a/drivers/pci/msi.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/drivers/pci/msi.h	Wed Apr 30 19:34:58 2008 +0800
@@ -84,6 +84,11 @@
 extern void (*interrupt[NR_IRQS])(void);
 extern int pci_vector_resources(int last, int nr_released);
 
+#ifdef CONFIG_XEN
+extern int unregister_msi_get_owner(int (*func)(struct pci_dev *dev));
+extern int register_msi_get_owner(int (*func)(struct pci_dev *dev));
+#endif
+
 /*
  * MSI-X Address Register
  */
diff -r f3d354a4b403 drivers/xen/pciback/Makefile
--- a/drivers/xen/pciback/Makefile	Fri Apr 25 00:01:08 2008 +0800
+++ b/drivers/xen/pciback/Makefile	Wed Apr 30 19:34:58 2008 +0800
@@ -6,6 +6,7 @@
 	     conf_space_capability_vpd.o \
 	     conf_space_capability_pm.o \
              conf_space_quirks.o
+pciback-$(CONFIG_PCI_MSI) += conf_space_capability_msi.o
 pciback-$(CONFIG_XEN_PCIDEV_BACKEND_VPCI) += vpci.o
 pciback-$(CONFIG_XEN_PCIDEV_BACKEND_SLOT) += slot.o
 pciback-$(CONFIG_XEN_PCIDEV_BACKEND_PASS) += passthrough.o
diff -r f3d354a4b403 drivers/xen/pciback/conf_space_capability_msi.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/drivers/xen/pciback/conf_space_capability_msi.c	Wed Apr 30 19:34:58 2008 +0800
@@ -0,0 +1,60 @@
+/*
+ * PCI Backend -- Configuration overlay for MSI capability
+ */
+#include <linux/pci.h>
+#include "conf_space.h"
+#include "conf_space_capability.h"
+#include <xen/interface/io/pciif.h>
+#include "pciback.h"
+
+int pciback_enable_msi(struct pciback_device *pdev,
+		struct pci_dev *dev, struct xen_pci_op *op)
+{
+	int otherend = pdev->xdev->otherend_id;
+	int irq;
+	int status;
+
+	status = pci_enable_msi(dev);
+
+	if (status) {
+		printk("error enable msi for guest %x status %x\n", otherend, status);
+		op->value = 0;
+		return XEN_PCI_ERR_op_failed;
+	}
+
+	op->value = dev->irq;
+	return 0;
+}
+
+int pciback_disable_msi(struct pciback_device *pdev,
+		struct pci_dev *dev, struct xen_pci_op *op)
+{
+	int old_irq = dev->irq;
+
+	pci_disable_msi(dev);
+
+	op->value = dev->irq;
+	return 0;
+}
+
+int pciback_enable_msix(struct pciback_device *pdev,
+		struct pci_dev *dev, struct xen_pci_op *op)
+{
+	int otherend = pdev->xdev->otherend_id, result, i;
+
+	result = pci_enable_msix(dev, op->msix_entries, op->value);
+
+	op->value = result;
+	return result;
+}
+
+int pciback_disable_msix(struct pciback_device *pdev,
+		struct pci_dev *dev, struct xen_pci_op *op)
+{
+
+	pci_disable_msix(dev);
+
+	op->value = dev->irq;
+	return 0;
+}
+
diff -r f3d354a4b403 drivers/xen/pciback/pci_stub.c
--- a/drivers/xen/pciback/pci_stub.c	Fri Apr 25 00:01:08 2008 +0800
+++ b/drivers/xen/pciback/pci_stub.c	Wed Apr 30 19:34:58 2008 +0800
@@ -805,6 +805,23 @@
 
 DRIVER_ATTR(permissive, S_IRUSR | S_IWUSR, permissive_show, permissive_add);
 
+#ifdef CONFIG_PCI_MSI
+
+int pciback_get_owner(struct pci_dev *dev)
+{
+	struct pcistub_device *psdev;
+
+	psdev = pcistub_device_find(pci_domain_nr(dev->bus), dev->bus->number,
+			PCI_SLOT(dev->devfn), PCI_FUNC(dev->devfn));
+	/* XXX will other domain has pciback support ??? */
+	if (!psdev || !psdev->pdev) {
+		printk(KERN_WARNING "no ownder\n");
+		return -1;
+	}
+	return psdev->pdev->xdev->otherend_id;
+}
+#endif
+
 static void pcistub_exit(void)
 {
 	driver_remove_file(&pciback_pci_driver.driver, &driver_attr_new_slot);
@@ -815,6 +832,9 @@
 	driver_remove_file(&pciback_pci_driver.driver, &driver_attr_permissive);
 
 	pci_unregister_driver(&pciback_pci_driver);
+#ifdef CONFIG_PCI_MSI
+	unregister_msi_get_owner(pciback_get_owner);
+#endif
 }
 
 static int __init pcistub_init(void)
@@ -872,6 +892,10 @@
 		err = driver_create_file(&pciback_pci_driver.driver,
 					 &driver_attr_permissive);
 
+#ifdef CONFIG_PCI_MSI
+	if (!err)
+		err = register_msi_get_owner(pciback_get_owner);
+#endif
 	if (err)
 		pcistub_exit();
 
diff -r f3d354a4b403 drivers/xen/pciback/pciback.h
--- a/drivers/xen/pciback/pciback.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/drivers/xen/pciback/pciback.h	Wed Apr 30 19:34:58 2008 +0800
@@ -93,5 +93,19 @@
 int pciback_xenbus_register(void);
 void pciback_xenbus_unregister(void);
 
+#ifdef CONFIG_PCI_MSI
+int pciback_enable_msi(struct pciback_device *pdev,
+                       struct pci_dev *dev, struct xen_pci_op *op);
+
+int pciback_disable_msi(struct pciback_device *pdev,
+                         struct pci_dev *dev, struct xen_pci_op *op);
+
+
+int pciback_enable_msix(struct pciback_device *pdev,
+                        struct pci_dev *dev, struct xen_pci_op *op);
+
+int pciback_disable_msix(struct pciback_device *pdev,
+                        struct pci_dev *dev, struct xen_pci_op *op);
+#endif
 extern int verbose_request;
 #endif
diff -r f3d354a4b403 drivers/xen/pciback/pciback_ops.c
--- a/drivers/xen/pciback/pciback_ops.c	Fri Apr 25 00:01:08 2008 +0800
+++ b/drivers/xen/pciback/pciback_ops.c	Wed Apr 30 19:34:58 2008 +0800
@@ -61,15 +61,37 @@
 
 	if (dev == NULL)
 		op->err = XEN_PCI_ERR_dev_not_found;
-	else if (op->cmd == XEN_PCI_OP_conf_read)
-		op->err = pciback_config_read(dev, op->offset, op->size,
-					      &op->value);
-	else if (op->cmd == XEN_PCI_OP_conf_write)
-		op->err = pciback_config_write(dev, op->offset, op->size,
-					       op->value);
 	else
-		op->err = XEN_PCI_ERR_not_implemented;
-
+	{
+		switch (op->cmd)
+		{
+			case XEN_PCI_OP_conf_read:
+				op->err = pciback_config_read(dev,
+					  op->offset, op->size, &op->value);
+				break;
+			case XEN_PCI_OP_conf_write:
+				op->err = pciback_config_write(dev,
+					  op->offset, op->size,	op->value);
+				break;
+#ifdef CONFIG_PCI_MSI
+			case XEN_PCI_OP_enable_msi:
+				op->err = pciback_enable_msi(pdev, dev, op);
+				break;
+			case XEN_PCI_OP_disable_msi:
+				op->err = pciback_disable_msi(pdev, dev, op);
+				break;
+			case XEN_PCI_OP_enable_msix:
+				op->err = pciback_enable_msix(pdev, dev, op);
+				break;
+			case XEN_PCI_OP_disable_msix:
+				op->err = pciback_disable_msix(pdev, dev, op);
+				break;
+#endif
+			default:
+				op->err = XEN_PCI_ERR_not_implemented;
+				break;
+		}
+	}
 	/* Tell the driver domain that we're done. */ 
 	wmb();
 	clear_bit(_XEN_PCIF_active, (unsigned long *)&pdev->sh_info->flags);
diff -r f3d354a4b403 drivers/xen/pcifront/pci_op.c
--- a/drivers/xen/pcifront/pci_op.c	Fri Apr 25 00:01:08 2008 +0800
+++ b/drivers/xen/pcifront/pci_op.c	Wed Apr 30 19:34:58 2008 +0800
@@ -277,6 +277,122 @@
 	.write = pcifront_bus_write,
 };
 
+#ifdef CONFIG_PCI_MSI
+int pci_frontend_enable_msix(struct pci_dev *dev,
+		struct msix_entry *entries,
+		int nvec)
+{
+	int err;
+	int i;
+	struct xen_pci_op op = {
+		.cmd    = XEN_PCI_OP_enable_msix,
+		.domain = pci_domain_nr(dev->bus),
+		.bus = dev->bus->number,
+		.devfn = dev->devfn,
+		.value = nvec,
+	};
+	struct pcifront_sd *sd = dev->bus->sysdata;
+	struct pcifront_device *pdev = pcifront_get_pdev(sd);
+
+	if (nvec > SH_INFO_MAX_VEC) {
+		printk("too much vector for pci frontend%x\n", nvec);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < nvec; i++) {
+		op.msix_entries[i].entry = entries[i].entry;
+		op.msix_entries[i].vector = entries[i].vector;
+	}
+
+	err = do_pci_op(pdev, &op);
+
+	if (!err) {
+		if (!op.value) {
+			/* we get the result */
+			for ( i = 0; i < nvec; i++)
+				entries[i].vector = op.msix_entries[i].vector;
+			return 0;
+		}
+		else {
+            printk("enable msix get value %x\n", op.value);
+			return op.value;
+		}
+	}
+	else {
+        printk("enable msix get err %x\n", err);
+		return err;
+	}
+}
+
+void pci_frontend_disable_msix(struct pci_dev* dev)
+{
+	int err;
+	struct xen_pci_op op = {
+		.cmd    = XEN_PCI_OP_disable_msix,
+		.domain = pci_domain_nr(dev->bus),
+		.bus = dev->bus->number,
+		.devfn = dev->devfn,
+	};
+	struct pcifront_sd *sd = dev->bus->sysdata;
+	struct pcifront_device *pdev = pcifront_get_pdev(sd);
+
+	err = do_pci_op(pdev, &op);
+
+	/* What should do for error ? */
+	if (err)
+		printk("pci_disable_msix get err %x\n", err);
+}
+
+int pci_frontend_enable_msi(struct pci_dev *dev)
+{
+	int err;
+	struct xen_pci_op op = {
+		.cmd    = XEN_PCI_OP_enable_msi,
+		.domain = pci_domain_nr(dev->bus),
+		.bus = dev->bus->number,
+		.devfn = dev->devfn,
+	};
+	struct pcifront_sd *sd = dev->bus->sysdata;
+	struct pcifront_device *pdev = pcifront_get_pdev(sd);
+
+	err = do_pci_op(pdev, &op);
+	if (likely(!err)) {
+		dev->irq = op.value;
+	}
+	else {
+		printk("pci frontend enable msi failed for dev %x:%x \n",
+				op.bus, op.devfn);
+		err = -EINVAL;
+	}
+	return err;
+}
+
+void pci_frontend_disable_msi(struct pci_dev* dev)
+{
+	int err;
+	struct xen_pci_op op = {
+		.cmd    = XEN_PCI_OP_disable_msi,
+		.domain = pci_domain_nr(dev->bus),
+		.bus = dev->bus->number,
+		.devfn = dev->devfn,
+	};
+	struct pcifront_sd *sd = dev->bus->sysdata;
+	struct pcifront_device *pdev = pcifront_get_pdev(sd);
+
+	err = do_pci_op(pdev, &op);
+	if (err == XEN_PCI_ERR_dev_not_found) {
+		/* XXX No response from backend, what shall we do? */
+		printk("get no response from backend for disable MSI\n");
+		return;
+	}
+	if (likely(!err))
+		dev->irq = op.value;
+	else
+		/* how can pciback notify us fail? */
+		printk("get fake response frombackend \n");
+}
+#endif /* CONFIG_PCI_MSI */
+
 /* Claim resources for the PCI frontend as-is, backend won't allow changes */
 static void pcifront_claim_resource(struct pci_dev *dev, void *data)
 {
diff -r f3d354a4b403 include/asm-i386/io_apic.h
--- a/include/asm-i386/io_apic.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/include/asm-i386/io_apic.h	Wed Apr 30 19:34:58 2008 +0800
@@ -12,7 +12,7 @@
 
 #ifdef CONFIG_X86_IO_APIC
 
-#ifdef CONFIG_PCI_MSI
+#if defined(CONFIG_PCI_MSI) && !defined(CONFIG_XEN)
 static inline int use_pci_vector(void)	{return 1;}
 static inline void disable_edge_ioapic_vector(unsigned int vector) { }
 static inline void mask_and_ack_level_ioapic_vector(unsigned int vector) { }
diff -r f3d354a4b403 include/asm-x86_64/io_apic.h
--- a/include/asm-x86_64/io_apic.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/include/asm-x86_64/io_apic.h	Wed Apr 30 19:34:58 2008 +0800
@@ -12,7 +12,7 @@
 
 #ifdef CONFIG_X86_IO_APIC
 
-#ifdef CONFIG_PCI_MSI
+#if defined(CONFIG_PCI_MSI) && !defined(CONFIG_XEN)
 static inline int use_pci_vector(void)	{return 1;}
 static inline void disable_edge_ioapic_vector(unsigned int vector) { }
 static inline void mask_and_ack_level_ioapic_vector(unsigned int vector) { }
diff -r f3d354a4b403 include/asm-x86_64/msi.h
--- a/include/asm-x86_64/msi.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/include/asm-x86_64/msi.h	Wed Apr 30 19:34:58 2008 +0800
@@ -7,14 +7,21 @@
 #define ASM_MSI_H
 
 #include <asm/desc.h>
+#ifndef CONFIG_XEN
 #include <asm/mach_apic.h>
+#endif
 #include <asm/smp.h>
 
+#ifndef CONFIG_XEN
 #define LAST_DEVICE_VECTOR	(FIRST_SYSTEM_VECTOR - 1)
+#else
+#define LAST_DYNAMIC_VECTOR 0xdf
+#define LAST_DEVICE_VECTOR	(LAST_DYNAMIC_VECTOR)
+#endif
+
 #define MSI_TARGET_CPU_SHIFT	12
 
 extern struct msi_ops msi_apic_ops;
-
 static inline int msi_arch_init(void)
 {
 	msi_register(&msi_apic_ops);
diff -r f3d354a4b403 include/linux/pci.h
--- a/include/linux/pci.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/include/linux/pci.h	Wed Apr 30 19:34:58 2008 +0800
@@ -152,6 +152,9 @@
 	 * directly, use the values stored here. They might be different!
 	 */
 	unsigned int	irq;
+#ifdef CONFIG_XEN
+	unsigned int    irq_old;
+#endif
 	struct resource resource[DEVICE_COUNT_RESOURCE]; /* I/O and memory regions + expansion ROMs */
 
 	/* These fields are used by common fixups */
diff -r f3d354a4b403 include/xen/evtchn.h
--- a/include/xen/evtchn.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/include/xen/evtchn.h	Wed Apr 30 19:34:58 2008 +0800
@@ -136,4 +136,18 @@
 void notify_remote_via_irq(int irq);
 int irq_to_evtchn_port(int irq);
 
+#define PIRQ_SET_MAPPING 0x0
+#define PIRQ_CLEAR_MAPPING 0x1
+#define PIRQ_GET_MAPPING 0x3
+int pirq_mapstatus(int pirq, int action);
+int set_pirq_hw_action(int pirq, int (*action)(int pirq, int action));
+int clear_pirq_hw_action(int pirq);
+
+#define PIRQ_STARTUP 1
+#define PIRQ_SHUTDOWN 2
+#define PIRQ_ENABLE 3
+#define PIRQ_DISABLE 4
+#define PIRQ_END 5
+#define PIRQ_ACK 6
+
 #endif /* __ASM_EVTCHN_H__ */
diff -r f3d354a4b403 include/xen/interface/io/pciif.h
--- a/include/xen/interface/io/pciif.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/include/xen/interface/io/pciif.h	Wed Apr 30 19:34:58 2008 +0800
@@ -34,6 +34,10 @@
 /* xen_pci_op commands */
 #define XEN_PCI_OP_conf_read    (0)
 #define XEN_PCI_OP_conf_write   (1)
+#define XEN_PCI_OP_enable_msi   (2)
+#define XEN_PCI_OP_enable_msix  (3)
+#define XEN_PCI_OP_disable_msi  (4)
+#define XEN_PCI_OP_disable_msix (5)
 
 /* xen_pci_op error numbers */
 #define XEN_PCI_ERR_success          (0)
@@ -43,6 +47,12 @@
 #define XEN_PCI_ERR_not_implemented (-4)
 /* XEN_PCI_ERR_op_failed - backend failed to complete the operation */
 #define XEN_PCI_ERR_op_failed       (-5)
+
+/*
+ * it should be PAGE_SIZE-sizeof(struct xen_pci_op))/sizeof(struct msix_entry))
+ * Should not exceed 128
+ */
+#define SH_INFO_MAX_VEC     128
 
 struct xen_pci_op {
     /* IN: what action to perform: XEN_PCI_OP_* */
@@ -62,6 +72,11 @@
 
     /* IN/OUT: Contains the result after a READ or the value to WRITE */
     uint32_t value;
+    /* IN: Contains extra infor for this operation */
+    uint32_t info;
+    /*IN:  param for msi-x */
+    struct msix_entry msix_entries[SH_INFO_MAX_VEC];
+
 };
 
 struct xen_pci_sharedinfo {
diff -r f3d354a4b403 include/xen/interface/physdev.h
--- a/include/xen/interface/physdev.h	Fri Apr 25 00:01:08 2008 +0800
+++ b/include/xen/interface/physdev.h	Wed Apr 30 19:34:58 2008 +0800
@@ -122,6 +122,38 @@
 typedef struct physdev_irq physdev_irq_t;
 DEFINE_XEN_GUEST_HANDLE(physdev_irq_t);
 
+#define MAP_PIRQ_TYPE_MSI               0x0
+#define MAP_PIRQ_TYPE_GSI               0x1
+#define MAP_PIRQ_TYPE_UNKNOWN           0x2
+
+#define PHYSDEVOP_map_pirq               13
+struct physdev_map_pirq {
+    domid_t domid;
+    /* IN */
+    int type;
+    /* IN */
+    int index;
+    /* IN or OUT */
+    int pirq;
+    /* msi info passed to VMM */
+    struct {
+        int bus, devfn, entry_nr;
+		int msi;  /* 0 - MSIX  1 - MSI */
+    } msi_info;
+};
+typedef struct physdev_map_pirq physdev_map_pirq_t;
+DEFINE_XEN_GUEST_HANDLE(physdev_map_pirq_t);
+
+#define PHYSDEVOP_unmap_pirq             14
+struct physdev_unmap_pirq {
+    domid_t domid;
+    /* IN */
+    int pirq;
+};
+
+typedef struct physdev_unmap_pirq physdev_unmap_pirq_t;
+DEFINE_XEN_GUEST_HANDLE(physdev_unmap_pirq_t);
+
 /*
  * Argument to physdev_op_compat() hypercall. Superceded by new physdev_op()
  * hypercall since 0x00030202.

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

RE: [PATCH 6/6] Add MSI support to XEN

[Jan Beulich]

>>> "Shan, Haitao" <haitao.shan@intel.com> 30.04.08 10:15 >>>
>Hi, Jan,
>I do not want to touch drivers/pci/msi.c. Sorry, this part of code is
>left in the patch duo to my mistake. The attached is the updated patch.
>The reason that I add a new file is that I did not find a clean way to
>use the original msi.c. There is code there managing vectors, which is
>definitly not needed in dom0. Also I do not want to insert too many
>"ifdef"s in that file.
>Do I understand your comments correctly?

Yes, I understand why you created a Xen clone of the original file,
and I indeed suspected the change to the original file just being a
leftover.

Thanks, Jan

Re: [PATCH 6/6] Add MSI support to XEN

[Neil Turton]

I tried the latest unstable tree on a NIC which supports both MSI and
MSI-X.  When the driver was configured to use MSI mode, everything
worked fine but when it was configured to use MSI-X, the call to
pci_enable_msix succeeded but the interrupt didn't work.  Have these
patches been seen to work on an MSI-X device?

In msix_capability_init (drivers/pci/msi-xen.c), there's a loop which
unmaps the old pirqs.  Shouldn't there be some similar code in
msi_capability_init?  I can't see any reason for the inconsistency
between the two functions.

Would it be possible to extend this code so that a PV backend driver
could map different MSI-X entries on a PCI device into different guests?
We already have frontend/backend drivers which allow multiple guests to
securely access the PCI memory of a single NIC and would like to extend
those drivers to map MSI-X interrupts directly to guests.  I
still don't understand enough about the new hypercalls to see how that
would work.

Cheers, Neil.

Re: [PATCH 6/6] Add MSI support to XEN

[Haitao Shan]

Hi, Neil,

Please see my comments embedded.

2008/5/13 Neil Turton <nturton@solarflare.com>:
> I tried the latest unstable tree on a NIC which supports both MSI and
>  MSI-X.  When the driver was configured to use MSI mode, everything
>  worked fine but when it was configured to use MSI-X, the call to
>  pci_enable_msix succeeded but the interrupt didn't work.  Have these
>  patches been seen to work on an MSI-X device?
I have tested the patch using a NIC with both MSI-X and MSX
capabilities. I can successfully enable MSI-X for the NIC both in dom0
and in domU. The drivers I used for test is igb from
e1000.sourceforge.net.
Can you tell me how to reproduce your problem? Maybe there are some
pathes that I do not take care of. So that I can fix the problem.

>
>  In msix_capability_init (drivers/pci/msi-xen.c), there's a loop which
>  unmaps the old pirqs.  Shouldn't there be some similar code in
>  msi_capability_init?  I can't see any reason for the inconsistency
>  between the two functions.
Agree. I will add that.

>
>  Would it be possible to extend this code so that a PV backend driver
>  could map different MSI-X entries on a PCI device into different guests?
>  We already have frontend/backend drivers which allow multiple guests to
>  securely access the PCI memory of a single NIC and would like to extend
>  those drivers to map MSI-X interrupts directly to guests.  I
>  still don't understand enough about the new hypercalls to see how that
>  would work.
Do you mean your MSI-X capable NIC is owned simultaneously by multi
guests? Actually, I have not thought of this usage model. Currently,
we rely on pcibackend to provide the owner of the device. Yes, all
MSI-X interrupts are mapped to the same domain to which the device
belongs. But I think by properly set the dom_id for hypercall
PHYSDEVOP_map_pirq, it can cope wth such situations.

>
>  Cheers, Neil.
>
>
>
>  _______________________________________________
>  Xen-devel mailing list
>  Xen-devel@lists.xensource.com
>  http://lists.xensource.com/xen-devel
>

Re: [PATCH 6/6] Add MSI support to XEN

[Neil Turton]

Haitao Shan wrote:
> Can you tell me how to reproduce your problem? Maybe there are some
> pathes that I do not take care of. So that I can fix the problem.

I've fixed it now, thanks.

> Do you mean your MSI-X capable NIC is owned simultaneously by multi
> guests?

Exactly.  Do I just call request_irq in the frontend?

Cheers, Neil.

Re: [PATCH 6/6] Add MSI support to XEN

[Haitao Shan]

2008/5/14 Neil Turton <nturton@solarflare.com>:
> Haitao Shan wrote:
>> Can you tell me how to reproduce your problem? Maybe there are some
>> pathes that I do not take care of. So that I can fix the problem.
>
> I've fixed it now, thanks.
OK. I see your patch. It is my fault to make such a mistake.

>
>> Do you mean your MSI-X capable NIC is owned simultaneously by multi
>> guests?
>
> Exactly.  Do I just call request_irq in the frontend?
I do not quite understand.
What I mean is call PHYSDEVOP_map_pirq with proper domain_id ( the
domain you want Xen to deliver this specific MSI to) in pci backend
and tell frontend the pirq number . Then request_irq in the frontend.
I think this should work if you want to deliver that MSI to some
specific domain.

>
> Cheers, Neil.
>

[PATCH] Not unmap all MSI-X pirqs when enabling MSI-X

[Shan, Haitao]

[-- Attachment #1: Type: text/plain, Size: 548 bytes --]

Hi, Keir,

This patch made some changes to msix_capability_init for kernel. 

Originally, all existing MSI-X pirqs of that device are unmapped before
mapping the required MSI-X entries. This is actually not right. This
function may be called several times, with each time requiring enabling
different parts of the device MSI-X entry. Former pirqs should not be
unmapped.

I made this patch to correct this. Thanks for Turton's comments on this
problem.

Signed-off-by: Shan Haitao <Haitao.shan@intel.com>

Best Regards
Shan Haitao

[-- Attachment #2: correctly_init_msix_pirqs.patch --]
[-- Type: application/octet-stream, Size: 2924 bytes --]

diff -r d59a4aaf8615 drivers/pci/msi-xen.c
--- a/drivers/pci/msi-xen.c	Wed May 14 13:53:06 2008 +0100
+++ b/drivers/pci/msi-xen.c	Thu May 15 18:36:20 2008 +0800
@@ -33,7 +33,6 @@
 }
 
 static LIST_HEAD(msi_dev_head);
-static int msi_dev_head_inited = 0;
 DEFINE_SPINLOCK(msi_dev_lock);
 
 struct msi_dev_list {
@@ -95,6 +94,23 @@
 	list_add_tail(&entry->list, &msi_dev_entry->pirq_list_head);
 	spin_unlock_irqrestore(&msi_dev_entry->pirq_list_lock, flags);
 	return 0;
+}
+
+static void detach_pirq_entry(int entry_nr,
+ 							struct msi_dev_list *msi_dev_entry)
+{
+	unsigned long flags;
+	struct msi_pirq_entry *pirq_entry;
+
+	list_for_each_entry(pirq_entry, &msi_dev_entry->pirq_list_head, list) {
+		if (pirq_entry->entry_nr == entry_nr) {
+			spin_lock_irqsave(&msi_dev_entry->pirq_list_lock, flags);
+			list_del(&pirq_entry->list);
+			spin_unlock_irqrestore(&msi_dev_entry->pirq_list_lock, flags);
+			kfree(pirq_entry);
+			return;
+		}
+	}
 }
 
 /*
@@ -379,40 +395,42 @@
 static int msix_capability_init(struct pci_dev *dev,
 				struct msix_entry *entries, int nvec)
 {
-	int pirq, i, pos;
+	int pirq, i, j, mapped, pos;
 	struct msi_dev_list *msi_dev_entry = get_msi_dev_pirq_list(dev);
-	struct msi_pirq_entry *pirq_entry, *tmp;
-	unsigned long flags;
+	struct msi_pirq_entry *pirq_entry;
 
 	if (!msi_dev_entry)
 		return -ENOMEM;
 
-	spin_lock_irqsave(&msi_dev_entry->pirq_list_lock, flags);
-	if (!list_empty(&msi_dev_entry->pirq_list_head))
-	{
-		printk(KERN_WARNING "msix pirqs for dev %02x:%02x:%01x are not freed \
-		       before acquire again.\n", dev->bus->number, PCI_SLOT(dev->devfn),
-			   PCI_FUNC(dev->devfn));
-		list_for_each_entry_safe(pirq_entry, tmp,
-		                         &msi_dev_entry->pirq_list_head, list) {
-			msi_unmap_pirq(dev, pirq_entry->pirq);
-			list_del(&pirq_entry->list);
-			kfree(pirq_entry);
-		}
-	}
-	spin_unlock_irqrestore(&msi_dev_entry->pirq_list_lock, flags);
-
 	/* MSI-X Table Initialization */
 	for (i = 0; i < nvec; i++) {
+		mapped = 0;
+		list_for_each_entry(pirq_entry, &msi_dev_entry->pirq_list_head, list) {
+			if (pirq_entry->entry_nr == entries[i].entry) {
+				printk(KERN_WARNING "msix entry %d for dev %02x:%02x:%01x are \
+				       not freed before acquire again.\n", entries[i].entry,
+					   dev->bus->number, PCI_SLOT(dev->devfn),
+					   PCI_FUNC(dev->devfn));
+				(entries + i)->vector = pirq_entry->pirq;
+				mapped = 1;
+				break;
+			}
+		}
+		if (mapped)
+			continue;
 		pirq = msi_map_vector(dev, entries[i].entry, 0);
 		if (pirq < 0)
 			break;
 		attach_pirq_entry(pirq, entries[i].entry, msi_dev_entry);
 		(entries + i)->vector = pirq;
 	}
+
 	if (i != nvec) {
-		msi_unmap_pirq(dev, dev->irq);
-		(entries + i)->vector = 0;
+		for (j = --i; j >= 0; j--) {
+			msi_unmap_pirq(dev, entries[j].vector);
+			detach_pirq_entry(entries[j].entry, msi_dev_entry);
+			entries[j].vector = 0;
+		}
 		return -EBUSY;
 	}
 

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [PATCH 6/6] Add MSI support to XEN

[Neil Turton]

Haitao Shan wrote:
> I do not quite understand.
> What I mean is call PHYSDEVOP_map_pirq with proper domain_id ( the
> domain you want Xen to deliver this specific MSI to) in pci backend
> and tell frontend the pirq number . Then request_irq in the frontend.

That answers my question.  Thanks for your help.

Cheers, Neil.