[pvops-dom0] [patch] Hook Linux's PCI probe and remove callbacks

[pvops-dom0] [patch] Hook Linux's PCI probe and remove callbacks

[Han, Weidong]

[-- Attachment #1: Type: text/plain, Size: 519 bytes --]

Hijack the pci_bus_type probe and remove callbacks. This option only
requires modification to the Xen specific part of Linux.

This is useful to add and remove pci device to Xen hypervisor when
load and remove its driver. For example, when VFs are created by PF,
they will be added to Xen hypervisor, and then can be assigned to guest.

This patch is based on xen-tip-master branch of jeremy's pv-ops tree.


Signed-off-by: Weidong Han <weidong.han@intel.com>
Signed-off-by: Allen Kay <allen.m.kay@intel.com>

[-- Attachment #2: 0001-Hook-Linux-s-PCI-probe-and-remove-callbacks.patch --]
[-- Type: application/octet-stream, Size: 4874 bytes --]

From ada197119ddc3ed692bdc085f97024c8e27730d1 Mon Sep 17 00:00:00 2001
From: Weidong Han <weidong.han@intel.com>
Date: Thu, 16 Jul 2009 20:37:02 +0800
Subject: [PATCH] Hook Linux's PCI probe and remove callbacks

Hijack the pci_bus_type probe and remove callbacks. This option only
requires modification to the Xen specific part of Linux.

This is useful to add and remove pci device to Xen hypervisor when
load and remove its driver. For example, when VFs are created by PF,
they will be added to Xen hypervisor, and then can be assigned to guest.

Signed-off-by: Weidong Han <weidong.han@intel.com>
Signed-off-by: Allen Kay <allen.m.kay@intel.com>
---
 drivers/xen/Makefile            |    3 +-
 drivers/xen/pci.c               |   84 +++++++++++++++++++++++++++++++++++++++
 include/xen/interface/physdev.h |   21 ++++++++++
 3 files changed, 107 insertions(+), 1 deletions(-)
 create mode 100644 drivers/xen/pci.c

diff --git a/drivers/xen/Makefile b/drivers/xen/Makefile
index 007aa99..a76dca5 100644
--- a/drivers/xen/Makefile
+++ b/drivers/xen/Makefile
@@ -1,6 +1,7 @@
 obj-y	+= grant-table.o features.o events.o manage.o biomerge.o
 obj-y	+= xenbus/
 
+obj-$(CONFIG_PCI)			+= pci.o
 obj-$(CONFIG_HOTPLUG_CPU)		+= cpu_hotplug.o
 obj-$(CONFIG_XEN_XENCOMM)		+= xencomm.o
 obj-$(CONFIG_XEN_BALLOON)		+= balloon.o
@@ -9,4 +10,4 @@ obj-$(CONFIG_XEN_GNTDEV)	+= gntdev.o
 obj-$(CONFIG_XEN_BLKDEV_BACKEND)	+= blkback/
 obj-$(CONFIG_XEN_NETDEV_BACKEND)	+= netback/
 obj-$(CONFIG_XENFS)			+= xenfs/
-obj-$(CONFIG_XEN_SYS_HYPERVISOR)	+= sys-hypervisor.o
\ No newline at end of file
+obj-$(CONFIG_XEN_SYS_HYPERVISOR)	+= sys-hypervisor.o
diff --git a/drivers/xen/pci.c b/drivers/xen/pci.c
new file mode 100644
index 0000000..8308b29
--- /dev/null
+++ b/drivers/xen/pci.c
@@ -0,0 +1,84 @@
+/*
+ * vim:shiftwidth=8:noexpandtab
+ */
+
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/pci.h>
+#include <xen/interface/physdev.h>
+#include <asm/xen/hypercall.h>
+#include "../pci/pci.h"
+
+static int (*pci_bus_probe)(struct device *dev);
+static int (*pci_bus_remove)(struct device *dev);
+
+static int pci_bus_probe_wrapper(struct device *dev)
+{
+	int r;
+	struct pci_dev *pci_dev = to_pci_dev(dev);
+	struct physdev_manage_pci manage_pci;
+	struct physdev_manage_pci_ext manage_pci_ext;
+
+#ifdef CONFIG_PCI_IOV
+	if (pci_dev->is_virtfn) {
+		memset(&manage_pci_ext, 0, sizeof(manage_pci_ext));
+		manage_pci_ext.bus = pci_dev->bus->number;
+		manage_pci_ext.devfn = pci_dev->devfn;
+		manage_pci_ext.is_virtfn = 1;
+		manage_pci_ext.physfn.bus = pci_dev->physfn->bus->number;
+		manage_pci_ext.physfn.devfn = pci_dev->physfn->devfn;
+		r = HYPERVISOR_physdev_op(PHYSDEVOP_manage_pci_add_ext,
+					  &manage_pci_ext);
+	} else
+#endif
+	if (pci_ari_enabled(pci_dev->bus) && PCI_SLOT(pci_dev->devfn)) {
+		memset(&manage_pci_ext, 0, sizeof(manage_pci_ext));
+		manage_pci_ext.bus = pci_dev->bus->number;
+		manage_pci_ext.devfn = pci_dev->devfn;
+		manage_pci_ext.is_extfn = 1;
+		r = HYPERVISOR_physdev_op(PHYSDEVOP_manage_pci_add_ext,
+					  &manage_pci_ext);
+	} else {
+		manage_pci.bus = pci_dev->bus->number;
+		manage_pci.devfn = pci_dev->devfn;
+		r = HYPERVISOR_physdev_op(PHYSDEVOP_manage_pci_add,
+					  &manage_pci);
+	}
+	if (r && r != -ENOSYS)
+		return r;
+
+	r = pci_bus_probe(dev);
+	return r;
+}
+
+static int pci_bus_remove_wrapper(struct device *dev)
+{
+	int r;
+	struct pci_dev *pci_dev = to_pci_dev(dev);
+	struct physdev_manage_pci manage_pci;
+	manage_pci.bus = pci_dev->bus->number;
+	manage_pci.devfn = pci_dev->devfn;
+
+	r = pci_bus_remove(dev);
+	/* dev and pci_dev are no longer valid!! */
+
+	WARN_ON(HYPERVISOR_physdev_op(PHYSDEVOP_manage_pci_remove,
+		&manage_pci));
+	return r;
+}
+
+static int __init hook_pci_bus(void)
+{
+	if (!xen_domain() || !xen_initial_domain())
+		return 0;
+
+	pci_bus_probe = pci_bus_type.probe;
+	pci_bus_type.probe = pci_bus_probe_wrapper;
+
+	pci_bus_remove = pci_bus_type.remove;
+	pci_bus_type.remove = pci_bus_remove_wrapper;
+
+	return 0;
+}
+
+core_initcall(hook_pci_bus);
diff --git a/include/xen/interface/physdev.h b/include/xen/interface/physdev.h
index cd69391..dd08c7d 100644
--- a/include/xen/interface/physdev.h
+++ b/include/xen/interface/physdev.h
@@ -106,6 +106,27 @@ struct physdev_irq {
 	uint32_t vector;
 };
 
+#define PHYSDEVOP_manage_pci_add         15
+#define PHYSDEVOP_manage_pci_remove      16
+struct physdev_manage_pci {
+    /* IN */
+    uint8_t bus;
+    uint8_t devfn;
+};
+
+#define PHYSDEVOP_manage_pci_add_ext     20
+struct physdev_manage_pci_ext {
+    /* IN */
+    uint8_t bus;
+    uint8_t devfn;
+    unsigned is_extfn;
+    unsigned is_virtfn;
+    struct {
+        uint8_t bus;
+        uint8_t devfn;
+    } physfn;
+};
+
 /*
  * Argument to physdev_op_compat() hypercall. Superceded by new physdev_op()
  * hypercall since 0x00030202.
-- 
1.6.0.4


[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

RE: [pvops-dom0] [patch] Hook Linux's PCI probe and remove callbacks

[Han, Weidong]

Forgot to mention this patch is backported from 2.6.18 dom0.

Regards,
Weidong

Han, Weidong wrote:
> Hijack the pci_bus_type probe and remove callbacks. This option only
> requires modification to the Xen specific part of Linux.
> 
> This is useful to add and remove pci device to Xen hypervisor when
> load and remove its driver. For example, when VFs are created by PF,
> they will be added to Xen hypervisor, and then can be assigned to
> guest. 
> 
> This patch is based on xen-tip-master branch of jeremy's pv-ops tree.
> 
> 
> Signed-off-by: Weidong Han <weidong.han@intel.com>
> Signed-off-by: Allen Kay <allen.m.kay@intel.com>

Re: [pvops-dom0] [patch] Hook Linux's PCI probe and remove callbacks

[Andy Burns]

2009/7/16 Han, Weidong <weidong.han@intel.com>:

> Hijack the pci_bus_type probe and remove callbacks. This option only
> requires modification to the Xen specific part of Linux.
>
> This is useful to add and remove pci device to Xen hypervisor when
> load and remove its driver. For example, when VFs are created by PF,
> they will be added to Xen hypervisor, and then can be assigned to guest.

Do I understand this patch is working towards adding pciback support
to pv_ops dom0?

Will it it allow device passthrough for CPUs without an IOMMU like
2.6.18 dom0 does?

Re: [pvops-dom0] [patch] Hook Linux's PCI probe and remove callbacks

[Jeremy Fitzhardinge]

On 07/15/09 22:15, Han, Weidong wrote:
> Hijack the pci_bus_type probe and remove callbacks. This option only
> requires modification to the Xen specific part of Linux.
>
> This is useful to add and remove pci device to Xen hypervisor when
> load and remove its driver. For example, when VFs are created by PF,
> they will be added to Xen hypervisor, and then can be assigned to guest.
>
> This patch is based on xen-tip-master branch of jeremy's pv-ops tree.
>   

Yunhong Jiang sent me a version of this patch as part of his MSI work. 
This kind of interception of pci_bus_type.probe is pretty ugly, and is
unlikely to be accepted upstream in this form (the patch may only touch
a Xen file, but it is changing a variable belonging to the PCI
subsystem).  We need to work with the Linux PCI maintainers to find some
other way of achieving what we need here.

Can you explain what the functional requirements are here.  Could we
defer registering the device with Xen until some other convenient hook,
or must it be done at this point in the code?

Thanks,
    J

RE: [pvops-dom0] [patch] Hook Linux's PCI probe and remove callbacks

[Kay, Allen M]

These hooks for two purposes:

1) Enabling of a SR-IOV virtual function.  The hypercall will tell xen to setup VT-d context entries for the BDF corresponding to the new virtual function.  This is required for SR-IOV to work.

2) PCI hot-plug support when devices are hot added/removed.  Vt-d context entries and data structures in Xen needs to be modified accordingly.

These hooks needs to be enabled before either of these functions are performed - I assume after dom0 boots.

Can you suggest a better way to do this?

-----Original Message-----
From: Jeremy Fitzhardinge [mailto:jeremy@goop.org] 
Sent: Friday, July 17, 2009 4:14 PM
To: Han, Weidong
Cc: 'xen-devel@lists.xensource.com'; 'Keir Fraser'; Kay, Allen M; Jiang, Yunhong
Subject: Re: [Xen-devel] [pvops-dom0] [patch] Hook Linux's PCI probe and remove callbacks

On 07/15/09 22:15, Han, Weidong wrote:
> Hijack the pci_bus_type probe and remove callbacks. This option only
> requires modification to the Xen specific part of Linux.
>
> This is useful to add and remove pci device to Xen hypervisor when
> load and remove its driver. For example, when VFs are created by PF,
> they will be added to Xen hypervisor, and then can be assigned to guest.
>
> This patch is based on xen-tip-master branch of jeremy's pv-ops tree.
>   

Yunhong Jiang sent me a version of this patch as part of his MSI work. 
This kind of interception of pci_bus_type.probe is pretty ugly, and is
unlikely to be accepted upstream in this form (the patch may only touch
a Xen file, but it is changing a variable belonging to the PCI
subsystem).  We need to work with the Linux PCI maintainers to find some
other way of achieving what we need here.

Can you explain what the functional requirements are here.  Could we
defer registering the device with Xen until some other convenient hook,
or must it be done at this point in the code?

Thanks,
    J

RE: [pvops-dom0] [patch] Hook Linux's PCI probe and remove callbacks

[Jiang, Yunhong]

[-- Attachment #1: Type: text/plain, Size: 3103 bytes --]

 One consideration to is to split them into two item:
a) For all device that is not hot-pluged, we don't need this hypercall, instead, Xen will do the scan and setup the mapping.
b) For those hot-plug device and SR-IOV devices, maybe we can use the BUS_NOTIFY_ADD_DEVICE notifier to achieve ths purpose. We can just add a notifier to it. I assume this is simlar to IOMMU in kernel, which should requires this notify also. So a Xen specific notifier should achieve our purpose without change to the PCI bus layer. 

I checked the kernel code, and noticed that arch/x86/kernel/amd_iommu.c register the notifier for BUS_NOTIFY_ADD_DEVICE, but a bit strange why Intel IOMMU has no such support, do you know if VT-d in kernel support device hotplug already?

Allen/Jeremy, any suggestion?

Thanks
Yunhong Jiang

>-----Original Message-----
>From: Kay, Allen M 
>Sent: 2009年7月18日 7:43
>To: Jeremy Fitzhardinge; Han, Weidong
>Cc: 'xen-devel@lists.xensource.com'; 'Keir Fraser'; Jiang, Yunhong
>Subject: RE: [Xen-devel] [pvops-dom0] [patch] Hook Linux's PCI 
>probe and remove callbacks
>
>These hooks for two purposes:
>
>1) Enabling of a SR-IOV virtual function.  The hypercall will 
>tell xen to setup VT-d context entries for the BDF 
>corresponding to the new virtual function.  This is required 
>for SR-IOV to work.
>
>2) PCI hot-plug support when devices are hot added/removed.  
>Vt-d context entries and data structures in Xen needs to be 
>modified accordingly.
>
>These hooks needs to be enabled before either of these 
>functions are performed - I assume after dom0 boots.
>
>Can you suggest a better way to do this?
>
>-----Original Message-----
>From: Jeremy Fitzhardinge [mailto:jeremy@goop.org] 
>Sent: Friday, July 17, 2009 4:14 PM
>To: Han, Weidong
>Cc: 'xen-devel@lists.xensource.com'; 'Keir Fraser'; Kay, Allen 
>M; Jiang, Yunhong
>Subject: Re: [Xen-devel] [pvops-dom0] [patch] Hook Linux's PCI 
>probe and remove callbacks
>
>On 07/15/09 22:15, Han, Weidong wrote:
>> Hijack the pci_bus_type probe and remove callbacks. This option only
>> requires modification to the Xen specific part of Linux.
>>
>> This is useful to add and remove pci device to Xen hypervisor when
>> load and remove its driver. For example, when VFs are created by PF,
>> they will be added to Xen hypervisor, and then can be 
>assigned to guest.
>>
>> This patch is based on xen-tip-master branch of jeremy's pv-ops tree.
>>   
>
>Yunhong Jiang sent me a version of this patch as part of his MSI work. 
>This kind of interception of pci_bus_type.probe is pretty ugly, and is
>unlikely to be accepted upstream in this form (the patch may only touch
>a Xen file, but it is changing a variable belonging to the PCI
>subsystem).  We need to work with the Linux PCI maintainers to 
>find some
>other way of achieving what we need here.
>
>Can you explain what the functional requirements are here.  Could we
>defer registering the device with Xen until some other convenient hook,
>or must it be done at this point in the code?
>
>Thanks,
>    J
>

[-- Attachment #2: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

RE: [pvops-dom0] [patch] Hook Linux's PCI probe and remove callbacks

[Han, Weidong]

[-- Attachment #1: Type: text/plain, Size: 3445 bytes --]

Jiang, Yunhong wrote:
>  One consideration to is to split them into two item:
> a) For all device that is not hot-pluged, we don't need this
> hypercall, instead, Xen will do the scan and setup the mapping. 
> b) For those hot-plug device and SR-IOV devices, maybe we can use the
> BUS_NOTIFY_ADD_DEVICE notifier to achieve ths purpose. We can just
> add a notifier to it. I assume this is simlar to IOMMU in kernel,
> which should requires this notify also. So a Xen specific notifier
> should achieve our purpose without change to the PCI bus layer.

This method looks good. It only needs to register a notifier call for Xen. It should be more possible to be accepted by upstream.

Regards,
Weidong
    
> 
> I checked the kernel code, and noticed that
> arch/x86/kernel/amd_iommu.c register the notifier for
> BUS_NOTIFY_ADD_DEVICE, but a bit strange why Intel IOMMU has no such
> support, do you know if VT-d in kernel support device hotplug
> already?   
> 
> Allen/Jeremy, any suggestion?
> 
> Thanks
> Yunhong Jiang
> 
>> -----Original Message-----
>> From: Kay, Allen M
>> Sent: 2009年7月18日 7:43
>> To: Jeremy Fitzhardinge; Han, Weidong
>> Cc: 'xen-devel@lists.xensource.com'; 'Keir Fraser'; Jiang, Yunhong
>> Subject: RE: [Xen-devel] [pvops-dom0] [patch] Hook Linux's PCI
>> probe and remove callbacks
>> 
>> These hooks for two purposes:
>> 
>> 1) Enabling of a SR-IOV virtual function.  The hypercall will
>> tell xen to setup VT-d context entries for the BDF
>> corresponding to the new virtual function.  This is required
>> for SR-IOV to work.
>> 
>> 2) PCI hot-plug support when devices are hot added/removed.
>> Vt-d context entries and data structures in Xen needs to be
>> modified accordingly.
>> 
>> These hooks needs to be enabled before either of these
>> functions are performed - I assume after dom0 boots.
>> 
>> Can you suggest a better way to do this?
>> 
>> -----Original Message-----
>> From: Jeremy Fitzhardinge [mailto:jeremy@goop.org]
>> Sent: Friday, July 17, 2009 4:14 PM
>> To: Han, Weidong
>> Cc: 'xen-devel@lists.xensource.com'; 'Keir Fraser'; Kay, Allen
>> M; Jiang, Yunhong
>> Subject: Re: [Xen-devel] [pvops-dom0] [patch] Hook Linux's PCI
>> probe and remove callbacks
>> 
>> On 07/15/09 22:15, Han, Weidong wrote:
>>> Hijack the pci_bus_type probe and remove callbacks. This option only
>>> requires modification to the Xen specific part of Linux.
>>> 
>>> This is useful to add and remove pci device to Xen hypervisor when
>>> load and remove its driver. For example, when VFs are created by PF,
>>> they will be added to Xen hypervisor, and then can be assigned to
>>> guest. 
>>> 
>>> This patch is based on xen-tip-master branch of jeremy's pv-ops
>>> tree. 
>>> 
>> 
>> Yunhong Jiang sent me a version of this patch as part of his MSI
>> work. This kind of interception of pci_bus_type.probe is pretty
>> ugly, and is unlikely to be accepted upstream in this form (the
>> patch may only touch a Xen file, but it is changing a variable
>> belonging to the PCI subsystem).  We need to work with the Linux PCI
>> maintainers to 
>> find some
>> other way of achieving what we need here.
>> 
>> Can you explain what the functional requirements are here.  Could we
>> defer registering the device with Xen until some other convenient
>> hook, or must it be done at this point in the code?
>> 
>> Thanks,
>>    J


[-- Attachment #2: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [pvops-dom0] [patch] Hook Linux's PCI probe and remove callbacks

[Jeremy Fitzhardinge]

On 07/17/09 16:42, Kay, Allen M wrote:
> These hooks for two purposes:
>
> 1) Enabling of a SR-IOV virtual function.  The hypercall will tell xen to setup VT-d context entries for the BDF corresponding to the new virtual function.  This is required for SR-IOV to work.
>
> 2) PCI hot-plug support when devices are hot added/removed.  Vt-d context entries and data structures in Xen needs to be modified accordingly.
>
> These hooks needs to be enabled before either of these functions are performed - I assume after dom0 boots.
>
> Can you suggest a better way to do this?
>   

The best person to ask is probably Jesse Barnes (cc:d).

In general the way to approach these things is to either find something
in the kernel which does something analogous and use the same
interfaces, possibly extending them a bit if necessary.

    J

Re: [pvops-dom0] [patch] Hook Linux's PCI probe and remove callbacks

[Jeremy Fitzhardinge]

On 07/19/09 04:55, Jiang, Yunhong wrote:
>  One consideration to is to split them into two item:
> a) For all device that is not hot-pluged, we don't need this hypercall, instead, Xen will do the scan and setup the mapping.
> b) For those hot-plug device and SR-IOV devices, maybe we can use the BUS_NOTIFY_ADD_DEVICE notifier to achieve ths purpose. We can just add a notifier to it. I assume this is simlar to IOMMU in kernel, which should requires this notify also. So a Xen specific notifier should achieve our purpose without change to the PCI bus layer. 
>
> I checked the kernel code, and noticed that arch/x86/kernel/amd_iommu.c register the notifier for BUS_NOTIFY_ADD_DEVICE, but a bit strange why Intel IOMMU has no such support, do you know if VT-d in kernel support device hotplug already?
>
> Allen/Jeremy, any suggestion?
>   

Yes, this is exactly the kind of answer I was looking for.

J
> Thanks
> Yunhong Jiang
>
>   
>> -----Original Message-----
>> From: Kay, Allen M 
>> Sent: 2009年7月18日 7:43
>> To: Jeremy Fitzhardinge; Han, Weidong
>> Cc: 'xen-devel@lists.xensource.com'; 'Keir Fraser'; Jiang, Yunhong
>> Subject: RE: [Xen-devel] [pvops-dom0] [patch] Hook Linux's PCI 
>> probe and remove callbacks
>>
>> These hooks for two purposes:
>>
>> 1) Enabling of a SR-IOV virtual function.  The hypercall will 
>> tell xen to setup VT-d context entries for the BDF 
>> corresponding to the new virtual function.  This is required 
>> for SR-IOV to work.
>>
>> 2) PCI hot-plug support when devices are hot added/removed.  
>> Vt-d context entries and data structures in Xen needs to be 
>> modified accordingly.
>>
>> These hooks needs to be enabled before either of these 
>> functions are performed - I assume after dom0 boots.
>>
>> Can you suggest a better way to do this?
>>
>> -----Original Message-----
>> From: Jeremy Fitzhardinge [mailto:jeremy@goop.org] 
>> Sent: Friday, July 17, 2009 4:14 PM
>> To: Han, Weidong
>> Cc: 'xen-devel@lists.xensource.com'; 'Keir Fraser'; Kay, Allen 
>> M; Jiang, Yunhong
>> Subject: Re: [Xen-devel] [pvops-dom0] [patch] Hook Linux's PCI 
>> probe and remove callbacks
>>
>> On 07/15/09 22:15, Han, Weidong wrote:
>>     
>>> Hijack the pci_bus_type probe and remove callbacks. This option only
>>> requires modification to the Xen specific part of Linux.
>>>
>>> This is useful to add and remove pci device to Xen hypervisor when
>>> load and remove its driver. For example, when VFs are created by PF,
>>> they will be added to Xen hypervisor, and then can be 
>>>       
>> assigned to guest.
>>     
>>> This patch is based on xen-tip-master branch of jeremy's pv-ops tree.
>>>   
>>>       
>> Yunhong Jiang sent me a version of this patch as part of his MSI work. 
>> This kind of interception of pci_bus_type.probe is pretty ugly, and is
>> unlikely to be accepted upstream in this form (the patch may only touch
>> a Xen file, but it is changing a variable belonging to the PCI
>> subsystem).  We need to work with the Linux PCI maintainers to 
>> find some
>> other way of achieving what we need here.
>>
>> Can you explain what the functional requirements are here.  Could we
>> defer registering the device with Xen until some other convenient hook,
>> or must it be done at this point in the code?
>>
>> Thanks,
>>    J
>>     
> >

Re: [pvops-dom0] [patch] Hook Linux's PCI probe and remove callbacks

[Jesse Barnes]

On Mon, 20 Jul 2009 10:01:37 -0700
Jeremy Fitzhardinge <jeremy@goop.org> wrote:

> On 07/19/09 04:55, Jiang, Yunhong wrote:
> >  One consideration to is to split them into two item:
> > a) For all device that is not hot-pluged, we don't need this
> > hypercall, instead, Xen will do the scan and setup the mapping. b)
> > For those hot-plug device and SR-IOV devices, maybe we can use the
> > BUS_NOTIFY_ADD_DEVICE notifier to achieve ths purpose. We can just
> > add a notifier to it. I assume this is simlar to IOMMU in kernel,
> > which should requires this notify also. So a Xen specific notifier
> > should achieve our purpose without change to the PCI bus layer. 
> >
> > I checked the kernel code, and noticed that
> > arch/x86/kernel/amd_iommu.c register the notifier for
> > BUS_NOTIFY_ADD_DEVICE, but a bit strange why Intel IOMMU has no
> > such support, do you know if VT-d in kernel support device hotplug
> > already?
> >
> > Allen/Jeremy, any suggestion?
> >   
> 
> Yes, this is exactly the kind of answer I was looking for.

Yeah if you can make that work for you it sounds like the best
approach.  If you have further questions or want some review on it,
please cc linux-pci@vger.kernel.org.  There's an active hotplug
development community who may be willing to help.

Thanks,
-- 
Jesse Barnes, Intel Open Source Technology Center