* [Qemu-devel] [RFC v9 01/18] vfio: extract vfio_get_hot_reset_info as a single function
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 02/18] vfio: squeeze out vfio_pci_do_hot_reset for support bus reset Chen Fan
` (16 subsequent siblings)
17 siblings, 0 replies; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
the function is used to get affected devices by bus reset.
so here extract it, and can used for aer soon.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/vfio/pci.c | 66 +++++++++++++++++++++++++++++++++++++++++++----------------
1 file changed, 48 insertions(+), 18 deletions(-)
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index e0e339a..4a97ccc 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -2641,6 +2641,51 @@ static void vfio_check_af_flr(VFIOPCIDevice *vdev, uint8_t pos)
}
}
+/*
+ * return negative with errno, return 0 on success.
+ * if success, the point of ret_info fill with the affected device reset info.
+ *
+ */
+static int vfio_get_hot_reset_info(VFIOPCIDevice *vdev,
+ struct vfio_pci_hot_reset_info **ret_info)
+{
+ struct vfio_pci_hot_reset_info *info;
+ int ret, count;
+
+ *ret_info = NULL;
+
+ info = g_malloc0(sizeof(*info));
+ info->argsz = sizeof(*info);
+
+ ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
+ if (ret && errno != ENOSPC) {
+ ret = -errno;
+ goto error;
+ }
+
+ count = info->count;
+
+ info = g_realloc(info, sizeof(*info) +
+ (count * sizeof(struct vfio_pci_dependent_device)));
+ info->argsz = sizeof(*info) +
+ (count * sizeof(struct vfio_pci_dependent_device));
+
+ ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
+ if (ret) {
+ ret = -errno;
+ error_report("vfio: hot reset info failed: %m");
+ goto error;
+ }
+
+ *ret_info = info;
+ info = NULL;
+
+ return 0;
+error:
+ g_free(info);
+ return ret;
+}
+
static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos)
{
PCIDevice *pdev = &vdev->pdev;
@@ -2780,7 +2825,7 @@ static bool vfio_pci_host_match(PCIHostDeviceAddress *host1,
static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
{
VFIOGroup *group;
- struct vfio_pci_hot_reset_info *info;
+ struct vfio_pci_hot_reset_info *info = NULL;
struct vfio_pci_dependent_device *devices;
struct vfio_pci_hot_reset *reset;
int32_t *fds;
@@ -2792,12 +2837,8 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
vfio_pci_pre_reset(vdev);
vdev->vbasedev.needs_reset = false;
- info = g_malloc0(sizeof(*info));
- info->argsz = sizeof(*info);
-
- ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
- if (ret && errno != ENOSPC) {
- ret = -errno;
+ ret = vfio_get_hot_reset_info(vdev, &info);
+ if (ret) {
if (!vdev->has_pm_reset) {
error_report("vfio: Cannot reset device %04x:%02x:%02x.%x, "
"no available reset mechanism.", vdev->host.domain,
@@ -2806,18 +2847,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
goto out_single;
}
- count = info->count;
- info = g_realloc(info, sizeof(*info) + (count * sizeof(*devices)));
- info->argsz = sizeof(*info) + (count * sizeof(*devices));
devices = &info->devices[0];
-
- ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
- if (ret) {
- ret = -errno;
- error_report("vfio: hot reset info failed: %m");
- goto out_single;
- }
-
trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name);
/* Verify that we have all the groups required */
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread* [Qemu-devel] [RFC v9 02/18] vfio: squeeze out vfio_pci_do_hot_reset for support bus reset
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 01/18] vfio: extract vfio_get_hot_reset_info as a single function Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 03/18] pcie: modify the capability size assert Chen Fan
` (15 subsequent siblings)
17 siblings, 0 replies; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
squeeze out vfio_pci_do_hot_reset to do host bus reset when AER recovery.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/vfio/pci.c | 75 +++++++++++++++++++++++++++++++++++------------------------
1 file changed, 44 insertions(+), 31 deletions(-)
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 4a97ccc..e056c49 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -2686,6 +2686,48 @@ error:
return ret;
}
+static int vfio_pci_do_hot_reset(VFIOPCIDevice *vdev,
+ struct vfio_pci_hot_reset_info *info)
+{
+ VFIOGroup *group;
+ struct vfio_pci_hot_reset *reset;
+ int32_t *fds;
+ int ret, i, count;
+ struct vfio_pci_dependent_device *devices;
+
+ /* Determine how many group fds need to be passed */
+ count = 0;
+ devices = &info->devices[0];
+ QLIST_FOREACH(group, &vfio_group_list, next) {
+ for (i = 0; i < info->count; i++) {
+ if (group->groupid == devices[i].group_id) {
+ count++;
+ break;
+ }
+ }
+ }
+
+ reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds)));
+ reset->argsz = sizeof(*reset) + (count * sizeof(*fds));
+ fds = &reset->group_fds[0];
+
+ /* Fill in group fds */
+ QLIST_FOREACH(group, &vfio_group_list, next) {
+ for (i = 0; i < info->count; i++) {
+ if (group->groupid == devices[i].group_id) {
+ fds[reset->count++] = group->fd;
+ break;
+ }
+ }
+ }
+
+ /* Bus reset! */
+ ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
+ g_free(reset);
+
+ return ret;
+}
+
static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos)
{
PCIDevice *pdev = &vdev->pdev;
@@ -2827,9 +2869,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
VFIOGroup *group;
struct vfio_pci_hot_reset_info *info = NULL;
struct vfio_pci_dependent_device *devices;
- struct vfio_pci_hot_reset *reset;
- int32_t *fds;
- int ret, i, count;
+ int ret, i;
bool multi = false;
trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
@@ -2908,34 +2948,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
goto out_single;
}
- /* Determine how many group fds need to be passed */
- count = 0;
- QLIST_FOREACH(group, &vfio_group_list, next) {
- for (i = 0; i < info->count; i++) {
- if (group->groupid == devices[i].group_id) {
- count++;
- break;
- }
- }
- }
-
- reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds)));
- reset->argsz = sizeof(*reset) + (count * sizeof(*fds));
- fds = &reset->group_fds[0];
-
- /* Fill in group fds */
- QLIST_FOREACH(group, &vfio_group_list, next) {
- for (i = 0; i < info->count; i++) {
- if (group->groupid == devices[i].group_id) {
- fds[reset->count++] = group->fd;
- break;
- }
- }
- }
-
- /* Bus reset! */
- ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
- g_free(reset);
+ ret = vfio_pci_do_hot_reset(vdev, info);
trace_vfio_pci_hot_reset_result(vdev->vbasedev.name,
ret ? "%m" : "Success");
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread* [Qemu-devel] [RFC v9 03/18] pcie: modify the capability size assert
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 01/18] vfio: extract vfio_get_hot_reset_info as a single function Chen Fan
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 02/18] vfio: squeeze out vfio_pci_do_hot_reset for support bus reset Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 04/18] vfio: make the 4 bytes aligned for capability size Chen Fan
` (14 subsequent siblings)
17 siblings, 0 replies; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
because the capabilities need to be DWORD aligned, so the size
should DWORD aligned too, and then the last capability size can
to be the greatest 0x1000. e.g. if I have a capability starting
4 bytes from the end, 0xFFC. The max size should be 4 bytes,
0x1000 - 0xFFC, not 3 bytes, 0xFFF - 0xFFC.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/pci/pcie.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/hw/pci/pcie.c b/hw/pci/pcie.c
index 1463e65..6cdd4a1 100644
--- a/hw/pci/pcie.c
+++ b/hw/pci/pcie.c
@@ -595,7 +595,7 @@ void pcie_add_capability(PCIDevice *dev,
assert(offset >= PCI_CONFIG_SPACE_SIZE);
assert(offset < offset + size);
- assert(offset + size < PCIE_CONFIG_SPACE_SIZE);
+ assert(offset + size <= PCIE_CONFIG_SPACE_SIZE);
assert(size >= 8);
assert(pci_is_express(dev));
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread* [Qemu-devel] [RFC v9 04/18] vfio: make the 4 bytes aligned for capability size
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
` (2 preceding siblings ...)
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 03/18] pcie: modify the capability size assert Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 05/18] vfio: add pcie extanded capability support Chen Fan
` (13 subsequent siblings)
17 siblings, 0 replies; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/vfio/pci.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index e056c49..52e8ad4 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -2481,7 +2481,8 @@ static void vfio_unmap_bars(VFIOPCIDevice *vdev)
*/
static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
{
- uint8_t tmp, next = 0xff;
+ uint8_t tmp;
+ uint16_t next = PCI_CONFIG_SPACE_SIZE;
for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp;
tmp = pdev->config[tmp + 1]) {
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread* [Qemu-devel] [RFC v9 05/18] vfio: add pcie extanded capability support
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
` (3 preceding siblings ...)
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 04/18] vfio: make the 4 bytes aligned for capability size Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
2015-06-09 21:22 ` Alex Williamson
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 06/18] aer: impove pcie_aer_init to support vfio device Chen Fan
` (12 subsequent siblings)
17 siblings, 1 reply; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
For vfio pcie device, we could expose the extended capability on
PCIE bus. in order to avoid config space broken, we introduce
a copy config for parsing extended caps. and rebuild the pcie
extended config space.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/vfio/pci.c | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 72 insertions(+), 1 deletion(-)
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 52e8ad4..54303d9 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -2494,6 +2494,21 @@ static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
return next - pos;
}
+
+static uint16_t vfio_ext_cap_max_size(const uint8_t *config, uint16_t pos)
+{
+ uint16_t tmp, next = PCIE_CONFIG_SPACE_SIZE;
+
+ for (tmp = PCI_CONFIG_SPACE_SIZE; tmp;
+ tmp = PCI_EXT_CAP_NEXT(pci_get_long(config + tmp))) {
+ if (tmp > pos && tmp < next) {
+ next = tmp;
+ }
+ }
+
+ return next - pos;
+}
+
static void vfio_set_word_bits(uint8_t *buf, uint16_t val, uint16_t mask)
{
pci_set_word(buf, (pci_get_word(buf) & ~mask) | val);
@@ -2804,16 +2819,72 @@ static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos)
return 0;
}
+static int vfio_add_ext_cap(VFIOPCIDevice *vdev, const uint8_t *config)
+{
+ PCIDevice *pdev = &vdev->pdev;
+ uint32_t header;
+ uint16_t cap_id, next, size;
+ uint8_t cap_ver;
+
+ for (next = PCI_CONFIG_SPACE_SIZE; next;
+ next = PCI_EXT_CAP_NEXT(pci_get_long(config + next))) {
+ header = pci_get_long(config + next);
+ cap_id = PCI_EXT_CAP_ID(header);
+ cap_ver = PCI_EXT_CAP_VER(header);
+
+ /*
+ * If it becomes important to configure extended capabilities to their
+ * actual size, use this as the default when it's something we don't
+ * recognize. Since QEMU doesn't actually handle many of the config
+ * accesses, exact size doesn't seem worthwhile.
+ */
+ size = vfio_ext_cap_max_size(config, next);
+
+ pcie_add_capability(pdev, cap_id, cap_ver, next, size);
+ if (next == PCI_CONFIG_SPACE_SIZE) {
+ /* Begin the rebuild, we should set the next offset zero. */
+ pci_set_long(pdev->config + next, PCI_EXT_CAP(cap_id, cap_ver, 0));
+ }
+
+ /* Use emulated header pointer to allow dropping extended caps */
+ pci_set_long(vdev->emulated_config_bits + next, 0xffffffff);
+ }
+
+ return 0;
+}
+
static int vfio_add_capabilities(VFIOPCIDevice *vdev)
{
PCIDevice *pdev = &vdev->pdev;
+ int ret;
+ uint8_t *config;
if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) ||
!pdev->config[PCI_CAPABILITY_LIST]) {
return 0; /* Nothing to add */
}
- return vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST]);
+ ret = vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST]);
+ if (ret) {
+ return ret;
+ }
+
+ /* on PCI bus, it doesn't make sense to expose extended capabilities. */
+ if (!pci_is_express(pdev) ||
+ !pci_bus_is_express(pdev->bus) ||
+ !pci_get_long(pdev->config + PCI_CONFIG_SPACE_SIZE)) {
+ return 0;
+ }
+
+ /*
+ * In order to avoid breaking config space, create a copy to
+ * use for parsing extended capabilities.
+ */
+ config = g_memdup(pdev->config, vdev->config_size);
+ ret = vfio_add_ext_cap(vdev, config);
+
+ g_free(config);
+ return ret;
}
static void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread* Re: [Qemu-devel] [RFC v9 05/18] vfio: add pcie extanded capability support
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 05/18] vfio: add pcie extanded capability support Chen Fan
@ 2015-06-09 21:22 ` Alex Williamson
0 siblings, 0 replies; 32+ messages in thread
From: Alex Williamson @ 2015-06-09 21:22 UTC (permalink / raw)
To: Chen Fan; +Cc: izumi.taku, qemu-devel
On Tue, 2015-06-09 at 11:37 +0800, Chen Fan wrote:
> For vfio pcie device, we could expose the extended capability on
> PCIE bus. in order to avoid config space broken, we introduce
> a copy config for parsing extended caps. and rebuild the pcie
> extended config space.
>
> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
> ---
> hw/vfio/pci.c | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
> 1 file changed, 72 insertions(+), 1 deletion(-)
>
> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> index 52e8ad4..54303d9 100644
> --- a/hw/vfio/pci.c
> +++ b/hw/vfio/pci.c
> @@ -2494,6 +2494,21 @@ static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
> return next - pos;
> }
>
> +
> +static uint16_t vfio_ext_cap_max_size(const uint8_t *config, uint16_t pos)
> +{
> + uint16_t tmp, next = PCIE_CONFIG_SPACE_SIZE;
> +
> + for (tmp = PCI_CONFIG_SPACE_SIZE; tmp;
> + tmp = PCI_EXT_CAP_NEXT(pci_get_long(config + tmp))) {
> + if (tmp > pos && tmp < next) {
> + next = tmp;
> + }
> + }
> +
> + return next - pos;
> +}
> +
> static void vfio_set_word_bits(uint8_t *buf, uint16_t val, uint16_t mask)
> {
> pci_set_word(buf, (pci_get_word(buf) & ~mask) | val);
> @@ -2804,16 +2819,72 @@ static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos)
> return 0;
> }
>
> +static int vfio_add_ext_cap(VFIOPCIDevice *vdev, const uint8_t *config)
> +{
> + PCIDevice *pdev = &vdev->pdev;
> + uint32_t header;
> + uint16_t cap_id, next, size;
> + uint8_t cap_ver;
> +
> + for (next = PCI_CONFIG_SPACE_SIZE; next;
> + next = PCI_EXT_CAP_NEXT(pci_get_long(config + next))) {
> + header = pci_get_long(config + next);
> + cap_id = PCI_EXT_CAP_ID(header);
> + cap_ver = PCI_EXT_CAP_VER(header);
> +
> + /*
> + * If it becomes important to configure extended capabilities to their
> + * actual size, use this as the default when it's something we don't
> + * recognize. Since QEMU doesn't actually handle many of the config
> + * accesses, exact size doesn't seem worthwhile.
> + */
> + size = vfio_ext_cap_max_size(config, next);
> +
> + pcie_add_capability(pdev, cap_id, cap_ver, next, size);
> + if (next == PCI_CONFIG_SPACE_SIZE) {
> + /* Begin the rebuild, we should set the next offset zero. */
> + pci_set_long(pdev->config + next, PCI_EXT_CAP(cap_id, cap_ver, 0));
> + }
> +
> + /* Use emulated header pointer to allow dropping extended caps */
> + pci_set_long(vdev->emulated_config_bits + next, 0xffffffff);
> + }
> +
> + return 0;
> +}
> +
> static int vfio_add_capabilities(VFIOPCIDevice *vdev)
> {
> PCIDevice *pdev = &vdev->pdev;
> + int ret;
> + uint8_t *config;
>
> if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) ||
> !pdev->config[PCI_CAPABILITY_LIST]) {
> return 0; /* Nothing to add */
> }
>
> - return vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST]);
> + ret = vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST]);
> + if (ret) {
> + return ret;
> + }
> +
> + /* on PCI bus, it doesn't make sense to expose extended capabilities. */
> + if (!pci_is_express(pdev) ||
> + !pci_bus_is_express(pdev->bus) ||
> + !pci_get_long(pdev->config + PCI_CONFIG_SPACE_SIZE)) {
> + return 0;
> + }
> +
> + /*
> + * In order to avoid breaking config space, create a copy to
> + * use for parsing extended capabilities.
> + */
> + config = g_memdup(pdev->config, vdev->config_size);
> + ret = vfio_add_ext_cap(vdev, config);
> +
> + g_free(config);
nit: it probably makes sense to move this memdup/free inside
vfio_add_ext_cap(). It's an implementation detail that doesn't need to
be exposed outside the function. This might be a left-over from a
version where the function was recursive.
> + return ret;
> }
>
> static void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
^ permalink raw reply [flat|nested] 32+ messages in thread
* [Qemu-devel] [RFC v9 06/18] aer: impove pcie_aer_init to support vfio device
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
` (4 preceding siblings ...)
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 05/18] vfio: add pcie extanded capability support Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 07/18] vfio: add aer support for " Chen Fan
` (11 subsequent siblings)
17 siblings, 0 replies; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
pcie_aer_init was used to emulate an aer capability for pcie device,
but for vfio device, the aer config space size is mutable and is not
always equal to PCI_ERR_SIZEOF(0x48). it depends on where the TLP Prefix
register required, so here we add a size argument.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/pci-bridge/ioh3420.c | 2 +-
hw/pci-bridge/xio3130_downstream.c | 2 +-
hw/pci-bridge/xio3130_upstream.c | 2 +-
hw/pci/pcie_aer.c | 4 ++--
include/hw/pci/pcie_aer.h | 2 +-
5 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/hw/pci-bridge/ioh3420.c b/hw/pci-bridge/ioh3420.c
index cce2fdd..4d9cd3f 100644
--- a/hw/pci-bridge/ioh3420.c
+++ b/hw/pci-bridge/ioh3420.c
@@ -129,7 +129,7 @@ static int ioh3420_initfn(PCIDevice *d)
goto err_pcie_cap;
}
pcie_cap_root_init(d);
- rc = pcie_aer_init(d, IOH_EP_AER_OFFSET);
+ rc = pcie_aer_init(d, IOH_EP_AER_OFFSET, PCI_ERR_SIZEOF);
if (rc < 0) {
goto err;
}
diff --git a/hw/pci-bridge/xio3130_downstream.c b/hw/pci-bridge/xio3130_downstream.c
index b3a6479..9737041 100644
--- a/hw/pci-bridge/xio3130_downstream.c
+++ b/hw/pci-bridge/xio3130_downstream.c
@@ -92,7 +92,7 @@ static int xio3130_downstream_initfn(PCIDevice *d)
goto err_pcie_cap;
}
pcie_cap_arifwd_init(d);
- rc = pcie_aer_init(d, XIO3130_AER_OFFSET);
+ rc = pcie_aer_init(d, XIO3130_AER_OFFSET, PCI_ERR_SIZEOF);
if (rc < 0) {
goto err;
}
diff --git a/hw/pci-bridge/xio3130_upstream.c b/hw/pci-bridge/xio3130_upstream.c
index eada582..4d7f894 100644
--- a/hw/pci-bridge/xio3130_upstream.c
+++ b/hw/pci-bridge/xio3130_upstream.c
@@ -81,7 +81,7 @@ static int xio3130_upstream_initfn(PCIDevice *d)
}
pcie_cap_flr_init(d);
pcie_cap_deverr_init(d);
- rc = pcie_aer_init(d, XIO3130_AER_OFFSET);
+ rc = pcie_aer_init(d, XIO3130_AER_OFFSET, PCI_ERR_SIZEOF);
if (rc < 0) {
goto err;
}
diff --git a/hw/pci/pcie_aer.c b/hw/pci/pcie_aer.c
index c8dea8e..6171a29 100644
--- a/hw/pci/pcie_aer.c
+++ b/hw/pci/pcie_aer.c
@@ -94,12 +94,12 @@ static void aer_log_clear_all_err(PCIEAERLog *aer_log)
aer_log->log_num = 0;
}
-int pcie_aer_init(PCIDevice *dev, uint16_t offset)
+int pcie_aer_init(PCIDevice *dev, uint16_t offset, uint16_t size)
{
PCIExpressDevice *exp;
pcie_add_capability(dev, PCI_EXT_CAP_ID_ERR, PCI_ERR_VER,
- offset, PCI_ERR_SIZEOF);
+ offset, size);
exp = &dev->exp;
exp->aer_cap = offset;
diff --git a/include/hw/pci/pcie_aer.h b/include/hw/pci/pcie_aer.h
index 2fb8388..156acb0 100644
--- a/include/hw/pci/pcie_aer.h
+++ b/include/hw/pci/pcie_aer.h
@@ -87,7 +87,7 @@ struct PCIEAERErr {
extern const VMStateDescription vmstate_pcie_aer_log;
-int pcie_aer_init(PCIDevice *dev, uint16_t offset);
+int pcie_aer_init(PCIDevice *dev, uint16_t offset, uint16_t size);
void pcie_aer_exit(PCIDevice *dev);
void pcie_aer_write_config(PCIDevice *dev,
uint32_t addr, uint32_t val, int len);
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread* [Qemu-devel] [RFC v9 07/18] vfio: add aer support for vfio device
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
` (5 preceding siblings ...)
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 06/18] aer: impove pcie_aer_init to support vfio device Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 08/18] vfio: improve vfio_get_group to support adding group without devices Chen Fan
` (10 subsequent siblings)
17 siblings, 0 replies; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
Calling pcie_aer_init to initilize aer related registers for
vfio device, then reload physical related registers to expose
device capability.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/vfio/pci.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 85 insertions(+), 1 deletion(-)
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 54303d9..7a3fad7 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -32,6 +32,7 @@
#include "hw/pci/msi.h"
#include "hw/pci/msix.h"
#include "hw/pci/pci.h"
+#include "hw/pci/pci_bridge.h"
#include "qemu-common.h"
#include "qemu/error-report.h"
#include "qemu/event_notifier.h"
@@ -160,6 +161,8 @@ typedef struct VFIOPCIDevice {
#define VFIO_FEATURE_ENABLE_VGA (1 << VFIO_FEATURE_ENABLE_VGA_BIT)
#define VFIO_FEATURE_ENABLE_REQ_BIT 1
#define VFIO_FEATURE_ENABLE_REQ (1 << VFIO_FEATURE_ENABLE_REQ_BIT)
+#define VFIO_FEATURE_ENABLE_AER_BIT 2
+#define VFIO_FEATURE_ENABLE_AER (1 << VFIO_FEATURE_ENABLE_AER_BIT)
int32_t bootindex;
uint8_t pm_cap;
bool has_vga;
@@ -2819,12 +2822,81 @@ static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos)
return 0;
}
+static int vfio_setup_aer(VFIOPCIDevice *vdev, uint8_t cap_ver,
+ int pos, uint16_t size)
+{
+ PCIDevice *pdev = &vdev->pdev;
+ uint8_t *exp_cap = pdev->config + pdev->exp.exp_cap;
+ PCIDevice *dev_iter;
+ uint8_t type;
+ uint32_t severity, errcap;
+ int ret;
+
+ if (!(vdev->features & VFIO_FEATURE_ENABLE_AER)) {
+ pcie_add_capability(pdev, PCI_EXT_CAP_ID_ERR,
+ cap_ver, pos, size);
+ return 0;
+ }
+
+ dev_iter = pci_bridge_get_device(pdev->bus);
+ if (!dev_iter) {
+ goto error;
+ }
+
+ while (dev_iter) {
+ type = pcie_cap_get_type(dev_iter);
+ if ((type != PCI_EXP_TYPE_ROOT_PORT &&
+ type != PCI_EXP_TYPE_UPSTREAM &&
+ type != PCI_EXP_TYPE_DOWNSTREAM)) {
+ goto error;
+ }
+
+ if (!dev_iter->exp.aer_cap) {
+ goto error;
+ }
+
+ dev_iter = pci_bridge_get_device(dev_iter->bus);
+ }
+
+ errcap = vfio_pci_read_config(pdev, pdev->exp.aer_cap + PCI_ERR_CAP, 4);
+ /*
+ * The ability to record multiple headers is depending on
+ * the state of the Multiple Header Recording Capable bit and
+ * enabled by the Multiple Header Recording Enable bit.
+ */
+ if ((errcap & PCI_ERR_CAP_MHRC) &&
+ (errcap & PCI_ERR_CAP_MHRE)) {
+ pdev->exp.aer_log.log_max = PCIE_AER_LOG_MAX_DEFAULT;
+ } else {
+ pdev->exp.aer_log.log_max = 0;
+ }
+
+ pcie_cap_deverr_init(pdev);
+ ret = pcie_aer_init(pdev, pos, size);
+ if (ret) {
+ return ret;
+ }
+
+ /* load physical registers */
+ severity = vfio_pci_read_config(pdev,
+ pdev->exp.aer_cap + PCI_ERR_UNCOR_SEVER, 4);
+ pci_long_test_and_clear_mask(exp_cap + PCI_ERR_UNCOR_SEVER, ~severity);
+
+ return 0;
+
+error:
+ error_report("vfio: Unable to enable AER for device %s, parent bus "
+ "does not support AER signaling", vdev->vbasedev.name);
+ return -1;
+}
+
static int vfio_add_ext_cap(VFIOPCIDevice *vdev, const uint8_t *config)
{
PCIDevice *pdev = &vdev->pdev;
uint32_t header;
uint16_t cap_id, next, size;
uint8_t cap_ver;
+ int ret = 0;
for (next = PCI_CONFIG_SPACE_SIZE; next;
next = PCI_EXT_CAP_NEXT(pci_get_long(config + next))) {
@@ -2840,7 +2912,19 @@ static int vfio_add_ext_cap(VFIOPCIDevice *vdev, const uint8_t *config)
*/
size = vfio_ext_cap_max_size(config, next);
- pcie_add_capability(pdev, cap_id, cap_ver, next, size);
+ switch (cap_id) {
+ case PCI_EXT_CAP_ID_ERR:
+ ret = vfio_setup_aer(vdev, cap_ver, next, size);
+ break;
+ default:
+ pcie_add_capability(pdev, cap_id, cap_ver, next, size);
+ break;
+ }
+
+ if (ret) {
+ return ret;
+ }
+
if (next == PCI_CONFIG_SPACE_SIZE) {
/* Begin the rebuild, we should set the next offset zero. */
pci_set_long(pdev->config + next, PCI_EXT_CAP(cap_id, cap_ver, 0));
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread* [Qemu-devel] [RFC v9 08/18] vfio: improve vfio_get_group to support adding group without devices
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
` (6 preceding siblings ...)
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 07/18] vfio: add aer support for " Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
2015-06-09 21:23 ` Alex Williamson
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 09/18] vfio: add ref for group to support own affected groups Chen Fan
` (9 subsequent siblings)
17 siblings, 1 reply; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
Pre-adding all affected groups for aer devices, it could
ensure the affected groups are owned in VM.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/vfio/common.c | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index b1045da..4230f83 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -793,8 +793,15 @@ VFIOGroup *vfio_get_group(int groupid, AddressSpace *as)
QLIST_FOREACH(group, &vfio_group_list, next) {
if (group->groupid == groupid) {
+ if (as && !group->container) {
+ if (vfio_connect_container(group, as)) {
+ error_report("vfio: failed to setup container for group %d",
+ groupid);
+ return NULL;
+ }
+ }
/* Found it. Now is it already in the right context? */
- if (group->container->space->as == as) {
+ if (!as || group->container->space->as == as) {
return group;
} else {
error_report("vfio: group %d used in multiple address spaces",
@@ -828,7 +835,7 @@ VFIOGroup *vfio_get_group(int groupid, AddressSpace *as)
group->groupid = groupid;
QLIST_INIT(&group->device_list);
- if (vfio_connect_container(group, as)) {
+ if (as && vfio_connect_container(group, as)) {
error_report("vfio: failed to setup container for group %d", groupid);
goto close_fd_exit;
}
@@ -859,7 +866,9 @@ void vfio_put_group(VFIOGroup *group)
}
vfio_kvm_device_del_group(group);
- vfio_disconnect_container(group);
+ if (group->container) {
+ vfio_disconnect_container(group);
+ }
QLIST_REMOVE(group, next);
trace_vfio_put_group(group->fd);
close(group->fd);
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread* Re: [Qemu-devel] [RFC v9 08/18] vfio: improve vfio_get_group to support adding group without devices
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 08/18] vfio: improve vfio_get_group to support adding group without devices Chen Fan
@ 2015-06-09 21:23 ` Alex Williamson
0 siblings, 0 replies; 32+ messages in thread
From: Alex Williamson @ 2015-06-09 21:23 UTC (permalink / raw)
To: Chen Fan; +Cc: izumi.taku, qemu-devel
On Tue, 2015-06-09 at 11:37 +0800, Chen Fan wrote:
> Pre-adding all affected groups for aer devices, it could
> ensure the affected groups are owned in VM.
>
> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
> ---
> hw/vfio/common.c | 15 ++++++++++++---
> 1 file changed, 12 insertions(+), 3 deletions(-)
>
> diff --git a/hw/vfio/common.c b/hw/vfio/common.c
> index b1045da..4230f83 100644
> --- a/hw/vfio/common.c
> +++ b/hw/vfio/common.c
> @@ -793,8 +793,15 @@ VFIOGroup *vfio_get_group(int groupid, AddressSpace *as)
>
> QLIST_FOREACH(group, &vfio_group_list, next) {
> if (group->groupid == groupid) {
> + if (as && !group->container) {
> + if (vfio_connect_container(group, as)) {
> + error_report("vfio: failed to setup container for group %d",
> + groupid);
> + return NULL;
> + }
> + }
> /* Found it. Now is it already in the right context? */
> - if (group->container->space->as == as) {
> + if (!as || group->container->space->as == as) {
> return group;
> } else {
> error_report("vfio: group %d used in multiple address spaces",
> @@ -828,7 +835,7 @@ VFIOGroup *vfio_get_group(int groupid, AddressSpace *as)
> group->groupid = groupid;
> QLIST_INIT(&group->device_list);
>
> - if (vfio_connect_container(group, as)) {
> + if (as && vfio_connect_container(group, as)) {
> error_report("vfio: failed to setup container for group %d", groupid);
> goto close_fd_exit;
> }
> @@ -859,7 +866,9 @@ void vfio_put_group(VFIOGroup *group)
> }
>
> vfio_kvm_device_del_group(group);
> - vfio_disconnect_container(group);
> + if (group->container) {
> + vfio_disconnect_container(group);
> + }
> QLIST_REMOVE(group, next);
> trace_vfio_put_group(group->fd);
> close(group->fd);
It's an interesting idea, but should we instead separate the container
from vfio_get/put_group() to be done as a separate step? Using the
AddressSpace pointer is a bit obfuscated.
Also, this gets you the group, but the vfio_group_get_external_user()
interface used by vfio-pci to do a bus reset requires that the group is
active, with an iommu. If it didn't, we could potentially be resetting
a non-viable group, with devices still owned by the host or not isolated
by the iommu. So you're a step closer, but if we go back to the example
of a dual port NIC with isolated functions, QEMU may be able to get the
group for the second port, it may even be a viable group, but if the
second port doesn't eventually get connected to the VM, the bus reset
won't be allowed.
^ permalink raw reply [flat|nested] 32+ messages in thread
* [Qemu-devel] [RFC v9 09/18] vfio: add ref for group to support own affected groups
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
` (7 preceding siblings ...)
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 08/18] vfio: improve vfio_get_group to support adding group without devices Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
2015-06-09 21:23 ` Alex Williamson
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 10/18] get all affected groups for each device support aer Chen Fan
` (8 subsequent siblings)
17 siblings, 1 reply; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/vfio/common.c | 12 ++++++++++++
include/hw/vfio/vfio-common.h | 1 +
2 files changed, 13 insertions(+)
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 4230f83..eff1930 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -832,6 +832,7 @@ VFIOGroup *vfio_get_group(int groupid, AddressSpace *as)
goto close_fd_exit;
}
+ group->ref = 0;
group->groupid = groupid;
QLIST_INIT(&group->device_list);
@@ -865,6 +866,15 @@ void vfio_put_group(VFIOGroup *group)
return;
}
+ if (group->ref) {
+ if (QLIST_EMPTY(&group->device_list)) {
+ if (group->container) {
+ vfio_disconnect_container(group);
+ }
+ }
+ return;
+ }
+
vfio_kvm_device_del_group(group);
if (group->container) {
vfio_disconnect_container(group);
@@ -901,6 +911,7 @@ int vfio_get_device(VFIOGroup *group, const char *name,
return ret;
}
+ group->ref++;
vbasedev->fd = fd;
vbasedev->group = group;
QLIST_INSERT_HEAD(&group->device_list, vbasedev, next);
@@ -921,6 +932,7 @@ void vfio_put_base_device(VFIODevice *vbasedev)
if (!vbasedev->group) {
return;
}
+ vbasedev->group->ref--;
QLIST_REMOVE(vbasedev, next);
vbasedev->group = NULL;
trace_vfio_put_base_device(vbasedev->fd);
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 0d1fb80..3eb042e 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -117,6 +117,7 @@ struct VFIODeviceOps {
typedef struct VFIOGroup {
int fd;
int groupid;
+ int ref;
VFIOContainer *container;
QLIST_HEAD(, VFIODevice) device_list;
QLIST_ENTRY(VFIOGroup) next;
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread* Re: [Qemu-devel] [RFC v9 09/18] vfio: add ref for group to support own affected groups
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 09/18] vfio: add ref for group to support own affected groups Chen Fan
@ 2015-06-09 21:23 ` Alex Williamson
0 siblings, 0 replies; 32+ messages in thread
From: Alex Williamson @ 2015-06-09 21:23 UTC (permalink / raw)
To: Chen Fan; +Cc: izumi.taku, qemu-devel
On Tue, 2015-06-09 at 11:37 +0800, Chen Fan wrote:
> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
> ---
> hw/vfio/common.c | 12 ++++++++++++
> include/hw/vfio/vfio-common.h | 1 +
> 2 files changed, 13 insertions(+)
>
> diff --git a/hw/vfio/common.c b/hw/vfio/common.c
> index 4230f83..eff1930 100644
> --- a/hw/vfio/common.c
> +++ b/hw/vfio/common.c
> @@ -832,6 +832,7 @@ VFIOGroup *vfio_get_group(int groupid, AddressSpace *as)
> goto close_fd_exit;
> }
>
> + group->ref = 0;
> group->groupid = groupid;
> QLIST_INIT(&group->device_list);
>
> @@ -865,6 +866,15 @@ void vfio_put_group(VFIOGroup *group)
> return;
> }
>
> + if (group->ref) {
> + if (QLIST_EMPTY(&group->device_list)) {
> + if (group->container) {
> + vfio_disconnect_container(group);
> + }
> + }
> + return;
> + }
Once we disconnect the container, our file descriptor for the group
isn't sufficient for a bus reset.
> +
> vfio_kvm_device_del_group(group);
> if (group->container) {
> vfio_disconnect_container(group);
> @@ -901,6 +911,7 @@ int vfio_get_device(VFIOGroup *group, const char *name,
> return ret;
> }
>
> + group->ref++;
> vbasedev->fd = fd;
> vbasedev->group = group;
> QLIST_INSERT_HEAD(&group->device_list, vbasedev, next);
> @@ -921,6 +932,7 @@ void vfio_put_base_device(VFIODevice *vbasedev)
> if (!vbasedev->group) {
> return;
> }
> + vbasedev->group->ref--;
> QLIST_REMOVE(vbasedev, next);
> vbasedev->group = NULL;
> trace_vfio_put_base_device(vbasedev->fd);
> diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
> index 0d1fb80..3eb042e 100644
> --- a/include/hw/vfio/vfio-common.h
> +++ b/include/hw/vfio/vfio-common.h
> @@ -117,6 +117,7 @@ struct VFIODeviceOps {
> typedef struct VFIOGroup {
> int fd;
> int groupid;
> + int ref;
> VFIOContainer *container;
> QLIST_HEAD(, VFIODevice) device_list;
> QLIST_ENTRY(VFIOGroup) next;
^ permalink raw reply [flat|nested] 32+ messages in thread
* [Qemu-devel] [RFC v9 10/18] get all affected groups for each device support aer
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
` (8 preceding siblings ...)
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 09/18] vfio: add ref for group to support own affected groups Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
2015-06-09 21:22 ` Alex Williamson
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 11/18] vfio: add check host bus reset is support or not Chen Fan
` (7 subsequent siblings)
17 siblings, 1 reply; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
Add the affected groups without any devices into VM,
it can keep the VM ownship the all groups. and use a
reference to make the group visible.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/vfio/pci.c | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++++++----
1 file changed, 108 insertions(+), 7 deletions(-)
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 7a3fad7..06006ce 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -2822,6 +2822,105 @@ static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos)
return 0;
}
+static bool vfio_pci_host_match(PCIHostDeviceAddress *host1,
+ PCIHostDeviceAddress *host2)
+{
+ return (host1->domain == host2->domain && host1->bus == host2->bus &&
+ host1->slot == host2->slot && host1->function == host2->function);
+}
+
+static int vfio_put_affected_groups(VFIOPCIDevice *vdev)
+{
+ struct vfio_pci_hot_reset_info *info = NULL;
+ struct vfio_pci_dependent_device *devices;
+ PCIHostDeviceAddress host;
+ VFIOGroup *group;
+ int ret, i;
+
+ ret = vfio_get_hot_reset_info(vdev, &info);
+ if (ret) {
+ goto out;
+ }
+
+ /* List all affected devices by bus reset */
+ devices = &info->devices[0];
+
+ /* Verify that we have all the groups required */
+ for (i = 0; i < info->count; i++) {
+ host.domain = devices[i].segment;
+ host.bus = devices[i].bus;
+ host.slot = PCI_SLOT(devices[i].devfn);
+ host.function = PCI_FUNC(devices[i].devfn);
+
+ /* Skip the current device */
+ if (vfio_pci_host_match(&host, &vdev->host)) {
+ continue;
+ }
+
+ /* Ensure we own the group of the affected device */
+ QLIST_FOREACH(group, &vfio_group_list, next) {
+ if (group->groupid == devices[i].group_id) {
+ break;
+ }
+ }
+
+ if (!group)
+ continue;
+
+ group->ref--;
+ vfio_put_group(group);
+ }
+
+ ret = 0;
+out:
+ g_free(info);
+ return ret;
+}
+
+static int vfio_get_affected_groups(VFIOPCIDevice *vdev)
+{
+ struct vfio_pci_hot_reset_info *info = NULL;
+ struct vfio_pci_dependent_device *devices;
+ PCIHostDeviceAddress host;
+ VFIOGroup *group;
+ int ret, i;
+
+ ret = vfio_get_hot_reset_info(vdev, &info);
+ if (ret) {
+ goto out;
+ }
+
+ /* List all affected devices by bus reset */
+ devices = &info->devices[0];
+
+ /* Verify that we have all the groups required */
+ for (i = 0; i < info->count; i++) {
+ host.domain = devices[i].segment;
+ host.bus = devices[i].bus;
+ host.slot = PCI_SLOT(devices[i].devfn);
+ host.function = PCI_FUNC(devices[i].devfn);
+
+ /* Skip the current device */
+ if (vfio_pci_host_match(&host, &vdev->host)) {
+ continue;
+ }
+
+ /* Get all affected groups into VM */
+ group = vfio_get_group(devices[i].group_id, NULL);
+ if (!group) {
+ error_report("vfio: failed to get affected group %d",
+ devices[i].group_id);
+ ret = -1;
+ goto out;
+ }
+ group->ref++;
+ }
+ ret = 0;
+out:
+ g_free(info);
+ return ret;
+}
+
static int vfio_setup_aer(VFIOPCIDevice *vdev, uint8_t cap_ver,
int pos, uint16_t size)
{
@@ -2858,6 +2957,12 @@ static int vfio_setup_aer(VFIOPCIDevice *vdev, uint8_t cap_ver,
dev_iter = pci_bridge_get_device(dev_iter->bus);
}
+ /* Ensure own all affected groups */
+ ret = vfio_get_affected_groups(vdev);
+ if (ret) {
+ goto error;
+ }
+
errcap = vfio_pci_read_config(pdev, pdev->exp.aer_cap + PCI_ERR_CAP, 4);
/*
* The ability to record multiple headers is depending on
@@ -3013,13 +3118,6 @@ static void vfio_pci_post_reset(VFIOPCIDevice *vdev)
vfio_enable_intx(vdev);
}
-static bool vfio_pci_host_match(PCIHostDeviceAddress *host1,
- PCIHostDeviceAddress *host2)
-{
- return (host1->domain == host2->domain && host1->bus == host2->bus &&
- host1->slot == host2->slot && host1->function == host2->function);
-}
-
static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
{
VFIOGroup *group;
@@ -3851,6 +3949,9 @@ static void vfio_instance_finalize(Object *obj)
g_free(vdev->rom);
vfio_put_device(vdev);
vfio_put_group(group);
+ if (vdev->features & VFIO_FEATURE_ENABLE_AER) {
+ vfio_put_affected_groups(vdev);
+ }
}
static void vfio_exitfn(PCIDevice *pdev)
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread* Re: [Qemu-devel] [RFC v9 10/18] get all affected groups for each device support aer
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 10/18] get all affected groups for each device support aer Chen Fan
@ 2015-06-09 21:22 ` Alex Williamson
0 siblings, 0 replies; 32+ messages in thread
From: Alex Williamson @ 2015-06-09 21:22 UTC (permalink / raw)
To: Chen Fan; +Cc: izumi.taku, qemu-devel
On Tue, 2015-06-09 at 11:37 +0800, Chen Fan wrote:
> Add the affected groups without any devices into VM,
> it can keep the VM ownship the all groups. and use a
> reference to make the group visible.
>
> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
> ---
> hw/vfio/pci.c | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++++++----
> 1 file changed, 108 insertions(+), 7 deletions(-)
>
> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> index 7a3fad7..06006ce 100644
> --- a/hw/vfio/pci.c
> +++ b/hw/vfio/pci.c
> @@ -2822,6 +2822,105 @@ static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos)
> return 0;
> }
>
> +static bool vfio_pci_host_match(PCIHostDeviceAddress *host1,
> + PCIHostDeviceAddress *host2)
> +{
> + return (host1->domain == host2->domain && host1->bus == host2->bus &&
> + host1->slot == host2->slot && host1->function == host2->function);
> +}
> +
> +static int vfio_put_affected_groups(VFIOPCIDevice *vdev)
> +{
> + struct vfio_pci_hot_reset_info *info = NULL;
> + struct vfio_pci_dependent_device *devices;
> + PCIHostDeviceAddress host;
> + VFIOGroup *group;
> + int ret, i;
> +
> + ret = vfio_get_hot_reset_info(vdev, &info);
> + if (ret) {
> + goto out;
> + }
> +
> + /* List all affected devices by bus reset */
> + devices = &info->devices[0];
> +
> + /* Verify that we have all the groups required */
> + for (i = 0; i < info->count; i++) {
> + host.domain = devices[i].segment;
> + host.bus = devices[i].bus;
> + host.slot = PCI_SLOT(devices[i].devfn);
> + host.function = PCI_FUNC(devices[i].devfn);
> +
> + /* Skip the current device */
> + if (vfio_pci_host_match(&host, &vdev->host)) {
> + continue;
> + }
> +
> + /* Ensure we own the group of the affected device */
> + QLIST_FOREACH(group, &vfio_group_list, next) {
> + if (group->groupid == devices[i].group_id) {
> + break;
> + }
> + }
> +
> + if (!group)
> + continue;
> +
> + group->ref--;
> + vfio_put_group(group);
> + }
> +
> + ret = 0;
> +out:
> + g_free(info);
> + return ret;
> +}
> +
> +static int vfio_get_affected_groups(VFIOPCIDevice *vdev)
> +{
> + struct vfio_pci_hot_reset_info *info = NULL;
> + struct vfio_pci_dependent_device *devices;
> + PCIHostDeviceAddress host;
> + VFIOGroup *group;
> + int ret, i;
> +
> + ret = vfio_get_hot_reset_info(vdev, &info);
> + if (ret) {
> + goto out;
> + }
> +
> + /* List all affected devices by bus reset */
> + devices = &info->devices[0];
> +
> + /* Verify that we have all the groups required */
> + for (i = 0; i < info->count; i++) {
> + host.domain = devices[i].segment;
> + host.bus = devices[i].bus;
> + host.slot = PCI_SLOT(devices[i].devfn);
> + host.function = PCI_FUNC(devices[i].devfn);
> +
> + /* Skip the current device */
> + if (vfio_pci_host_match(&host, &vdev->host)) {
> + continue;
> + }
> +
> + /* Get all affected groups into VM */
> + group = vfio_get_group(devices[i].group_id, NULL);
> + if (!group) {
> + error_report("vfio: failed to get affected group %d",
> + devices[i].group_id);
> + ret = -1;
> + goto out;
There needs to be an unwind here, if the error occurs during hotplug,
we've just left QEMU in a state where some groups have extra references.
> + }
> + group->ref++;
If we're going to have a group reference, get_group should increment it
and put_group should decrement it, we shouldn't modify it externally
from those functions.
> + }
> + ret = 0;
> +out:
> + g_free(info);
> + return ret;
> +}
> +
> static int vfio_setup_aer(VFIOPCIDevice *vdev, uint8_t cap_ver,
> int pos, uint16_t size)
> {
> @@ -2858,6 +2957,12 @@ static int vfio_setup_aer(VFIOPCIDevice *vdev, uint8_t cap_ver,
> dev_iter = pci_bridge_get_device(dev_iter->bus);
> }
>
> + /* Ensure own all affected groups */
> + ret = vfio_get_affected_groups(vdev);
> + if (ret) {
> + goto error;
> + }
> +
> errcap = vfio_pci_read_config(pdev, pdev->exp.aer_cap + PCI_ERR_CAP, 4);
> /*
> * The ability to record multiple headers is depending on
> @@ -3013,13 +3118,6 @@ static void vfio_pci_post_reset(VFIOPCIDevice *vdev)
> vfio_enable_intx(vdev);
> }
>
> -static bool vfio_pci_host_match(PCIHostDeviceAddress *host1,
> - PCIHostDeviceAddress *host2)
> -{
> - return (host1->domain == host2->domain && host1->bus == host2->bus &&
> - host1->slot == host2->slot && host1->function == host2->function);
> -}
> -
> static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
> {
> VFIOGroup *group;
> @@ -3851,6 +3949,9 @@ static void vfio_instance_finalize(Object *obj)
> g_free(vdev->rom);
> vfio_put_device(vdev);
> vfio_put_group(group);
> + if (vdev->features & VFIO_FEATURE_ENABLE_AER) {
> + vfio_put_affected_groups(vdev);
> + }
> }
>
> static void vfio_exitfn(PCIDevice *pdev)
^ permalink raw reply [flat|nested] 32+ messages in thread
* [Qemu-devel] [RFC v9 11/18] vfio: add check host bus reset is support or not
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
` (9 preceding siblings ...)
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 10/18] get all affected groups for each device support aer Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
2015-06-09 21:23 ` Alex Williamson
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 12/18] pci: add bus reset_notifiers callbacks for host bus reset Chen Fan
` (6 subsequent siblings)
17 siblings, 1 reply; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
we introduce a has_bus_reset capability to sign the vfio
devices if support host bus reset.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/vfio/pci.c | 169 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 169 insertions(+)
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 06006ce..97af143 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -32,6 +32,7 @@
#include "hw/pci/msi.h"
#include "hw/pci/msix.h"
#include "hw/pci/pci.h"
+#include "hw/pci/pci_bus.h"
#include "hw/pci/pci_bridge.h"
#include "qemu-common.h"
#include "qemu/error-report.h"
@@ -2921,6 +2922,169 @@ out:
return ret;
}
+struct VFIODeviceFind {
+ PCIDevice *pdev;
+ bool found;
+};
+
+static void find_devices(PCIBus *bus, void *opaque)
+{
+ struct VFIODeviceFind *find = opaque;
+ int i;
+
+ if (find->found) {
+ return;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(bus->devices); i++) {
+ if (!bus->devices[i]) {
+ continue;
+ }
+
+ if (bus->devices[i] == find->pdev) {
+ find->found = true;
+ break;
+ }
+ }
+}
+
+static void vfio_check_device_reset(PCIBus *bus, void *opaque)
+{
+ int i;
+ PCIDevice *dev;
+ VFIOPCIDevice *vdev;
+ struct VFIODeviceFind *find = opaque;
+
+ if (find->found) {
+ return;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(bus->devices); i++) {
+ if (!bus->devices[i]) {
+ continue;
+ }
+ dev = bus->devices[i];
+ if (!object_dynamic_cast(OBJECT(dev), "vfio-pci")) {
+ continue;
+ }
+ vdev = DO_UPCAST(VFIOPCIDevice, pdev, dev);
+ if (!(vdev->features & VFIO_FEATURE_ENABLE_AER) &&
+ !vdev->vbasedev.reset_works &&
+ !vdev->has_flr &&
+ !vdev->has_pm_reset) {
+ find->pdev = dev;
+ find->found = true;
+ break;
+ }
+ }
+}
+
+static int vfio_check_host_bus_reset(VFIOPCIDevice *vdev)
+{
+ PCIBus *bus = vdev->pdev.bus;
+ struct vfio_pci_hot_reset_info *info = NULL;
+ struct vfio_pci_dependent_device *devices;
+ VFIOGroup *group;
+ int ret, i;
+ struct VFIODeviceFind find;
+
+ ret = vfio_get_hot_reset_info(vdev, &info);
+ if (ret) {
+ goto out;
+ }
+
+ /* List all affected devices by bus reset */
+ devices = &info->devices[0];
+
+ /* Verify that we have all the groups required */
+ for (i = 0; i < info->count; i++) {
+ PCIHostDeviceAddress host;
+ VFIOPCIDevice *tmp;
+ VFIODevice *vbasedev_iter;
+
+ host.domain = devices[i].segment;
+ host.bus = devices[i].bus;
+ host.slot = PCI_SLOT(devices[i].devfn);
+ host.function = PCI_FUNC(devices[i].devfn);
+
+ /* Skip the current device */
+ if (vfio_pci_host_match(&host, &vdev->host)) {
+ continue;
+ }
+
+ /* Ensure we own the group of the affected device */
+ QLIST_FOREACH(group, &vfio_group_list, next) {
+ if (group->groupid == devices[i].group_id) {
+ break;
+ }
+ }
+
+ if (!group) {
+ ret = -1;
+ goto out;
+ }
+
+ /* Ensure affected devices for reset under the same bus */
+ QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
+ if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
+ continue;
+ }
+ tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
+ if (vfio_pci_host_match(&host, &tmp->host)) {
+ find.pdev = &tmp->pdev;
+ find.found = false;
+
+ pci_for_each_bus(bus, find_devices, &find);
+ if (!find.found) {
+ ret = -1;
+ goto out;
+ }
+ break;
+ }
+ }
+ }
+
+ /*
+ * Check the all vfio pci devices on or below the target bus
+ * have a reset mechanism at least.
+ */
+ find.pdev = NULL;
+ find.found = false;
+ pci_for_each_bus(bus, vfio_check_device_reset, &find);
+ if (find.found) {
+ ret = -1;
+ goto out;
+ }
+
+ ret = 0;
+out:
+ g_free(info);
+ return ret;
+}
+
+static int vfio_check_devices_host_bus_reset(void)
+{
+ VFIOGroup *group;
+ VFIODevice *vbasedev;
+ VFIOPCIDevice *vdev;
+
+ /* Check All vfio-pci devices if have bus reset capability */
+ QLIST_FOREACH(group, &vfio_group_list, next) {
+ QLIST_FOREACH(vbasedev, &group->device_list, next) {
+ vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
+ if ((vdev->features & VFIO_FEATURE_ENABLE_AER) &&
+ vfio_check_host_bus_reset(vdev)) {
+ error_report("vfio: Cannot enable AER for device %s, "
+ "which does not support host bus reset.",
+ vdev->vbasedev.name);
+ return -1;
+ }
+ }
+ }
+
+ return 0;
+}
+
static int vfio_setup_aer(VFIOPCIDevice *vdev, uint8_t cap_ver,
int pos, uint16_t size)
{
@@ -2963,6 +3127,11 @@ static int vfio_setup_aer(VFIOPCIDevice *vdev, uint8_t cap_ver,
goto error;
}
+ /* Make sure this devices does conflict the existing aer topology */
+ if (vfio_check_devices_host_bus_reset()) {
+ goto error;
+ }
+
errcap = vfio_pci_read_config(pdev, pdev->exp.aer_cap + PCI_ERR_CAP, 4);
/*
* The ability to record multiple headers is depending on
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread* Re: [Qemu-devel] [RFC v9 11/18] vfio: add check host bus reset is support or not
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 11/18] vfio: add check host bus reset is support or not Chen Fan
@ 2015-06-09 21:23 ` Alex Williamson
0 siblings, 0 replies; 32+ messages in thread
From: Alex Williamson @ 2015-06-09 21:23 UTC (permalink / raw)
To: Chen Fan; +Cc: izumi.taku, qemu-devel
On Tue, 2015-06-09 at 11:37 +0800, Chen Fan wrote:
> we introduce a has_bus_reset capability to sign the vfio
> devices if support host bus reset.
Old description, no sign of has_bus_reset here.
>
> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
> ---
> hw/vfio/pci.c | 169 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 169 insertions(+)
>
> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> index 06006ce..97af143 100644
> --- a/hw/vfio/pci.c
> +++ b/hw/vfio/pci.c
> @@ -32,6 +32,7 @@
> #include "hw/pci/msi.h"
> #include "hw/pci/msix.h"
> #include "hw/pci/pci.h"
> +#include "hw/pci/pci_bus.h"
> #include "hw/pci/pci_bridge.h"
> #include "qemu-common.h"
> #include "qemu/error-report.h"
> @@ -2921,6 +2922,169 @@ out:
> return ret;
> }
>
> +struct VFIODeviceFind {
> + PCIDevice *pdev;
> + bool found;
> +};
> +
> +static void find_devices(PCIBus *bus, void *opaque)
> +{
> + struct VFIODeviceFind *find = opaque;
> + int i;
> +
> + if (find->found) {
> + return;
> + }
> +
> + for (i = 0; i < ARRAY_SIZE(bus->devices); i++) {
> + if (!bus->devices[i]) {
> + continue;
> + }
> +
> + if (bus->devices[i] == find->pdev) {
> + find->found = true;
> + break;
> + }
> + }
> +}
> +
> +static void vfio_check_device_reset(PCIBus *bus, void *opaque)
> +{
> + int i;
> + PCIDevice *dev;
> + VFIOPCIDevice *vdev;
> + struct VFIODeviceFind *find = opaque;
> +
> + if (find->found) {
> + return;
> + }
> +
> + for (i = 0; i < ARRAY_SIZE(bus->devices); i++) {
> + if (!bus->devices[i]) {
> + continue;
> + }
> + dev = bus->devices[i];
> + if (!object_dynamic_cast(OBJECT(dev), "vfio-pci")) {
> + continue;
> + }
> + vdev = DO_UPCAST(VFIOPCIDevice, pdev, dev);
> + if (!(vdev->features & VFIO_FEATURE_ENABLE_AER) &&
> + !vdev->vbasedev.reset_works &&
> + !vdev->has_flr &&
> + !vdev->has_pm_reset) {
I don't think these last two are necessary, these are internal
differentiation of the reset mechanism used, not separate reset
mechanisms. We don't require FLR and PM reset.
> + find->pdev = dev;
> + find->found = true;
> + break;
> + }
> + }
> +}
> +
> +static int vfio_check_host_bus_reset(VFIOPCIDevice *vdev)
> +{
> + PCIBus *bus = vdev->pdev.bus;
> + struct vfio_pci_hot_reset_info *info = NULL;
> + struct vfio_pci_dependent_device *devices;
> + VFIOGroup *group;
> + int ret, i;
> + struct VFIODeviceFind find;
> +
> + ret = vfio_get_hot_reset_info(vdev, &info);
> + if (ret) {
> + goto out;
> + }
> +
> + /* List all affected devices by bus reset */
> + devices = &info->devices[0];
> +
> + /* Verify that we have all the groups required */
> + for (i = 0; i < info->count; i++) {
> + PCIHostDeviceAddress host;
> + VFIOPCIDevice *tmp;
> + VFIODevice *vbasedev_iter;
> +
> + host.domain = devices[i].segment;
> + host.bus = devices[i].bus;
> + host.slot = PCI_SLOT(devices[i].devfn);
> + host.function = PCI_FUNC(devices[i].devfn);
> +
> + /* Skip the current device */
> + if (vfio_pci_host_match(&host, &vdev->host)) {
> + continue;
> + }
> +
> + /* Ensure we own the group of the affected device */
> + QLIST_FOREACH(group, &vfio_group_list, next) {
> + if (group->groupid == devices[i].group_id) {
> + break;
> + }
> + }
> +
> + if (!group) {
> + ret = -1;
> + goto out;
> + }
> +
> + /* Ensure affected devices for reset under the same bus */
> + QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
> + if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
> + continue;
> + }
> + tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
> + if (vfio_pci_host_match(&host, &tmp->host)) {
> + find.pdev = &tmp->pdev;
> + find.found = false;
> +
> + pci_for_each_bus(bus, find_devices, &find);
> + if (!find.found) {
> + ret = -1;
> + goto out;
> + }
> + break;
> + }
> + }
> + }
> +
> + /*
> + * Check the all vfio pci devices on or below the target bus
> + * have a reset mechanism at least.
> + */
> + find.pdev = NULL;
> + find.found = false;
> + pci_for_each_bus(bus, vfio_check_device_reset, &find);
> + if (find.found) {
> + ret = -1;
> + goto out;
> + }
> +
> + ret = 0;
> +out:
> + g_free(info);
> + return ret;
> +}
> +
> +static int vfio_check_devices_host_bus_reset(void)
> +{
> + VFIOGroup *group;
> + VFIODevice *vbasedev;
> + VFIOPCIDevice *vdev;
> +
> + /* Check All vfio-pci devices if have bus reset capability */
> + QLIST_FOREACH(group, &vfio_group_list, next) {
> + QLIST_FOREACH(vbasedev, &group->device_list, next) {
> + vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
> + if ((vdev->features & VFIO_FEATURE_ENABLE_AER) &&
> + vfio_check_host_bus_reset(vdev)) {
> + error_report("vfio: Cannot enable AER for device %s, "
> + "which does not support host bus reset.",
> + vdev->vbasedev.name);
> + return -1;
> + }
> + }
> + }
> +
> + return 0;
> +}
> +
> static int vfio_setup_aer(VFIOPCIDevice *vdev, uint8_t cap_ver,
> int pos, uint16_t size)
> {
> @@ -2963,6 +3127,11 @@ static int vfio_setup_aer(VFIOPCIDevice *vdev, uint8_t cap_ver,
> goto error;
> }
>
> + /* Make sure this devices does conflict the existing aer topology */
> + if (vfio_check_devices_host_bus_reset()) {
> + goto error;
> + }
> +
> errcap = vfio_pci_read_config(pdev, pdev->exp.aer_cap + PCI_ERR_CAP, 4);
> /*
> * The ability to record multiple headers is depending on
^ permalink raw reply [flat|nested] 32+ messages in thread
* [Qemu-devel] [RFC v9 12/18] pci: add bus reset_notifiers callbacks for host bus reset
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
` (10 preceding siblings ...)
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 11/18] vfio: add check host bus reset is support or not Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 13/18] vfio: add sec_bus_reset notifier to notify physical bus reset is needed Chen Fan
` (5 subsequent siblings)
17 siblings, 0 replies; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
Particularly, For vfio devices, Once need to recovery devices
by bus reset such as AER, we always need to reset the host bus
to recovery the devices under the bus, so we need to add pci bus
callbacks to specify to do host bus reset.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/pci/pci.c | 16 ++++++++++++++++
hw/pci/pci_bridge.c | 6 ++++++
include/hw/pci/pci.h | 4 ++++
include/hw/pci/pci_bus.h | 2 ++
4 files changed, 28 insertions(+)
diff --git a/hw/pci/pci.c b/hw/pci/pci.c
index 3423c3a..3bd954e 100644
--- a/hw/pci/pci.c
+++ b/hw/pci/pci.c
@@ -74,11 +74,27 @@ static const VMStateDescription vmstate_pcibus = {
}
};
+void pci_bus_add_reset_notifier(PCIBus *bus, Notifier *notify)
+{
+ notifier_list_add(&bus->reset_notifiers, notify);
+}
+
+void pci_bus_remove_reset_notifier(Notifier *notify)
+{
+ notifier_remove(notify);
+}
+
+void pci_bus_run_reset_notifier(PCIBus *bus, void *opaque)
+{
+ notifier_list_notify(&bus->reset_notifiers, opaque);
+}
+
static void pci_bus_realize(BusState *qbus, Error **errp)
{
PCIBus *bus = PCI_BUS(qbus);
vmstate_register(NULL, -1, &vmstate_pcibus, bus);
+ notifier_list_init(&bus->reset_notifiers);
}
static void pci_bus_unrealize(BusState *qbus, Error **errp)
diff --git a/hw/pci/pci_bridge.c b/hw/pci/pci_bridge.c
index 40c97b1..a8e3f57 100644
--- a/hw/pci/pci_bridge.c
+++ b/hw/pci/pci_bridge.c
@@ -267,6 +267,12 @@ void pci_bridge_write_config(PCIDevice *d,
newctl = pci_get_word(d->config + PCI_BRIDGE_CONTROL);
if (~oldctl & newctl & PCI_BRIDGE_CTL_BUS_RESET) {
+ /*
+ * Notify all vfio-pci devices under the bus
+ * to do physical bus reset.
+ */
+ pci_for_each_bus(&s->sec_bus,
+ pci_bus_run_reset_notifier, d);
/* Trigger hot reset on 0->1 transition. */
qbus_reset_all(&s->sec_bus.qbus);
}
diff --git a/include/hw/pci/pci.h b/include/hw/pci/pci.h
index 6c2af0d..d353c9d 100644
--- a/include/hw/pci/pci.h
+++ b/include/hw/pci/pci.h
@@ -7,6 +7,7 @@
#include "exec/memory.h"
#include "sysemu/dma.h"
#include "qapi/error.h"
+#include "qemu/notify.h"
/* PCI includes legacy ISA access. */
#include "hw/isa/isa.h"
@@ -377,6 +378,9 @@ void pci_bus_fire_intx_routing_notifier(PCIBus *bus);
void pci_device_set_intx_routing_notifier(PCIDevice *dev,
PCIINTxRoutingNotifier notifier);
void pci_device_reset(PCIDevice *dev);
+void pci_bus_add_reset_notifier(PCIBus *bus, Notifier *notify);
+void pci_bus_remove_reset_notifier(Notifier *notify);
+void pci_bus_run_reset_notifier(PCIBus *bus, void *opaque);
PCIDevice *pci_nic_init_nofail(NICInfo *nd, PCIBus *rootbus,
const char *default_model,
diff --git a/include/hw/pci/pci_bus.h b/include/hw/pci/pci_bus.h
index fabaeee..3b551d7 100644
--- a/include/hw/pci/pci_bus.h
+++ b/include/hw/pci/pci_bus.h
@@ -29,6 +29,8 @@ struct PCIBus {
Keep a count of the number of devices with raised IRQs. */
int nirq;
int *irq_count;
+
+ NotifierList reset_notifiers;
};
typedef struct PCIBridgeWindows PCIBridgeWindows;
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread* [Qemu-devel] [RFC v9 13/18] vfio: add sec_bus_reset notifier to notify physical bus reset is needed
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
` (11 preceding siblings ...)
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 12/18] pci: add bus reset_notifiers callbacks for host bus reset Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
2015-06-09 21:23 ` Alex Williamson
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 14/18] vfio: improve vfio_pci_hot_reset to support more case Chen Fan
` (4 subsequent siblings)
17 siblings, 1 reply; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/vfio/pci.c | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 81 insertions(+)
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 97af143..a4e8658 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -156,6 +156,7 @@ typedef struct VFIOPCIDevice {
PCIHostDeviceAddress host;
EventNotifier err_notifier;
EventNotifier req_notifier;
+ Notifier sec_bus_reset_notifier;
int (*resetfn)(struct VFIOPCIDevice *);
uint32_t features;
#define VFIO_FEATURE_ENABLE_VGA_BIT 0
@@ -171,6 +172,7 @@ typedef struct VFIOPCIDevice {
bool req_enabled;
bool has_flr;
bool has_pm_reset;
+ bool needs_bus_reset;
bool rom_read_failed;
} VFIOPCIDevice;
@@ -3085,6 +3087,81 @@ static int vfio_check_devices_host_bus_reset(void)
return 0;
}
+/* Update all the reset state of the affected devices in VM */
+static void vfio_pci_host_needs_bus_reset(Notifier *n, void *opaque)
+{
+ VFIOPCIDevice *vdev = container_of(n, VFIOPCIDevice, sec_bus_reset_notifier);
+ VFIODevice *vbasedev = &vdev->vbasedev;
+ PCIDevice *parent = opaque;
+ VFIOGroup *group;
+ struct vfio_pci_hot_reset_info *info = NULL;
+ struct vfio_pci_dependent_device *devices;
+ int i, ret;
+
+ ret = vfio_get_hot_reset_info(vdev, &info);
+ if (ret) {
+ goto out;
+ }
+
+ vdev->needs_bus_reset = true;
+ /*
+ * qbus_reset_all always reset the devices from the depth level,
+ * here only need to do the reset for the device encounter the aer,
+ * in order to avoid reset the affected device first below the bus,
+ * we only set the top level devices as needs_reset.
+ */
+ if (parent == pci_bridge_get_device(vdev->pdev.bus)) {
+ vbasedev->needs_reset = true;
+ }
+
+ /* List all affected devices by bus reset */
+ devices = &info->devices[0];
+
+ /* Verify that we have all the groups required */
+ for (i = 0; i < info->count; i++) {
+ PCIHostDeviceAddress host;
+ VFIOPCIDevice *tmp;
+ VFIODevice *vbasedev_iter;
+
+ host.domain = devices[i].segment;
+ host.bus = devices[i].bus;
+ host.slot = PCI_SLOT(devices[i].devfn);
+ host.function = PCI_FUNC(devices[i].devfn);
+
+ /* Skip the current device */
+ if (vfio_pci_host_match(&host, &vdev->host)) {
+ continue;
+ }
+
+ /* Ensure we own the group of the affected device */
+ QLIST_FOREACH(group, &vfio_group_list, next) {
+ if (group->groupid == devices[i].group_id) {
+ break;
+ }
+ }
+
+ if (!group) {
+ goto out;
+ }
+
+ /* Ensure affected devices for reset under the same bus */
+ QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
+ if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
+ continue;
+ }
+ tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
+ if (vfio_pci_host_match(&host, &tmp->host)) {
+ tmp->needs_bus_reset = true;
+ tmp->vbasedev.needs_reset = false;
+ break;
+ }
+ }
+ }
+
+out:
+ g_free(info);
+}
+
static int vfio_setup_aer(VFIOPCIDevice *vdev, uint8_t cap_ver,
int pos, uint16_t size)
{
@@ -3156,6 +3233,9 @@ static int vfio_setup_aer(VFIOPCIDevice *vdev, uint8_t cap_ver,
pdev->exp.aer_cap + PCI_ERR_UNCOR_SEVER, 4);
pci_long_test_and_clear_mask(exp_cap + PCI_ERR_UNCOR_SEVER, ~severity);
+ vdev->sec_bus_reset_notifier.notify = vfio_pci_host_needs_bus_reset;
+ pci_bus_add_reset_notifier(pdev->bus, &vdev->sec_bus_reset_notifier);
+
return 0;
error:
@@ -4129,6 +4209,7 @@ static void vfio_exitfn(PCIDevice *pdev)
vfio_unregister_req_notifier(vdev);
vfio_unregister_err_notifier(vdev);
+ pci_bus_remove_reset_notifier(&vdev->sec_bus_reset_notifier);
pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
vfio_disable_interrupts(vdev);
if (vdev->intx.mmap_timer) {
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread* Re: [Qemu-devel] [RFC v9 13/18] vfio: add sec_bus_reset notifier to notify physical bus reset is needed
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 13/18] vfio: add sec_bus_reset notifier to notify physical bus reset is needed Chen Fan
@ 2015-06-09 21:23 ` Alex Williamson
0 siblings, 0 replies; 32+ messages in thread
From: Alex Williamson @ 2015-06-09 21:23 UTC (permalink / raw)
To: Chen Fan; +Cc: izumi.taku, qemu-devel
On Tue, 2015-06-09 at 11:37 +0800, Chen Fan wrote:
> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
> ---
> hw/vfio/pci.c | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 81 insertions(+)
>
> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> index 97af143..a4e8658 100644
> --- a/hw/vfio/pci.c
> +++ b/hw/vfio/pci.c
> @@ -156,6 +156,7 @@ typedef struct VFIOPCIDevice {
> PCIHostDeviceAddress host;
> EventNotifier err_notifier;
> EventNotifier req_notifier;
> + Notifier sec_bus_reset_notifier;
> int (*resetfn)(struct VFIOPCIDevice *);
> uint32_t features;
> #define VFIO_FEATURE_ENABLE_VGA_BIT 0
> @@ -171,6 +172,7 @@ typedef struct VFIOPCIDevice {
> bool req_enabled;
> bool has_flr;
> bool has_pm_reset;
> + bool needs_bus_reset;
> bool rom_read_failed;
> } VFIOPCIDevice;
>
> @@ -3085,6 +3087,81 @@ static int vfio_check_devices_host_bus_reset(void)
> return 0;
> }
>
> +/* Update all the reset state of the affected devices in VM */
> +static void vfio_pci_host_needs_bus_reset(Notifier *n, void *opaque)
> +{
> + VFIOPCIDevice *vdev = container_of(n, VFIOPCIDevice, sec_bus_reset_notifier);
> + VFIODevice *vbasedev = &vdev->vbasedev;
> + PCIDevice *parent = opaque;
> + VFIOGroup *group;
> + struct vfio_pci_hot_reset_info *info = NULL;
> + struct vfio_pci_dependent_device *devices;
> + int i, ret;
> +
> + ret = vfio_get_hot_reset_info(vdev, &info);
> + if (ret) {
> + goto out;
> + }
> +
> + vdev->needs_bus_reset = true;
> + /*
> + * qbus_reset_all always reset the devices from the depth level,
> + * here only need to do the reset for the device encounter the aer,
> + * in order to avoid reset the affected device first below the bus,
> + * we only set the top level devices as needs_reset.
> + */
> + if (parent == pci_bridge_get_device(vdev->pdev.bus)) {
> + vbasedev->needs_reset = true;
> + }
> +
> + /* List all affected devices by bus reset */
> + devices = &info->devices[0];
> +
> + /* Verify that we have all the groups required */
> + for (i = 0; i < info->count; i++) {
> + PCIHostDeviceAddress host;
> + VFIOPCIDevice *tmp;
> + VFIODevice *vbasedev_iter;
> +
> + host.domain = devices[i].segment;
> + host.bus = devices[i].bus;
> + host.slot = PCI_SLOT(devices[i].devfn);
> + host.function = PCI_FUNC(devices[i].devfn);
> +
> + /* Skip the current device */
> + if (vfio_pci_host_match(&host, &vdev->host)) {
> + continue;
> + }
> +
> + /* Ensure we own the group of the affected device */
> + QLIST_FOREACH(group, &vfio_group_list, next) {
> + if (group->groupid == devices[i].group_id) {
> + break;
> + }
This is insufficient now that groups can be opened without a container.
> + }
> +
> + if (!group) {
> + goto out;
> + }
> +
> + /* Ensure affected devices for reset under the same bus */
> + QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
> + if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
> + continue;
> + }
> + tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
> + if (vfio_pci_host_match(&host, &tmp->host)) {
> + tmp->needs_bus_reset = true;
> + tmp->vbasedev.needs_reset = false;
> + break;
> + }
> + }
> + }
> +
> +out:
> + g_free(info);
> +}
> +
> static int vfio_setup_aer(VFIOPCIDevice *vdev, uint8_t cap_ver,
> int pos, uint16_t size)
> {
> @@ -3156,6 +3233,9 @@ static int vfio_setup_aer(VFIOPCIDevice *vdev, uint8_t cap_ver,
> pdev->exp.aer_cap + PCI_ERR_UNCOR_SEVER, 4);
> pci_long_test_and_clear_mask(exp_cap + PCI_ERR_UNCOR_SEVER, ~severity);
>
> + vdev->sec_bus_reset_notifier.notify = vfio_pci_host_needs_bus_reset;
> + pci_bus_add_reset_notifier(pdev->bus, &vdev->sec_bus_reset_notifier);
> +
> return 0;
>
> error:
> @@ -4129,6 +4209,7 @@ static void vfio_exitfn(PCIDevice *pdev)
>
> vfio_unregister_req_notifier(vdev);
> vfio_unregister_err_notifier(vdev);
> + pci_bus_remove_reset_notifier(&vdev->sec_bus_reset_notifier);
> pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
> vfio_disable_interrupts(vdev);
> if (vdev->intx.mmap_timer) {
^ permalink raw reply [flat|nested] 32+ messages in thread
* [Qemu-devel] [RFC v9 14/18] vfio: improve vfio_pci_hot_reset to support more case
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
` (12 preceding siblings ...)
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 13/18] vfio: add sec_bus_reset notifier to notify physical bus reset is needed Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
2015-06-09 21:24 ` Alex Williamson
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 15/18] vfio: do hot bus reset when do virtual secondary bus reset Chen Fan
` (3 subsequent siblings)
17 siblings, 1 reply; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
the vfio_pci_hot_reset differentiate the single and multi in-used
devices for reset. but sometimes we own the group without any devices,
that also should support hot reset.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/vfio/pci.c | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index a4e8658..6507f39 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -3398,6 +3398,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
PCIHostDeviceAddress host;
VFIOPCIDevice *tmp;
VFIODevice *vbasedev_iter;
+ bool found;
host.domain = devices[i].segment;
host.bus = devices[i].bus;
@@ -3427,6 +3428,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
goto out;
}
+ found = false;
/* Prep dependent devices for reset and clear our marker. */
QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
@@ -3438,12 +3440,21 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
ret = -EINVAL;
goto out_single;
}
+ found = true;
vfio_pci_pre_reset(tmp);
tmp->vbasedev.needs_reset = false;
multi = true;
break;
}
}
+
+ /*
+ * If we own the group but does not own the device, we also
+ * should call hot reset with multi.
+ */
+ if (!single && !found) {
+ multi = true;
+ }
}
if (!single && !multi) {
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread* Re: [Qemu-devel] [RFC v9 14/18] vfio: improve vfio_pci_hot_reset to support more case
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 14/18] vfio: improve vfio_pci_hot_reset to support more case Chen Fan
@ 2015-06-09 21:24 ` Alex Williamson
2015-06-16 8:10 ` Chen Fan
0 siblings, 1 reply; 32+ messages in thread
From: Alex Williamson @ 2015-06-09 21:24 UTC (permalink / raw)
To: Chen Fan; +Cc: izumi.taku, qemu-devel
On Tue, 2015-06-09 at 11:37 +0800, Chen Fan wrote:
> the vfio_pci_hot_reset differentiate the single and multi in-used
> devices for reset. but sometimes we own the group without any devices,
> that also should support hot reset.
Nope, did you try it? It can be done, but the group still needs to be
connected to a container for isolation.
> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
> ---
> hw/vfio/pci.c | 11 +++++++++++
> 1 file changed, 11 insertions(+)
>
> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> index a4e8658..6507f39 100644
> --- a/hw/vfio/pci.c
> +++ b/hw/vfio/pci.c
> @@ -3398,6 +3398,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
> PCIHostDeviceAddress host;
> VFIOPCIDevice *tmp;
> VFIODevice *vbasedev_iter;
> + bool found;
>
> host.domain = devices[i].segment;
> host.bus = devices[i].bus;
> @@ -3427,6 +3428,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
> goto out;
> }
>
> + found = false;
> /* Prep dependent devices for reset and clear our marker. */
> QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
> if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
> @@ -3438,12 +3440,21 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
> ret = -EINVAL;
> goto out_single;
> }
> + found = true;
> vfio_pci_pre_reset(tmp);
> tmp->vbasedev.needs_reset = false;
> multi = true;
> break;
> }
> }
> +
> + /*
> + * If we own the group but does not own the device, we also
> + * should call hot reset with multi.
> + */
> + if (!single && !found) {
> + multi = true;
> + }
> }
>
> if (!single && !multi) {
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [Qemu-devel] [RFC v9 14/18] vfio: improve vfio_pci_hot_reset to support more case
2015-06-09 21:24 ` Alex Williamson
@ 2015-06-16 8:10 ` Chen Fan
2015-06-16 14:08 ` Alex Williamson
0 siblings, 1 reply; 32+ messages in thread
From: Chen Fan @ 2015-06-16 8:10 UTC (permalink / raw)
To: Alex Williamson; +Cc: izumi.taku, qemu-devel
On 06/10/2015 05:24 AM, Alex Williamson wrote:
> On Tue, 2015-06-09 at 11:37 +0800, Chen Fan wrote:
>> the vfio_pci_hot_reset differentiate the single and multi in-used
>> devices for reset. but sometimes we own the group without any devices,
>> that also should support hot reset.
> Nope, did you try it? It can be done, but the group still needs to be
> connected to a container for isolation.
I'm sorry for that. because I have no such host in hand. but I think if
we can keep connect container for each affected group, we also able
to use this method to do host bus reset.
Thanks,
Chen
>
>> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
>> ---
>> hw/vfio/pci.c | 11 +++++++++++
>> 1 file changed, 11 insertions(+)
>>
>> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
>> index a4e8658..6507f39 100644
>> --- a/hw/vfio/pci.c
>> +++ b/hw/vfio/pci.c
>> @@ -3398,6 +3398,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
>> PCIHostDeviceAddress host;
>> VFIOPCIDevice *tmp;
>> VFIODevice *vbasedev_iter;
>> + bool found;
>>
>> host.domain = devices[i].segment;
>> host.bus = devices[i].bus;
>> @@ -3427,6 +3428,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
>> goto out;
>> }
>>
>> + found = false;
>> /* Prep dependent devices for reset and clear our marker. */
>> QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
>> if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
>> @@ -3438,12 +3440,21 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
>> ret = -EINVAL;
>> goto out_single;
>> }
>> + found = true;
>> vfio_pci_pre_reset(tmp);
>> tmp->vbasedev.needs_reset = false;
>> multi = true;
>> break;
>> }
>> }
>> +
>> + /*
>> + * If we own the group but does not own the device, we also
>> + * should call hot reset with multi.
>> + */
>> + if (!single && !found) {
>> + multi = true;
>> + }
>> }
>>
>> if (!single && !multi) {
>
>
>
>
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [Qemu-devel] [RFC v9 14/18] vfio: improve vfio_pci_hot_reset to support more case
2015-06-16 8:10 ` Chen Fan
@ 2015-06-16 14:08 ` Alex Williamson
2015-06-17 6:28 ` Chen Fan
0 siblings, 1 reply; 32+ messages in thread
From: Alex Williamson @ 2015-06-16 14:08 UTC (permalink / raw)
To: Chen Fan; +Cc: izumi.taku, qemu-devel
On Tue, 2015-06-16 at 16:10 +0800, Chen Fan wrote:
> On 06/10/2015 05:24 AM, Alex Williamson wrote:
> > On Tue, 2015-06-09 at 11:37 +0800, Chen Fan wrote:
> >> the vfio_pci_hot_reset differentiate the single and multi in-used
> >> devices for reset. but sometimes we own the group without any devices,
> >> that also should support hot reset.
> > Nope, did you try it? It can be done, but the group still needs to be
> > connected to a container for isolation.
> I'm sorry for that. because I have no such host in hand. but I think if
> we can keep connect container for each affected group, we also able
> to use this method to do host bus reset.
All you need is a dual-port card with isolation, which includes all
Intel 1G NICs (igb & e1000e) as of the quirks that are currently in
linux-next to be pushed for v4.2. Intel 10G NICs are already quirked
upstream. There are certainly ways to fake isolation for testing as
well. Thanks,
Alex
> >> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
> >> ---
> >> hw/vfio/pci.c | 11 +++++++++++
> >> 1 file changed, 11 insertions(+)
> >>
> >> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> >> index a4e8658..6507f39 100644
> >> --- a/hw/vfio/pci.c
> >> +++ b/hw/vfio/pci.c
> >> @@ -3398,6 +3398,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
> >> PCIHostDeviceAddress host;
> >> VFIOPCIDevice *tmp;
> >> VFIODevice *vbasedev_iter;
> >> + bool found;
> >>
> >> host.domain = devices[i].segment;
> >> host.bus = devices[i].bus;
> >> @@ -3427,6 +3428,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
> >> goto out;
> >> }
> >>
> >> + found = false;
> >> /* Prep dependent devices for reset and clear our marker. */
> >> QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
> >> if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
> >> @@ -3438,12 +3440,21 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
> >> ret = -EINVAL;
> >> goto out_single;
> >> }
> >> + found = true;
> >> vfio_pci_pre_reset(tmp);
> >> tmp->vbasedev.needs_reset = false;
> >> multi = true;
> >> break;
> >> }
> >> }
> >> +
> >> + /*
> >> + * If we own the group but does not own the device, we also
> >> + * should call hot reset with multi.
> >> + */
> >> + if (!single && !found) {
> >> + multi = true;
> >> + }
> >> }
> >>
> >> if (!single && !multi) {
> >
> >
> >
> >
>
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [Qemu-devel] [RFC v9 14/18] vfio: improve vfio_pci_hot_reset to support more case
2015-06-16 14:08 ` Alex Williamson
@ 2015-06-17 6:28 ` Chen Fan
2015-06-17 15:23 ` Alex Williamson
0 siblings, 1 reply; 32+ messages in thread
From: Chen Fan @ 2015-06-17 6:28 UTC (permalink / raw)
To: Alex Williamson; +Cc: izumi.taku, qemu-devel
On 06/16/2015 10:08 PM, Alex Williamson wrote:
> On Tue, 2015-06-16 at 16:10 +0800, Chen Fan wrote:
>> On 06/10/2015 05:24 AM, Alex Williamson wrote:
>>> On Tue, 2015-06-09 at 11:37 +0800, Chen Fan wrote:
>>>> the vfio_pci_hot_reset differentiate the single and multi in-used
>>>> devices for reset. but sometimes we own the group without any devices,
>>>> that also should support hot reset.
>>> Nope, did you try it? It can be done, but the group still needs to be
>>> connected to a container for isolation.
>> I'm sorry for that. because I have no such host in hand. but I think if
>> we can keep connect container for each affected group, we also able
>> to use this method to do host bus reset.
>
> All you need is a dual-port card with isolation, which includes all
> Intel 1G NICs (igb & e1000e) as of the quirks that are currently in
> linux-next to be pushed for v4.2. Intel 10G NICs are already quirked
> upstream. There are certainly ways to fake isolation for testing as
> well. Thanks,
I just have a Intel Corporation 82576 dual-port card, but how can I fake
isolation group for this card in linux-next kernel? can you tell me the
document link?
Thanks,
Chen
>
> Alex
>
>>>> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
>>>> ---
>>>> hw/vfio/pci.c | 11 +++++++++++
>>>> 1 file changed, 11 insertions(+)
>>>>
>>>> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
>>>> index a4e8658..6507f39 100644
>>>> --- a/hw/vfio/pci.c
>>>> +++ b/hw/vfio/pci.c
>>>> @@ -3398,6 +3398,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
>>>> PCIHostDeviceAddress host;
>>>> VFIOPCIDevice *tmp;
>>>> VFIODevice *vbasedev_iter;
>>>> + bool found;
>>>>
>>>> host.domain = devices[i].segment;
>>>> host.bus = devices[i].bus;
>>>> @@ -3427,6 +3428,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
>>>> goto out;
>>>> }
>>>>
>>>> + found = false;
>>>> /* Prep dependent devices for reset and clear our marker. */
>>>> QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
>>>> if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
>>>> @@ -3438,12 +3440,21 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
>>>> ret = -EINVAL;
>>>> goto out_single;
>>>> }
>>>> + found = true;
>>>> vfio_pci_pre_reset(tmp);
>>>> tmp->vbasedev.needs_reset = false;
>>>> multi = true;
>>>> break;
>>>> }
>>>> }
>>>> +
>>>> + /*
>>>> + * If we own the group but does not own the device, we also
>>>> + * should call hot reset with multi.
>>>> + */
>>>> + if (!single && !found) {
>>>> + multi = true;
>>>> + }
>>>> }
>>>>
>>>> if (!single && !multi) {
>>>
>>>
>>>
>
>
>
> .
>
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [Qemu-devel] [RFC v9 14/18] vfio: improve vfio_pci_hot_reset to support more case
2015-06-17 6:28 ` Chen Fan
@ 2015-06-17 15:23 ` Alex Williamson
2015-06-18 10:27 ` Chen Fan
0 siblings, 1 reply; 32+ messages in thread
From: Alex Williamson @ 2015-06-17 15:23 UTC (permalink / raw)
To: Chen Fan; +Cc: izumi.taku, qemu-devel
On Wed, 2015-06-17 at 14:28 +0800, Chen Fan wrote:
> On 06/16/2015 10:08 PM, Alex Williamson wrote:
> > On Tue, 2015-06-16 at 16:10 +0800, Chen Fan wrote:
> >> On 06/10/2015 05:24 AM, Alex Williamson wrote:
> >>> On Tue, 2015-06-09 at 11:37 +0800, Chen Fan wrote:
> >>>> the vfio_pci_hot_reset differentiate the single and multi in-used
> >>>> devices for reset. but sometimes we own the group without any devices,
> >>>> that also should support hot reset.
> >>> Nope, did you try it? It can be done, but the group still needs to be
> >>> connected to a container for isolation.
> >> I'm sorry for that. because I have no such host in hand. but I think if
> >> we can keep connect container for each affected group, we also able
> >> to use this method to do host bus reset.
> >
> > All you need is a dual-port card with isolation, which includes all
> > Intel 1G NICs (igb & e1000e) as of the quirks that are currently in
> > linux-next to be pushed for v4.2. Intel 10G NICs are already quirked
> > upstream. There are certainly ways to fake isolation for testing as
> > well. Thanks,
>
> I just have a Intel Corporation 82576 dual-port card, but how can I fake
> isolation group for this card in linux-next kernel? can you tell me the
> document link?
If you're running linux-next and have the card installed under a root
port that provides isolation then each port should be in a separate
iommu group. Nearly all Intel PCH root ports should have quirks to
enable isolation. If you're installing it in a processor root port
slot, you need to use a Xeon E5 or better CPU or else the lack of
isolation at the root port will negate the isolation capabilities of the
endpoint. Thanks,
Alex
> >>>> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
> >>>> ---
> >>>> hw/vfio/pci.c | 11 +++++++++++
> >>>> 1 file changed, 11 insertions(+)
> >>>>
> >>>> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> >>>> index a4e8658..6507f39 100644
> >>>> --- a/hw/vfio/pci.c
> >>>> +++ b/hw/vfio/pci.c
> >>>> @@ -3398,6 +3398,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
> >>>> PCIHostDeviceAddress host;
> >>>> VFIOPCIDevice *tmp;
> >>>> VFIODevice *vbasedev_iter;
> >>>> + bool found;
> >>>>
> >>>> host.domain = devices[i].segment;
> >>>> host.bus = devices[i].bus;
> >>>> @@ -3427,6 +3428,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
> >>>> goto out;
> >>>> }
> >>>>
> >>>> + found = false;
> >>>> /* Prep dependent devices for reset and clear our marker. */
> >>>> QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
> >>>> if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
> >>>> @@ -3438,12 +3440,21 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
> >>>> ret = -EINVAL;
> >>>> goto out_single;
> >>>> }
> >>>> + found = true;
> >>>> vfio_pci_pre_reset(tmp);
> >>>> tmp->vbasedev.needs_reset = false;
> >>>> multi = true;
> >>>> break;
> >>>> }
> >>>> }
> >>>> +
> >>>> + /*
> >>>> + * If we own the group but does not own the device, we also
> >>>> + * should call hot reset with multi.
> >>>> + */
> >>>> + if (!single && !found) {
> >>>> + multi = true;
> >>>> + }
> >>>> }
> >>>>
> >>>> if (!single && !multi) {
> >>>
> >>>
> >>>
> >
> >
> >
> > .
> >
>
^ permalink raw reply [flat|nested] 32+ messages in thread* Re: [Qemu-devel] [RFC v9 14/18] vfio: improve vfio_pci_hot_reset to support more case
2015-06-17 15:23 ` Alex Williamson
@ 2015-06-18 10:27 ` Chen Fan
0 siblings, 0 replies; 32+ messages in thread
From: Chen Fan @ 2015-06-18 10:27 UTC (permalink / raw)
To: Alex Williamson; +Cc: izumi.taku, qemu-devel
On 06/17/2015 11:23 PM, Alex Williamson wrote:
> On Wed, 2015-06-17 at 14:28 +0800, Chen Fan wrote:
>> On 06/16/2015 10:08 PM, Alex Williamson wrote:
>>> On Tue, 2015-06-16 at 16:10 +0800, Chen Fan wrote:
>>>> On 06/10/2015 05:24 AM, Alex Williamson wrote:
>>>>> On Tue, 2015-06-09 at 11:37 +0800, Chen Fan wrote:
>>>>>> the vfio_pci_hot_reset differentiate the single and multi in-used
>>>>>> devices for reset. but sometimes we own the group without any devices,
>>>>>> that also should support hot reset.
>>>>> Nope, did you try it? It can be done, but the group still needs to be
>>>>> connected to a container for isolation.
>>>> I'm sorry for that. because I have no such host in hand. but I think if
>>>> we can keep connect container for each affected group, we also able
>>>> to use this method to do host bus reset.
>>> All you need is a dual-port card with isolation, which includes all
>>> Intel 1G NICs (igb & e1000e) as of the quirks that are currently in
>>> linux-next to be pushed for v4.2. Intel 10G NICs are already quirked
>>> upstream. There are certainly ways to fake isolation for testing as
>>> well. Thanks,
>> I just have a Intel Corporation 82576 dual-port card, but how can I fake
>> isolation group for this card in linux-next kernel? can you tell me the
>> document link?
> If you're running linux-next and have the card installed under a root
> port that provides isolation then each port should be in a separate
> iommu group. Nearly all Intel PCH root ports should have quirks to
> enable isolation. If you're installing it in a processor root port
> slot, you need to use a Xeon E5 or better CPU or else the lack of
> isolation at the root port will negate the isolation capabilities of the
> endpoint. Thanks,
Hi Alex,
I had test the case with isolation groups in latest linux-next,
I can see the dual-port devices in host looks like:
#readlink /sys/bus/pci/devices/0000\:06\:00.0/iommu_group
../../../../kernel/iommu_groups/30
#readlink /sys/bus/pci/devices/0000\:06\:00.1/iommu_group
../../../../kernel/iommu_groups/31
I used my v10 qemu code that added affected groups to VM to
test the case with qemu command:
qemu-system-x86_64 -M q35 -device
ioh3420,bus=pcie.0,addr=1c.0,port=1,id=bridge1
-device vfio-pci,host=06:00.0,bus=bridge1,aer=true --enable-kvm
then I used to emulate the aer with aer-inject in host. I could find the aer
recovery successful in guest. and the 6:00.0 NIC can reuse by
network-manage.
but when I re-binding the dual devices to host. the host show error:
Jun 18 19:13:36 TX300I kernel: [<ffffffff81666913>] dump_stack+0x45/0x57
Jun 18 19:13:36 TX300I kernel: [<ffffffff8107986a>]
warn_slowpath_common+0x8a/0xc0
Jun 18 19:13:36 TX300I kernel: [<ffffffff810798f5>]
warn_slowpath_fmt+0x55/0x70
Jun 18 19:13:36 TX300I kernel: [<ffffffff81326818>] bad_io_access+0x38/0x40
Jun 18 19:13:36 TX300I kernel: [<ffffffff81326a57>] pci_iounmap+0x27/0x40
Jun 18 19:13:36 TX300I kernel: [<ffffffffa01254ad>]
igb_probe+0xafd/0x1280 [igb]
Jun 18 19:13:36 TX300I kernel: [<ffffffff8134a6d5>]
local_pci_probe+0x45/0xa0
Jun 18 19:13:36 TX300I kernel: [<ffffffff8134b884>] ?
pci_match_device+0xf4/0x120
Jun 18 19:13:36 TX300I kernel: [<ffffffff8134b9d9>]
pci_device_probe+0xe9/0x130
Jun 18 19:13:36 TX300I kernel: [<ffffffff8143547f>]
driver_probe_device+0x14f/0x420
Jun 18 19:13:36 TX300I kernel: [<ffffffff8143389c>] bind_store+0xdc/0x120
Jun 18 19:13:36 TX300I kernel: [<ffffffff81432f74>] drv_attr_store+0x24/0x30
Jun 18 19:13:36 TX300I kernel: [<ffffffff8126de9a>] sysfs_kf_write+0x3a/0x50
Jun 18 19:13:36 TX300I kernel: [<ffffffff8126d520>]
kernfs_fop_write+0x120/0x170
Jun 18 19:13:36 TX300I kernel: [<ffffffff811f13a7>] __vfs_write+0x37/0x100
Jun 18 19:13:36 TX300I kernel: [<ffffffff811f40d8>] ?
__sb_start_write+0x58/0x110
Jun 18 19:13:36 TX300I kernel: [<ffffffff811f1aa9>] vfs_write+0xa9/0x190
Jun 18 19:13:36 TX300I kernel: [<ffffffff81023556>] ?
do_audit_syscall_entry+0x66/0x70
Jun 18 19:13:36 TX300I kernel: [<ffffffff811f28a5>] SyS_write+0x55/0xc0
Jun 18 19:13:36 TX300I kernel: [<ffffffff8166d72e>]
entry_SYSCALL_64_fastpath+0x12/0x71
Jun 18 19:13:36 TX300I kernel: ---[ end trace b312cb051751fac4 ]---
Jun 18 19:13:36 TX300I kernel: igb: probe of 0000:06:00.1 failed with
error -5
the 06:00.0 device can be initialized by igb driver. but the affected
06:00.1 can't be initialized by igb driver.
Is not the 06:00.1 device state initialized by reset ?
Thanks,
Chen
>
> Alex
>
>>>>>> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
>>>>>> ---
>>>>>> hw/vfio/pci.c | 11 +++++++++++
>>>>>> 1 file changed, 11 insertions(+)
>>>>>>
>>>>>> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
>>>>>> index a4e8658..6507f39 100644
>>>>>> --- a/hw/vfio/pci.c
>>>>>> +++ b/hw/vfio/pci.c
>>>>>> @@ -3398,6 +3398,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
>>>>>> PCIHostDeviceAddress host;
>>>>>> VFIOPCIDevice *tmp;
>>>>>> VFIODevice *vbasedev_iter;
>>>>>> + bool found;
>>>>>>
>>>>>> host.domain = devices[i].segment;
>>>>>> host.bus = devices[i].bus;
>>>>>> @@ -3427,6 +3428,7 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
>>>>>> goto out;
>>>>>> }
>>>>>>
>>>>>> + found = false;
>>>>>> /* Prep dependent devices for reset and clear our marker. */
>>>>>> QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
>>>>>> if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
>>>>>> @@ -3438,12 +3440,21 @@ static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
>>>>>> ret = -EINVAL;
>>>>>> goto out_single;
>>>>>> }
>>>>>> + found = true;
>>>>>> vfio_pci_pre_reset(tmp);
>>>>>> tmp->vbasedev.needs_reset = false;
>>>>>> multi = true;
>>>>>> break;
>>>>>> }
>>>>>> }
>>>>>> +
>>>>>> + /*
>>>>>> + * If we own the group but does not own the device, we also
>>>>>> + * should call hot reset with multi.
>>>>>> + */
>>>>>> + if (!single && !found) {
>>>>>> + multi = true;
>>>>>> + }
>>>>>> }
>>>>>>
>>>>>> if (!single && !multi) {
>>>>>
>>>>>
>>>
>>>
>>> .
>>>
>
>
> .
>
^ permalink raw reply [flat|nested] 32+ messages in thread
* [Qemu-devel] [RFC v9 15/18] vfio: do hot bus reset when do virtual secondary bus reset
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
` (13 preceding siblings ...)
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 14/18] vfio: improve vfio_pci_hot_reset to support more case Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
2015-06-09 21:23 ` Alex Williamson
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 16/18] pcie_aer: expose pcie_aer_msg() interface Chen Fan
` (2 subsequent siblings)
17 siblings, 1 reply; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
when do virtual secondary bus reset, the vfio device under
this bus need to do host bus reset to reset the device.
so add this case.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/vfio/pci.c | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 6507f39..655ac59 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -4239,6 +4239,15 @@ static void vfio_pci_reset(DeviceState *dev)
vfio_pci_pre_reset(vdev);
+ if (vdev->needs_bus_reset) {
+ vdev->needs_bus_reset = false;
+ /* Avoid duplicate bus reset */
+ if (vdev->vbasedev.needs_reset) {
+ vfio_pci_hot_reset(vdev, false);
+ }
+ return;
+ }
+
if (vdev->resetfn && !vdev->resetfn(vdev)) {
goto post_reset;
}
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread* Re: [Qemu-devel] [RFC v9 15/18] vfio: do hot bus reset when do virtual secondary bus reset
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 15/18] vfio: do hot bus reset when do virtual secondary bus reset Chen Fan
@ 2015-06-09 21:23 ` Alex Williamson
0 siblings, 0 replies; 32+ messages in thread
From: Alex Williamson @ 2015-06-09 21:23 UTC (permalink / raw)
To: Chen Fan; +Cc: izumi.taku, qemu-devel
On Tue, 2015-06-09 at 11:37 +0800, Chen Fan wrote:
> when do virtual secondary bus reset, the vfio device under
> this bus need to do host bus reset to reset the device.
> so add this case.
>
> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
> ---
> hw/vfio/pci.c | 9 +++++++++
> 1 file changed, 9 insertions(+)
>
> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> index 6507f39..655ac59 100644
> --- a/hw/vfio/pci.c
> +++ b/hw/vfio/pci.c
> @@ -4239,6 +4239,15 @@ static void vfio_pci_reset(DeviceState *dev)
>
> vfio_pci_pre_reset(vdev);
>
> + if (vdev->needs_bus_reset) {
> + vdev->needs_bus_reset = false;
> + /* Avoid duplicate bus reset */
> + if (vdev->vbasedev.needs_reset) {
> + vfio_pci_hot_reset(vdev, false);
> + }
> + return;
> + }
So if we call hot reset, then we do pre-reset twice, once here, once in
the hot reset function. If we don't call hot reset, we fail to call
post-reset for cleanup.
> +
> if (vdev->resetfn && !vdev->resetfn(vdev)) {
> goto post_reset;
> }
^ permalink raw reply [flat|nested] 32+ messages in thread
* [Qemu-devel] [RFC v9 16/18] pcie_aer: expose pcie_aer_msg() interface
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
` (14 preceding siblings ...)
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 15/18] vfio: do hot bus reset when do virtual secondary bus reset Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 17/18] vfio-pci: pass the aer error to guest Chen Fan
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 18/18] vfio: add 'aer' property to expose aercap Chen Fan
17 siblings, 0 replies; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
For vfio device, we need to propagate the aer error to
Guest OS. we use the pcie_aer_msg() to send aer error
to guest.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/pci/pcie_aer.c | 2 +-
include/hw/pci/pcie_aer.h | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/hw/pci/pcie_aer.c b/hw/pci/pcie_aer.c
index 6171a29..373d1f2 100644
--- a/hw/pci/pcie_aer.c
+++ b/hw/pci/pcie_aer.c
@@ -370,7 +370,7 @@ static void pcie_aer_msg_root_port(PCIDevice *dev, const PCIEAERMsg *msg)
*
* Walk up the bus tree from the device, propagate the error message.
*/
-static void pcie_aer_msg(PCIDevice *dev, const PCIEAERMsg *msg)
+void pcie_aer_msg(PCIDevice *dev, const PCIEAERMsg *msg)
{
uint8_t type;
diff --git a/include/hw/pci/pcie_aer.h b/include/hw/pci/pcie_aer.h
index 156acb0..c2ee4e2 100644
--- a/include/hw/pci/pcie_aer.h
+++ b/include/hw/pci/pcie_aer.h
@@ -102,5 +102,6 @@ void pcie_aer_root_write_config(PCIDevice *dev,
/* error injection */
int pcie_aer_inject_error(PCIDevice *dev, const PCIEAERErr *err);
+void pcie_aer_msg(PCIDevice *dev, const PCIEAERMsg *msg);
#endif /* QEMU_PCIE_AER_H */
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread* [Qemu-devel] [RFC v9 17/18] vfio-pci: pass the aer error to guest
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
` (15 preceding siblings ...)
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 16/18] pcie_aer: expose pcie_aer_msg() interface Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 18/18] vfio: add 'aer' property to expose aercap Chen Fan
17 siblings, 0 replies; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
when the vfio device encounters an uncorrectable error in host,
the vfio_pci driver will signal the eventfd registered by this
vfio device, the results in the qemu eventfd handler getting
invoked.
this patch is to pass the error to guest and have the guest driver
recover from the error.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/vfio/pci.c | 45 +++++++++++++++++++++++++++++++++++++++------
1 file changed, 39 insertions(+), 6 deletions(-)
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 655ac59..b2e7601 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -3690,18 +3690,51 @@ static void vfio_put_device(VFIOPCIDevice *vdev)
static void vfio_err_notifier_handler(void *opaque)
{
VFIOPCIDevice *vdev = opaque;
+ PCIDevice *dev = &vdev->pdev;
+ PCIEAERMsg msg = {
+ .severity = 0,
+ .source_id = (pci_bus_num(dev->bus) << 8) | dev->devfn,
+ };
if (!event_notifier_test_and_clear(&vdev->err_notifier)) {
return;
}
/*
- * TBD. Retrieve the error details and decide what action
- * needs to be taken. One of the actions could be to pass
- * the error to the guest and have the guest driver recover
- * from the error. This requires that PCIe capabilities be
- * exposed to the guest. For now, we just terminate the
- * guest to contain the error.
+ * in case the real hardware configration has been changed,
+ * here we should recheck the bus reset capability.
+ */
+ if ((vdev->features & VFIO_FEATURE_ENABLE_AER) &&
+ vfio_check_host_bus_reset(vdev)) {
+ goto stop;
+ }
+ /*
+ * we should read the error details from the real hardware
+ * configuration spaces, here we only need to do is signaling
+ * to guest an uncorrectable error has occurred.
+ */
+ if ((vdev->features & VFIO_FEATURE_ENABLE_AER) &&
+ dev->exp.aer_cap) {
+ uint8_t *aer_cap = dev->config + dev->exp.aer_cap;
+ uint32_t uncor_status;
+ bool isfatal;
+
+ uncor_status = vfio_pci_read_config(dev,
+ dev->exp.aer_cap + PCI_ERR_UNCOR_STATUS, 4);
+
+ isfatal = uncor_status & pci_get_long(aer_cap + PCI_ERR_UNCOR_SEVER);
+
+ msg.severity = isfatal ? PCI_ERR_ROOT_CMD_FATAL_EN :
+ PCI_ERR_ROOT_CMD_NONFATAL_EN;
+
+ pcie_aer_msg(dev, &msg);
+ return;
+ }
+
+stop:
+ /*
+ * If the aer capability is not exposed to the guest. we just
+ * terminate the guest to contain the error.
*/
error_report("%s(%04x:%02x:%02x.%x) Unrecoverable error detected. "
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread* [Qemu-devel] [RFC v9 18/18] vfio: add 'aer' property to expose aercap
2015-06-09 3:37 [Qemu-devel] [RFC v9 00/18] vfio-pci: pass the aer error to guest Chen Fan
` (16 preceding siblings ...)
2015-06-09 3:37 ` [Qemu-devel] [RFC v9 17/18] vfio-pci: pass the aer error to guest Chen Fan
@ 2015-06-09 3:37 ` Chen Fan
17 siblings, 0 replies; 32+ messages in thread
From: Chen Fan @ 2015-06-09 3:37 UTC (permalink / raw)
To: qemu-devel; +Cc: izumi.taku, alex.williamson
add 'aer' property to let user able to decide whether expose
the aer capability. by default we should disable aer feature,
because it needs configuration restrictions.
Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
---
hw/vfio/pci.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index b2e7601..cf5d46d 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -4326,6 +4326,8 @@ static Property vfio_pci_dev_properties[] = {
VFIO_FEATURE_ENABLE_VGA_BIT, false),
DEFINE_PROP_BIT("x-req", VFIOPCIDevice, features,
VFIO_FEATURE_ENABLE_REQ_BIT, true),
+ DEFINE_PROP_BIT("aer", VFIOPCIDevice, features,
+ VFIO_FEATURE_ENABLE_AER_BIT, false),
DEFINE_PROP_INT32("bootindex", VFIOPCIDevice, bootindex, -1),
DEFINE_PROP_BOOL("x-mmap", VFIOPCIDevice, vbasedev.allow_mmap, true),
/*
--
1.9.3
^ permalink raw reply related [flat|nested] 32+ messages in thread