Re: [Qemu-devel] [RFC v9 13/18] vfio: add sec_bus_reset notifier to notify physical bus reset is needed

[Alex Williamson <alex.williamson@redhat.com>]

On Tue, 2015-06-09 at 11:37 +0800, Chen Fan wrote:
> Signed-off-by: Chen Fan <chen.fan.fnst@cn.fujitsu.com>
> ---
>  hw/vfio/pci.c | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 81 insertions(+)
> 
> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> index 97af143..a4e8658 100644
> --- a/hw/vfio/pci.c
> +++ b/hw/vfio/pci.c
> @@ -156,6 +156,7 @@ typedef struct VFIOPCIDevice {
>      PCIHostDeviceAddress host;
>      EventNotifier err_notifier;
>      EventNotifier req_notifier;
> +    Notifier sec_bus_reset_notifier;
>      int (*resetfn)(struct VFIOPCIDevice *);
>      uint32_t features;
>  #define VFIO_FEATURE_ENABLE_VGA_BIT 0
> @@ -171,6 +172,7 @@ typedef struct VFIOPCIDevice {
>      bool req_enabled;
>      bool has_flr;
>      bool has_pm_reset;
> +    bool needs_bus_reset;
>      bool rom_read_failed;
>  } VFIOPCIDevice;
>  
> @@ -3085,6 +3087,81 @@ static int vfio_check_devices_host_bus_reset(void)
>      return 0;
>  }
>  
> +/* Update all the reset state of the affected devices in VM */
> +static void vfio_pci_host_needs_bus_reset(Notifier *n, void *opaque)
> +{
> +    VFIOPCIDevice *vdev = container_of(n, VFIOPCIDevice, sec_bus_reset_notifier);
> +    VFIODevice *vbasedev = &vdev->vbasedev;
> +    PCIDevice *parent = opaque;
> +    VFIOGroup *group;
> +    struct vfio_pci_hot_reset_info *info = NULL;
> +    struct vfio_pci_dependent_device *devices;
> +    int i, ret;
> +
> +    ret = vfio_get_hot_reset_info(vdev, &info);
> +    if (ret) {
> +        goto out;
> +    }
> +
> +    vdev->needs_bus_reset = true;
> +    /*
> +     * qbus_reset_all always reset the devices from the depth level,
> +     * here only need to do the reset for the device encounter the aer,
> +     * in order to avoid reset the affected device first below the bus,
> +     * we only set the top level devices as needs_reset.
> +     */
> +    if (parent == pci_bridge_get_device(vdev->pdev.bus)) {
> +        vbasedev->needs_reset = true;
> +    }
> +
> +    /* List all affected devices by bus reset */
> +    devices = &info->devices[0];
> +
> +    /* Verify that we have all the groups required */
> +    for (i = 0; i < info->count; i++) {
> +        PCIHostDeviceAddress host;
> +        VFIOPCIDevice *tmp;
> +        VFIODevice *vbasedev_iter;
> +
> +        host.domain = devices[i].segment;
> +        host.bus = devices[i].bus;
> +        host.slot = PCI_SLOT(devices[i].devfn);
> +        host.function = PCI_FUNC(devices[i].devfn);
> +
> +        /* Skip the current device */
> +        if (vfio_pci_host_match(&host, &vdev->host)) {
> +            continue;
> +        }
> +
> +        /* Ensure we own the group of the affected device */
> +        QLIST_FOREACH(group, &vfio_group_list, next) {
> +            if (group->groupid == devices[i].group_id) {
> +                break;
> +            }

This is insufficient now that groups can be opened without a container.

> +        }
> +
> +        if (!group) {
> +            goto out;
> +        }
> +
> +        /* Ensure affected devices for reset under the same bus */
> +        QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
> +            if (vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
> +                continue;
> +            }
> +            tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
> +            if (vfio_pci_host_match(&host, &tmp->host)) {
> +                tmp->needs_bus_reset = true;
> +                tmp->vbasedev.needs_reset = false;
> +                break;
> +            }
> +        }
> +    }
> +
> +out:
> +    g_free(info);
> +}
> +
>  static int vfio_setup_aer(VFIOPCIDevice *vdev, uint8_t cap_ver,
>                            int pos, uint16_t size)
>  {
> @@ -3156,6 +3233,9 @@ static int vfio_setup_aer(VFIOPCIDevice *vdev, uint8_t cap_ver,
>                     pdev->exp.aer_cap + PCI_ERR_UNCOR_SEVER, 4);
>      pci_long_test_and_clear_mask(exp_cap + PCI_ERR_UNCOR_SEVER, ~severity);
>  
> +    vdev->sec_bus_reset_notifier.notify = vfio_pci_host_needs_bus_reset;
> +    pci_bus_add_reset_notifier(pdev->bus, &vdev->sec_bus_reset_notifier);
> +
>      return 0;
>  
>  error:
> @@ -4129,6 +4209,7 @@ static void vfio_exitfn(PCIDevice *pdev)
>  
>      vfio_unregister_req_notifier(vdev);
>      vfio_unregister_err_notifier(vdev);
> +    pci_bus_remove_reset_notifier(&vdev->sec_bus_reset_notifier);
>      pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
>      vfio_disable_interrupts(vdev);
>      if (vdev->intx.mmap_timer) {