From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from eggs.gnu.org ([2001:4830:134:3::10]:59696) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1XQqwo-0003Y1-78 for qemu-devel@nongnu.org; Mon, 08 Sep 2014 00:53:11 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1XQqwb-0004vf-Ir for qemu-devel@nongnu.org; Mon, 08 Sep 2014 00:53:02 -0400 Received: from e23smtp06.au.ibm.com ([202.81.31.148]:49321) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1XQqwa-0004sq-Ur for qemu-devel@nongnu.org; Mon, 08 Sep 2014 00:52:49 -0400 Received: from /spool/local by e23smtp06.au.ibm.com with IBM ESMTP SMTP Gateway: Authorized Use Only! Violators will be prosecuted for from ; Mon, 8 Sep 2014 14:52:44 +1000 Received: from d23relay08.au.ibm.com (d23relay08.au.ibm.com [9.185.71.33]) by d23dlp02.au.ibm.com (Postfix) with ESMTP id 8EB592BB0040 for ; Mon, 8 Sep 2014 14:52:41 +1000 (EST) Received: from d23av04.au.ibm.com (d23av04.au.ibm.com [9.190.235.139]) by d23relay08.au.ibm.com (8.14.9/8.14.9/NCO v10.0) with ESMTP id s884rwD130736566 for ; Mon, 8 Sep 2014 14:53:58 +1000 Received: from d23av04.au.ibm.com (localhost [127.0.0.1]) by d23av04.au.ibm.com (8.14.4/8.14.4/NCO v10.0 AVout) with ESMTP id s884qejg019574 for ; Mon, 8 Sep 2014 14:52:41 +1000 From: Gavin Shan Date: Mon, 8 Sep 2014 14:52:40 +1000 Message-Id: <1410151960-18199-3-git-send-email-gwshan@linux.vnet.ibm.com> In-Reply-To: <1410151960-18199-1-git-send-email-gwshan@linux.vnet.ibm.com> References: <1410151960-18199-1-git-send-email-gwshan@linux.vnet.ibm.com> Subject: [Qemu-devel] [RFC PATCH v3 2/2] VFIO: Clear stale interrupt vectors during reset List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: qemu-devel@nongnu.org Cc: aik@ozlabs.ru, alex.williamson@redhat.com, Gavin Shan As to one particular VFIO device, there are 4 kinds of resets as follows: FLR (Function Level Reset), secondary bus reset, machine reset, EEH reset. For the first 2 cases, PCI core helps keeping consistent interrupt vectors. For the later 2 cases, we have to drop the stale interrupt (MSI or MSIx) vectors. Otherwise, we will potentially lose chance to restore MSI or MSIx vectors when reenabling MSI or MSIx interrupts after reset. The patch clears stale MSI or MSIx vectors before machine/EEH reset so that MSI or MSIx vectors could be restored properly after EEH PE reset. Signed-off-by: Gavin Shan --- hw/misc/vfio.c | 66 ++++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 55 insertions(+), 11 deletions(-) diff --git a/hw/misc/vfio.c b/hw/misc/vfio.c index 1a3e7eb..026da77 100644 --- a/hw/misc/vfio.c +++ b/hw/misc/vfio.c @@ -2724,6 +2724,22 @@ static void vfio_disable_interrupts(VFIODevice *vdev) } } +static void vfio_disable_and_reset_interrupts(VFIODevice *vdev) +{ + int interrupt = vdev->interrupt; + + vfio_disable_interrupts(vdev); + + switch (interrupt) { + case VFIO_INT_MSI: + msi_reset(&vdev->pdev); + break; + case VFIO_INT_MSIX: + msix_reset(&vdev->pdev); + break; + } +} + static int vfio_setup_msi(VFIODevice *vdev, int pos) { uint16_t ctrl; @@ -3298,12 +3314,16 @@ static int vfio_add_capabilities(VFIODevice *vdev) return vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST]); } -static void vfio_pci_pre_reset(VFIODevice *vdev) +static void vfio_pci_pre_reset(VFIODevice *vdev, bool reset_interrupt) { PCIDevice *pdev = &vdev->pdev; uint16_t cmd; - vfio_disable_interrupts(vdev); + if (reset_interrupt) { + vfio_disable_and_reset_interrupts(vdev); + } else { + vfio_disable_interrupts(vdev); + } /* Make sure the device is in D0 */ if (vdev->pm_cap) { @@ -3347,7 +3367,8 @@ static bool vfio_pci_host_match(PCIHostDeviceAddress *host1, host1->slot == host2->slot && host1->function == host2->function); } -static int vfio_pci_hot_reset(VFIODevice *vdev, bool single) +static int vfio_pci_hot_reset(VFIODevice *vdev, + bool single, bool reset_interrupt) { VFIOGroup *group; struct vfio_pci_hot_reset_info *info; @@ -3361,7 +3382,7 @@ static int vfio_pci_hot_reset(VFIODevice *vdev, bool single) vdev->host.bus, vdev->host.slot, vdev->host.function, single ? "one" : "multi"); - vfio_pci_pre_reset(vdev); + vfio_pci_pre_reset(vdev, reset_interrupt); vdev->needs_reset = false; info = g_malloc0(sizeof(*info)); @@ -3438,7 +3459,7 @@ static int vfio_pci_hot_reset(VFIODevice *vdev, bool single) ret = -EINVAL; goto out_single; } - vfio_pci_pre_reset(tmp); + vfio_pci_pre_reset(tmp, reset_interrupt); tmp->needs_reset = false; multi = true; break; @@ -3541,12 +3562,12 @@ out_single: */ static int vfio_pci_hot_reset_one(VFIODevice *vdev) { - return vfio_pci_hot_reset(vdev, true); + return vfio_pci_hot_reset(vdev, true, false); } -static int vfio_pci_hot_reset_multi(VFIODevice *vdev) +static int vfio_pci_hot_reset_multi_and_interrupts(VFIODevice *vdev) { - return vfio_pci_hot_reset(vdev, false); + return vfio_pci_hot_reset(vdev, false, true); } static void vfio_pci_reset_handler(void *opaque) @@ -3565,7 +3586,7 @@ static void vfio_pci_reset_handler(void *opaque) QLIST_FOREACH(group, &group_list, next) { QLIST_FOREACH(vdev, &group->device_list, next) { if (vdev->needs_reset) { - vfio_pci_hot_reset_multi(vdev); + vfio_pci_hot_reset_multi_and_interrupts(vdev); } } } @@ -4334,6 +4355,11 @@ static void vfio_exitfn(PCIDevice *pdev) vfio_put_group(group); } +/* + * The function is invoked when issuing FLR or secondary bus reset. + * In the case, the PCI core should drop stale MSI or MSIx vectors + * if applicable. So we needn't do that by ourselves. + */ static void vfio_pci_reset(DeviceState *dev) { PCIDevice *pdev = DO_UPCAST(PCIDevice, qdev, dev); @@ -4342,7 +4368,7 @@ static void vfio_pci_reset(DeviceState *dev) DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain, vdev->host.bus, vdev->host.slot, vdev->host.function); - vfio_pci_pre_reset(vdev); + vfio_pci_pre_reset(vdev, false); if (vdev->reset_works && (vdev->has_flr || !vdev->has_pm_reset) && !ioctl(vdev->fd, VFIO_DEVICE_RESET)) { @@ -4442,8 +4468,26 @@ int vfio_container_ioctl(AddressSpace *as, int32_t groupid, switch (req) { case VFIO_CHECK_EXTENSION: case VFIO_IOMMU_SPAPR_TCE_GET_INFO: - case VFIO_EEH_PE_OP: break; + case VFIO_EEH_PE_OP: { + VFIODevice *vdev; + struct vfio_eeh_pe_op *arg = (struct vfio_eeh_pe_op *)param; + + switch (arg->op) { + case VFIO_EEH_PE_RESET_HOT: + case VFIO_EEH_PE_RESET_FUNDAMENTAL: + /* + * The stale MSI or MSIx information, including controlling + * information and vectors should be dropped so that they + * can be restored properly after reset. + */ + QLIST_FOREACH(vdev, &group->device_list, next) { + vfio_disable_and_reset_interrupts(vdev); + } + } + + break; + } default: /* Return an error on unknown requests */ error_report("vfio: unsupported ioctl %X", req); -- 1.8.3.2