[Qemu-devel] [PATCH v2 0/3] ivshmem: MSI bug fixes

[Qemu-devel] [PATCH v2 0/3] ivshmem: MSI bug fixes

[Ladi Prosek]

Fixes bugs in the ivshmem device implementation uncovered with the new
Windows ivshmem driver:
https://github.com/virtio-win/kvm-guest-drivers-windows/tree/master/ivshmem

v1->v2:
* Patch 1 - added reproducer info to commit message (Markus)
* Patch 2 - restructured conditionals, fixed comment formatting (Markus)
* Patch 3 - added reproducer info to commit message (Markus)

Ladi Prosek (3):
  ivshmem: Don't update non-existent MSI routes
  ivshmem: Always remove irqfd notifiers
  ivshmem: Improve MSI irqfd error handling

 hw/misc/ivshmem.c | 77 +++++++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 58 insertions(+), 19 deletions(-)

-- 
2.13.5

[Qemu-devel] [PATCH v2 1/3] ivshmem: Don't update non-existent MSI routes

[Ladi Prosek]

As of commit 660c97eef6f8 ("ivshmem: use kvm irqfd for msi notifications"),
QEMU crashes with:

  kvm_irqchip_commit_routes: Assertion `ret == 0' failed.

if the ivshmem device is configured with more vectors than what the server
supports. This is caused by the ivshmem_vector_unmask() being called on
vectors that have not been initialized by ivshmem_add_kvm_msi_virq().

This commit fixes it by adding a simple check to the mask and unmask
callbacks.

Note that the opposite mismatch, if the server supplies more vectors than
what the device is configured for, is already handled and leads to output
like:

  Too many eventfd received, device has 1 vectors

To reproduce the assert, run:

  ivshmem-server -n 0

and QEMU with:

  -device ivshmem-doorbell,chardev=iv
  -chardev socket,path=/tmp/ivshmem_socket,id=iv

then load the Windows driver, at the time of writing available at:

https://github.com/virtio-win/kvm-guest-drivers-windows/tree/master/ivshmem

The issue is believed to have been masked by other guest drivers, notably
Linux ones, not enabling MSI-X on the device.

Fixes: 660c97eef6f8 ("ivshmem: use kvm irqfd for msi notifications")
Signed-off-by: Ladi Prosek <lprosek@redhat.com>
Reviewed-by: Marc-André Lureau <marcandre.lureau@redhat.com>
Reviewed-by: Markus Armbruster <armbru@redhat.com>
---
 hw/misc/ivshmem.c | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/hw/misc/ivshmem.c b/hw/misc/ivshmem.c
index a5a46827fe..6e46669744 100644
--- a/hw/misc/ivshmem.c
+++ b/hw/misc/ivshmem.c
@@ -317,6 +317,10 @@ static int ivshmem_vector_unmask(PCIDevice *dev, unsigned vector,
     int ret;
 
     IVSHMEM_DPRINTF("vector unmask %p %d\n", dev, vector);
+    if (!v->pdev) {
+        error_report("ivshmem: vector %d route does not exist", vector);
+        return -EINVAL;
+    }
 
     ret = kvm_irqchip_update_msi_route(kvm_state, v->virq, msg, dev);
     if (ret < 0) {
@@ -331,12 +335,16 @@ static void ivshmem_vector_mask(PCIDevice *dev, unsigned vector)
 {
     IVShmemState *s = IVSHMEM_COMMON(dev);
     EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
+    MSIVector *v = &s->msi_vectors[vector];
     int ret;
 
     IVSHMEM_DPRINTF("vector mask %p %d\n", dev, vector);
+    if (!v->pdev) {
+        error_report("ivshmem: vector %d route does not exist", vector);
+        return;
+    }
 
-    ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n,
-                                                s->msi_vectors[vector].virq);
+    ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, v->virq);
     if (ret != 0) {
         error_report("remove_irqfd_notifier_gsi failed");
     }
-- 
2.13.5

[Qemu-devel] [PATCH v2 2/3] ivshmem: Always remove irqfd notifiers

[Ladi Prosek]

As of commit 660c97eef6f8 ("ivshmem: use kvm irqfd for msi notifications"),
QEMU crashes with:

ivshmem: msix_set_vector_notifiers failed
msix_unset_vector_notifiers: Assertion `dev->msix_vector_use_notifier && dev->msix_vector_release_notifier' failed.

if MSI-X is repeatedly enabled and disabled on the ivshmem device, for example
by loading and unloading the Windows ivshmem driver. This is because
msix_unset_vector_notifiers() doesn't call any of the release notifier callbacks
since MSI-X is already disabled at that point (msix_enabled() returning false
is how this transition is detected in the first place). Thus ivshmem_vector_mask()
doesn't run and when MSI-X is subsequently enabled again ivshmem_vector_unmask()
fails.

This is fixed by keeping track of unmasked vectors and making sure that
ivshmem_vector_mask() always runs on MSI-X disable.

Fixes: 660c97eef6f8 ("ivshmem: use kvm irqfd for msi notifications")
Signed-off-by: Ladi Prosek <lprosek@redhat.com>
Reviewed-by: Markus Armbruster <armbru@redhat.com>
---
 hw/misc/ivshmem.c | 32 ++++++++++++++++++++++++++------
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/hw/misc/ivshmem.c b/hw/misc/ivshmem.c
index 6e46669744..91364d8364 100644
--- a/hw/misc/ivshmem.c
+++ b/hw/misc/ivshmem.c
@@ -77,6 +77,7 @@ typedef struct Peer {
 typedef struct MSIVector {
     PCIDevice *pdev;
     int virq;
+    bool unmasked;
 } MSIVector;
 
 typedef struct IVShmemState {
@@ -321,6 +322,7 @@ static int ivshmem_vector_unmask(PCIDevice *dev, unsigned vector,
         error_report("ivshmem: vector %d route does not exist", vector);
         return -EINVAL;
     }
+    assert(!v->unmasked);
 
     ret = kvm_irqchip_update_msi_route(kvm_state, v->virq, msg, dev);
     if (ret < 0) {
@@ -328,7 +330,13 @@ static int ivshmem_vector_unmask(PCIDevice *dev, unsigned vector,
     }
     kvm_irqchip_commit_routes(kvm_state);
 
-    return kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, v->virq);
+    ret = kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, v->virq);
+    if (ret < 0) {
+        return ret;
+    }
+    v->unmasked = true;
+
+    return 0;
 }
 
 static void ivshmem_vector_mask(PCIDevice *dev, unsigned vector)
@@ -343,11 +351,14 @@ static void ivshmem_vector_mask(PCIDevice *dev, unsigned vector)
         error_report("ivshmem: vector %d route does not exist", vector);
         return;
     }
+    assert(v->unmasked);
 
     ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, v->virq);
-    if (ret != 0) {
+    if (ret < 0) {
         error_report("remove_irqfd_notifier_gsi failed");
+        return;
     }
+    v->unmasked = false;
 }
 
 static void ivshmem_vector_poll(PCIDevice *dev,
@@ -817,11 +828,20 @@ static void ivshmem_disable_irqfd(IVShmemState *s)
     PCIDevice *pdev = PCI_DEVICE(s);
     int i;
 
-    for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) {
-        ivshmem_remove_kvm_msi_virq(s, i);
-    }
-
     msix_unset_vector_notifiers(pdev);
+
+    for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) {
+        /*
+         * MSI-X is already disabled here so msix_unset_vector_notifiers()
+         * didn't call our release notifier.  Do it now to keep our masks and
+         * unmasks balanced.
+         */
+        if (s->msi_vectors[i].unmasked) {
+            ivshmem_vector_mask(pdev, i);
+        }
+        ivshmem_remove_kvm_msi_virq(s, i);
+    }
+
 }
 
 static void ivshmem_write_config(PCIDevice *pdev, uint32_t address,
-- 
2.13.5

[Qemu-devel] [PATCH v2 3/3] ivshmem: Improve MSI irqfd error handling

[Ladi Prosek]

Adds a rollback path to ivshmem_enable_irqfd() and fixes
ivshmem_disable_irqfd() to bail if irqfd has not been enabled.

To reproduce, run:

  ivshmem-server -n 0

and QEMU with:

  -device ivshmem-doorbell,chardev=iv
  -chardev socket,path=/tmp/ivshmem_socket,id=iv

then load, unload, and load again the Windows driver, at the time of writing
available at:

https://github.com/virtio-win/kvm-guest-drivers-windows/tree/master/ivshmem

The issue is believed to have been masked by other guest drivers, notably
Linux ones, not enabling MSI-X on the device.

Signed-off-by: Ladi Prosek <lprosek@redhat.com>
Reviewed-by: Markus Armbruster <armbru@redhat.com>
---
 hw/misc/ivshmem.c | 37 ++++++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/hw/misc/ivshmem.c b/hw/misc/ivshmem.c
index 91364d8364..d1bb246d12 100644
--- a/hw/misc/ivshmem.c
+++ b/hw/misc/ivshmem.c
@@ -786,6 +786,20 @@ static int ivshmem_setup_interrupts(IVShmemState *s, Error **errp)
     return 0;
 }
 
+static void ivshmem_remove_kvm_msi_virq(IVShmemState *s, int vector)
+{
+    IVSHMEM_DPRINTF("ivshmem_remove_kvm_msi_virq vector:%d\n", vector);
+
+    if (s->msi_vectors[vector].pdev == NULL) {
+        return;
+    }
+
+    /* it was cleaned when masked in the frontend. */
+    kvm_irqchip_release_virq(kvm_state, s->msi_vectors[vector].virq);
+
+    s->msi_vectors[vector].pdev = NULL;
+}
+
 static void ivshmem_enable_irqfd(IVShmemState *s)
 {
     PCIDevice *pdev = PCI_DEVICE(s);
@@ -797,7 +811,7 @@ static void ivshmem_enable_irqfd(IVShmemState *s)
         ivshmem_add_kvm_msi_virq(s, i, &err);
         if (err) {
             error_report_err(err);
-            /* TODO do we need to handle the error? */
+            goto undo;
         }
     }
 
@@ -806,21 +820,14 @@ static void ivshmem_enable_irqfd(IVShmemState *s)
                                   ivshmem_vector_mask,
                                   ivshmem_vector_poll)) {
         error_report("ivshmem: msix_set_vector_notifiers failed");
+        goto undo;
     }
-}
+    return;
 
-static void ivshmem_remove_kvm_msi_virq(IVShmemState *s, int vector)
-{
-    IVSHMEM_DPRINTF("ivshmem_remove_kvm_msi_virq vector:%d\n", vector);
-
-    if (s->msi_vectors[vector].pdev == NULL) {
-        return;
+undo:
+    while (--i >= 0) {
+        ivshmem_remove_kvm_msi_virq(s, i);
     }
-
-    /* it was cleaned when masked in the frontend. */
-    kvm_irqchip_release_virq(kvm_state, s->msi_vectors[vector].virq);
-
-    s->msi_vectors[vector].pdev = NULL;
 }
 
 static void ivshmem_disable_irqfd(IVShmemState *s)
@@ -828,6 +835,10 @@ static void ivshmem_disable_irqfd(IVShmemState *s)
     PCIDevice *pdev = PCI_DEVICE(s);
     int i;
 
+    if (!pdev->msix_vector_use_notifier) {
+        return;
+    }
+
     msix_unset_vector_notifiers(pdev);
 
     for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) {
-- 
2.13.5

Re: [Qemu-devel] [PATCH v2 0/3] ivshmem: MSI bug fixes

[geoff]

I just updated to the latest build and applied this patch set, now on VM 
reset the qemu crashes with the following assert:

ivshmem.c:467: ivshmem_add_kvm_msi_virq: Assertion 
`!s->msi_vectors[vector].pdev' failed.

On 2017-11-15 18:31, Ladi Prosek wrote:
> Fixes bugs in the ivshmem device implementation uncovered with the new
> Windows ivshmem driver:
> https://github.com/virtio-win/kvm-guest-drivers-windows/tree/master/ivshmem
> 
> v1->v2:
> * Patch 1 - added reproducer info to commit message (Markus)
> * Patch 2 - restructured conditionals, fixed comment formatting 
> (Markus)
> * Patch 3 - added reproducer info to commit message (Markus)
> 
> Ladi Prosek (3):
>   ivshmem: Don't update non-existent MSI routes
>   ivshmem: Always remove irqfd notifiers
>   ivshmem: Improve MSI irqfd error handling
> 
>  hw/misc/ivshmem.c | 77 
> +++++++++++++++++++++++++++++++++++++++++--------------
>  1 file changed, 58 insertions(+), 19 deletions(-)

Re: [Qemu-devel] [PATCH v2 0/3] ivshmem: MSI bug fixes

[Ladi Prosek]

On Sun, Nov 19, 2017 at 9:39 PM,  <geoff@hostfission.com> wrote:
> I just updated to the latest build and applied this patch set, now on VM
> reset the qemu crashes with the following assert:
>
> ivshmem.c:467: ivshmem_add_kvm_msi_virq: Assertion
> `!s->msi_vectors[vector].pdev' failed.

I see asserts too. Even with v1 on top of QEMU v2.10.0 so I must have
missed something.

Looking. And, needless to say, these patches should not be applied just yet :)

Thanks!
Ladi

> On 2017-11-15 18:31, Ladi Prosek wrote:
>>
>> Fixes bugs in the ivshmem device implementation uncovered with the new
>> Windows ivshmem driver:
>>
>> https://github.com/virtio-win/kvm-guest-drivers-windows/tree/master/ivshmem
>>
>> v1->v2:
>> * Patch 1 - added reproducer info to commit message (Markus)
>> * Patch 2 - restructured conditionals, fixed comment formatting (Markus)
>> * Patch 3 - added reproducer info to commit message (Markus)
>>
>> Ladi Prosek (3):
>>   ivshmem: Don't update non-existent MSI routes
>>   ivshmem: Always remove irqfd notifiers
>>   ivshmem: Improve MSI irqfd error handling
>>
>>  hw/misc/ivshmem.c | 77
>> +++++++++++++++++++++++++++++++++++++++++--------------
>>  1 file changed, 58 insertions(+), 19 deletions(-)
>
>

Re: [Qemu-devel] [PATCH v2 0/3] ivshmem: MSI bug fixes

[Ladi Prosek]

On Mon, Nov 20, 2017 at 10:07 AM, Ladi Prosek <lprosek@redhat.com> wrote:
> On Sun, Nov 19, 2017 at 9:39 PM,  <geoff@hostfission.com> wrote:
>> I just updated to the latest build and applied this patch set, now on VM
>> reset the qemu crashes with the following assert:
>>
>> ivshmem.c:467: ivshmem_add_kvm_msi_virq: Assertion
>> `!s->msi_vectors[vector].pdev' failed.
>
> I see asserts too. Even with v1 on top of QEMU v2.10.0 so I must have
> missed something.
>
> Looking. And, needless to say, these patches should not be applied just yet :)

Ok, here goes it.

1)
ivshmem.c:467: ivshmem_add_kvm_msi_virq: Assertion
`!s->msi_vectors[vector].pdev' failed.

Is caused by the ivshmem device not undoing the effects of
ivshmem_enable_irqfd() on reset.

This fix works for me:

--- a/hw/misc/ivshmem.c
+++ b/hw/misc/ivshmem.c
@@ -758,10 +758,15 @@ static void ivshmem_msix_vector_use(IVShmemState *s)
     }
 }

+
+static void ivshmem_disable_irqfd(IVShmemState *s);
+
 static void ivshmem_reset(DeviceState *d)
 {
     IVShmemState *s = IVSHMEM_COMMON(d);

+    ivshmem_disable_irqfd(s);
+
     s->intrstatus = 0;
     s->intrmask = 0;
     if (ivshmem_has_feature(s, IVSHMEM_MSI)) {


2)
ivshmem.c:354: ivshmem_vector_mask: Assertion `v->unmasked' failed.

which I've been also getting after I enabled Driver Verifier and
Windows started crashing
(https://github.com/virtio-win/kvm-guest-drivers-windows/pull/199), is
caused by the MSI-X code masking already masked vectors on reset. I'm
going to post a patch similar to this:

--- a/hw/pci/msix.c
+++ b/hw/pci/msix.c
@@ -500,11 +500,12 @@ void msix_reset(PCIDevice *dev)
         return;
     }
     msix_clear_all_vectors(dev);
+    msix_mask_all(dev, dev->msix_entries_nr);
     dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &=
            ~dev->wmask[dev->msix_cap + MSIX_CONTROL_OFFSET];
     memset(dev->msix_table, 0, dev->msix_entries_nr * PCI_MSIX_ENTRY_SIZE);
     memset(dev->msix_pba, 0, QEMU_ALIGN_UP(dev->msix_entries_nr, 64) / 8);
-    msix_mask_all(dev, dev->msix_entries_nr);
+    msix_update_function_masked(dev);
 }


Then either no further changes to this patchset are necessary. Or, if
relying on unmasks/masks (or
msix_vector_use_notifier/msix_vector_release_notifier as it's called
in msix.c) always being balanced is not recommended, the assert will
simply change into an if.

> Thanks!
> Ladi
>
>> On 2017-11-15 18:31, Ladi Prosek wrote:
>>>
>>> Fixes bugs in the ivshmem device implementation uncovered with the new
>>> Windows ivshmem driver:
>>>
>>> https://github.com/virtio-win/kvm-guest-drivers-windows/tree/master/ivshmem
>>>
>>> v1->v2:
>>> * Patch 1 - added reproducer info to commit message (Markus)
>>> * Patch 2 - restructured conditionals, fixed comment formatting (Markus)
>>> * Patch 3 - added reproducer info to commit message (Markus)
>>>
>>> Ladi Prosek (3):
>>>   ivshmem: Don't update non-existent MSI routes
>>>   ivshmem: Always remove irqfd notifiers
>>>   ivshmem: Improve MSI irqfd error handling
>>>
>>>  hw/misc/ivshmem.c | 77
>>> +++++++++++++++++++++++++++++++++++++++++--------------
>>>  1 file changed, 58 insertions(+), 19 deletions(-)
>>
>>

Re: [Qemu-devel] [PATCH v2 0/3] ivshmem: MSI bug fixes

[Ladi Prosek]

On Mon, Nov 20, 2017 at 1:49 PM, Ladi Prosek <lprosek@redhat.com> wrote:
> On Mon, Nov 20, 2017 at 10:07 AM, Ladi Prosek <lprosek@redhat.com> wrote:
>> On Sun, Nov 19, 2017 at 9:39 PM,  <geoff@hostfission.com> wrote:
>>> I just updated to the latest build and applied this patch set, now on VM
>>> reset the qemu crashes with the following assert:
>>>
>>> ivshmem.c:467: ivshmem_add_kvm_msi_virq: Assertion
>>> `!s->msi_vectors[vector].pdev' failed.
>>
>> I see asserts too. Even with v1 on top of QEMU v2.10.0 so I must have
>> missed something.
>>
>> Looking. And, needless to say, these patches should not be applied just yet :)
>
> Ok, here goes it.
>
> 1)
> ivshmem.c:467: ivshmem_add_kvm_msi_virq: Assertion
> `!s->msi_vectors[vector].pdev' failed.
>
> Is caused by the ivshmem device not undoing the effects of
> ivshmem_enable_irqfd() on reset.
>
> This fix works for me:
>
> --- a/hw/misc/ivshmem.c
> +++ b/hw/misc/ivshmem.c
> @@ -758,10 +758,15 @@ static void ivshmem_msix_vector_use(IVShmemState *s)
>      }
>  }
>
> +
> +static void ivshmem_disable_irqfd(IVShmemState *s);
> +
>  static void ivshmem_reset(DeviceState *d)
>  {
>      IVShmemState *s = IVSHMEM_COMMON(d);
>
> +    ivshmem_disable_irqfd(s);
> +
>      s->intrstatus = 0;
>      s->intrmask = 0;
>      if (ivshmem_has_feature(s, IVSHMEM_MSI)) {

I have added this to v3 as patch 4.

> 2)
> ivshmem.c:354: ivshmem_vector_mask: Assertion `v->unmasked' failed.
>
> which I've been also getting after I enabled Driver Verifier and
> Windows started crashing
> (https://github.com/virtio-win/kvm-guest-drivers-windows/pull/199), is
> caused by the MSI-X code masking already masked vectors on reset. I'm
> going to post a patch similar to this:
>
> --- a/hw/pci/msix.c
> +++ b/hw/pci/msix.c
> @@ -500,11 +500,12 @@ void msix_reset(PCIDevice *dev)
>          return;
>      }
>      msix_clear_all_vectors(dev);
> +    msix_mask_all(dev, dev->msix_entries_nr);
>      dev->config[dev->msix_cap + MSIX_CONTROL_OFFSET] &=
>             ~dev->wmask[dev->msix_cap + MSIX_CONTROL_OFFSET];
>      memset(dev->msix_table, 0, dev->msix_entries_nr * PCI_MSIX_ENTRY_SIZE);
>      memset(dev->msix_pba, 0, QEMU_ALIGN_UP(dev->msix_entries_nr, 64) / 8);
> -    msix_mask_all(dev, dev->msix_entries_nr);
> +    msix_update_function_masked(dev);
>  }
>
>
> Then either no further changes to this patchset are necessary. Or, if
> relying on unmasks/masks (or
> msix_vector_use_notifier/msix_vector_release_notifier as it's called
> in msix.c) always being balanced is not recommended, the assert will
> simply change into an if.

This is fixed in "msix: don't mask already masked vectors on reset":
https://lists.gnu.org/archive/html/qemu-devel/2017-12/msg01362.html