Re: [Qemu-devel] [PATCH v2] virtio-pci: fix migration for pci bus master

["Michael S. Tsirkin" <mst@redhat.com>]

On Wed, Oct 15, 2014 at 05:34:25PM +0200, Greg Kurz wrote:
> On Tue, 14 Oct 2014 19:55:23 +0300
> "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > Current support for bus master (clearing OK bit) together with the need to
> > support guests which do not enable PCI bus mastering, leads to extra state in
> > VIRTIO_PCI_FLAG_BUS_MASTER_BUG bit, which isn't robust in case of cross-version
> > migration for the case when guests use the device before setting DRIVER_OK.
> > 
> > Rip out this code, and replace it:
> > -   Modern QEMU doesn't need VIRTIO_PCI_FLAG_BUS_MASTER_BUG
> >     so just drop it for latest machine type.
> > -   For compat machine types, set PCI_COMMAND if DRIVER_OK
> >     is set.
> > 
> > As this is needed for 2.1 for both pc and ppc, move PC_COMPAT macros from pc.h
> > to a new common header.
> > 
> > Cc: Greg Kurz <gkurz@linux.vnet.ibm.com>
> > Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
> > ---
> 
> Just a question, see below.
> 
> Apart from that, it looks good.
> 
> Reviewed-by: Greg Kurz <gkurz@linux.vnet.ibm.com>
> Tested-by: Greg Kurz <gkurz@linux.vnet.ibm.com>
> 
> >  hw/virtio/virtio-pci.h |  5 +++++
> >  include/hw/compat.h    | 16 ++++++++++++++++
> >  include/hw/i386/pc.h   | 10 ++--------
> >  hw/i386/pc_piix.c      |  2 +-
> >  hw/i386/pc_q35.c       |  2 +-
> >  hw/ppc/spapr.c         |  9 ++++++++-
> >  hw/virtio/virtio-pci.c | 29 +++++++++++------------------
> >  7 files changed, 44 insertions(+), 29 deletions(-)
> >  create mode 100644 include/hw/compat.h
> > 
> > diff --git a/hw/virtio/virtio-pci.h b/hw/virtio/virtio-pci.h
> > index 1cea157..8873b6d 100644
> > --- a/hw/virtio/virtio-pci.h
> > +++ b/hw/virtio/virtio-pci.h
> > @@ -53,6 +53,11 @@ typedef struct VirtioBusClass VirtioPCIBusClass;
> >  #define VIRTIO_PCI_BUS_CLASS(klass) \
> >          OBJECT_CLASS_CHECK(VirtioPCIBusClass, klass, TYPE_VIRTIO_PCI_BUS)
> > 
> > +/* Need to activate work-arounds for buggy guests at vmstate load. */
> > +#define VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT  0
> > +#define VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION \
> > +    (1 << VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT)
> > +
> >  /* Performance improves when virtqueue kick processing is decoupled from the
> >   * vcpu thread using ioeventfd for some devices. */
> >  #define VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT 1
> > diff --git a/include/hw/compat.h b/include/hw/compat.h
> > new file mode 100644
> > index 0000000..47f6ff5
> > --- /dev/null
> > +++ b/include/hw/compat.h
> > @@ -0,0 +1,16 @@
> > +#ifndef HW_COMPAT_H
> > +#define HW_COMPAT_H
> > +
> > +#define HW_COMPAT_2_1 \
> > +        {\
> > +            .driver   = "intel-hda",\
> > +            .property = "old_msi_addr",\
> > +            .value    = "on",\
> > +        },\
> > +        {\
> > +            .driver   = "virtio-pci",\
> > +            .property = "virtio-pci-bus-master-bug-migration",\
> > +            .value    = "on",\
> > +        }
> > +
> > +#endif /* HW_COMPAT_H */
> > diff --git a/include/hw/i386/pc.h b/include/hw/i386/pc.h
> > index bae023a..82ad046 100644
> > --- a/include/hw/i386/pc.h
> > +++ b/include/hw/i386/pc.h
> > @@ -14,6 +14,7 @@
> >  #include "sysemu/sysemu.h"
> >  #include "hw/pci/pci.h"
> >  #include "hw/boards.h"
> > +#include "hw/compat.h"
> > 
> >  #define HPET_INTCAP "hpet-intcap"
> > 
> > @@ -307,15 +308,8 @@ int e820_add_entry(uint64_t, uint64_t, uint32_t);
> >  int e820_get_num_entries(void);
> >  bool e820_get_entry(int, uint32_t, uint64_t *, uint64_t *);
> > 
> > -#define PC_COMPAT_2_1 \
> > -        {\
> > -            .driver   = "intel-hda",\
> > -            .property = "old_msi_addr",\
> > -            .value    = "on",\
> > -        }
> > -
> >  #define PC_COMPAT_2_0 \
> > -        PC_COMPAT_2_1, \
> > +        HW_COMPAT_2_1, \
> >          {\
> >              .driver   = "virtio-scsi-pci",\
> >              .property = "any_layout",\
> > diff --git a/hw/i386/pc_piix.c b/hw/i386/pc_piix.c
> > index 553afdd..a1634ab 100644
> > --- a/hw/i386/pc_piix.c
> > +++ b/hw/i386/pc_piix.c
> > @@ -502,7 +502,7 @@ static QEMUMachine pc_i440fx_machine_v2_1 = {
> >      .name = "pc-i440fx-2.1",
> >      .init = pc_init_pci,
> >      .compat_props = (GlobalProperty[]) {
> > -        PC_COMPAT_2_1,
> > +        HW_COMPAT_2_1,
> >          { /* end of list */ }
> >      },
> >  };
> > diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c
> > index a199043..f330f7a 100644
> > --- a/hw/i386/pc_q35.c
> > +++ b/hw/i386/pc_q35.c
> > @@ -365,7 +365,7 @@ static QEMUMachine pc_q35_machine_v2_1 = {
> >      .name = "pc-q35-2.1",
> >      .init = pc_q35_init,
> >      .compat_props = (GlobalProperty[]) {
> > -        PC_COMPAT_2_1,
> > +        HW_COMPAT_2_1,
> >          { /* end of list */ }
> >      },
> >  };
> > diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
> > index 2becc9f..8bc597f 100644
> > --- a/hw/ppc/spapr.c
> > +++ b/hw/ppc/spapr.c
> > @@ -57,6 +57,8 @@
> >  #include "trace.h"
> >  #include "hw/nmi.h"
> > 
> > +#include "hw/compat.h"
> > +
> >  #include <libfdt.h>
> > 
> >  /* SLOF memory layout:
> > @@ -1689,10 +1691,15 @@ static const TypeInfo spapr_machine_info = {
> >  static void spapr_machine_2_1_class_init(ObjectClass *oc, void *data)
> >  {
> >      MachineClass *mc = MACHINE_CLASS(oc);
> > +    static GlobalProperty compat_props[] = {
> > +        HW_COMPAT_2_1,
> > +        { /* end of list */ }
> > +    };
> > 
> >      mc->name = "pseries-2.1";
> >      mc->desc = "pSeries Logical Partition (PAPR compliant) v2.1";
> > -    mc->is_default = 0;
> > +    mc->is_default = -1;
> 
> I could find no other place where we set is_default to -1. What's the point ?

Leftover from unrelated code changes.
I'll drop this, good catch!

> > +    mc->compat_props = compat_props;
> >  }
> > 
> >  static const TypeInfo spapr_machine_2_1_info = {
> > diff --git a/hw/virtio/virtio-pci.c b/hw/virtio/virtio-pci.c
> > index a827cd4..a499a3c 100644
> > --- a/hw/virtio/virtio-pci.c
> > +++ b/hw/virtio/virtio-pci.c
> > @@ -86,9 +86,6 @@
> >   * 12 is historical, and due to x86 page size. */
> >  #define VIRTIO_PCI_QUEUE_ADDR_SHIFT    12
> > 
> > -/* Flags track per-device state like workarounds for quirks in older guests. */
> > -#define VIRTIO_PCI_FLAG_BUS_MASTER_BUG  (1 << 0)
> > -
> >  static void virtio_pci_bus_new(VirtioBusState *bus, size_t bus_size,
> >                                 VirtIOPCIProxy *dev);
> > 
> > @@ -323,14 +320,6 @@ static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
> >                                       proxy->pci_dev.config[PCI_COMMAND] |
> >                                       PCI_COMMAND_MASTER, 1);
> >          }
> > -
> > -        /* Linux before 2.6.34 sets the device as OK without enabling
> > -           the PCI device bus master bit. In this case we need to disable
> > -           some safety checks. */
> > -        if ((val & VIRTIO_CONFIG_S_DRIVER_OK) &&
> > -            !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
> > -            proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
> > -        }
> >          break;
> >      case VIRTIO_MSI_CONFIG_VECTOR:
> >          msix_vector_unuse(&proxy->pci_dev, vdev->config_vector);
> > @@ -483,8 +472,7 @@ static void virtio_write_config(PCIDevice *pci_dev, uint32_t address,
> >      pci_default_write_config(pci_dev, address, val, len);
> > 
> >      if (range_covers_byte(address, len, PCI_COMMAND) &&
> > -        !(pci_dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER) &&
> > -        !(proxy->flags & VIRTIO_PCI_FLAG_BUS_MASTER_BUG)) {
> > +        !(pci_dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
> >          virtio_pci_stop_ioeventfd(proxy);
> >          virtio_set_status(vdev, vdev->status & ~VIRTIO_CONFIG_S_DRIVER_OK);
> >      }
> > @@ -895,11 +883,15 @@ static void virtio_pci_vmstate_change(DeviceState *d, bool running)
> >      VirtIODevice *vdev = virtio_bus_get_device(&proxy->bus);
> > 
> >      if (running) {
> > -        /* Try to find out if the guest has bus master disabled, but is
> > -           in ready state. Then we have a buggy guest OS. */
> > -        if ((vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) &&
> > +        /* Old QEMU versions did not set bus master enable on status write.
> > +         * Detect DRIVER set and enable it.
> > +         */
> > +        if ((proxy->flags & VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION) &&
> > +            (vdev->status & VIRTIO_CONFIG_S_DRIVER) &&
> >              !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) {
> > -            proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
> > +            pci_default_write_config(&proxy->pci_dev, PCI_COMMAND,
> > +                                     proxy->pci_dev.config[PCI_COMMAND] |
> > +                                     PCI_COMMAND_MASTER, 1);
> >          }
> >          virtio_pci_start_ioeventfd(proxy);
> >      } else {
> > @@ -1040,10 +1032,11 @@ static void virtio_pci_reset(DeviceState *qdev)
> >      virtio_pci_stop_ioeventfd(proxy);
> >      virtio_bus_reset(bus);
> >      msix_unuse_all_vectors(&proxy->pci_dev);
> > -    proxy->flags &= ~VIRTIO_PCI_FLAG_BUS_MASTER_BUG;
> >  }
> > 
> >  static Property virtio_pci_properties[] = {
> > +    DEFINE_PROP_BIT("virtio-pci-bus-master-bug-migration", VirtIOPCIProxy, flags,
> > +                    VIRTIO_PCI_FLAG_BUS_MASTER_BUG_MIGRATION_BIT, false),
> >      DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
> >      DEFINE_PROP_END_OF_LIST(),
> >  };