[Qemu-devel] [PATCH 0/6] xhci: add live migration support

[Qemu-devel] [PATCH 0/6] xhci: add live migration support

[Gerd Hoffmann]

  Hi,

This patch series adds live migration support to the xhci usb host
adapter.  Very first cut, survived light testing, not stressed yet.

/me looks for review comments, especially on patch #1 for better
msix live migration.

cheers,
  Gerd

Gerd Hoffmann (6):
  pci: add VMSTATE_MSIX
  xhci: add XHCISlot->addressed
  xhci: add xhci_alloc_epctx
  xhci: add xhci_init_epctx
  xhci: add vmstate
  [debug] xhci: remove unmigratable flag

 hw/pci/msix.c         |   33 ++++++++
 hw/usb/hcd-xhci.c     |  208 +++++++++++++++++++++++++++++++++++++++++++------
 include/hw/pci/msix.h |   11 +++
 3 files changed, 229 insertions(+), 23 deletions(-)

-- 
1.7.9.7

[Qemu-devel] [PATCH 1/6] pci: add VMSTATE_MSIX

[Gerd Hoffmann]

Using a trick cut+pasted from vmstate_scsi_device
to wind up msix_save and msix_load.
---
 hw/pci/msix.c         |   33 +++++++++++++++++++++++++++++++++
 include/hw/pci/msix.h |   11 +++++++++++
 2 files changed, 44 insertions(+)

diff --git a/hw/pci/msix.c b/hw/pci/msix.c
index e231a0d..6da75ec 100644
--- a/hw/pci/msix.c
+++ b/hw/pci/msix.c
@@ -569,3 +569,36 @@ void msix_unset_vector_notifiers(PCIDevice *dev)
     dev->msix_vector_release_notifier = NULL;
     dev->msix_vector_poll_notifier = NULL;
 }
+
+static void put_msix_state(QEMUFile *f, void *pv, size_t size)
+{
+    msix_save(pv, f);
+}
+
+static int get_msix_state(QEMUFile *f, void *pv, size_t size)
+{
+    msix_load(pv, f);
+    return 0;
+}
+
+static VMStateInfo vmstate_info_msix = {
+    .name = "msix state",
+    .get  = get_msix_state,
+    .put  = put_msix_state,
+};
+
+const VMStateDescription vmstate_msix = {
+    .name = "msix",
+    .fields = (VMStateField[]) {
+        {
+            .name         = "msix",
+            .version_id   = 0,
+            .field_exists = NULL,
+            .size         = 0,   /* ouch */
+            .info         = &vmstate_info_msix,
+            .flags        = VMS_SINGLE,
+            .offset       = 0,
+        },
+        VMSTATE_END_OF_LIST()
+    }
+};
diff --git a/include/hw/pci/msix.h b/include/hw/pci/msix.h
index e648410..954d82b 100644
--- a/include/hw/pci/msix.h
+++ b/include/hw/pci/msix.h
@@ -43,4 +43,15 @@ int msix_set_vector_notifiers(PCIDevice *dev,
                               MSIVectorReleaseNotifier release_notifier,
                               MSIVectorPollNotifier poll_notifier);
 void msix_unset_vector_notifiers(PCIDevice *dev);
+
+extern const VMStateDescription vmstate_msix;
+
+#define VMSTATE_MSIX(_field, _state) {                               \
+    .name       = (stringify(_field)),                               \
+    .size       = sizeof(PCIDevice),                                 \
+    .vmsd       = &vmstate_msix,                                     \
+    .flags      = VMS_STRUCT,                                        \
+    .offset     = vmstate_offset_value(_state, _field, PCIDevice),   \
+}
+
 #endif
-- 
1.7.9.7

[Qemu-devel] [PATCH 2/6] xhci: add XHCISlot->addressed

[Gerd Hoffmann]

Preparing for live-migration support, post_load will need that.

Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
---
 hw/usb/hcd-xhci.c |    3 +++
 1 file changed, 3 insertions(+)

diff --git a/hw/usb/hcd-xhci.c b/hw/usb/hcd-xhci.c
index 8813bdf..ac683ce 100644
--- a/hw/usb/hcd-xhci.c
+++ b/hw/usb/hcd-xhci.c
@@ -405,6 +405,7 @@ struct XHCIEPContext {
 
 typedef struct XHCISlot {
     bool enabled;
+    bool addressed;
     dma_addr_t ctx;
     USBPort *uport;
     XHCIEPContext * eps[31];
@@ -2041,6 +2042,7 @@ static TRBCCode xhci_disable_slot(XHCIState *xhci, unsigned int slotid)
     }
 
     xhci->slots[slotid-1].enabled = 0;
+    xhci->slots[slotid-1].addressed = 0;
     return CC_SUCCESS;
 }
 
@@ -2167,6 +2169,7 @@ static TRBCCode xhci_address_slot(XHCIState *xhci, unsigned int slotid,
     xhci_dma_write_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
     xhci_dma_write_u32s(xhci, octx+32, ep0_ctx, sizeof(ep0_ctx));
 
+    xhci->slots[slotid-1].addressed = 1;
     return res;
 }
 
-- 
1.7.9.7

[Qemu-devel] [PATCH 3/6] xhci: add xhci_alloc_epctx

[Gerd Hoffmann]

Factor out endpoint context allocation to a separate function.
xhci live migration will need that too, in post_load.

Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
---
 hw/usb/hcd-xhci.c |   32 +++++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/hw/usb/hcd-xhci.c b/hw/usb/hcd-xhci.c
index ac683ce..5084e52 100644
--- a/hw/usb/hcd-xhci.c
+++ b/hw/usb/hcd-xhci.c
@@ -1198,6 +1198,26 @@ static void xhci_ep_kick_timer(void *opaque)
     xhci_kick_ep(epctx->xhci, epctx->slotid, epctx->epid, 0);
 }
 
+static XHCIEPContext *xhci_alloc_epctx(XHCIState *xhci,
+                                       unsigned int slotid,
+                                       unsigned int epid)
+{
+    XHCIEPContext *epctx;
+    int i;
+
+    epctx = g_new0(XHCIEPContext, 1);
+    epctx->xhci = xhci;
+    epctx->slotid = slotid;
+    epctx->epid = epid;
+
+    for (i = 0; i < ARRAY_SIZE(epctx->transfers); i++) {
+        usb_packet_init(&epctx->transfers[i].packet);
+    }
+    epctx->kick_timer = qemu_new_timer_ns(vm_clock, xhci_ep_kick_timer, epctx);
+
+    return epctx;
+}
+
 static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,
                                unsigned int epid, dma_addr_t pctx,
                                uint32_t *ctx)
@@ -1205,7 +1225,6 @@ static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,
     XHCISlot *slot;
     XHCIEPContext *epctx;
     dma_addr_t dequeue;
-    int i;
 
     trace_usb_xhci_ep_enable(slotid, epid);
     assert(slotid >= 1 && slotid <= xhci->numslots);
@@ -1216,12 +1235,7 @@ static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,
         xhci_disable_ep(xhci, slotid, epid);
     }
 
-    epctx = g_malloc(sizeof(XHCIEPContext));
-    memset(epctx, 0, sizeof(XHCIEPContext));
-    epctx->xhci = xhci;
-    epctx->slotid = slotid;
-    epctx->epid = epid;
-
+    epctx = xhci_alloc_epctx(xhci, slotid, epid);
     slot->eps[epid-1] = epctx;
 
     dequeue = xhci_addr64(ctx[2] & ~0xf, ctx[3]);
@@ -1241,13 +1255,9 @@ static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,
         xhci_ring_init(xhci, &epctx->ring, dequeue);
         epctx->ring.ccs = ctx[2] & 1;
     }
-    for (i = 0; i < ARRAY_SIZE(epctx->transfers); i++) {
-        usb_packet_init(&epctx->transfers[i].packet);
-    }
 
     epctx->interval = 1 << (ctx[0] >> 16) & 0xff;
     epctx->mfindex_last = 0;
-    epctx->kick_timer = qemu_new_timer_ns(vm_clock, xhci_ep_kick_timer, epctx);
 
     epctx->state = EP_RUNNING;
     ctx[0] &= ~EP_STATE_MASK;
-- 
1.7.9.7

[Qemu-devel] [PATCH 4/6] xhci: add xhci_init_epctx

[Gerd Hoffmann]

Factor out endpoint context initialization to a separate function.
xhci live migration will need that too, in post_load.

Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
---
 hw/usb/hcd-xhci.c |   43 +++++++++++++++++++++++++------------------
 1 file changed, 25 insertions(+), 18 deletions(-)

diff --git a/hw/usb/hcd-xhci.c b/hw/usb/hcd-xhci.c
index 5084e52..9b90067 100644
--- a/hw/usb/hcd-xhci.c
+++ b/hw/usb/hcd-xhci.c
@@ -1218,26 +1218,11 @@ static XHCIEPContext *xhci_alloc_epctx(XHCIState *xhci,
     return epctx;
 }
 
-static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,
-                               unsigned int epid, dma_addr_t pctx,
-                               uint32_t *ctx)
+static void xhci_init_epctx(XHCIEPContext *epctx,
+                            dma_addr_t pctx, uint32_t *ctx)
 {
-    XHCISlot *slot;
-    XHCIEPContext *epctx;
     dma_addr_t dequeue;
 
-    trace_usb_xhci_ep_enable(slotid, epid);
-    assert(slotid >= 1 && slotid <= xhci->numslots);
-    assert(epid >= 1 && epid <= 31);
-
-    slot = &xhci->slots[slotid-1];
-    if (slot->eps[epid-1]) {
-        xhci_disable_ep(xhci, slotid, epid);
-    }
-
-    epctx = xhci_alloc_epctx(xhci, slotid, epid);
-    slot->eps[epid-1] = epctx;
-
     dequeue = xhci_addr64(ctx[2] & ~0xf, ctx[3]);
 
     epctx->type = (ctx[1] >> EP_TYPE_SHIFT) & EP_TYPE_MASK;
@@ -1252,11 +1237,33 @@ static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,
     if (epctx->max_pstreams) {
         xhci_alloc_streams(epctx, dequeue);
     } else {
-        xhci_ring_init(xhci, &epctx->ring, dequeue);
+        xhci_ring_init(epctx->xhci, &epctx->ring, dequeue);
         epctx->ring.ccs = ctx[2] & 1;
     }
 
     epctx->interval = 1 << (ctx[0] >> 16) & 0xff;
+}
+
+static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,
+                               unsigned int epid, dma_addr_t pctx,
+                               uint32_t *ctx)
+{
+    XHCISlot *slot;
+    XHCIEPContext *epctx;
+
+    trace_usb_xhci_ep_enable(slotid, epid);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
+    assert(epid >= 1 && epid <= 31);
+
+    slot = &xhci->slots[slotid-1];
+    if (slot->eps[epid-1]) {
+        xhci_disable_ep(xhci, slotid, epid);
+    }
+
+    epctx = xhci_alloc_epctx(xhci, slotid, epid);
+    slot->eps[epid-1] = epctx;
+    xhci_init_epctx(epctx, pctx, ctx);
+
     epctx->mfindex_last = 0;
 
     epctx->state = EP_RUNNING;
-- 
1.7.9.7

[Qemu-devel] [PATCH 5/6] xhci: add vmstate

[Gerd Hoffmann]

Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
---
 hw/usb/hcd-xhci.c |  145 ++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 144 insertions(+), 1 deletion(-)

diff --git a/hw/usb/hcd-xhci.c b/hw/usb/hcd-xhci.c
index 9b90067..426478c 100644
--- a/hw/usb/hcd-xhci.c
+++ b/hw/usb/hcd-xhci.c
@@ -3386,9 +3386,152 @@ static int usb_xhci_initfn(struct PCIDevice *dev)
     return 0;
 }
 
+static int usb_xhci_post_load(void *opaque, int version_id)
+{
+    XHCIState *xhci = opaque;
+    XHCISlot *slot;
+    XHCIEPContext *epctx;
+    dma_addr_t dcbaap, pctx;
+    uint32_t slot_ctx[4];
+    uint32_t ep_ctx[5];
+    int slotid, epid, state, intr;
+
+    dcbaap = xhci_addr64(xhci->dcbaap_low, xhci->dcbaap_high);
+
+    for (slotid = 1; slotid <= xhci->numslots; slotid++) {
+        slot = &xhci->slots[slotid-1];
+        if (!slot->addressed) {
+            continue;
+        }
+        slot->ctx =
+            xhci_mask64(ldq_le_pci_dma(&xhci->pci_dev, dcbaap + 8*slotid));
+        xhci_dma_read_u32s(xhci, slot->ctx, slot_ctx, sizeof(slot_ctx));
+        slot->uport = xhci_lookup_uport(xhci, slot_ctx);
+        assert(slot->uport && slot->uport->dev);
+
+        for (epid = 1; epid <= 32; epid++) {
+            pctx = slot->ctx + 32 * epid;
+            xhci_dma_read_u32s(xhci, pctx, ep_ctx, sizeof(ep_ctx));
+            state = ep_ctx[0] & EP_STATE_MASK;
+            if (state == EP_DISABLED) {
+                continue;
+            }
+            epctx = xhci_alloc_epctx(xhci, slotid, epid);
+            slot->eps[epid-1] = epctx;
+            xhci_init_epctx(epctx, pctx, ep_ctx);
+            epctx->state = state;
+            if (state == EP_RUNNING) {
+                /* kick endpoint after vmload is finished */
+                qemu_mod_timer(epctx->kick_timer, qemu_get_clock_ns(vm_clock));
+            }
+        }
+    }
+
+    for (intr = 0; intr < xhci->numintrs; intr++) {
+        if (xhci->intr[intr].msix_used) {
+            msix_vector_use(&xhci->pci_dev, intr);
+        } else {
+            msix_vector_unuse(&xhci->pci_dev, intr);
+        }
+    }
+
+    return 0;
+}
+
+static const VMStateDescription vmstate_xhci_ring = {
+    .name = "xhci-ring",
+    .version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(dequeue, XHCIRing),
+        VMSTATE_BOOL(ccs, XHCIRing),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_xhci_port = {
+    .name = "xhci-port",
+    .version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(portsc, XHCIPort),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_xhci_slot = {
+    .name = "xhci-slot",
+    .version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_BOOL(enabled,   XHCISlot),
+        VMSTATE_BOOL(addressed, XHCISlot),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_xhci_intr = {
+    .name = "xhci-intr",
+    .version_id = 1,
+    .fields = (VMStateField[]) {
+        /* registers */
+        VMSTATE_UINT32(iman,          XHCIInterrupter),
+        VMSTATE_UINT32(imod,          XHCIInterrupter),
+        VMSTATE_UINT32(erstsz,        XHCIInterrupter),
+        VMSTATE_UINT32(erstba_low,    XHCIInterrupter),
+        VMSTATE_UINT32(erstba_high,   XHCIInterrupter),
+        VMSTATE_UINT32(erdp_low,      XHCIInterrupter),
+        VMSTATE_UINT32(erdp_high,     XHCIInterrupter),
+
+        /* state */
+        VMSTATE_BOOL(msix_used,       XHCIInterrupter),
+        VMSTATE_BOOL(er_pcs,          XHCIInterrupter),
+        VMSTATE_UINT64(er_start,      XHCIInterrupter),
+        VMSTATE_UINT32(er_size,       XHCIInterrupter),
+        VMSTATE_UINT32(er_ep_idx,     XHCIInterrupter),
+
+#if 0
+        /* event queue (used if ring is full) */
+        VMSTATE_BOOL(er_full,         XHCIInterrupter),
+        VMSTATE_UINT32(ev_buffer_put, XHCIInterrupter),
+        VMSTATE_UINT32(ev_buffer_get, XHCIInterrupter),
+        /* TODO */ XHCIEvent ev_buffer[EV_QUEUE];
+#endif
+
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static const VMStateDescription vmstate_xhci = {
     .name = "xhci",
-    .unmigratable = 1,
+    .unmigratable = 1,  /* not finished yet */
+    .version_id = 1,
+    .post_load = usb_xhci_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCIE_DEVICE(pci_dev, XHCIState),
+        VMSTATE_MSIX(pci_dev, XHCIState),
+
+        VMSTATE_STRUCT_ARRAY(ports, XHCIState, MAXPORTS, 1,
+                             vmstate_xhci_port, XHCIPort),
+        VMSTATE_STRUCT_ARRAY(slots, XHCIState, MAXSLOTS, 1,
+                             vmstate_xhci_slot, XHCISlot),
+        VMSTATE_STRUCT_ARRAY(intr, XHCIState, MAXINTRS, 1,
+                             vmstate_xhci_intr, XHCIInterrupter),
+
+        /* Operational Registers */
+        VMSTATE_UINT32(usbcmd,        XHCIState),
+        VMSTATE_UINT32(usbsts,        XHCIState),
+        VMSTATE_UINT32(dnctrl,        XHCIState),
+        VMSTATE_UINT32(crcr_low,      XHCIState),
+        VMSTATE_UINT32(crcr_high,     XHCIState),
+        VMSTATE_UINT32(dcbaap_low,    XHCIState),
+        VMSTATE_UINT32(dcbaap_high,   XHCIState),
+        VMSTATE_UINT32(config,        XHCIState),
+
+        /* Runtime Registers & state */
+        VMSTATE_INT64(mfindex_start,  XHCIState),
+        VMSTATE_TIMER(mfwrap_timer,   XHCIState),
+        VMSTATE_STRUCT(cmd_ring, XHCIState, 1, vmstate_xhci_ring, XHCIRing),
+
+        VMSTATE_END_OF_LIST()
+    }
 };
 
 static Property xhci_properties[] = {
-- 
1.7.9.7

[Qemu-devel] [PATCH 6/6] [debug] xhci: remove unmigratable flag

[Gerd Hoffmann]

For testing only, migration support isn't finished yet.

Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
---
 hw/usb/hcd-xhci.c |    1 -
 1 file changed, 1 deletion(-)

diff --git a/hw/usb/hcd-xhci.c b/hw/usb/hcd-xhci.c
index 426478c..4ab138a 100644
--- a/hw/usb/hcd-xhci.c
+++ b/hw/usb/hcd-xhci.c
@@ -3501,7 +3501,6 @@ static const VMStateDescription vmstate_xhci_intr = {
 
 static const VMStateDescription vmstate_xhci = {
     .name = "xhci",
-    .unmigratable = 1,  /* not finished yet */
     .version_id = 1,
     .post_load = usb_xhci_post_load,
     .fields = (VMStateField[]) {
-- 
1.7.9.7

Re: [Qemu-devel] [PATCH 1/6] pci: add VMSTATE_MSIX

[Michael S. Tsirkin]

On Tue, May 07, 2013 at 03:34:31PM +0200, Gerd Hoffmann wrote:
> Using a trick cut+pasted from vmstate_scsi_device
> to wind up msix_save and msix_load.

Any reason this is not signed-off by you?

> ---
>  hw/pci/msix.c         |   33 +++++++++++++++++++++++++++++++++
>  include/hw/pci/msix.h |   11 +++++++++++
>  2 files changed, 44 insertions(+)
> 
> diff --git a/hw/pci/msix.c b/hw/pci/msix.c
> index e231a0d..6da75ec 100644
> --- a/hw/pci/msix.c
> +++ b/hw/pci/msix.c
> @@ -569,3 +569,36 @@ void msix_unset_vector_notifiers(PCIDevice *dev)
>      dev->msix_vector_release_notifier = NULL;
>      dev->msix_vector_poll_notifier = NULL;
>  }
> +
> +static void put_msix_state(QEMUFile *f, void *pv, size_t size)
> +{
> +    msix_save(pv, f);
> +}
> +
> +static int get_msix_state(QEMUFile *f, void *pv, size_t size)
> +{
> +    msix_load(pv, f);
> +    return 0;
> +}
> +
> +static VMStateInfo vmstate_info_msix = {
> +    .name = "msix state",
> +    .get  = get_msix_state,
> +    .put  = put_msix_state,
> +};
> +
> +const VMStateDescription vmstate_msix = {
> +    .name = "msix",
> +    .fields = (VMStateField[]) {
> +        {
> +            .name         = "msix",
> +            .version_id   = 0,
> +            .field_exists = NULL,
> +            .size         = 0,   /* ouch */
> +            .info         = &vmstate_info_msix,
> +            .flags        = VMS_SINGLE,
> +            .offset       = 0,
> +        },
> +        VMSTATE_END_OF_LIST()
> +    }
> +};
> diff --git a/include/hw/pci/msix.h b/include/hw/pci/msix.h
> index e648410..954d82b 100644
> --- a/include/hw/pci/msix.h
> +++ b/include/hw/pci/msix.h
> @@ -43,4 +43,15 @@ int msix_set_vector_notifiers(PCIDevice *dev,
>                                MSIVectorReleaseNotifier release_notifier,
>                                MSIVectorPollNotifier poll_notifier);
>  void msix_unset_vector_notifiers(PCIDevice *dev);
> +
> +extern const VMStateDescription vmstate_msix;
> +
> +#define VMSTATE_MSIX(_field, _state) {                               \
> +    .name       = (stringify(_field)),                               \
> +    .size       = sizeof(PCIDevice),                                 \
> +    .vmsd       = &vmstate_msix,                                     \
> +    .flags      = VMS_STRUCT,                                        \
> +    .offset     = vmstate_offset_value(_state, _field, PCIDevice),   \
> +}
> +
>  #endif
> -- 
> 1.7.9.7
> 

Re: [Qemu-devel] [PATCH 5/6] xhci: add vmstate

[Michael S. Tsirkin]

On Tue, May 07, 2013 at 03:34:35PM +0200, Gerd Hoffmann wrote:
> Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
> ---
>  hw/usb/hcd-xhci.c |  145 ++++++++++++++++++++++++++++++++++++++++++++++++++++-
>  1 file changed, 144 insertions(+), 1 deletion(-)
> 
> diff --git a/hw/usb/hcd-xhci.c b/hw/usb/hcd-xhci.c
> index 9b90067..426478c 100644
> --- a/hw/usb/hcd-xhci.c
> +++ b/hw/usb/hcd-xhci.c
> @@ -3386,9 +3386,152 @@ static int usb_xhci_initfn(struct PCIDevice *dev)
>      return 0;
>  }
>  
> +static int usb_xhci_post_load(void *opaque, int version_id)
> +{
> +    XHCIState *xhci = opaque;
> +    XHCISlot *slot;
> +    XHCIEPContext *epctx;
> +    dma_addr_t dcbaap, pctx;
> +    uint32_t slot_ctx[4];
> +    uint32_t ep_ctx[5];
> +    int slotid, epid, state, intr;
> +
> +    dcbaap = xhci_addr64(xhci->dcbaap_low, xhci->dcbaap_high);
> +
> +    for (slotid = 1; slotid <= xhci->numslots; slotid++) {
> +        slot = &xhci->slots[slotid-1];
> +        if (!slot->addressed) {
> +            continue;
> +        }
> +        slot->ctx =
> +            xhci_mask64(ldq_le_pci_dma(&xhci->pci_dev, dcbaap + 8*slotid));
> +        xhci_dma_read_u32s(xhci, slot->ctx, slot_ctx, sizeof(slot_ctx));
> +        slot->uport = xhci_lookup_uport(xhci, slot_ctx);
> +        assert(slot->uport && slot->uport->dev);
> +
> +        for (epid = 1; epid <= 32; epid++) {
> +            pctx = slot->ctx + 32 * epid;
> +            xhci_dma_read_u32s(xhci, pctx, ep_ctx, sizeof(ep_ctx));
> +            state = ep_ctx[0] & EP_STATE_MASK;
> +            if (state == EP_DISABLED) {
> +                continue;
> +            }
> +            epctx = xhci_alloc_epctx(xhci, slotid, epid);
> +            slot->eps[epid-1] = epctx;
> +            xhci_init_epctx(epctx, pctx, ep_ctx);
> +            epctx->state = state;
> +            if (state == EP_RUNNING) {
> +                /* kick endpoint after vmload is finished */
> +                qemu_mod_timer(epctx->kick_timer, qemu_get_clock_ns(vm_clock));
> +            }
> +        }
> +    }
> +
> +    for (intr = 0; intr < xhci->numintrs; intr++) {
> +        if (xhci->intr[intr].msix_used) {
> +            msix_vector_use(&xhci->pci_dev, intr);
> +        } else {
> +            msix_vector_unuse(&xhci->pci_dev, intr);
> +        }
> +    }
> +
> +    return 0;
> +}
> +
> +static const VMStateDescription vmstate_xhci_ring = {
> +    .name = "xhci-ring",
> +    .version_id = 1,
> +    .fields = (VMStateField[]) {
> +        VMSTATE_UINT64(dequeue, XHCIRing),
> +        VMSTATE_BOOL(ccs, XHCIRing),
> +        VMSTATE_END_OF_LIST()
> +    }
> +};
> +
> +static const VMStateDescription vmstate_xhci_port = {
> +    .name = "xhci-port",
> +    .version_id = 1,
> +    .fields = (VMStateField[]) {
> +        VMSTATE_UINT32(portsc, XHCIPort),
> +        VMSTATE_END_OF_LIST()
> +    }
> +};
> +
> +static const VMStateDescription vmstate_xhci_slot = {
> +    .name = "xhci-slot",
> +    .version_id = 1,
> +    .fields = (VMStateField[]) {
> +        VMSTATE_BOOL(enabled,   XHCISlot),
> +        VMSTATE_BOOL(addressed, XHCISlot),
> +        VMSTATE_END_OF_LIST()
> +    }
> +};
> +
> +static const VMStateDescription vmstate_xhci_intr = {
> +    .name = "xhci-intr",
> +    .version_id = 1,
> +    .fields = (VMStateField[]) {
> +        /* registers */
> +        VMSTATE_UINT32(iman,          XHCIInterrupter),
> +        VMSTATE_UINT32(imod,          XHCIInterrupter),
> +        VMSTATE_UINT32(erstsz,        XHCIInterrupter),
> +        VMSTATE_UINT32(erstba_low,    XHCIInterrupter),
> +        VMSTATE_UINT32(erstba_high,   XHCIInterrupter),
> +        VMSTATE_UINT32(erdp_low,      XHCIInterrupter),
> +        VMSTATE_UINT32(erdp_high,     XHCIInterrupter),
> +
> +        /* state */
> +        VMSTATE_BOOL(msix_used,       XHCIInterrupter),
> +        VMSTATE_BOOL(er_pcs,          XHCIInterrupter),
> +        VMSTATE_UINT64(er_start,      XHCIInterrupter),
> +        VMSTATE_UINT32(er_size,       XHCIInterrupter),
> +        VMSTATE_UINT32(er_ep_idx,     XHCIInterrupter),
> +
> +#if 0

should have a comment explaining why is this
commented out.
Or just drop this section.

> +        /* event queue (used if ring is full) */
> +        VMSTATE_BOOL(er_full,         XHCIInterrupter),
> +        VMSTATE_UINT32(ev_buffer_put, XHCIInterrupter),
> +        VMSTATE_UINT32(ev_buffer_get, XHCIInterrupter),
> +        /* TODO */ XHCIEvent ev_buffer[EV_QUEUE];
> +#endif
> +
> +        VMSTATE_END_OF_LIST()
> +    }
> +};
> +
>  static const VMStateDescription vmstate_xhci = {
>      .name = "xhci",
> -    .unmigratable = 1,
> +    .unmigratable = 1,  /* not finished yet */
> +    .version_id = 1,
> +    .post_load = usb_xhci_post_load,
> +    .fields = (VMStateField[]) {
> +        VMSTATE_PCIE_DEVICE(pci_dev, XHCIState),
> +        VMSTATE_MSIX(pci_dev, XHCIState),
> +
> +        VMSTATE_STRUCT_ARRAY(ports, XHCIState, MAXPORTS, 1,
> +                             vmstate_xhci_port, XHCIPort),
> +        VMSTATE_STRUCT_ARRAY(slots, XHCIState, MAXSLOTS, 1,
> +                             vmstate_xhci_slot, XHCISlot),
> +        VMSTATE_STRUCT_ARRAY(intr, XHCIState, MAXINTRS, 1,
> +                             vmstate_xhci_intr, XHCIInterrupter),
> +
> +        /* Operational Registers */
> +        VMSTATE_UINT32(usbcmd,        XHCIState),
> +        VMSTATE_UINT32(usbsts,        XHCIState),
> +        VMSTATE_UINT32(dnctrl,        XHCIState),
> +        VMSTATE_UINT32(crcr_low,      XHCIState),
> +        VMSTATE_UINT32(crcr_high,     XHCIState),
> +        VMSTATE_UINT32(dcbaap_low,    XHCIState),
> +        VMSTATE_UINT32(dcbaap_high,   XHCIState),
> +        VMSTATE_UINT32(config,        XHCIState),
> +
> +        /* Runtime Registers & state */
> +        VMSTATE_INT64(mfindex_start,  XHCIState),
> +        VMSTATE_TIMER(mfwrap_timer,   XHCIState),
> +        VMSTATE_STRUCT(cmd_ring, XHCIState, 1, vmstate_xhci_ring, XHCIRing),
> +
> +        VMSTATE_END_OF_LIST()
> +    }
>  };
>  
>  static Property xhci_properties[] = {
> -- 
> 1.7.9.7
> 

Re: [Qemu-devel] [PATCH 1/6] pci: add VMSTATE_MSIX

[Gerd Hoffmann]

On 05/07/13 16:05, Michael S. Tsirkin wrote:
> On Tue, May 07, 2013 at 03:34:31PM +0200, Gerd Hoffmann wrote:
>> > Using a trick cut+pasted from vmstate_scsi_device
>> > to wind up msix_save and msix_load.
> Any reason this is not signed-off by you?
> 
Just forgot it, here we go:

Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>

cheers,
  Gerd

Re: [Qemu-devel] [PATCH 1/6] pci: add VMSTATE_MSIX

[Michael S. Tsirkin]

On Tue, May 07, 2013 at 04:08:46PM +0200, Gerd Hoffmann wrote:
> On 05/07/13 16:05, Michael S. Tsirkin wrote:
> > On Tue, May 07, 2013 at 03:34:31PM +0200, Gerd Hoffmann wrote:
> >> > Using a trick cut+pasted from vmstate_scsi_device
> >> > to wind up msix_save and msix_load.
> > Any reason this is not signed-off by you?
> > 
> Just forgot it, here we go:
> 
> Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
> 
> cheers,
>   Gerd

I'm fine with the patch itself.

Acked-by: Michael S. Tsirkin <mst@redhat.com>

Re: [Qemu-devel] [PATCH 5/6] xhci: add vmstate

[Gerd Hoffmann]

On 05/07/13 16:06, Michael S. Tsirkin wrote:
>> +#if 0
> should have a comment explaining why is this
> commented out.
> Or just drop this section.
> 
>> > +        /* event queue (used if ring is full) */
>> > +        VMSTATE_BOOL(er_full,         XHCIInterrupter),
>> > +        VMSTATE_UINT32(ev_buffer_put, XHCIInterrupter),
>> > +        VMSTATE_UINT32(ev_buffer_get, XHCIInterrupter),
>> > +        /* TODO */ XHCIEvent ev_buffer[EV_QUEUE];
>> > +#endif
>> > 

xhci buffers events in case the ring is full.  Doesn't happen under
normal circumstances, thats why I didn't bother finishing it (yet).

Doing that and stress-testing xhci migration are the todo list items
before I'll go merge this series.

cheers,
  Gerd

Re: [Qemu-devel] [PATCH 5/6] xhci: add vmstate

[Michael S. Tsirkin]

On Tue, May 07, 2013 at 04:14:01PM +0200, Gerd Hoffmann wrote:
> On 05/07/13 16:06, Michael S. Tsirkin wrote:
> >> +#if 0
> > should have a comment explaining why is this
> > commented out.
> > Or just drop this section.
> > 
> >> > +        /* event queue (used if ring is full) */
> >> > +        VMSTATE_BOOL(er_full,         XHCIInterrupter),
> >> > +        VMSTATE_UINT32(ev_buffer_put, XHCIInterrupter),
> >> > +        VMSTATE_UINT32(ev_buffer_get, XHCIInterrupter),
> >> > +        /* TODO */ XHCIEvent ev_buffer[EV_QUEUE];
> >> > +#endif
> >> > 
> 
> xhci buffers events in case the ring is full.  Doesn't happen under
> normal circumstances, thats why I didn't bother finishing it (yet).
> 
> Doing that and stress-testing xhci migration are the todo list items
> before I'll go merge this series.
> 
> cheers,
>   Gerd

'PATCH notformerge' or something like this in the subj would be helpful,
in the future.


-- 
MST