[Qemu-devel] [PATCH 5/5] Migration: migrate pending_events of spapr state

[Jianjun Duan <duanj@linux.vnet.ibm.com>]

In racing situations between hotplug events and migration operation,
a rtas hotplug event could have not yet be delivered to the source
guest when migration is started. In this case the pending_events of
spapr state need be transmitted to the target so that the hotplug
event can be finished on the target.

All the different fields of the events are encoded as defined by
PAPR. We can migrate them as uint8_t binary stream without any
concerns about data padding or endianess. We tracked the size of
pending_events queue, and introduced a dynamic cache for
pending_events to get around having to create VMSD for the queue.

We also have existence test in place for the newly added
pending_events related fields in spapr state VMSD to make sure
forward migration is not broken.

Signed-off-by: Jianjun Duan <duanj@linux.vnet.ibm.com>
---
 hw/ppc/spapr.c         | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++
 hw/ppc/spapr_events.c  | 24 ++++++++++++++---------
 include/hw/ppc/spapr.h | 14 ++++++++++++-
 3 files changed, 81 insertions(+), 10 deletions(-)

diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index eab95f0..73e4401 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -1248,9 +1248,24 @@ static bool spapr_vga_init(PCIBus *pci_bus, Error **errp)
 static void spapr_pre_save(void *opaque)
 {
     sPAPRMachineState *spapr = (sPAPRMachineState *)opaque;
+    sPAPREventLogEntry *event;
+    sPAPREventLogEntryCache *event_cache;
     sPAPRConfigureConnectorState *ccs;
     sPAPRConfigureConnectorStateCache *ccs_cache;
 
+    /* Copy pending_events to pending_events_cache */
+    spapr->pending_events_cache = g_new0(sPAPREventLogEntryCache,
+                                         spapr->pending_events_num);
+    event_cache = spapr->pending_events_cache;
+    QTAILQ_FOREACH(event, &spapr->pending_events, next) {
+        event_cache->log_type = event->log_type;
+        event_cache->exception = event->exception;
+        event_cache->data_size = event->data_size;
+        event_cache->data = g_malloc0(event_cache->data_size);
+        memcpy(event_cache->data, event->data, event_cache->data_size);
+        event_cache++;
+    }
+
     /* Copy ccs_list to ccs_list_cache */
     spapr->ccs_list_cache = g_new0(sPAPRConfigureConnectorStateCache,
                                    spapr->ccs_list_num);
@@ -1267,6 +1282,8 @@ static int spapr_post_load(void *opaque, int version_id)
 {
     sPAPRMachineState *spapr = (sPAPRMachineState *)opaque;
     int err = 0;
+    sPAPREventLogEntry *event;
+    sPAPREventLogEntryCache *event_cache = spapr->pending_events_cache;
     sPAPRConfigureConnectorState *ccs;
     sPAPRConfigureConnectorStateCache *ccs_cache = spapr->ccs_list_cache;
     int index = 0;
@@ -1282,6 +1299,18 @@ static int spapr_post_load(void *opaque, int version_id)
     if (version_id < 4) {
         return err;
     }
+    /* Copy pending_events_cache to pending_events */
+    for (index = 0; index < spapr->pending_events_num; index++) {
+        event = g_new0(sPAPREventLogEntry, 1);
+        event->log_type = (event_cache + index)->log_type;
+        event->exception = (event_cache + index)->exception;
+        event->data_size = (event_cache + index)->data_size;
+        event->data = g_malloc0(event->data_size);
+        memcpy(event->data, (event_cache + index)->data, event->data_size);
+        QTAILQ_INSERT_TAIL(&spapr->pending_events, event, next);
+    }
+    g_free(spapr->pending_events_cache);
+
     /* Copy ccs_list_cache to ccs_list */
     for (index = 0; index < spapr->ccs_list_num; index ++) {
         ccs = g_new0(sPAPRConfigureConnectorState, 1);
@@ -1305,6 +1334,20 @@ static bool version_ge_4(void *opaque, int version_id)
     return version_id >= 4;
 }
 
+static const VMStateDescription vmstate_spapr_event_cache = {
+    .name = "spapreventlogentrycache",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_INT32(log_type, sPAPREventLogEntryCache),
+        VMSTATE_BOOL(exception, sPAPREventLogEntryCache),
+        VMSTATE_UINT32(data_size, sPAPREventLogEntryCache),
+        VMSTATE_VARRAY_UINT32_ALLOC(data, sPAPREventLogEntryCache, data_size,
+                                    0, vmstate_info_uint8, uint8_t),
+        VMSTATE_END_OF_LIST()
+  },
+};
+
 static const VMStateDescription vmstate_spapr_ccs_cache = {
     .name = "spaprconfigureconnectorstate",
     .version_id = 1,
@@ -1331,6 +1374,16 @@ static const VMStateDescription vmstate_spapr = {
         VMSTATE_UINT64_TEST(rtc_offset, sPAPRMachineState, version_before_3),
 
         VMSTATE_PPC_TIMEBASE_V(tb, sPAPRMachineState, 2),
+
+        /* Pending sPAPR event list*/
+        VMSTATE_INT32_TEST(pending_events_num, sPAPRMachineState,
+                           version_ge_4),
+        VMSTATE_STRUCT_VARRAY_ALLOC_TEST(pending_events_cache,
+                                         sPAPRMachineState,
+                                         version_ge_4,
+                                         pending_events_num, 1,
+                                         vmstate_spapr_event_cache,
+                                         sPAPREventLogEntryCache),
         /* RTAS state */
         VMSTATE_INT32_TEST(ccs_list_num, sPAPRMachineState, version_ge_4),
         VMSTATE_STRUCT_VARRAY_ALLOC_TEST(ccs_list_cache, sPAPRMachineState,
diff --git a/hw/ppc/spapr_events.c b/hw/ppc/spapr_events.c
index 269ab7e..1bf228e 100644
--- a/hw/ppc/spapr_events.c
+++ b/hw/ppc/spapr_events.c
@@ -239,7 +239,8 @@ void spapr_events_fdt_skel(void *fdt, uint32_t check_exception_irq)
     _FDT((fdt_end_node(fdt)));
 }
 
-static void rtas_event_log_queue(int log_type, void *data, bool exception)
+static void rtas_event_log_queue(int log_type, void *data, bool exception,
+				 int data_size)
 {
     sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
     sPAPREventLogEntry *entry = g_new(sPAPREventLogEntry, 1);
@@ -248,7 +249,9 @@ static void rtas_event_log_queue(int log_type, void *data, bool exception)
     entry->log_type = log_type;
     entry->exception = exception;
     entry->data = data;
+    entry->data_size = data_size;
     QTAILQ_INSERT_TAIL(&spapr->pending_events, entry, next);
+    spapr->pending_events_num++;
 }
 
 static sPAPREventLogEntry *rtas_event_log_dequeue(uint32_t event_mask,
@@ -276,6 +279,7 @@ static sPAPREventLogEntry *rtas_event_log_dequeue(uint32_t event_mask,
 
     if (entry) {
         QTAILQ_REMOVE(&spapr->pending_events, entry, next);
+        spapr->pending_events_num--;
     }
 
     return entry;
@@ -350,6 +354,7 @@ static void spapr_powerdown_req(Notifier *n, void *opaque)
     struct rtas_event_log_v6_mainb *mainb;
     struct rtas_event_log_v6_epow *epow;
     struct epow_log_full *new_epow;
+    uint32_t data_size;
 
     new_epow = g_malloc0(sizeof(*new_epow));
     hdr = &new_epow->hdr;
@@ -358,13 +363,13 @@ static void spapr_powerdown_req(Notifier *n, void *opaque)
     mainb = &new_epow->mainb;
     epow = &new_epow->epow;
 
+    data_size = sizeof(*new_epow);
     hdr->summary = cpu_to_be32(RTAS_LOG_VERSION_6
                                | RTAS_LOG_SEVERITY_EVENT
                                | RTAS_LOG_DISPOSITION_NOT_RECOVERED
                                | RTAS_LOG_OPTIONAL_PART_PRESENT
                                | RTAS_LOG_TYPE_EPOW);
-    hdr->extended_length = cpu_to_be32(sizeof(*new_epow)
-                                       - sizeof(new_epow->hdr));
+    hdr->extended_length = cpu_to_be32(data_size - sizeof(new_epow->hdr));
 
     spapr_init_v6hdr(v6hdr);
     spapr_init_maina(maina, 3 /* Main-A, Main-B and EPOW */);
@@ -384,7 +389,7 @@ static void spapr_powerdown_req(Notifier *n, void *opaque)
     epow->event_modifier = RTAS_LOG_V6_EPOW_MODIFIER_NORMAL;
     epow->extended_modifier = RTAS_LOG_V6_EPOW_XMODIFIER_PARTITION_SPECIFIC;
 
-    rtas_event_log_queue(RTAS_LOG_TYPE_EPOW, new_epow, true);
+    rtas_event_log_queue(RTAS_LOG_TYPE_EPOW, new_epow, true, data_size);
 
     qemu_irq_pulse(xics_get_qirq(spapr->icp, spapr->check_exception_irq));
 }
@@ -407,6 +412,7 @@ static void spapr_hotplug_req_event(uint8_t hp_id, uint8_t hp_action,
     struct rtas_event_log_v6_maina *maina;
     struct rtas_event_log_v6_mainb *mainb;
     struct rtas_event_log_v6_hp *hp;
+    uint32_t data_size;
 
     new_hp = g_malloc0(sizeof(struct hp_log_full));
     hdr = &new_hp->hdr;
@@ -415,14 +421,14 @@ static void spapr_hotplug_req_event(uint8_t hp_id, uint8_t hp_action,
     mainb = &new_hp->mainb;
     hp = &new_hp->hp;
 
+    data_size = sizeof(*new_hp);
     hdr->summary = cpu_to_be32(RTAS_LOG_VERSION_6
                                | RTAS_LOG_SEVERITY_EVENT
                                | RTAS_LOG_DISPOSITION_NOT_RECOVERED
                                | RTAS_LOG_OPTIONAL_PART_PRESENT
                                | RTAS_LOG_INITIATOR_HOTPLUG
                                | RTAS_LOG_TYPE_HOTPLUG);
-    hdr->extended_length = cpu_to_be32(sizeof(*new_hp)
-                                       - sizeof(new_hp->hdr));
+    hdr->extended_length = cpu_to_be32(data_size - sizeof(new_hp->hdr));
 
     spapr_init_v6hdr(v6hdr);
     spapr_init_maina(maina, 3 /* Main-A, Main-B, HP */);
@@ -460,7 +466,7 @@ static void spapr_hotplug_req_event(uint8_t hp_id, uint8_t hp_action,
         hp->drc.index = cpu_to_be32(drc);
     }
 
-    rtas_event_log_queue(RTAS_LOG_TYPE_HOTPLUG, new_hp, true);
+    rtas_event_log_queue(RTAS_LOG_TYPE_HOTPLUG, new_hp, true, data_size);
 
     if (hp->hotplug_action == RTAS_LOG_V6_HP_ACTION_ADD) {
         spapr_hotplug_set_signalled(drc);
@@ -531,7 +537,7 @@ static void check_exception(PowerPCCPU *cpu, sPAPRMachineState *spapr,
         goto out_no_events;
     }
 
-    hdr = event->data;
+    hdr = (struct rtas_error_log *)event->data;
     event_len = be32_to_cpu(hdr->extended_length) + sizeof(*hdr);
 
     if (event_len < len) {
@@ -581,7 +587,7 @@ static void event_scan(PowerPCCPU *cpu, sPAPRMachineState *spapr,
         goto out_no_events;
     }
 
-    hdr = event->data;
+    hdr = (struct rtas_error_log *)event->data;
     event_len = be32_to_cpu(hdr->extended_length) + sizeof(*hdr);
 
     if (event_len < len) {
diff --git a/include/hw/ppc/spapr.h b/include/hw/ppc/spapr.h
index c8be926..12419c2 100644
--- a/include/hw/ppc/spapr.h
+++ b/include/hw/ppc/spapr.h
@@ -14,6 +14,7 @@ typedef struct sPAPRConfigureConnectorState sPAPRConfigureConnectorState;
 typedef struct sPAPRConfigureConnectorStateCache
         sPAPRConfigureConnectorStateCache;
 typedef struct sPAPREventLogEntry sPAPREventLogEntry;
+typedef struct sPAPREventLogEntryCache sPAPREventLogEntryCache;
 
 #define HPTE64_V_HPTE_DIRTY     0x0000000000000040ULL
 #define SPAPR_ENTRY_POINT       0x100
@@ -69,6 +70,9 @@ struct sPAPRMachineState {
     uint32_t check_exception_irq;
     Notifier epow_notifier;
     QTAILQ_HEAD(, sPAPREventLogEntry) pending_events;
+    /* Temporary cache for migration purposes */
+    int32_t pending_events_num;
+    sPAPREventLogEntryCache *pending_events_cache;
 
     /* Migration state */
     int htab_save_index;
@@ -557,10 +561,18 @@ sPAPRTCETable *spapr_tce_find_by_liobn(target_ulong liobn);
 struct sPAPREventLogEntry {
     int log_type;
     bool exception;
-    void *data;
+    uint32_t data_size;
+    uint8_t *data;
     QTAILQ_ENTRY(sPAPREventLogEntry) next;
 };
 
+struct sPAPREventLogEntryCache {
+    int log_type;
+    bool exception;
+    uint32_t data_size;
+    uint8_t *data;
+};
+
 void spapr_events_init(sPAPRMachineState *sm);
 void spapr_events_fdt_skel(void *fdt, uint32_t epow_irq);
 int spapr_h_cas_compose_response(sPAPRMachineState *sm,
-- 
1.9.1