[Qemu-devel] [QEMU RFC PATCH v3 4/6] Migration: migrate QTAILQ

[Jianjun Duan <duanj@linux.vnet.ibm.com>]

Currently we cannot directly transfer a QTAILQ instance because of the
limitation in the migration code. Here we introduce an approach to
transfer such structures. In our approach such a structure is tagged
with VMS_CSTM. We then modified vmstate_save_state and vmstate_load_state
so that when VMS_CSTM is encountered, put and get from VMStateInfo are
called respectively. This approach will be used to transfer pending_events
and ccs_list in spapr state.

We also create some macros in qemu/queue.h to access a QTAILQ using pointer
arithmetic. This ensures that we do not depend on the implementation
details about QTAILQ in the migration code.

Signed-off-by: Jianjun Duan <duanj@linux.vnet.ibm.com>
---
 include/migration/vmstate.h | 22 +++++++++++++
 include/qemu/queue.h        | 32 ++++++++++++++++++
 migration/vmstate.c         | 79 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 133 insertions(+)

diff --git a/include/migration/vmstate.h b/include/migration/vmstate.h
index 56a4171..da4ef7f 100644
--- a/include/migration/vmstate.h
+++ b/include/migration/vmstate.h
@@ -185,6 +185,8 @@ enum VMStateFlags {
      * to determine the number of entries in the array. Only valid in
      * combination with one of VMS_VARRAY*. */
     VMS_MULTIPLY_ELEMENTS = 0x4000,
+    /* For fields which need customized handling, such as QTAILQ in queue.h*/
+    VMS_CSTM            = 0x8000,
 };
 
 struct VMStateField {
@@ -245,6 +247,7 @@ extern const VMStateInfo vmstate_info_timer;
 extern const VMStateInfo vmstate_info_buffer;
 extern const VMStateInfo vmstate_info_unused_buffer;
 extern const VMStateInfo vmstate_info_bitmap;
+extern const VMStateInfo vmstate_info_qtailq;
 
 #define type_check_2darray(t1,t2,n,m) ((t1(*)[n][m])0 - (t2*)0)
 #define type_check_array(t1,t2,n) ((t1(*)[n])0 - (t2*)0)
@@ -656,6 +659,25 @@ extern const VMStateInfo vmstate_info_bitmap;
     .offset       = offsetof(_state, _field),                        \
 }
 
+/* For QTAILQ that need customized handling
+ * _type: type of QTAILQ element
+ * _next: name of QTAILQ entry field in QTAILQ element
+ * _vmsd: VMSD for QTAILQ element
+ * size: size of QTAILQ element
+ * start: offset of QTAILQ entry in QTAILQ element
+ */
+#define VMSTATE_QTAILQ_V(_field, _state, _version, _vmsd, _type, _next)  \
+{                                                                        \
+    .name         = (stringify(_field)),                                 \
+    .version_id   = (_version),                                          \
+    .vmsd         = &(_vmsd),                                            \
+    .size         = sizeof(_type),                                       \
+    .info         = &vmstate_info_qtailq,                                \
+    .flags        = VMS_CSTM,                                            \
+    .offset       = offsetof(_state, _field),                            \
+    .start        = offsetof(_type, _next),                              \
+}
+
 /* _f : field name
    _f_n : num of elements field_name
    _n : num of elements
diff --git a/include/qemu/queue.h b/include/qemu/queue.h
index f781aa2..003e368 100644
--- a/include/qemu/queue.h
+++ b/include/qemu/queue.h
@@ -437,3 +437,35 @@ struct {                                                                \
         (*(((struct headname *)((elm)->field.tqe_prev))->tqh_last))
 
 #endif  /* !QEMU_SYS_QUEUE_H_ */
+
+/*
+ * Offsets of layout of a tail queue head.
+ */
+#define QTAILQ_FIRST_OFFSET 0
+#define QTAILQ_LAST_OFFSET (sizeof(void *))
+
+/*
+ * Offsets of layout of a tail queue element.
+ */
+#define QTAILQ_NEXT_OFFSET 0
+#define QTAILQ_PREV_OFFSET (sizeof(void *))
+
+/*
+ * Tail queue tranversal using pointer arithmetic.
+ */
+#define QTAILQ_RAW_FOREACH(elm, head, entry)                                   \
+        for ((elm) = *((void **) ((char *) (head) + QTAILQ_FIRST_OFFSET));     \
+             (elm);                                                            \
+             (elm) =                                                           \
+                 *((void **) ((char *) (elm) + (entry) + QTAILQ_NEXT_OFFSET)))
+/*
+ * Tail queue insertion using pointer arithmetic.
+ */
+#define QTAILQ_RAW_INSERT_TAIL(head, elm, entry) do {                          \
+        *((void **) ((char *) (elm) + (entry) + QTAILQ_NEXT_OFFSET)) = NULL;   \
+        *((void **) ((char *) (elm) + (entry) + QTAILQ_PREV_OFFSET)) =         \
+            *((void **) ((char *) (head) +QTAILQ_LAST_OFFSET));                \
+        **((void ***)((char *) (head) +QTAILQ_LAST_OFFSET)) = (elm);           \
+        *((void **) ((char *) (head) + QTAILQ_LAST_OFFSET)) =                  \
+            (void *) ((char *) (elm) + (entry) + QTAILQ_NEXT_OFFSET);          \
+} while (/*CONSTCOND*/0)
diff --git a/migration/vmstate.c b/migration/vmstate.c
index 644ba1f..ff56650 100644
--- a/migration/vmstate.c
+++ b/migration/vmstate.c
@@ -5,7 +5,9 @@
 #include "migration/vmstate.h"
 #include "qemu/bitops.h"
 #include "qemu/error-report.h"
+#include "qemu/queue.h"
 #include "trace.h"
+#include "migration/qjson.h"
 
 static void vmstate_subsection_save(QEMUFile *f, const VMStateDescription *vmsd,
                                     void *opaque, QJSON *vmdesc);
@@ -120,6 +122,8 @@ int vmstate_load_state(QEMUFile *f, const VMStateDescription *vmsd,
                 if (field->flags & VMS_STRUCT) {
                     ret = vmstate_load_state(f, field->vmsd, addr,
                                              field->vmsd->version_id);
+                } else if (field->flags & VMS_CSTM) {
+                    ret = field->info->get(f, addr, size, field);
                 } else {
                     ret = field->info->get(f, addr, size, NULL);
 
@@ -192,6 +196,8 @@ static const char *vmfield_get_type_name(VMStateField *field)
 
     if (field->flags & VMS_STRUCT) {
         type = "struct";
+    } else if (field->flags & VMS_CSTM) {
+        type = "customized";
     } else if (field->info->name) {
         type = field->info->name;
     }
@@ -326,6 +332,8 @@ void vmstate_save_state(QEMUFile *f, const VMStateDescription *vmsd,
                 }
                 if (field->flags & VMS_STRUCT) {
                     vmstate_save_state(f, field->vmsd, addr, vmdesc_loop);
+                } else if  (field->flags & VMS_CSTM) {
+                    field->info->put(f, addr, size, field, vmdesc_loop);
                 } else {
                     field->info->put(f, addr, size, NULL, NULL);
                 }
@@ -938,3 +946,74 @@ const VMStateInfo vmstate_info_bitmap = {
     .get = get_bitmap,
     .put = put_bitmap,
 };
+
+/*get for QTAILQ */
+static int get_qtailq(QEMUFile *f, void *pv, size_t unused_size,
+                      VMStateField *field)
+{
+    bool link;
+    int ret = 0;
+    const VMStateDescription *vmsd = field->vmsd;
+    size_t size = field->size;
+    size_t entry = field->start;
+    int version_id = field->version_id;
+    void *elm;
+
+    trace_vmstate_load_state(vmsd->name, version_id);
+    if (version_id > vmsd->version_id) {
+        trace_vmstate_load_state_end(vmsd->name, "too new", -EINVAL);
+        return -EINVAL;
+    }
+    if (version_id < vmsd->minimum_version_id) {
+        trace_vmstate_load_state_end(vmsd->name, "too old", -EINVAL);
+        return -EINVAL;
+    }
+
+    while (true) {
+        vmstate_info_bool.get(f, &link, sizeof(bool), NULL);
+        if (!link) {
+            break;
+        }
+
+        elm =  g_malloc(size);
+        ret = vmstate_load_state(f, vmsd, elm, version_id);
+        if (ret) {
+            return ret;
+        }
+        QTAILQ_RAW_INSERT_TAIL(pv, elm, entry);
+    }
+
+    trace_vmstate_load_state_end(vmsd->name, "end", ret);
+    return ret;
+}
+
+/* put for QTAILQ */
+static void put_qtailq(QEMUFile *f, void *pv, size_t unused_size,
+                       VMStateField *field, QJSON *vmdesc)
+{
+    bool link = true;
+    const VMStateDescription *vmsd = field->vmsd;
+    size_t entry = field->start;
+    void *elm;
+
+    if (vmdesc) {
+        json_prop_str(vmdesc, "vmsd_name", vmsd->name);
+        json_prop_int(vmdesc, "version", vmsd->version_id);
+        json_start_array(vmdesc, "fields");
+    }
+
+    QTAILQ_RAW_FOREACH(elm, pv, entry) {
+        vmstate_info_bool.put(f, &link, sizeof(bool), NULL, NULL);
+        vmstate_save_state(f, vmsd, elm, vmdesc);
+    }
+    link = false;
+    vmstate_info_bool.put(f, &link, sizeof(bool), NULL, NULL);
+    if (vmdesc) {
+        json_end_array(vmdesc);
+    }
+}
+const VMStateInfo vmstate_info_qtailq = {
+    .name = "qtailq",
+    .get  = get_qtailq,
+    .put  = put_qtailq,
+};
-- 
1.9.1