Re: [Qemu-devel] [QEMU RFC PATCH v2 4/6] Migration: migrate QTAILQ

[David Gibson <david@gibson.dropbear.id.au>]

[-- Attachment #1: Type: text/plain, Size: 12570 bytes --]

On Tue, May 24, 2016 at 10:55:07AM -0700, Jianjun Duan wrote:
> A recursive structure has elements of the same type in itself. Currently
> we cannot directly transfer a QTAILQ instance, or any recursive
> structure such as lists in migration because of the limitation in the
> migration code. Here we introduce a general approach to transfer such
> structures. In our approach a recursive structure is tagged with VMS_CSTM.
> We extend VMStateField with 3 fields: meta_data to store the meta data
> about the recursive structure in question, extend_get to load the structure
> from migration stream to memory, and extend_put to dump the structure into
> the migration stream. This extension mirrors VMStateInfo. We then modify
> vmstate_save_state and vmstate_load_state so that when VMS_CSTM is
> encountered, extend_put and extend_get are called respectively with the
> knowledge of the meta data.

All the talk about recursive structures just obscures the issue.  The
point is you're building a way of migrating QTAILQs - just stick to
that.

> To make it work for QTAILQ in qemu/queue.h, we created the meta data
> format, extend_get and extend_put for it. We will use this approach to
> transfer pending_events and ccs_list in spapr state.
> 
> We also create some macros in qemu/queue.h to get the layout information
> about QTAILQ. This ensures that we do not depend on the implementation
> details about QTAILQ in the migration code.

I'm interested to see what Juan and Dave Gilbert think about this.


> 
> Signed-off-by: Jianjun Duan <duanj@linux.vnet.ibm.com>
> ---
>  include/migration/vmstate.h | 59 +++++++++++++++++++++++++++++++
>  include/qemu/queue.h        | 38 ++++++++++++++++++++
>  migration/vmstate.c         | 84 +++++++++++++++++++++++++++++++++++++++++++++
>  3 files changed, 181 insertions(+)
> 
> diff --git a/include/migration/vmstate.h b/include/migration/vmstate.h
> index 1622638..bf57b25 100644
> --- a/include/migration/vmstate.h
> +++ b/include/migration/vmstate.h
> @@ -183,6 +183,8 @@ enum VMStateFlags {
>       * to determine the number of entries in the array. Only valid in
>       * combination with one of VMS_VARRAY*. */
>      VMS_MULTIPLY_ELEMENTS = 0x4000,
> +    /* For fields which need customized handling, such as QTAILQ in queue.h*/
> +    VMS_CSTM            = 0x8000,
>  };
>  
>  typedef struct {
> @@ -198,6 +200,14 @@ typedef struct {
>      const VMStateDescription *vmsd;
>      int version_id;
>      bool (*field_exists)(void *opaque, int version_id);
> +    /*
> +     * Following 3 fields are for VMStateField which needs customized handling,
> +     * such as QTAILQ in qemu/queue.h, lists, and tree.
> +     */
> +    const void *meta_data;
> +    int (*extend_get)(QEMUFile *f, const void *metadata, void *opaque);
> +    void (*extend_put)(QEMUFile *f, const void *metadata, void *opaque,
> +                       QJSON *vmdesc);
>  } VMStateField;
>  
>  struct VMStateDescription {
> @@ -654,6 +664,18 @@ extern const VMStateInfo vmstate_info_bitmap;
>      .offset       = offsetof(_state, _field),                        \
>  }
>  
> +/* For fields that need customized handling, such as queue, list */
> +#define VMSTATE_CSTM_V(_field, _state, _version, _meta_data, _get, _put)  \
> +{                                                                         \
> +    .name         = (stringify(_field)),                                  \
> +    .version_id   = (_version),                                           \
> +    .flags        = VMS_CSTM,                                             \
> +    .offset       = offsetof(_state, _field),                             \
> +    .meta_data    = &(_meta_data),                                        \
> +    .extend_get   = (_get),                                               \
> +    .extend_put   = (_put),                                               \
> +}
> +
>  /* _f : field name
>     _f_n : num of elements field_name
>     _n : num of elements
> @@ -970,4 +992,41 @@ int64_t self_announce_delay(int round)
>  
>  void dump_vmstate_json_to_file(FILE *out_fp);
>  
> +
> +/* Meta data for QTAILQ */
> +typedef struct QTAILQMetaData {
> +    /* the offset of tqh_first in QTAILQ_HEAD */
> +    size_t first;
> +    /* the offset of tqh_last in QTAILQ_HEAD */
> +    size_t last;
> +    /* the offset of tqe_next in QTAILQ_ENTRY */
> +    size_t next;
> +    /* the offset of tqe_prev in QTAILQ_ENTRY */
> +    size_t prev;
> +    /* the offset of QTAILQ_ENTRY in a QTAILQ element*/
> +    size_t entry;
> +    /* size of a QTAILQ element */
> +    size_t size;
> +    /* vmsd of a QTAILQ element */
> +    const VMStateDescription *vmsd;
> +} QTAILQMetaData;
> +
> +#define VMSTATE_QTAILQ_METADATA(_field, _state, _type, _next, _vmsd) { \
> +    .first = QTAILQ_FIRST_OFFSET(typeof_field(_state, _field)),        \
> +    .last =  QTAILQ_LAST_OFFSET(typeof_field(_state, _field)),         \
> +    .next = QTAILQ_NEXT_OFFSET(_type, _next),                          \
> +    .prev = QTAILQ_PREV_OFFSET(_type, _next),                          \
> +    .entry = offsetof(_type, _next),                                   \
> +    .size = sizeof(_type),                                             \
> +    .vmsd = &(_vmsd),                                                  \
> +}
> +
> +int vmstate_get_qtailq(QEMUFile *f, const void *metadata, void *opaque);
> +void vmstate_put_qtailq(QEMUFile *f, const void *metadata,
> +                         void *opaque, QJSON *vmdesc);
> +
> +/* VMStateField for QTAILQ field */
> +#define VMSTATE_QTAILQ_V(_field, _state, _version, _meta_data)               \
> +    VMSTATE_CSTM_V(_field, _state, _version, _meta_data, vmstate_get_qtailq, \
> +                   vmstate_put_qtailq)
>  #endif
> diff --git a/include/qemu/queue.h b/include/qemu/queue.h
> index f781aa2..46962d7 100644
> --- a/include/qemu/queue.h
> +++ b/include/qemu/queue.h
> @@ -437,3 +437,41 @@ struct {                                                                \
>          (*(((struct headname *)((elm)->field.tqe_prev))->tqh_last))
>  
>  #endif  /* !QEMU_SYS_QUEUE_H_ */
> +
> +/*
> + * Offsets of layout of a tail queue head.
> + */
> +#define QTAILQ_FIRST_OFFSET(head_type) \
> +        ((size_t) ((char *) &((head_type *)0)->tqh_first - (char *)0))
> +
> +#define QTAILQ_LAST_OFFSET(head_type) \
> +        ((size_t) ((char *) &((head_type *)0)->tqh_last - (char *)0))
> +
> +/*
> + * Offsets of layout of a tail queue element.
> + */
> +#define QTAILQ_NEXT_OFFSET(ele_type, field)                            \
> +        ((size_t) ((char *) &((ele_type *)0)->field.tqe_next -         \
> +                   (char *) &((ele_type *)0)->field))
> +
> +#define QTAILQ_PREV_OFFSET(ele_type, field) \
> +        ((size_t) ((char *) &((ele_type *)0)->field.tqe_prev -         \
> +                   (char *) &((ele_type *)0)->field))
> +/*
> + * Tail queue tranversal using pointer arithmetic.
> + */
> +#define QTAILQ_RAW_FOREACH(elm, head, entry, first, next)              \
> +        for ((elm) = *((void **) ((char *) (head) + (first)));         \
> +             (elm);                                                    \
> +             (elm) = *((void **) ((char *) (elm) + (entry) + (next))))
> +/*
> + * Tail queue insertion using pointer arithmetic.
> + */
> +#define QTAILQ_RAW_INSERT_TAIL(head, elm, entry, first, last, next, prev) do { \
> +            *((void **) ((char *) (elm) + (entry) + (next))) = NULL;           \
> +            *((void **) ((char *) (elm) + (entry) + (prev))) =                 \
> +                *((void **) ((char *) (head) + (last)));                       \
> +            **((void ***)((char *) (head) + (last))) = (elm);                  \
> +            *((void **) ((char *) (head) + (last))) =                          \
> +                (void *) ((char *) (elm) + (entry) + (next));                  \
> +} while (/*CONSTCOND*/0)
> diff --git a/migration/vmstate.c b/migration/vmstate.c
> index bf3d5db..47cd052 100644
> --- a/migration/vmstate.c
> +++ b/migration/vmstate.c
> @@ -5,6 +5,7 @@
>  #include "migration/vmstate.h"
>  #include "qemu/bitops.h"
>  #include "qemu/error-report.h"
> +#include "qemu/queue.h"
>  #include "trace.h"
>  #include "qjson.h"
>  
> @@ -121,6 +122,8 @@ int vmstate_load_state(QEMUFile *f, const VMStateDescription *vmsd,
>                  if (field->flags & VMS_STRUCT) {
>                      ret = vmstate_load_state(f, field->vmsd, addr,
>                                               field->vmsd->version_id);
> +                } else if (field->flags & VMS_CSTM) {
> +                    ret = field->extend_get(f, field->meta_data, addr);
>                  } else {
>                      ret = field->info->get(f, addr, size);
>  
> @@ -193,6 +196,8 @@ static const char *vmfield_get_type_name(VMStateField *field)
>  
>      if (field->flags & VMS_STRUCT) {
>          type = "struct";
> +    } else if (field->flags & VMS_CSTM) {
> +        type = "customized";
>      } else if (field->info->name) {
>          type = field->info->name;
>      }
> @@ -327,6 +332,8 @@ void vmstate_save_state(QEMUFile *f, const VMStateDescription *vmsd,
>                  }
>                  if (field->flags & VMS_STRUCT) {
>                      vmstate_save_state(f, field->vmsd, addr, vmdesc_loop);
> +                } else if  (field->flags & VMS_CSTM) {
> +                    field->extend_put(f, field->meta_data, addr, vmdesc_loop);
>                  } else {
>                      field->info->put(f, addr, size);
>                  }
> @@ -916,3 +923,80 @@ const VMStateInfo vmstate_info_bitmap = {
>      .get = get_bitmap,
>      .put = put_bitmap,
>  };
> +
> +/* extend_get for QTAILQ */
> +int vmstate_get_qtailq(QEMUFile *f, const void *metadata, void *opaque)
> +{
> +    bool link;
> +    int ret = 0;
> +    size_t first, last, next, prev, entry, size;
> +    QTAILQMetaData *data = (QTAILQMetaData *)metadata;
> +    const VMStateDescription *vmsd = data->vmsd;
> +    int version_id = vmsd->version_id;
> +    void *elm;
> +
> +    first = data->first;
> +    last = data->last;
> +    next = data->next;
> +    prev = data->prev;
> +    entry = data->entry;
> +    size = data->size;
> +
> +    trace_vmstate_load_state(vmsd->name, version_id);
> +    if (version_id > vmsd->version_id) {
> +        trace_vmstate_load_state_end(vmsd->name, "too new", -EINVAL);
> +        return -EINVAL;
> +    }
> +    if (version_id < vmsd->minimum_version_id) {
> +        trace_vmstate_load_state_end(vmsd->name, "too old", -EINVAL);
> +        return -EINVAL;
> +    }
> +
> +    while (true) {
> +        vmstate_info_bool.get(f, &link, sizeof(bool));
> +        if (!link) {
> +            break;
> +        }
> +
> +        elm =  g_malloc(size);
> +        ret = vmstate_load_state(f, vmsd, elm, version_id);
> +        if (ret) {
> +            return ret;
> +        }
> +        QTAILQ_RAW_INSERT_TAIL(opaque, elm, entry, first, last, next, prev);
> +    }
> +
> +    trace_vmstate_load_state_end(vmsd->name, "end", ret);
> +    return ret;
> +}
> +
> +/* extend_get for QTAILQ */
> +void vmstate_put_qtailq(QEMUFile *f, const void *metadata, void *opaque,
> +                         QJSON *vmdesc)
> +{
> +    bool link = true;
> +    size_t first, next, entry;
> +    QTAILQMetaData *data = (QTAILQMetaData *)metadata;
> +    const VMStateDescription *vmsd = data->vmsd;
> +    void *elm;
> +
> +    first = data->first;
> +    next = data->next;
> +    entry = data->entry;
> +
> +    if (vmdesc) {
> +        json_prop_str(vmdesc, "vmsd_name", vmsd->name);
> +        json_prop_int(vmdesc, "version", vmsd->version_id);
> +        json_start_array(vmdesc, "fields");
> +    }
> +
> +    QTAILQ_RAW_FOREACH(elm, opaque, entry, first, next) {
> +        vmstate_info_bool.put(f, &link, sizeof(bool));
> +        vmstate_save_state(f, vmsd, elm, vmdesc);
> +    }
> +    link = false;
> +    vmstate_info_bool.put(f, &link, sizeof(bool));
> +    if (vmdesc) {
> +        json_end_array(vmdesc);
> +    }
> +}

-- 
David Gibson			| I'll have my music baroque, and my code
david AT gibson.dropbear.id.au	| minimalist, thank you.  NOT _the_ _other_
				| _way_ _around_!
http://www.ozlabs.org/~dgibson

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 819 bytes --]