Re: [Qemu-devel] [QEMU PATCH v6 2/2] migration: migrate QTAILQ

["Dr. David Alan Gilbert" <dgilbert@redhat.com>]

* Jianjun Duan (duanj@linux.vnet.ibm.com) wrote:
> Currently we cannot directly transfer a QTAILQ instance because of the
> limitation in the migration code. Here we introduce an approach to
> transfer such structures. We created VMStateInfo vmstate_info_qtailq
> for QTAILQ. Similar VMStateInfo can be created for other data structures
> such as list.
> 
> This approach will be used to transfer pending_events and ccs_list in spapr
> state.
> 
> We also create some macros in qemu/queue.h to access a QTAILQ using pointer
> arithmetic. This ensures that we do not depend on the implementation
> details about QTAILQ in the migration code.
> 
> Signed-off-by: Jianjun Duan <duanj@linux.vnet.ibm.com>
> ---
>  include/migration/vmstate.h | 20 +++++++++++++++
>  include/qemu/queue.h        | 32 ++++++++++++++++++++++++
>  migration/trace-events      |  4 +++
>  migration/vmstate.c         | 59 +++++++++++++++++++++++++++++++++++++++++++++
>  4 files changed, 115 insertions(+)
> 
> diff --git a/include/migration/vmstate.h b/include/migration/vmstate.h
> index d0e37b5..4dd0aed 100644
> --- a/include/migration/vmstate.h
> +++ b/include/migration/vmstate.h
> @@ -251,6 +251,7 @@ extern const VMStateInfo vmstate_info_timer;
>  extern const VMStateInfo vmstate_info_buffer;
>  extern const VMStateInfo vmstate_info_unused_buffer;
>  extern const VMStateInfo vmstate_info_bitmap;
> +extern const VMStateInfo vmstate_info_qtailq;
>  
>  #define type_check_2darray(t1,t2,n,m) ((t1(*)[n][m])0 - (t2*)0)
>  #define type_check_array(t1,t2,n) ((t1(*)[n])0 - (t2*)0)
> @@ -662,6 +663,25 @@ extern const VMStateInfo vmstate_info_bitmap;
>      .offset       = offsetof(_state, _field),                        \
>  }
>  
> +/* For QTAILQ that need customized handling
> + * _type: type of QTAILQ element
> + * _next: name of QTAILQ entry field in QTAILQ element
> + * _vmsd: VMSD for QTAILQ element
> + * size: size of QTAILQ element
> + * start: offset of QTAILQ entry in QTAILQ element
> + */
> +#define VMSTATE_QTAILQ_V(_field, _state, _version, _vmsd, _type, _next)  \
> +{                                                                        \
> +    .name         = (stringify(_field)),                                 \
> +    .version_id   = (_version),                                          \
> +    .vmsd         = &(_vmsd),                                            \
> +    .size         = sizeof(_type),                                       \
> +    .info         = &vmstate_info_qtailq,                                \
> +    .flags        = VMS_LINKED,                                          \

Hang on - you killed VMS_LINKED off in the previous patch; so this
doesn't work.
(Adding a test to tests/test-vmstate.c would have found that)

> +    .offset       = offsetof(_state, _field),                            \
> +    .start        = offsetof(_type, _next),                              \
> +}
> +
>  /* _f : field name
>     _f_n : num of elements field_name
>     _n : num of elements
> diff --git a/include/qemu/queue.h b/include/qemu/queue.h
> index 342073f..d672ae0 100644
> --- a/include/qemu/queue.h
> +++ b/include/qemu/queue.h
> @@ -438,4 +438,36 @@ struct {                                                                \
>  #define QTAILQ_PREV(elm, headname, field) \
>          (*(((struct headname *)((elm)->field.tqe_prev))->tqh_last))
>  
> +/*
> + * Offsets of layout of a tail queue head.
> + */
> +#define QTAILQ_FIRST_OFFSET 0
> +#define QTAILQ_LAST_OFFSET (sizeof(void *))
> +
> +/*
> + * Offsets of layout of a tail queue element.
> + */
> +#define QTAILQ_NEXT_OFFSET 0
> +#define QTAILQ_PREV_OFFSET (sizeof(void *))
> +
> +/*
> + * Tail queue tranversal using pointer arithmetic.
> + */
> +#define QTAILQ_RAW_FOREACH(elm, head, entry)                                   \
> +        for ((elm) = *((void **) ((char *) (head) + QTAILQ_FIRST_OFFSET));     \
> +             (elm);                                                            \
> +             (elm) =                                                           \
> +                 *((void **) ((char *) (elm) + (entry) + QTAILQ_NEXT_OFFSET)))
> +/*
> + * Tail queue insertion using pointer arithmetic.
> + */
> +#define QTAILQ_RAW_INSERT_TAIL(head, elm, entry) do {                          \
> +        *((void **) ((char *) (elm) + (entry) + QTAILQ_NEXT_OFFSET)) = NULL;   \
> +        *((void **) ((char *) (elm) + (entry) + QTAILQ_PREV_OFFSET)) =         \
> +            *((void **) ((char *) (head) + QTAILQ_LAST_OFFSET));               \
> +        **((void ***)((char *) (head) + QTAILQ_LAST_OFFSET)) = (elm);          \
> +        *((void **) ((char *) (head) + QTAILQ_LAST_OFFSET)) =                  \
> +            (void *) ((char *) (elm) + (entry) + QTAILQ_NEXT_OFFSET);          \
> +} while (/*CONSTCOND*/0)

I wonder if there's a simpler way to do this; I'm not sure this works, but something like:

struct QTAILQDummy {
    char dummy;
};

QTAILQ_HEAD(QTAILQRawHead, struct QTAILQDummy)
typedef QTAILQ_ENTRY(struct QTAILQDummy) QTAILQRawEntry;

#define QTAILQ_RAW_FOREACH(elm, head, entry)                                   \
        for ((elm) = ((struct QTAILQRawHead *)head)->tqh_first)                \
             (elm);                                                            \
             (elm) =                                                           \
             (elm) = ((QTAILQRawEntry *)((char *) (elm) + (entry)))->tqh_next

and then I think elm gets declared as a struct QTAILQDummy.
But it does avoid those FIRST_OFFSET/LAST_OFFSET/NEXT_OFFSET/PREV_OFFSET calculations.

Would that work?

>  #endif /* QEMU_SYS_QUEUE_H */
> diff --git a/migration/trace-events b/migration/trace-events
> index dfee75a..9a6ec59 100644
> --- a/migration/trace-events
> +++ b/migration/trace-events
> @@ -52,6 +52,10 @@ vmstate_n_elems(const char *name, int n_elems) "%s: %d"
>  vmstate_subsection_load(const char *parent) "%s"
>  vmstate_subsection_load_bad(const char *parent,  const char *sub, const char *sub2) "%s: %s/%s"
>  vmstate_subsection_load_good(const char *parent) "%s"
> +get_qtailq(const char *name, int version_id) "%s v%d"
> +get_qtailq_end(const char *name, const char *reason, int val) "%s %s/%d"
> +put_qtailq(const char *name, int version_id) "%s v%d"
> +put_qtailq_end(const char *name, const char *reason) "%s %s"
>  
>  # migration/qemu-file.c
>  qemu_file_fclose(void) ""
> diff --git a/migration/vmstate.c b/migration/vmstate.c
> index e8eff3f..eb68fc6 100644
> --- a/migration/vmstate.c
> +++ b/migration/vmstate.c
> @@ -5,6 +5,7 @@
>  #include "migration/vmstate.h"
>  #include "qemu/bitops.h"
>  #include "qemu/error-report.h"
> +#include "qemu/queue.h"
>  #include "trace.h"
>  #include "migration/qjson.h"
>  
> @@ -946,3 +947,61 @@ const VMStateInfo vmstate_info_bitmap = {
>      .get = get_bitmap,
>      .put = put_bitmap,
>  };
> +
> +/*get for QTAILQ */
> +static int get_qtailq(QEMUFile *f, void *pv, size_t unused_size,
> +                      VMStateField *field)
> +{
> +    int ret = 0;
> +    const VMStateDescription *vmsd = field->vmsd;
> +    size_t size = field->size;
> +    size_t entry = field->start;

Please comment these two here; I'd prefer entry_offset or something
as a name to make clear what it is.

> +    int version_id = field->version_id;
> +    void *elm;
> +
> +    trace_get_qtailq(vmsd->name, version_id);
> +    if (version_id > vmsd->version_id) {
> +        trace_get_qtailq_end(vmsd->name, "too new", -EINVAL);
> +        return -EINVAL;
> +    }
> +    if (version_id < vmsd->minimum_version_id) {
> +        trace_get_qtailq_end(vmsd->name, "too old", -EINVAL);
> +        return -EINVAL;
> +    }

Please make those error_report's - if anything fails migration
I like to see it in the stderr log.

> +    while (qemu_get_byte(f)) {
> +        elm =  g_malloc(size);
> +        ret = vmstate_load_state(f, vmsd, elm, version_id);
> +        if (ret) {
> +            return ret;
> +        }
> +        QTAILQ_RAW_INSERT_TAIL(pv, elm, entry);
> +    }
> +
> +    trace_get_qtailq_end(vmsd->name, "end", ret);
> +    return ret;
> +}
> +
> +/* put for QTAILQ */
> +static void put_qtailq(QEMUFile *f, void *pv, size_t unused_size,
> +                       VMStateField *field, QJSON *vmdesc)
> +{
> +    const VMStateDescription *vmsd = field->vmsd;
> +    size_t entry = field->start;
> +    void *elm;
> +
> +    trace_put_qtailq(vmsd->name, vmsd->version_id);
> +
> +    QTAILQ_RAW_FOREACH(elm, pv, entry) {
> +        qemu_put_byte(f, true);
> +        vmstate_save_state(f, vmsd, elm, vmdesc);
> +    }
> +    qemu_put_byte(f, false);
> +
> +    trace_put_qtailq_end(vmsd->name, "end");
> +}
> +const VMStateInfo vmstate_info_qtailq = {
> +    .name = "qtailq",
> +    .get  = get_qtailq,
> +    .put  = put_qtailq,
> +};
> -- 
> 1.9.1

Dave

> 
--
Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK