Re: [PATCH RFC 11/21] migration: Add hugetlb-doublemap cap

["Dr. David Alan Gilbert" <dgilbert@redhat.com>]

* Peter Xu (peterx@redhat.com) wrote:
> Add a new cap to allow mapping hugetlbfs backed RAMs in small page sizes.
> 
> Signed-off-by: Peter Xu <peterx@redhat.com>


Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>

although, I'm curious if the protocol actually changes - or whether
a doublepage enabled destination would work with an unmodified source?
I guess potentially you can get away without the dirty clearing of the
partially sent hugepages that the source normally does.

Dave

> ---
>  migration/migration.c | 48 ++++++++++++++++++++++++++++++++++++++++++-
>  migration/migration.h |  1 +
>  qapi/migration.json   |  7 ++++++-
>  3 files changed, 54 insertions(+), 2 deletions(-)
> 
> diff --git a/migration/migration.c b/migration/migration.c
> index 64f74534e2..b174f2af92 100644
> --- a/migration/migration.c
> +++ b/migration/migration.c
> @@ -17,6 +17,7 @@
>  #include "qemu/cutils.h"
>  #include "qemu/error-report.h"
>  #include "qemu/main-loop.h"
> +#include "qemu/madvise.h"
>  #include "migration/blocker.h"
>  #include "exec.h"
>  #include "fd.h"
> @@ -62,6 +63,7 @@
>  #include "sysemu/cpus.h"
>  #include "yank_functions.h"
>  #include "sysemu/qtest.h"
> +#include "exec/ramblock.h"
>  
>  #define MAX_THROTTLE  (128 << 20)      /* Migration transfer speed throttling */
>  
> @@ -1363,12 +1365,47 @@ static bool migrate_caps_check(bool *cap_list,
>                     "Zero copy only available for non-compressed non-TLS multifd migration");
>          return false;
>      }
> +
> +    if (cap_list[MIGRATION_CAPABILITY_HUGETLB_DOUBLEMAP]) {
> +        RAMBlock *rb;
> +
> +        /* Check whether the platform/binary supports the new madvise()s */
> +
> +#if QEMU_MADV_SPLIT == QEMU_MADV_INVALID
> +        error_setg(errp, "MADV_SPLIT is not supported by the QEMU binary");
> +        return false;
> +#endif
> +
> +#if QEMU_MADV_COLLAPSE == QEMU_MADV_INVALID
> +        error_setg(errp, "MADV_COLLAPSE is not supported by the QEMU binary");
> +        return false;
> +#endif
> +
> +        /*
> +         * Check against kernel support of MADV_SPLIT is not easy, delay
> +         * that until we have all the hugetlb mappings ready on dest node,
> +         * meanwhile do the best effort check here because doublemap
> +         * requires the hugetlb ramblocks to be shared first.
> +         */
> +        RAMBLOCK_FOREACH_NOT_IGNORED(rb) {
> +            if (qemu_ram_is_hugetlb(rb) && !qemu_ram_is_shared(rb)) {
> +                error_setg(errp, "RAMBlock '%s' needs to be shared for doublemap",
> +                           rb->idstr);
> +                return false;
> +            }
> +        }
> +    }
>  #else
>      if (cap_list[MIGRATION_CAPABILITY_ZERO_COPY_SEND]) {
>          error_setg(errp,
>                     "Zero copy currently only available on Linux");
>          return false;
>      }
> +
> +    if (cap_list[MIGRATION_CAPABILITY_HUGETLB_DOUBLEMAP]) {
> +        error_setg(errp, "Hugetlb doublemap is only supported on Linux");
> +        return false;
> +    }
>  #endif
>  
>      if (cap_list[MIGRATION_CAPABILITY_POSTCOPY_PREEMPT]) {
> @@ -2792,6 +2829,13 @@ bool migrate_postcopy_preempt(void)
>      return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_PREEMPT];
>  }
>  
> +bool migrate_hugetlb_doublemap(void)
> +{
> +    MigrationState *s = migrate_get_current();
> +
> +    return s->enabled_capabilities[MIGRATION_CAPABILITY_HUGETLB_DOUBLEMAP];
> +}
> +
>  /* migration thread support */
>  /*
>   * Something bad happened to the RP stream, mark an error
> @@ -4472,7 +4516,9 @@ static Property migration_properties[] = {
>      DEFINE_PROP_MIG_CAP("x-return-path", MIGRATION_CAPABILITY_RETURN_PATH),
>      DEFINE_PROP_MIG_CAP("x-multifd", MIGRATION_CAPABILITY_MULTIFD),
>      DEFINE_PROP_MIG_CAP("x-background-snapshot",
> -            MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT),
> +                        MIGRATION_CAPABILITY_BACKGROUND_SNAPSHOT),
> +    DEFINE_PROP_MIG_CAP("hugetlb-doublemap",
> +                        MIGRATION_CAPABILITY_HUGETLB_DOUBLEMAP),
>  #ifdef CONFIG_LINUX
>      DEFINE_PROP_MIG_CAP("x-zero-copy-send",
>              MIGRATION_CAPABILITY_ZERO_COPY_SEND),
> diff --git a/migration/migration.h b/migration/migration.h
> index 5674a13876..bbd610a2d5 100644
> --- a/migration/migration.h
> +++ b/migration/migration.h
> @@ -447,6 +447,7 @@ bool migrate_use_events(void);
>  bool migrate_postcopy_blocktime(void);
>  bool migrate_background_snapshot(void);
>  bool migrate_postcopy_preempt(void);
> +bool migrate_hugetlb_doublemap(void);
>  
>  /* Sending on the return path - generic and then for each message type */
>  void migrate_send_rp_shut(MigrationIncomingState *mis,
> diff --git a/qapi/migration.json b/qapi/migration.json
> index 88ecf86ac8..b23516e75e 100644
> --- a/qapi/migration.json
> +++ b/qapi/migration.json
> @@ -477,6 +477,11 @@
>  #                    will be handled faster.  This is a performance feature and
>  #                    should not affect the correctness of postcopy migration.
>  #                    (since 7.1)
> +# @hugetlb-doublemap: If enabled, the migration process will allow postcopy
> +#                     to handle page faults based on small pages even if
> +#                     hugetlb is used.  This will drastically reduce page
> +#                     fault latencies when hugetlb is used as the guest RAM
> +#                     backends. (since 7.3)
>  #
>  # Features:
>  # @unstable: Members @x-colo and @x-ignore-shared are experimental.
> @@ -492,7 +497,7 @@
>             'dirty-bitmaps', 'postcopy-blocktime', 'late-block-activate',
>             { 'name': 'x-ignore-shared', 'features': [ 'unstable' ] },
>             'validate-uuid', 'background-snapshot',
> -           'zero-copy-send', 'postcopy-preempt'] }
> +           'zero-copy-send', 'postcopy-preempt', 'hugetlb-doublemap'] }
>  
>  ##
>  # @MigrationCapabilityStatus:
> -- 
> 2.37.3
> 
-- 
Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK