Re: [PATCH] Adding support for hyper-v enlightments hv-extcalls and hv-memory-hint

["Michael S. Tsirkin" <mst@redhat.com>]

Thanks for the patch! yet something to improve:

On Mon, Nov 02, 2026 at 05:58:04PM +0200, Shmuel Fomberg wrote:
> hv-memory-hint (hypercall HvExtCallMemoryHeatHint) allow a guest to
> notify host which memory pages will be needed in the near future.
> This is not a replacement for the balloon functionality, but can
> help the kernel

given this is supposed to be an optimization, I expect to see
some performance numbers showing it actually helps, and some
documentation explaining when it does and when it doesn't.

e.g. does it work with vhost-user (shmem)? hugetlb?

pls document which configs were tested, with linux and windows guests.

> make better decisions when memory is scare.

scarse?

> 
> Signed-off-by: Shmuel Fomberg <shmuelfomberg@gmail.com>
> ---
>  docs/system/i386/hyperv.rst      |  14 +++
>  hw/hyperv/hyperv.c               | 158 +++++++++++++++++++++++++++++++
>  include/hw/hyperv/hyperv-proto.h |  25 +++++
>  include/hw/hyperv/hyperv.h       |  13 +++
>  include/qemu/madvise.h           |   7 ++
>  target/i386/cpu.c                |   3 +
>  target/i386/cpu.h                |   2 +
>  target/i386/kvm/hyperv-proto.h   |   1 +
>  target/i386/kvm/hyperv.c         |  24 +++++
>  target/i386/kvm/kvm.c            |   7 ++
>  10 files changed, 254 insertions(+)
> 
> diff --git a/docs/system/i386/hyperv.rst b/docs/system/i386/hyperv.rst
> index e1babffb0f..1e164f1863 100644
> --- a/docs/system/i386/hyperv.rst
> +++ b/docs/system/i386/hyperv.rst
> @@ -259,6 +259,15 @@ Existing enlightenments
>  
>    Recommended: ``hv-evmcs`` (Intel)
>  
> +``hv-extcalls``
> +  Advertises extended Hyper-V hypercalls (call codes 0x8000 and above) to
> +  the guest.
> +
> +``hv-memory-hint``
> +  Enables guest memory heat hints (HvExtCallMemoryHeatHint).
> +
> +  Requires: ``hv-extcalls``
> +
>  Supplementary features
>  ----------------------
>  
> @@ -276,6 +285,11 @@ Supplementary features
>    Windows guests from booting when used without proper configuration. If needed,
>    ``hv-syndbg`` can be enabled additionally.
>  
> +  Note: ``hv-passthrough`` enables ``hv-extcalls`` when the host KVM advertises
> +  it. It does not enable ``hv-memory-hint``, which is implemented in QEMU and
> +  has no KVM CPUID bit. Enable ``hv-memory-hint`` explicitly when guest heat
> +  hints should reach the host.
> +

I'm not sure what all this means.
"KVM CPUID" is also obscure.

Can you pls rewrite? Maybe along the lines of:

if .... and .... then ... results in ...
if .... and .... then ... is ignored




>    Note: ``hv-passthrough`` effectively prevents migration as the list of enabled
>    enlightenments may differ between target and destination hosts.
>  
> diff --git a/hw/hyperv/hyperv.c b/hw/hyperv/hyperv.c
> index 900ff80213..b3fce7b563 100644
> --- a/hw/hyperv/hyperv.c
> +++ b/hw/hyperv/hyperv.c
> @@ -20,7 +20,9 @@
>  #include "system/physmem.h"
>  #include "qemu/bitops.h"
>  #include "qemu/error-report.h"
> +#include "qemu/int128.h"
>  #include "qemu/lockable.h"
> +#include "qemu/madvise.h"
>  #include "qemu/queue.h"
>  #include "qemu/rcu.h"
>  #include "qemu/rcu_queue.h"
> @@ -646,6 +648,162 @@ unmap:
>      return ret;
>  }
>  
> +uint16_t hyperv_hcall_ext_capabilities(uint64_t outgpa, bool fast,
> +                                       bool memory_hint)
> +{
> +    uint16_t ret;
> +    hwaddr len;
> +    uint64_t *caps = NULL;
> +
> +    if (fast) {
> +        return HV_STATUS_INVALID_HYPERCALL_CODE;
> +    }
> +    if (outgpa & (__alignof__(*caps) - 1)) {
> +        return HV_STATUS_INVALID_ALIGNMENT;
> +    }
> +
> +    len = sizeof(*caps);
> +    caps = physical_memory_map(outgpa, &len, 1);
> +    if (!caps || len < sizeof(*caps)) {
> +        ret = HV_STATUS_INSUFFICIENT_MEMORY;
> +        goto unmap;
> +    }
> +
> +    *caps = 0;
> +    if (memory_hint) {
> +        *caps = HV_EXT_CAP_MEMORY_HEAT_HINT;
> +        if (QEMU_MADV_COLD != QEMU_MADV_INVALID) {
> +            *caps |= HV_EXT_CAP_MEMORY_HEAT_HINT_COLD;
> +        }
> +    }
> +    ret = HV_STATUS_SUCCESS;
> +
> +unmap:
> +    if (caps) {
> +        physical_memory_unmap(caps, len, 1, ret == HV_STATUS_SUCCESS ? len : 0);
> +    }
> +    return ret;
> +}
> +
> +static bool hv_decode_gpa_page_range(uint64_t raw, hwaddr *gpa, hwaddr *len)
> +{
> +    uint64_t pages = (raw & HV_GPA_PAGE_RANGE_ADDITIONAL_MASK) + 1;

shouldn't we check alignment and error out rather than force it?


> +
> +    if (raw & HV_GPA_PAGE_RANGE_LARGEPAGE) {
> +        if (raw & HV_GPA_PAGE_RANGE_PAGE_SIZE_1GB) {
> +            *gpa = raw & HV_GPA_PAGE_RANGE_1G_GPA_MASK;
> +            *len = pages << 30;
> +        } else {
> +            *gpa = raw & HV_GPA_PAGE_RANGE_2M_GPA_MASK;
> +            *len = pages << 21;
> +        }
> +    } else {
> +        *gpa = raw & HV_GPA_PAGE_RANGE_4K_GPA_MASK;
> +        *len = pages << 12;
> +    }
> +
> +    return *len != 0;

how can len be 0?




> +}
> +
> +static void hv_madvise_gpa_range(hwaddr gpa, hwaddr len, int advice)
> +{
> +    while (len) {
> +        MemoryRegionSection section;
> +        hwaddr chunk, start, skip;
> +        void *host;
> +
> +        section = memory_region_find(get_system_memory(), gpa, len);
> +        if (!section.mr) {
> +            break;

why break? we could have an unmapped hole followed by a mapped one.

> +        }
> +
> +        start = section.offset_within_address_space;
> +        chunk = int128_get64(section.size);
> +
> +        if (!chunk || start < gpa) {


is this dead code? we got the mr from find can start be before?

> +            memory_region_unref(section.mr);
> +            break;
> +        }
> +
> +        skip = start - gpa;
> +        if (skip >= len || chunk > len - skip) {

can this trigger? or more dead code?

> +            memory_region_unref(section.mr);
> +            break;
> +        }
> +
> +        gpa = start + chunk;
> +        len -= skip + chunk;
> +
> +        if (memory_region_is_ram(section.mr) &&
> +            !memory_region_is_rom(section.mr) &&
> +            !memory_region_is_romd(section.mr)) {
> +            host = memory_region_get_ram_ptr(section.mr) +
> +                   section.offset_within_region;
> +            qemu_madvise(host, chunk, advice);
> +        }
> +        memory_region_unref(section.mr);
> +    }
> +}
> +
> +uint16_t hyperv_hcall_memory_hint(uint64_t in_gpa, uint16_t rep_count,
> +                                  bool fast)
> +{
> +    uint16_t ret = HV_STATUS_SUCCESS;
> +    hwaddr in_len, needed;
> +    struct hyperv_memory_heat_hint *in = NULL;
> +    unsigned int i;
> +    uint64_t heat_type;
> +    int advice;
> +
> +    if (fast) {
> +        return HV_STATUS_INVALID_HYPERCALL_CODE;
> +    }
> +    if (rep_count < 1 || rep_count > HV_EXT_MEMORY_HEAT_HINT_MAX_RANGES) {
> +        return HV_STATUS_INVALID_HYPERCALL_INPUT;
> +    }
> +    if (in_gpa & (__alignof__(*in) - 1)) {
> +        return HV_STATUS_INVALID_ALIGNMENT;
> +    }
> +
> +    needed = sizeof(in->heat_type) + sizeof(in->ranges[0]) * rep_count;
> +    in_len = TARGET_PAGE_SIZE;
> +    in = physical_memory_map(in_gpa, &in_len, 0);
> +    if (!in || in_len < needed) {
> +        ret = HV_STATUS_INSUFFICIENT_MEMORY;
> +        goto unmap;

needed can be 4096 with rep_count 511, can it not? if the input
is not page aligned or end of page is not ram, will this not fail?
why not just map needed bytes?

> +    }
> +
> +    heat_type = in->heat_type & 3;

I would add
	#define HV_EXTMEM_HEAT_HINT_MASK      3
or
	(HV_EXTMEM_HEAT_HINT_HOT|HV_EXTMEM_HEAT_HINT_COLD|HV_EXTMEM_HEAT_HINT_COLD_DISCARD)

also are you really supposed to just ignore rest of bits?
add a code comment explaining why.


> +    switch (heat_type) {
> +    case HV_EXTMEM_HEAT_HINT_HOT:
> +        advice = QEMU_MADV_WILLNEED;
> +        break;
> +    case HV_EXTMEM_HEAT_HINT_COLD:
> +    case HV_EXTMEM_HEAT_HINT_COLD_DISCARD:
> +        advice = QEMU_MADV_COLD;
> +        break;
> +    default:
> +        ret = HV_STATUS_INVALID_PARAMETER;
> +        goto unmap;
> +    }

Pls document what are these things supposed to do, with code comments.

For example "discard" sounds like MADV_DONTNEED, MADV_REMOVE or
MADV_FREE or a combination of these to me?

"COLD" sounds kinda like MADV_COLD but there's also MADV_PAGEOUT
which is same but more aggressive.



> +
> +    for (i = 0; i < rep_count; i++) {
> +        hwaddr gpa, len;
> +        uint64_t raw = in->ranges[i];
> +
> +        if (!hv_decode_gpa_page_range(raw, &gpa, &len)) {
> +            continue;

what's the point of this? does it ever trigger?

> +        }
> +        hv_madvise_gpa_range(gpa, len, advice);
> +    }
> +
> +unmap:
> +    if (in) {
> +        physical_memory_unmap(in, in_len, 0, 0);
> +    }
> +    return ret;
> +}
> +
>  static int set_event_flag_handler(uint32_t conn_id, EventNotifier *notifier)
>  {
>      int ret;
> diff --git a/include/hw/hyperv/hyperv-proto.h b/include/hw/hyperv/hyperv-proto.h
> index fffc5ce342..67e6f72a90 100644
> --- a/include/hw/hyperv/hyperv-proto.h
> +++ b/include/hw/hyperv/hyperv-proto.h
> @@ -35,7 +35,32 @@
>  #define HV_POST_DEBUG_DATA                    0x0069
>  #define HV_RETRIEVE_DEBUG_DATA                0x006a
>  #define HV_RESET_DEBUG_SESSION                0x006b
> +#define HV_EXT_CALL_QUERY_CAPABILITIES        0x8001
> +#define HV_EXT_CALL_MEMORY_HEAT_HINT          0x8003
>  #define HV_HYPERCALL_FAST                     (1u << 16)
> +#define HV_HYPERCALL_REP_COMP_OFFSET          32
> +#define HV_HYPERCALL_REP_COMP_MASK            0xfff
> +
> +#define HV_EXT_CAP_MEMORY_HEAT_HINT           (1ull << 1)
> +#define HV_EXT_CAP_MEMORY_HEAT_HINT_COLD      (1ull << 2)

Here:
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/plain/include/hyperv/hvgdk_ext.h


I see:
#define HV_EXT_CAPABILITY_MEMORY_COLD_DISCARD_HINT BIT(8)



> +#define HV_EXT_MEMORY_HEAT_HINT_MAX_RANGES    511
> +
> +#define HV_EXTMEM_HEAT_HINT_COLD              0
> +#define HV_EXTMEM_HEAT_HINT_HOT               1
> +#define HV_EXTMEM_HEAT_HINT_COLD_DISCARD      2
> +
> +struct hyperv_memory_heat_hint {
> +    uint64_t heat_type;
> +    uint64_t ranges[HV_EXT_MEMORY_HEAT_HINT_MAX_RANGES];
> +};
> +
> +/* HV_GPA_PAGE_RANGE in each heat-hint range qword */
> +#define HV_GPA_PAGE_RANGE_ADDITIONAL_MASK     0x7ffull
> +#define HV_GPA_PAGE_RANGE_LARGEPAGE           (1ull << 11)
> +#define HV_GPA_PAGE_RANGE_PAGE_SIZE_1GB       (1ull << 12)
> +#define HV_GPA_PAGE_RANGE_4K_GPA_MASK         (~0xfffull)
> +#define HV_GPA_PAGE_RANGE_2M_GPA_MASK         (~0x1fffffull)
> +#define HV_GPA_PAGE_RANGE_1G_GPA_MASK         (~0x3fffffffull)
>  

Can you try to move the linux bits out to a uapi file and import from there,
instead of duplicating?

>  /*
>   * Message size
> diff --git a/include/hw/hyperv/hyperv.h b/include/hw/hyperv/hyperv.h
> index 23091301d0..8cccbb91ba 100644
> --- a/include/hw/hyperv/hyperv.h
> +++ b/include/hw/hyperv/hyperv.h
> @@ -70,6 +70,19 @@ uint16_t hyperv_hcall_post_message(uint64_t param, bool fast);
>   * the connection as specified in @param.
>   */
>  uint16_t hyperv_hcall_signal_event(uint64_t param, bool fast);
> +/*
> + * Process HV_EXT_CALL_QUERY_CAPABILITIES hypercall: list supported ext hcall
> + * fill the capabilities bit array in @outgpa.
> + */
> +uint16_t hyperv_hcall_ext_capabilities(uint64_t outgpa, bool fast,
> +                                       bool memory_hint);
> +/*
> + * Process HV_EXT_CALL_MEMORY_HEAT_HINT hypercall: receive a list of hot/cold
> + * pages and call madvice on them.


madvise

> + */
> +uint16_t hyperv_hcall_memory_hint(uint64_t in_gpa, uint16_t rep_count,
> +                                  bool fast);
> +
>  
>  static inline uint32_t hyperv_vp_index(CPUState *cs)
>  {
> diff --git a/include/qemu/madvise.h b/include/qemu/madvise.h
> index e155f59a0d..fd638c2254 100644
> --- a/include/qemu/madvise.h
> +++ b/include/qemu/madvise.h
> @@ -59,6 +59,11 @@
>  #else
>  #define QEMU_MADV_POPULATE_WRITE QEMU_MADV_INVALID
>  #endif
> +#ifdef MADV_COLD
> +#define QEMU_MADV_COLD MADV_COLD
> +#else
> +#define QEMU_MADV_COLD QEMU_MADV_INVALID
> +#endif
>  
>  #elif defined(CONFIG_POSIX_MADVISE)
>  
> @@ -73,6 +78,7 @@
>  #define QEMU_MADV_NOHUGEPAGE  QEMU_MADV_INVALID
>  #define QEMU_MADV_REMOVE QEMU_MADV_DONTNEED
>  #define QEMU_MADV_POPULATE_WRITE QEMU_MADV_INVALID
> +#define QEMU_MADV_COLD QEMU_MADV_INVALID
>  
>  #else /* no-op */
>  
> @@ -87,6 +93,7 @@
>  #define QEMU_MADV_NOHUGEPAGE  QEMU_MADV_INVALID
>  #define QEMU_MADV_REMOVE QEMU_MADV_INVALID
>  #define QEMU_MADV_POPULATE_WRITE QEMU_MADV_INVALID
> +#define QEMU_MADV_COLD QEMU_MADV_INVALID
>  
>  #endif
>  
> diff --git a/target/i386/cpu.c b/target/i386/cpu.c
> index e65eca0459..263523d052 100644
> --- a/target/i386/cpu.c
> +++ b/target/i386/cpu.c
> @@ -10824,6 +10824,9 @@ static const Property x86_cpu_properties[] = {
>                        HYPERV_FEAT_TLBFLUSH_EXT, 0),
>      DEFINE_PROP_BIT64("hv-tlbflush-direct", X86CPU, hyperv_features,
>                        HYPERV_FEAT_TLBFLUSH_DIRECT, 0),
> +    DEFINE_PROP_BIT64("hv-extcalls", X86CPU, hyperv_features,
> +                      HYPERV_FEAT_EXT_HYPERCALLS, 0),
> +    DEFINE_PROP_BOOL("hv-memory-hint", X86CPU, hyperv_memory_hint, false),
>      DEFINE_PROP_ON_OFF_AUTO("hv-no-nonarch-coresharing", X86CPU,
>                              hyperv_no_nonarch_cs, ON_OFF_AUTO_OFF),
>  #ifdef CONFIG_SYNDBG
> diff --git a/target/i386/cpu.h b/target/i386/cpu.h
> index 3945e90371..3c9da61e17 100644
> --- a/target/i386/cpu.h
> +++ b/target/i386/cpu.h
> @@ -1489,6 +1489,7 @@ uint64_t x86_cpu_get_supported_feature_word(X86CPU *cpu, FeatureWord w);
>  #define HYPERV_FEAT_XMM_INPUT           18
>  #define HYPERV_FEAT_TLBFLUSH_EXT        19
>  #define HYPERV_FEAT_TLBFLUSH_DIRECT     20
> +#define HYPERV_FEAT_EXT_HYPERCALLS      21
>  
>  #ifndef HYPERV_SPINLOCK_NEVER_NOTIFY
>  #define HYPERV_SPINLOCK_NEVER_NOTIFY             0xFFFFFFFF
> @@ -2374,6 +2375,7 @@ struct ArchCPU {
>      uint32_t hyperv_spinlock_attempts;
>      char *hyperv_vendor;
>      uint64_t hyperv_features;
> +    bool hyperv_memory_hint;
>      bool hyperv_passthrough;
>      OnOffAuto hyperv_no_nonarch_cs;
>      uint32_t hyperv_vendor_id[3];
> diff --git a/target/i386/kvm/hyperv-proto.h b/target/i386/kvm/hyperv-proto.h
> index a9f056f2f3..e88913fd37 100644
> --- a/target/i386/kvm/hyperv-proto.h
> +++ b/target/i386/kvm/hyperv-proto.h
> @@ -66,6 +66,7 @@
>   * HV_CPUID_FEATURES.EBX bits
>   */
>  #define HV_PARTITION_DEBUGGING_ALLOWED          (1u << 12)
> +#define HV_ENABLE_EXTENDED_HYPERCALLS           (1u << 20)
>  
>  /*
>   * HV_CPUID_ENLIGHTMENT_INFO.EAX bits
> diff --git a/target/i386/kvm/hyperv.c b/target/i386/kvm/hyperv.c
> index 420c76b5ff..9d97f5cb4e 100644
> --- a/target/i386/kvm/hyperv.c
> +++ b/target/i386/kvm/hyperv.c
> @@ -97,6 +97,13 @@ int kvm_hv_handle_exit(X86CPU *cpu, struct kvm_hyperv_exit *exit)
>          bool fast = exit->u.hcall.input & HV_HYPERCALL_FAST;
>          uint64_t in_param = exit->u.hcall.params[0];
>          uint64_t out_param = exit->u.hcall.params[1];
> +        uint64_t rep_count = (exit->u.hcall.input >> 32) & 0xFFF;
> +
> +        if ((code & 0x8000) &&
> +            !hyperv_feat_enabled(cpu, HYPERV_FEAT_EXT_HYPERCALLS)) {
> +            exit->u.hcall.result = HV_STATUS_INVALID_HYPERCALL_CODE;
> +            return 0;
> +        }
>  
>          switch (code) {
>          case HV_POST_MESSAGE:
> @@ -117,6 +124,23 @@ int kvm_hv_handle_exit(X86CPU *cpu, struct kvm_hyperv_exit *exit)
>              exit->u.hcall.result =
>                  hyperv_hcall_reset_dbg_session(out_param);
>              break;
> +        case HV_EXT_CALL_QUERY_CAPABILITIES:
> +            exit->u.hcall.result = hyperv_hcall_ext_capabilities(
> +                out_param, fast, cpu->hyperv_memory_hint);
> +            break;
> +        case HV_EXT_CALL_MEMORY_HEAT_HINT:
> +            if (!cpu->hyperv_memory_hint) {
> +                exit->u.hcall.result = HV_STATUS_INVALID_HYPERCALL_CODE;
> +                break;
> +            }
> +            exit->u.hcall.result =
> +                hyperv_hcall_memory_hint(in_param, rep_count, fast);
> +            if (exit->u.hcall.result == HV_STATUS_SUCCESS) {
> +                exit->u.hcall.result |=
> +                    (rep_count & HV_HYPERCALL_REP_COMP_MASK) <<
> +                    HV_HYPERCALL_REP_COMP_OFFSET;
> +            }
> +            break;
>          default:
>              exit->u.hcall.result = HV_STATUS_INVALID_HYPERCALL_CODE;
>          }
> diff --git a/target/i386/kvm/kvm.c b/target/i386/kvm/kvm.c
> index 072da8b9f1..46c983f963 100644
> --- a/target/i386/kvm/kvm.c
> +++ b/target/i386/kvm/kvm.c
> @@ -1137,6 +1137,13 @@ static struct {
>          },
>          .dependencies = BIT(HYPERV_FEAT_VAPIC)
>      },
> +    [HYPERV_FEAT_EXT_HYPERCALLS] = {
> +        .desc = "enable extended hypercalls (hv-extcalls)",
> +        .flags = {
> +            {.func = HV_CPUID_FEATURES, .reg = R_EBX,
> +             .bits = HV_ENABLE_EXTENDED_HYPERCALLS}
> +        }
> +    },
>  };
>  
>  static struct kvm_cpuid2 *try_get_hv_cpuid(CPUState *cs, int max,
> -- 
> 2.54.0 (Apple Git-157)
> 
>