[PATCH v6 0/4] Expose HW APIC virtualization support to HVM guests

[PATCH v6 0/4] Expose HW APIC virtualization support to HVM guests

[Boris Ostrovsky]

Version 6:
* Drop generic support for changing hypervisor leaves, only allow modifying max number of leaves, ignore all other bit change requests.

Version 5:
* Remember to deal with Viridian leaves in xc_cpuid_constrain()

Version 4:
* asm changes per Jan's suggestions
* Don't pass leaf index number to handlers, check for sub_idx==0
* Added (possibly somewhat overwrought) interface to constrain user-requested
  CPUID to what is allowed (specifically, for leaf 0x40000000 only EAX[7:0]
  can be ovverwritten).

Version 3:
* Removed sysctl for querying hypervisor leaf from libxc, replaced it with
  prefixed CPUID instruction
* Limited ability change hypervisor leaves to 0x40000000
* Fixed IS_HYPERVISOR_LEAF macro

Version 2:
* Added ability to specify hypervisor CPUID leaves in config file (this requires
  new sysctl)
* Use 2 bits to indicate what is supported --- one for APIC memory access and the
  other for x2APIC. Still not sure whether virtual interrupt delivery should be
  exposed as well.



HVM guests running on HW that supports HW APIC virtualization features
(APIC-register virtualization, virtual interrupt delivery, etc) may
want to use APIC instead of hvm_pirqs. Since we are not guaranteed to
have these features on VMX (for example, there is a boot option to
turn it off) and there is no such support on SVM we need to make the
guest aware that its APIC accesses may not be so bad.

CPUID seems to be a good way to provide this info to the guest.

Having a guest switch to APIC shows fairly good impact on number of
VMEXITs. For example, with a pass-through NIC, netperf sees almost
half as many. Here are results for 'xentrace -e 0x00083fff -c 2 -D -T 2'

(The guest here essentially turned off XENFEAT_hvm_pirqs but we may
want to use APIC for MSI interrupts only and leave pirqs for gsi).


[root@ovs105 virt]#  cat orig  |xentrace_format  ~/xen/tools/xentrace/formats | awk  '{print $5}' | sort  | uniq -c
     94 cpu_change
  13944 HLT
  26341 INJ_VIRQ
  12054 INTR
  30784 INTR_WINDOW
  10126 TRAP
 124783 VMENTRY
 124782 VMEXIT
  59217 VMMCALL
     35 wrap_buffer
 
 
[root@ovs105 virt]#  cat apicv  |xentrace_format  ~/xen/tools/xentrace/formats | awk  '{print $5}' | sort  | uniq -c
     49 cpu_change
  16157 HLT
     31 INJ_VIRQ
  10652 INTR
     38 INTR_WINDOW
     10 NPF
  10286 TRAP
  71269 VMENTRY
  71269 VMEXIT
  34129 VMMCALL
     15 wrap_buffer

The difference is even larger when the guest is busy.

These results are in line with what has been reported for KVM. For example
http://events.linuxfoundation.org/sites/events/files/cojp13_natapov.pdf
http://www.linuxplumbersconf.org/2012/wp-content/uploads/2012/09/2012-lpc-virt-intel-vt-feat-nakajima.pdf

I am also not sure whether (cpu_has_vmx_apic_reg_virt &
cpu_has_vmx_virtualize_x2apic_mode) is sufficient to declare full HW
APIC support to a guest. The tests show ~95K VMEXITs when virtual
interrupt delivery and posted interrupts are turned off so there
appears to still be some benefit. I suppose we can use another CPUID
bit for these two (although I am not particularly eager to do this).



Boris Ostrovsky (4):
  xen/libxc: Allow changing max number of hypervisor cpuid leaves
  x86/hvm: Revert 80ecb40362365ba77e68fc609de8bd3b7208ae19
  x86/hvm: Add HVM-specific hypervisor CPUID leaf
  x86/hvm: Indicate avaliability of HW support of APIC virtualization
    to HVM guests

 tools/libxc/xc_cpuid_x86.c          |   11 +++++++++++
 xen/arch/x86/hvm/hvm.c              |    9 +++++++++
 xen/arch/x86/hvm/vmx/vmx.c          |   15 +++++++++++++++
 xen/arch/x86/traps.c                |   18 ++++++++++++------
 xen/include/asm-x86/hvm/hvm.h       |    7 +++++++
 xen/include/public/arch-x86/cpuid.h |   10 ++++++++++
 6 files changed, 64 insertions(+), 6 deletions(-)

-- 
1.7.10.4

[PATCH v6 1/4] xen/libxc: Allow changing max number of hypervisor cpuid leaves

[Boris Ostrovsky]

Add support for changing max number of hypervisor leaves from configuration
file.

This number can be specified using xl's standard 'cpuid' option. Only lowest
8 bits of leaf's 0x40000x00 eax register are processed, all others are ignored.

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 tools/libxc/xc_cpuid_x86.c |   11 +++++++++++
 xen/arch/x86/traps.c       |   18 ++++++++++++------
 2 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/tools/libxc/xc_cpuid_x86.c b/tools/libxc/xc_cpuid_x86.c
index bbbf9b8..21ba68b 100644
--- a/tools/libxc/xc_cpuid_x86.c
+++ b/tools/libxc/xc_cpuid_x86.c
@@ -555,6 +555,17 @@ static int xc_cpuid_policy(
 {
     xc_dominfo_t        info;
 
+    /*
+     * For hypervisor leaves (0x4000XXXX) only 0x40000x00.EAX[7:0] bits (max
+     * number of leaves) can be set by user. Hypervisor will enforce this so
+     * all other bits are don't-care and we can set them to zero.
+     */
+    if ( (input[0] & 0xffff0000) == 0x40000000 )
+    {
+        regs[0] = regs[1] = regs[2] = regs[3] = 0;
+        return 0;
+    }
+
     if ( xc_domain_getinfo(xch, domid, 1, &info) == 0 )
         return -EINVAL;
 
diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c
index c462317..65c34f3 100644
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -677,15 +677,21 @@ int cpuid_hypervisor_leaves( uint32_t idx, uint32_t sub_idx,
     struct domain *d = current->domain;
     /* Optionally shift out of the way of Viridian architectural leaves. */
     uint32_t base = is_viridian_domain(d) ? 0x40000100 : 0x40000000;
-    uint32_t limit;
+    uint32_t limit, dummy;
 
     idx -= base;
+    if ( idx > 3 )
+        return 0; /* Avoid unnecessary pass through domain_cpuid() */
 
-    /*
-     * Some Solaris PV drivers fail if max > base + 2. Help them out by
-     * hiding the PVRDTSCP leaf if PVRDTSCP is disabled.
-     */
-    limit = (d->arch.tsc_mode < TSC_MODE_PVRDTSCP) ? 2 : 3;
+    /* Number of leaves may be user-specified */
+    domain_cpuid(d, base, 0, &limit, &dummy, &dummy, &dummy);
+    limit &= 0xff;
+    if ( (limit < 2) || (limit > 3) )
+        /*
+         * Some Solaris PV drivers fail if max > base + 2. Help them out by
+         * hiding the PVRDTSCP leaf if PVRDTSCP is disabled.
+         */
+        limit = (d->arch.tsc_mode < TSC_MODE_PVRDTSCP) ? 2 : 3;
 
     if ( idx > limit ) 
         return 0;
-- 
1.7.10.4

Re: [PATCH v6 1/4] xen/libxc: Allow changing max number of hypervisor cpuid leaves

[Jan Beulich]

>>> On 21.03.14 at 04:51, Boris Ostrovsky <boris.ostrovsky@oracle.com> wrote:
> --- a/tools/libxc/xc_cpuid_x86.c
> +++ b/tools/libxc/xc_cpuid_x86.c
> @@ -555,6 +555,17 @@ static int xc_cpuid_policy(
>  {
>      xc_dominfo_t        info;
>  
> +    /*
> +     * For hypervisor leaves (0x4000XXXX) only 0x40000x00.EAX[7:0] bits (max

0x4000xx00.EAX[7:0] (also in the description).

> +     * number of leaves) can be set by user. Hypervisor will enforce this so
> +     * all other bits are don't-care and we can set them to zero.
> +     */
> +    if ( (input[0] & 0xffff0000) == 0x40000000 )
> +    {
> +        regs[0] = regs[1] = regs[2] = regs[3] = 0;
> +        return 0;
> +    }

Except that x in the config won't work anymore (not that I consider
this particularly useful here, but you never know what people come
up with - read: I'm not sure this needs dealing with).

> --- a/xen/arch/x86/traps.c
> +++ b/xen/arch/x86/traps.c
> @@ -677,15 +677,21 @@ int cpuid_hypervisor_leaves( uint32_t idx, uint32_t sub_idx,
>      struct domain *d = current->domain;
>      /* Optionally shift out of the way of Viridian architectural leaves. */
>      uint32_t base = is_viridian_domain(d) ? 0x40000100 : 0x40000000;
> -    uint32_t limit;
> +    uint32_t limit, dummy;
>  
>      idx -= base;
> +    if ( idx > 3 )
> +        return 0; /* Avoid unnecessary pass through domain_cpuid() */
>  
> -    /*
> -     * Some Solaris PV drivers fail if max > base + 2. Help them out by
> -     * hiding the PVRDTSCP leaf if PVRDTSCP is disabled.
> -     */
> -    limit = (d->arch.tsc_mode < TSC_MODE_PVRDTSCP) ? 2 : 3;
> +    /* Number of leaves may be user-specified */
> +    domain_cpuid(d, base, 0, &limit, &dummy, &dummy, &dummy);
> +    limit &= 0xff;
> +    if ( (limit < 2) || (limit > 3) )

So all of the sudden you also enforce a lower limit? That doesn't
seem to belong here (and at a first glance it doesn't seem to be
right either).

There are two things you want enforce
- limit never becoming zero due to user override (at once taking
  care of the no override case)
- limit never getting set higher than the hypervisor internal limit
but that's unrelated to the Solaris specifics here, so should be
handled separately (and in fact there's no point in having the
fake revert in the next patch then: do the right thing here right
away, as what the next patch does isn't really a revert anymore
with the changes here in place).

Jan

> +        /*
> +         * Some Solaris PV drivers fail if max > base + 2. Help them out by
> +         * hiding the PVRDTSCP leaf if PVRDTSCP is disabled.
> +         */
> +        limit = (d->arch.tsc_mode < TSC_MODE_PVRDTSCP) ? 2 : 3;
>  
>      if ( idx > limit ) 
>          return 0;

Re: [PATCH v6 1/4] xen/libxc: Allow changing max number of hypervisor cpuid leaves

[Boris Ostrovsky]

On 03/21/2014 06:54 AM, Jan Beulich wrote:
>>>> On 21.03.14 at 04:51, Boris Ostrovsky <boris.ostrovsky@oracle.com> wrote:
>> --- a/tools/libxc/xc_cpuid_x86.c
>> +++ b/tools/libxc/xc_cpuid_x86.c
>> @@ -555,6 +555,17 @@ static int xc_cpuid_policy(
>>   {
>>       xc_dominfo_t        info;
>>   
>> +    /*
>> +     * For hypervisor leaves (0x4000XXXX) only 0x40000x00.EAX[7:0] bits (max
> 0x4000xx00.EAX[7:0] (also in the description).
>
>> +     * number of leaves) can be set by user. Hypervisor will enforce this so
>> +     * all other bits are don't-care and we can set them to zero.
>> +     */
>> +    if ( (input[0] & 0xffff0000) == 0x40000000 )
>> +    {
>> +        regs[0] = regs[1] = regs[2] = regs[3] = 0;
>> +        return 0;
>> +    }
> Except that x in the config won't work anymore (not that I consider
> this particularly useful here, but you never know what people come
> up with - read: I'm not sure this needs dealing with).

Yes, I did think about 'x' but didn't feel that it's worth adding back
pv cpuid (which would address this issue) since specifying 'x' only
for some bits in the number of leaves field seemed somewhat strange.

>> --- a/xen/arch/x86/traps.c
>> +++ b/xen/arch/x86/traps.c
>> @@ -677,15 +677,21 @@ int cpuid_hypervisor_leaves( uint32_t idx, uint32_t sub_idx,
>>       struct domain *d = current->domain;
>>       /* Optionally shift out of the way of Viridian architectural leaves. */
>>       uint32_t base = is_viridian_domain(d) ? 0x40000100 : 0x40000000;
>> -    uint32_t limit;
>> +    uint32_t limit, dummy;
>>   
>>       idx -= base;
>> +    if ( idx > 3 )
>> +        return 0; /* Avoid unnecessary pass through domain_cpuid() */
>>   
>> -    /*
>> -     * Some Solaris PV drivers fail if max > base + 2. Help them out by
>> -     * hiding the PVRDTSCP leaf if PVRDTSCP is disabled.
>> -     */
>> -    limit = (d->arch.tsc_mode < TSC_MODE_PVRDTSCP) ? 2 : 3;
>> +    /* Number of leaves may be user-specified */
>> +    domain_cpuid(d, base, 0, &limit, &dummy, &dummy, &dummy);
>> +    limit &= 0xff;
>> +    if ( (limit < 2) || (limit > 3) )
> So all of the sudden you also enforce a lower limit? That doesn't
> seem to belong here (and at a first glance it doesn't seem to be
> right either).
>
> There are two things you want enforce
> - limit never becoming zero due to user override (at once taking
>    care of the no override case)

You think not having leaf 1 is something we should allow? I can see why
2 perhaps may be omitted.

> - limit never getting set higher than the hypervisor internal limit
> but that's unrelated to the Solaris specifics here, so should be
> handled separately (and in fact there's no point in having the
> fake revert in the next patch then: do the right thing here right
> away, as what the next patch does isn't really a revert anymore
> with the changes here in place).

Yes, I thought that reverting Solaris fix as a standalone patch may be too
much work for no benefit. I'll merge it into the first patch.

-boris

Re: [PATCH v6 1/4] xen/libxc: Allow changing max number of hypervisor cpuid leaves

[Jan Beulich]

>>> On 21.03.14 at 16:13, Boris Ostrovsky <boris.ostrovsky@oracle.com> wrote:
> On 03/21/2014 06:54 AM, Jan Beulich wrote:
>>>>> On 21.03.14 at 04:51, Boris Ostrovsky <boris.ostrovsky@oracle.com> wrote:
>>> -    /*
>>> -     * Some Solaris PV drivers fail if max > base + 2. Help them out by
>>> -     * hiding the PVRDTSCP leaf if PVRDTSCP is disabled.
>>> -     */
>>> -    limit = (d->arch.tsc_mode < TSC_MODE_PVRDTSCP) ? 2 : 3;
>>> +    /* Number of leaves may be user-specified */
>>> +    domain_cpuid(d, base, 0, &limit, &dummy, &dummy, &dummy);
>>> +    limit &= 0xff;
>>> +    if ( (limit < 2) || (limit > 3) )
>> So all of the sudden you also enforce a lower limit? That doesn't
>> seem to belong here (and at a first glance it doesn't seem to be
>> right either).
>>
>> There are two things you want enforce
>> - limit never becoming zero due to user override (at once taking
>>    care of the no override case)
> 
> You think not having leaf 1 is something we should allow? I can see why
> 2 perhaps may be omitted.

If we force the lower limit that can be asked for to 1, leaf 1 will always
be visible.

Jan

Re: [PATCH v6 1/4] xen/libxc: Allow changing max number of hypervisor cpuid leaves

[Boris Ostrovsky]

On 03/21/2014 11:15 AM, Jan Beulich wrote:
>>
>> There are two things you want enforce
>> - limit never becoming zero due to user override (at once taking
>>     care of the no override case)
>> You think not having leaf 1 is something we should allow? I can see why
>> 2 perhaps may be omitted.
> If we force the lower limit that can be asked for to 1, leaf 1 will always
> be visible.

Oh, I clearly have issues with reading comprehension (among other 
things). Sorry.

-boris

[PATCH v6 2/4] x86/hvm: Revert 80ecb40362365ba77e68fc609de8bd3b7208ae19

[Boris Ostrovsky]

The Solaris bug that commit 80ecb40362365ba77e68fc609de8bd3b7208ae19
addressed has been fixed and backported to earlier releases.

Those still using those releases can specify number of hypervisor leaves
explicitly via 'cpuid' xl config option.

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 xen/arch/x86/traps.c |    6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c
index 65c34f3..b4fadf0 100644
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -687,11 +687,7 @@ int cpuid_hypervisor_leaves( uint32_t idx, uint32_t sub_idx,
     domain_cpuid(d, base, 0, &limit, &dummy, &dummy, &dummy);
     limit &= 0xff;
     if ( (limit < 2) || (limit > 3) )
-        /*
-         * Some Solaris PV drivers fail if max > base + 2. Help them out by
-         * hiding the PVRDTSCP leaf if PVRDTSCP is disabled.
-         */
-        limit = (d->arch.tsc_mode < TSC_MODE_PVRDTSCP) ? 2 : 3;
+        limit = 3;
 
     if ( idx > limit ) 
         return 0;
-- 
1.7.10.4

[PATCH v6 3/4] x86/hvm: Add HVM-specific hypervisor CPUID leaf

[Boris Ostrovsky]

CPUID leaf 0x40000004 is for HVM-specific features.

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 xen/arch/x86/hvm/hvm.c        |    9 +++++++++
 xen/arch/x86/traps.c          |   10 +++++++---
 xen/include/asm-x86/hvm/hvm.h |    7 +++++++
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/xen/arch/x86/hvm/hvm.c b/xen/arch/x86/hvm/hvm.c
index ae24211..b07f11e 100644
--- a/xen/arch/x86/hvm/hvm.c
+++ b/xen/arch/x86/hvm/hvm.c
@@ -2980,6 +2980,15 @@ unsigned long copy_from_user_hvm(void *to, const void *from, unsigned len)
     return rc ? len : 0; /* fake a copy_from_user() return code */
 }
 
+void hvm_hypervisor_cpuid_leaf(uint32_t sub_idx,
+                               uint32_t *eax, uint32_t *ebx,
+                               uint32_t *ecx, uint32_t *edx)
+{
+    *eax = *ebx = *ecx = *edx = 0;
+    if ( hvm_funcs.hypervisor_cpuid_leaf )
+        hvm_funcs.hypervisor_cpuid_leaf(sub_idx, eax, ebx, ecx, edx);
+}
+
 void hvm_cpuid(unsigned int input, unsigned int *eax, unsigned int *ebx,
                                    unsigned int *ecx, unsigned int *edx)
 {
diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c
index b4fadf0..be78932 100644
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -680,14 +680,14 @@ int cpuid_hypervisor_leaves( uint32_t idx, uint32_t sub_idx,
     uint32_t limit, dummy;
 
     idx -= base;
-    if ( idx > 3 )
+    if ( idx > 4 )
         return 0; /* Avoid unnecessary pass through domain_cpuid() */
 
     /* Number of leaves may be user-specified */
     domain_cpuid(d, base, 0, &limit, &dummy, &dummy, &dummy);
     limit &= 0xff;
-    if ( (limit < 2) || (limit > 3) )
-        limit = 3;
+    if ( (limit < 2) || (limit > 4) )
+        limit = 4;
 
     if ( idx > limit ) 
         return 0;
@@ -724,6 +724,10 @@ int cpuid_hypervisor_leaves( uint32_t idx, uint32_t sub_idx,
         cpuid_time_leaf( sub_idx, eax, ebx, ecx, edx );
         break;
 
+    case 4:
+        hvm_hypervisor_cpuid_leaf(sub_idx, eax, ebx, ecx, edx);
+        break;
+
     default:
         BUG();
     }
diff --git a/xen/include/asm-x86/hvm/hvm.h b/xen/include/asm-x86/hvm/hvm.h
index dcc3483..a030ea4 100644
--- a/xen/include/asm-x86/hvm/hvm.h
+++ b/xen/include/asm-x86/hvm/hvm.h
@@ -200,6 +200,10 @@ struct hvm_function_table {
                                 paddr_t *L1_gpa, unsigned int *page_order,
                                 uint8_t *p2m_acc, bool_t access_r,
                                 bool_t access_w, bool_t access_x);
+
+    void (*hypervisor_cpuid_leaf)(uint32_t sub_idx,
+                                  uint32_t *eax, uint32_t *ebx,
+                                  uint32_t *ecx, uint32_t *edx);
 };
 
 extern struct hvm_function_table hvm_funcs;
@@ -336,6 +340,9 @@ static inline unsigned long hvm_get_shadow_gs_base(struct vcpu *v)
 #define is_viridian_domain(_d)                                             \
  (is_hvm_domain(_d) && ((_d)->arch.hvm_domain.params[HVM_PARAM_VIRIDIAN]))
 
+void hvm_hypervisor_cpuid_leaf(uint32_t sub_idx,
+                               uint32_t *eax, uint32_t *ebx,
+                               uint32_t *ecx, uint32_t *edx);
 void hvm_cpuid(unsigned int input, unsigned int *eax, unsigned int *ebx,
                                    unsigned int *ecx, unsigned int *edx);
 void hvm_migrate_timers(struct vcpu *v);
-- 
1.7.10.4

[PATCH v6 4/4] x86/hvm: Indicate avaliability of HW support of APIC virtualization to HVM guests

[Boris Ostrovsky]

Set bits in hypervisor CPUID leaf indicating that HW provides (and the
hypervisor enables) HW support for APIC and x2APIC virtualization.

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 xen/arch/x86/hvm/vmx/vmx.c          |   15 +++++++++++++++
 xen/include/public/arch-x86/cpuid.h |   10 ++++++++++
 2 files changed, 25 insertions(+)

diff --git a/xen/arch/x86/hvm/vmx/vmx.c b/xen/arch/x86/hvm/vmx/vmx.c
index 8395e86..a59bac6 100644
--- a/xen/arch/x86/hvm/vmx/vmx.c
+++ b/xen/arch/x86/hvm/vmx/vmx.c
@@ -57,6 +57,7 @@
 #include <asm/apic.h>
 #include <asm/hvm/nestedhvm.h>
 #include <asm/event.h>
+#include <public/arch-x86/cpuid.h>
 
 enum handler_return { HNDL_done, HNDL_unhandled, HNDL_exception_raised };
 
@@ -1646,6 +1647,19 @@ static void vmx_handle_eoi(u8 vector)
     __vmwrite(GUEST_INTR_STATUS, status);
 }
 
+void vmx_hypervisor_cpuid_leaf(uint32_t sub_idx,
+                               uint32_t *eax, uint32_t *ebx,
+                               uint32_t *ecx, uint32_t *edx)
+{
+    if ( sub_idx != 0 )
+        return;
+
+    if ( cpu_has_vmx_apic_reg_virt )
+        *eax |= XEN_HVM_CPUID_APIC_ACCESS_VIRT;
+    if ( cpu_has_vmx_virtualize_x2apic_mode )
+        *eax |= XEN_HVM_CPUID_X2APIC_VIRT;
+}
+
 static struct hvm_function_table __initdata vmx_function_table = {
     .name                 = "VMX",
     .cpu_up_prepare       = vmx_cpu_up_prepare,
@@ -1703,6 +1717,7 @@ static struct hvm_function_table __initdata vmx_function_table = {
     .sync_pir_to_irr      = vmx_sync_pir_to_irr,
     .handle_eoi           = vmx_handle_eoi,
     .nhvm_hap_walk_L1_p2m = nvmx_hap_walk_L1_p2m,
+    .hypervisor_cpuid_leaf= vmx_hypervisor_cpuid_leaf,
 };
 
 const struct hvm_function_table * __init start_vmx(void)
diff --git a/xen/include/public/arch-x86/cpuid.h b/xen/include/public/arch-x86/cpuid.h
index d9bd627..7118760 100644
--- a/xen/include/public/arch-x86/cpuid.h
+++ b/xen/include/public/arch-x86/cpuid.h
@@ -65,4 +65,14 @@
 #define _XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD 0
 #define XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD  (1u<<0)
 
+/*
+ * Leaf 5 (0x40000004)
+ * HVM-specific features
+ */
+
+/* EAX Features */
+#define XEN_HVM_CPUID_APIC_ACCESS_VIRT (1u << 0) /* Virtualized APIC registers */
+#define XEN_HVM_CPUID_X2APIC_VIRT      (1u << 1) /* Virtualized x2APIC accesses */
+
+
 #endif /* __XEN_PUBLIC_ARCH_X86_CPUID_H__ */
-- 
1.7.10.4