[PATCH 00 of 20] NestedVMX support

[PATCH 00 of 20] NestedVMX support

[Eddie Dong]

This patch series enable the nestedvmx support.

patch 1-2 is a pre-cleanup.
patch 3 is for nested vmx data structure
patch 4 for nestedhvm ops
patch 5, 7,8,9, 10, 11 is for VMX instruction emulation
patch 6 for virtual VMCS data structure and access APIs.
patch 12 for VMCS switching
Patch 13 for vmreseume/launch emulation
patch 14 for shadow control VMCS fields
patch 15 for n1/n2 guest VMCS switch 
patch 16 for interrupt/exceptions
patch 17 for nested vm exit
patch 18 for lazy FPU and patch 19 VMXE bits in CR4
patch 20 for MSR handling and capability


Thanks, Eddie

[PATCH 01 of 20] pre-cleanup1: Extend nhvm_vmcx_guest_intercepts_trap to include errcode to

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003600 -28800
# Node ID 332616c43f52f85893f41537e9e6324a3fa01a57
# Parent  0c446850d85e654dfde039a0a1a5acd4e6b3c278
pre-cleanup1: Extend nhvm_vmcx_guest_intercepts_trap to include errcode to
assist decision of TRAP_page_fault in VMX.

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r 0c446850d85e -r 332616c43f52 xen/arch/x86/hvm/hvm.c
--- a/xen/arch/x86/hvm/hvm.c	Wed May 11 12:58:04 2011 +0100
+++ b/xen/arch/x86/hvm/hvm.c	Thu Jun 02 16:33:20 2011 +0800
@@ -1152,7 +1152,7 @@ void hvm_inject_exception(unsigned int t
         return;
     }
 
-    if ( nhvm_vmcx_guest_intercepts_trap(v, trapnr) )
+    if ( nhvm_vmcx_guest_intercepts_trap(v, trapnr, errcode) )
     {
         enum nestedhvm_vmexits nsret;
 
@@ -4175,10 +4175,10 @@ uint32_t nhvm_vcpu_asid(struct vcpu *v)
     return -EOPNOTSUPP;
 }
 
-int nhvm_vmcx_guest_intercepts_trap(struct vcpu *v, unsigned int trap)
+int nhvm_vmcx_guest_intercepts_trap(struct vcpu *v, unsigned int trap, int errcode)
 {
     if (hvm_funcs.nhvm_vmcx_guest_intercepts_trap)
-        return hvm_funcs.nhvm_vmcx_guest_intercepts_trap(v, trap);
+        return hvm_funcs.nhvm_vmcx_guest_intercepts_trap(v, trap, errcode);
     return -EOPNOTSUPP;
 }
 
diff -r 0c446850d85e -r 332616c43f52 xen/arch/x86/hvm/svm/nestedsvm.c
--- a/xen/arch/x86/hvm/svm/nestedsvm.c	Wed May 11 12:58:04 2011 +0100
+++ b/xen/arch/x86/hvm/svm/nestedsvm.c	Thu Jun 02 16:33:20 2011 +0800
@@ -895,7 +895,7 @@ nsvm_vmcb_guest_intercepts_exitcode(stru
 }
 
 int
-nsvm_vmcb_guest_intercepts_trap(struct vcpu *v, unsigned int trapnr)
+nsvm_vmcb_guest_intercepts_trap(struct vcpu *v, unsigned int trapnr, int errcode)
 {
     return nsvm_vmcb_guest_intercepts_exitcode(v,
         guest_cpu_user_regs(), VMEXIT_EXCEPTION_DE + trapnr);
diff -r 0c446850d85e -r 332616c43f52 xen/include/asm-x86/hvm/hvm.h
--- a/xen/include/asm-x86/hvm/hvm.h	Wed May 11 12:58:04 2011 +0100
+++ b/xen/include/asm-x86/hvm/hvm.h	Thu Jun 02 16:33:20 2011 +0800
@@ -164,7 +164,8 @@ struct hvm_function_table {
     uint64_t (*nhvm_vcpu_guestcr3)(struct vcpu *v);
     uint64_t (*nhvm_vcpu_hostcr3)(struct vcpu *v);
     uint32_t (*nhvm_vcpu_asid)(struct vcpu *v);
-    int (*nhvm_vmcx_guest_intercepts_trap)(struct vcpu *v, unsigned int trapnr);
+    int (*nhvm_vmcx_guest_intercepts_trap)(struct vcpu *v, 
+                               unsigned int trapnr, int errcode);
 
     bool_t (*nhvm_vmcx_hap_enabled)(struct vcpu *v);
 
@@ -443,7 +444,8 @@ uint64_t nhvm_vcpu_hostcr3(struct vcpu *
 uint32_t nhvm_vcpu_asid(struct vcpu *v);
 
 /* returns true, when l1 guest intercepts the specified trap */
-int nhvm_vmcx_guest_intercepts_trap(struct vcpu *v, unsigned int trapnr);
+int nhvm_vmcx_guest_intercepts_trap(struct vcpu *v, 
+                                    unsigned int trapnr, int errcode);
 
 /* returns true when l1 guest wants to use hap to run l2 guest */
 bool_t nhvm_vmcx_hap_enabled(struct vcpu *v);
diff -r 0c446850d85e -r 332616c43f52 xen/include/asm-x86/hvm/svm/nestedsvm.h
--- a/xen/include/asm-x86/hvm/svm/nestedsvm.h	Wed May 11 12:58:04 2011 +0100
+++ b/xen/include/asm-x86/hvm/svm/nestedsvm.h	Thu Jun 02 16:33:20 2011 +0800
@@ -114,7 +114,8 @@ uint64_t nsvm_vcpu_hostcr3(struct vcpu *
 uint32_t nsvm_vcpu_asid(struct vcpu *v);
 int nsvm_vmcb_guest_intercepts_exitcode(struct vcpu *v,
     struct cpu_user_regs *regs, uint64_t exitcode);
-int nsvm_vmcb_guest_intercepts_trap(struct vcpu *v, unsigned int trapnr);
+int nsvm_vmcb_guest_intercepts_trap(struct vcpu *v, unsigned int trapnr,
+                                    int errcode);
 bool_t nsvm_vmcb_hap_enabled(struct vcpu *v);
 enum hvm_intblk nsvm_intr_blocked(struct vcpu *v);

[PATCH 02 of 20] pre-cleanup2: Move IDT_VECTORING processing code out of intr_assist

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003600 -28800
# Node ID ce6ed8ca4ebd2f2fb96627e61f7d2ef737e7193d
# Parent  332616c43f52f85893f41537e9e6324a3fa01a57
pre-cleanup2: Move IDT_VECTORING processing code out of intr_assist.

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r 332616c43f52 -r ce6ed8ca4ebd xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:20 2011 +0800
@@ -2098,6 +2098,33 @@ static int vmx_handle_eoi_write(void)
     return 0;
 }
 
+static void vmx_idtv_reinject(unsigned long idtv_info)
+{
+
+    /* Event delivery caused this intercept? Queue for redelivery. */
+    if ( unlikely(idtv_info & INTR_INFO_VALID_MASK) )
+    {
+        if ( hvm_event_needs_reinjection((idtv_info>>8)&7, idtv_info&0xff) )
+        {
+            /* See SDM 3B 25.7.1.1 and .2 for info about masking resvd bits. */
+            __vmwrite(VM_ENTRY_INTR_INFO,
+                      idtv_info & ~INTR_INFO_RESVD_BITS_MASK);
+            if ( idtv_info & INTR_INFO_DELIVER_CODE_MASK )
+                __vmwrite(VM_ENTRY_EXCEPTION_ERROR_CODE,
+                          __vmread(IDT_VECTORING_ERROR_CODE));
+        }
+
+        /*
+         * Clear NMI-blocking interruptibility info if an NMI delivery faulted.
+         * Re-delivery will re-set it (see SDM 3B 25.7.1.2).
+         */
+        if ( (idtv_info & INTR_INFO_INTR_TYPE_MASK) == (X86_EVENTTYPE_NMI<<8) )
+            __vmwrite(GUEST_INTERRUPTIBILITY_INFO,
+                      __vmread(GUEST_INTERRUPTIBILITY_INFO) &
+                      ~VMX_INTR_SHADOW_NMI);
+    }
+}
+
 asmlinkage void vmx_vmexit_handler(struct cpu_user_regs *regs)
 {
     unsigned int exit_reason, idtv_info, intr_info = 0, vector = 0;
@@ -2187,30 +2214,9 @@ asmlinkage void vmx_vmexit_handler(struc
 
     hvm_maybe_deassert_evtchn_irq();
 
-    /* Event delivery caused this intercept? Queue for redelivery. */
     idtv_info = __vmread(IDT_VECTORING_INFO);
-    if ( unlikely(idtv_info & INTR_INFO_VALID_MASK) &&
-         (exit_reason != EXIT_REASON_TASK_SWITCH) )
-    {
-        if ( hvm_event_needs_reinjection((idtv_info>>8)&7, idtv_info&0xff) )
-        {
-            /* See SDM 3B 25.7.1.1 and .2 for info about masking resvd bits. */
-            __vmwrite(VM_ENTRY_INTR_INFO,
-                      idtv_info & ~INTR_INFO_RESVD_BITS_MASK);
-            if ( idtv_info & INTR_INFO_DELIVER_CODE_MASK )
-                __vmwrite(VM_ENTRY_EXCEPTION_ERROR_CODE,
-                          __vmread(IDT_VECTORING_ERROR_CODE));
-        }
-
-        /*
-         * Clear NMI-blocking interruptibility info if an NMI delivery faulted.
-         * Re-delivery will re-set it (see SDM 3B 25.7.1.2).
-         */
-        if ( (idtv_info & INTR_INFO_INTR_TYPE_MASK) == (X86_EVENTTYPE_NMI<<8) )
-            __vmwrite(GUEST_INTERRUPTIBILITY_INFO,
-                      __vmread(GUEST_INTERRUPTIBILITY_INFO) &
-                      ~VMX_INTR_SHADOW_NMI);
-    }
+    if ( exit_reason != EXIT_REASON_TASK_SWITCH )
+        vmx_idtv_reinject(idtv_info);
 
     switch ( exit_reason )
     {

[PATCH 03 of 20] Add data structure for nestedvmx

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003600 -28800
# Node ID 4bbf0eaec85c764c7872d1cfc1c59c419dfabe0a
# Parent  ce6ed8ca4ebd2f2fb96627e61f7d2ef737e7193d
Add data structure for nestedvmx

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r ce6ed8ca4ebd -r 4bbf0eaec85c xen/include/asm-x86/hvm/vcpu.h
--- a/xen/include/asm-x86/hvm/vcpu.h	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/include/asm-x86/hvm/vcpu.h	Thu Jun 02 16:33:20 2011 +0800
@@ -24,6 +24,7 @@
 #include <asm/hvm/io.h>
 #include <asm/hvm/vlapic.h>
 #include <asm/hvm/vmx/vmcs.h>
+#include <asm/hvm/vmx/vvmx.h>
 #include <asm/hvm/svm/vmcb.h>
 #include <asm/hvm/svm/nestedsvm.h>
 #include <asm/mtrr.h>
@@ -57,6 +58,7 @@ struct nestedvcpu {
     /* SVM/VMX arch specific */
     union {
         struct nestedsvm nsvm;
+        struct nestedvmx nvmx;
     } u;
 
     bool_t nv_flushp2m; /* True, when p2m table must be flushed */
diff -r ce6ed8ca4ebd -r 4bbf0eaec85c xen/include/asm-x86/hvm/vmx/vvmx.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:20 2011 +0800
@@ -0,0 +1,38 @@
+
+/*
+ * vvmx.h: Support virtual VMX for nested virtualization.
+ *
+ * Copyright (c) 2010, Intel Corporation.
+ * Author: Qing He <qing.he@intel.com>
+ *         Eddie Dong <eddie.dong@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ *
+ */
+#ifndef __ASM_X86_HVM_VVMX_H__
+#define __ASM_X86_HVM_VVMX_H__
+
+struct nestedvmx {
+    paddr_t    vmxon_region_pa;
+    void       *iobitmap[2];		/* map (va) of L1 guest I/O bitmap */
+    /* deferred nested interrupt */
+    struct {
+        unsigned long intr_info;
+        u32           error_code;
+    } intr;
+};
+
+#define vcpu_2_nvmx(v)	(vcpu_nestedhvm(v).u.nvmx)
+#endif /* __ASM_X86_HVM_VVMX_H__ */
+

[PATCH 04 of 20] Add APIs for nestedhvm_ops

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003600 -28800
# Node ID 4e094881883f10f94575a6f69194a2393e16b7d1
# Parent  4bbf0eaec85c764c7872d1cfc1c59c419dfabe0a
Add APIs for nestedhvm_ops.

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r 4bbf0eaec85c -r 4e094881883f xen/arch/x86/hvm/hvm.c
--- a/xen/arch/x86/hvm/hvm.c	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/hvm.c	Thu Jun 02 16:33:20 2011 +0800
@@ -3502,7 +3502,7 @@ long do_hvm_op(unsigned long op, XEN_GUE
                 /* Remove the check below once we have
                  * shadow-on-shadow.
                  */
-                if ( !paging_mode_hap(d) && a.value )
+                if ( cpu_has_svm && !paging_mode_hap(d) && a.value )
                     rc = -EINVAL;
                 /* Set up NHVM state for any vcpus that are already up */
                 if ( !d->arch.hvm_domain.params[HVM_PARAM_NESTEDHVM] )
diff -r 4bbf0eaec85c -r 4e094881883f xen/arch/x86/hvm/vmx/Makefile
--- a/xen/arch/x86/hvm/vmx/Makefile	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/Makefile	Thu Jun 02 16:33:20 2011 +0800
@@ -4,3 +4,4 @@ obj-y += realmode.o
 obj-y += vmcs.o
 obj-y += vmx.o
 obj-y += vpmu_core2.o
+obj-y += vvmx.o
diff -r 4bbf0eaec85c -r 4e094881883f xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:20 2011 +0800
@@ -1407,7 +1407,13 @@ static struct hvm_function_table __read_
     .invlpg_intercept     = vmx_invlpg_intercept,
     .set_uc_mode          = vmx_set_uc_mode,
     .set_info_guest       = vmx_set_info_guest,
-    .set_rdtsc_exiting    = vmx_set_rdtsc_exiting
+    .set_rdtsc_exiting    = vmx_set_rdtsc_exiting,
+    .nhvm_vcpu_initialise = nvmx_vcpu_initialise,
+    .nhvm_vcpu_destroy    = nvmx_vcpu_destroy,
+    .nhvm_vcpu_reset      = nvmx_vcpu_reset,
+    .nhvm_vcpu_guestcr3   = nvmx_vcpu_guestcr3,
+    .nhvm_vcpu_hostcr3    = nvmx_vcpu_hostcr3,
+    .nhvm_vcpu_asid       = nvmx_vcpu_asid
 };
 
 struct hvm_function_table * __init start_vmx(void)
diff -r 4bbf0eaec85c -r 4e094881883f xen/arch/x86/hvm/vmx/vvmx.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:20 2011 +0800
@@ -0,0 +1,93 @@
+/*
+ * vvmx.c: Support virtual VMX for nested virtualization.
+ *
+ * Copyright (c) 2010, Intel Corporation.
+ * Author: Qing He <qing.he@intel.com>
+ *         Eddie Dong <eddie.dong@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+ * Place - Suite 330, Boston, MA 02111-1307 USA.
+ *
+ */
+
+#include <xen/config.h>
+#include <asm/types.h>
+#include <asm/p2m.h>
+#include <asm/hvm/vmx/vmx.h>
+#include <asm/hvm/vmx/vvmx.h>
+
+int nvmx_vcpu_initialise(struct vcpu *v)
+{
+    struct nestedvmx *nvmx = &vcpu_2_nvmx(v);
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+
+    nvcpu->nv_n2vmcx = alloc_xenheap_page();
+    if ( !nvcpu->nv_n2vmcx )
+    {
+        gdprintk(XENLOG_ERR, "nest: allocation for shadow vmcs failed\n");
+	goto out;
+    }
+    nvmx->vmxon_region_pa = 0;
+    nvcpu->nv_vvmcx = NULL;
+    nvcpu->nv_vvmcxaddr = VMCX_EADDR;
+    nvmx->intr.intr_info = 0;
+    nvmx->intr.error_code = 0;
+    nvmx->iobitmap[0] = NULL;
+    nvmx->iobitmap[1] = NULL;
+    return 0;
+out:
+    return -ENOMEM;
+}
+ 
+void nvmx_vcpu_destroy(struct vcpu *v)
+{
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+
+    if ( nvcpu->nv_n2vmcx ) {
+        __vmpclear(virt_to_maddr(nvcpu->nv_n2vmcx));
+        free_xenheap_page(nvcpu->nv_n2vmcx);
+        nvcpu->nv_n2vmcx = NULL;
+    }
+    if ( nvcpu->nv_vvmcx ) {
+        unmap_domain_page_global(nvcpu->nv_vvmcx);
+        nvcpu->nv_vvmcx == NULL;
+    }
+    nvcpu->nv_vvmcxaddr = VMCX_EADDR;
+}
+ 
+int nvmx_vcpu_reset(struct vcpu *v)
+{
+    return 0;
+}
+
+uint64_t nvmx_vcpu_guestcr3(struct vcpu *v)
+{
+    /* TODO */
+    ASSERT(0);
+    return 0;
+}
+
+uint64_t nvmx_vcpu_hostcr3(struct vcpu *v)
+{
+    /* TODO */
+    ASSERT(0);
+    return 0;
+}
+
+uint32_t nvmx_vcpu_asid(struct vcpu *v)
+{
+    /* TODO */
+    ASSERT(0);
+    return 0;
+}
+
diff -r 4bbf0eaec85c -r 4e094881883f xen/include/asm-x86/hvm/vmx/vvmx.h
--- a/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:20 2011 +0800
@@ -34,5 +34,13 @@ struct nestedvmx {
 };
 
 #define vcpu_2_nvmx(v)	(vcpu_nestedhvm(v).u.nvmx)
+
+int nvmx_vcpu_initialise(struct vcpu *v);
+void nvmx_vcpu_destroy(struct vcpu *v);
+int nvmx_vcpu_reset(struct vcpu *v);
+uint64_t nvmx_vcpu_guestcr3(struct vcpu *v);
+uint64_t nvmx_vcpu_hostcr3(struct vcpu *v);
+uint32_t nvmx_vcpu_asid(struct vcpu *v);
+
 #endif /* __ASM_X86_HVM_VVMX_H__ */

[PATCH 05 of 20] Emulation of guest VMXON/OFF instruction

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003600 -28800
# Node ID c8812151acfd6d9468f3407bc6a1a278cd764567
# Parent  4e094881883f10f94575a6f69194a2393e16b7d1
Emulation of guest VMXON/OFF instruction.

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r 4e094881883f -r c8812151acfd xen/arch/x86/hvm/vmx/Makefile
--- a/xen/arch/x86/hvm/vmx/Makefile	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/Makefile	Thu Jun 02 16:33:20 2011 +0800
@@ -5,3 +5,4 @@ obj-y += vmcs.o
 obj-y += vmx.o
 obj-y += vpmu_core2.o
 obj-y += vvmx.o
+obj-y += vvmx.o
diff -r 4e094881883f -r c8812151acfd xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:20 2011 +0800
@@ -2434,6 +2434,16 @@ asmlinkage void vmx_vmexit_handler(struc
         break;
     }
 
+    case EXIT_REASON_VMXOFF:
+        if ( nvmx_handle_vmxoff(regs) == X86EMUL_OKAY )
+            update_guest_eip();
+        break;
+
+    case EXIT_REASON_VMXON:
+        if ( nvmx_handle_vmxon(regs) == X86EMUL_OKAY )
+            update_guest_eip();
+        break;
+
     case EXIT_REASON_MWAIT_INSTRUCTION:
     case EXIT_REASON_MONITOR_INSTRUCTION:
     case EXIT_REASON_VMCLEAR:
@@ -2443,8 +2453,6 @@ asmlinkage void vmx_vmexit_handler(struc
     case EXIT_REASON_VMREAD:
     case EXIT_REASON_VMRESUME:
     case EXIT_REASON_VMWRITE:
-    case EXIT_REASON_VMXOFF:
-    case EXIT_REASON_VMXON:
     case EXIT_REASON_GETSEC:
     case EXIT_REASON_INVEPT:
     case EXIT_REASON_INVVPID:
diff -r 4e094881883f -r c8812151acfd xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:20 2011 +0800
@@ -91,3 +91,228 @@ uint32_t nvmx_vcpu_asid(struct vcpu *v)
     return 0;
 }
 
+enum x86_segment sreg_to_index[] = {
+    [VMX_SREG_ES] = x86_seg_es,
+    [VMX_SREG_CS] = x86_seg_cs,
+    [VMX_SREG_SS] = x86_seg_ss,
+    [VMX_SREG_DS] = x86_seg_ds,
+    [VMX_SREG_FS] = x86_seg_fs,
+    [VMX_SREG_GS] = x86_seg_gs,
+};
+
+struct vmx_inst_decoded {
+#define VMX_INST_MEMREG_TYPE_MEMORY 0
+#define VMX_INST_MEMREG_TYPE_REG    1
+    int type;
+    union {
+        struct {
+            unsigned long mem;
+            unsigned int  len;
+        };
+        enum vmx_regs_enc reg1;
+    };
+
+    enum vmx_regs_enc reg2;
+};
+
+enum vmx_ops_result {
+    VMSUCCEED,
+    VMFAIL_VALID,
+    VMFAIL_INVALID,
+};
+
+#define CASE_GET_REG(REG, reg)      \
+    case VMX_REG_ ## REG: value = regs->reg; break
+
+static unsigned long reg_read(struct cpu_user_regs *regs,
+                              enum vmx_regs_enc index)
+{
+    unsigned long value = 0;
+
+    switch ( index ) {
+    CASE_GET_REG(RAX, eax);
+    CASE_GET_REG(RCX, ecx);
+    CASE_GET_REG(RDX, edx);
+    CASE_GET_REG(RBX, ebx);
+    CASE_GET_REG(RBP, ebp);
+    CASE_GET_REG(RSI, esi);
+    CASE_GET_REG(RDI, edi);
+    CASE_GET_REG(RSP, esp);
+#ifdef CONFIG_X86_64
+    CASE_GET_REG(R8, r8);
+    CASE_GET_REG(R9, r9);
+    CASE_GET_REG(R10, r10);
+    CASE_GET_REG(R11, r11);
+    CASE_GET_REG(R12, r12);
+    CASE_GET_REG(R13, r13);
+    CASE_GET_REG(R14, r14);
+    CASE_GET_REG(R15, r15);
+#endif
+    default:
+        break;
+    }
+
+    return value;
+}
+
+static int vmx_inst_check_privilege(struct cpu_user_regs *regs, int vmxop_check)
+{
+    struct vcpu *v = current;
+    struct segment_register cs;
+
+    hvm_get_segment_register(v, x86_seg_cs, &cs);
+
+    if ( vmxop_check )
+    {
+        if ( !(v->arch.hvm_vcpu.guest_cr[0] & X86_CR0_PE) ||
+             !(v->arch.hvm_vcpu.guest_cr[4] & X86_CR4_VMXE) )
+            goto invalid_op;
+    }
+    else if ( !vcpu_2_nvmx(v).vmxon_region_pa )
+        goto invalid_op;
+
+    if ( (regs->eflags & X86_EFLAGS_VM) ||
+         (hvm_long_mode_enabled(v) && cs.attr.fields.l == 0) )
+        goto invalid_op;
+    /* TODO: check vmx operation mode */
+
+    if ( (cs.sel & 3) > 0 )
+        goto gp_fault;
+
+    return X86EMUL_OKAY;
+
+invalid_op:
+    gdprintk(XENLOG_ERR, "vmx_inst_check_privilege: invalid_op\n");
+    hvm_inject_exception(TRAP_invalid_op, 0, 0);
+    return X86EMUL_EXCEPTION;
+
+gp_fault:
+    gdprintk(XENLOG_ERR, "vmx_inst_check_privilege: gp_fault\n");
+    hvm_inject_exception(TRAP_gp_fault, 0, 0);
+    return X86EMUL_EXCEPTION;
+}
+
+static int decode_vmx_inst(struct cpu_user_regs *regs,
+                           struct vmx_inst_decoded *decode,
+                           unsigned long *poperandS, int vmxon_check)
+{
+    struct vcpu *v = current;
+    union vmx_inst_info info;
+    struct segment_register seg;
+    unsigned long base, index, seg_base, disp, offset;
+    int scale, size;
+
+    if ( vmx_inst_check_privilege(regs, vmxon_check) != X86EMUL_OKAY )
+        return X86EMUL_EXCEPTION;
+
+    info.word = __vmread(VMX_INSTRUCTION_INFO);
+
+    if ( info.fields.memreg ) {
+        decode->type = VMX_INST_MEMREG_TYPE_REG;
+        decode->reg1 = info.fields.reg1;
+        if ( poperandS != NULL )
+            *poperandS = reg_read(regs, decode->reg1);
+    }
+    else
+    {
+        decode->type = VMX_INST_MEMREG_TYPE_MEMORY;
+        hvm_get_segment_register(v, sreg_to_index[info.fields.segment], &seg);
+        /* TODO: segment type check */
+        seg_base = seg.base;
+
+        base = info.fields.base_reg_invalid ? 0 :
+            reg_read(regs, info.fields.base_reg);
+
+        index = info.fields.index_reg_invalid ? 0 :
+            reg_read(regs, info.fields.index_reg);
+
+        scale = 1 << info.fields.scaling;
+
+        disp = __vmread(EXIT_QUALIFICATION);
+
+        size = 1 << (info.fields.addr_size + 1);
+
+        offset = base + index * scale + disp;
+        if ( (offset > seg.limit || offset + size > seg.limit) &&
+            (!hvm_long_mode_enabled(v) || info.fields.segment == VMX_SREG_GS) )
+            goto gp_fault;
+
+        if ( poperandS != NULL &&
+             hvm_copy_from_guest_virt(poperandS, seg_base + offset, size, 0)
+                  != HVMCOPY_okay )
+            return X86EMUL_EXCEPTION;
+        decode->mem = seg_base + offset;
+        decode->len = size;
+    }
+
+    decode->reg2 = info.fields.reg2;
+
+    return X86EMUL_OKAY;
+
+gp_fault:
+    hvm_inject_exception(TRAP_gp_fault, 0, 0);
+    return X86EMUL_EXCEPTION;
+}
+
+static void vmreturn(struct cpu_user_regs *regs, enum vmx_ops_result ops_res)
+{
+    unsigned long eflags = regs->eflags;
+    unsigned long mask = X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
+                         X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF;
+
+    eflags &= ~mask;
+
+    switch ( ops_res ) {
+    case VMSUCCEED:
+        break;
+    case VMFAIL_VALID:
+        /* TODO: error number, useful for guest VMM debugging */
+        eflags |= X86_EFLAGS_ZF;
+        break;
+    case VMFAIL_INVALID:
+    default:
+        eflags |= X86_EFLAGS_CF;
+        break;
+    }
+
+    regs->eflags = eflags;
+}
+
+/*
+ * VMX instructions handling
+ */
+
+int nvmx_handle_vmxon(struct cpu_user_regs *regs)
+{
+    struct vcpu *v=current;
+    struct nestedvmx *nvmx = &vcpu_2_nvmx(v);
+    struct vmx_inst_decoded decode;
+    unsigned long gpa = 0;
+    int rc;
+
+    rc = decode_vmx_inst(regs, &decode, &gpa, 1);
+    if ( rc != X86EMUL_OKAY )
+        return rc;
+
+    nvmx->vmxon_region_pa = gpa;
+    vmreturn(regs, VMSUCCEED);
+
+    return X86EMUL_OKAY;
+}
+
+int nvmx_handle_vmxoff(struct cpu_user_regs *regs)
+{
+    struct vcpu *v=current;
+    struct nestedvmx *nvmx = &vcpu_2_nvmx(v);
+    int rc;
+
+    rc = vmx_inst_check_privilege(regs, 0);
+    if ( rc != X86EMUL_OKAY )
+        return rc;
+
+    nvmx->vmxon_region_pa = 0;
+
+    vmreturn(regs, VMSUCCEED);
+    return X86EMUL_OKAY;
+}
+
diff -r 4e094881883f -r c8812151acfd xen/include/asm-x86/hvm/vmx/vvmx.h
--- a/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:20 2011 +0800
@@ -35,6 +35,58 @@ struct nestedvmx {
 
 #define vcpu_2_nvmx(v)	(vcpu_nestedhvm(v).u.nvmx)
 
+/*
+ * Encode of VMX instructions base on Table 24-11 & 24-12 of SDM 3B
+ */
+
+enum vmx_regs_enc {
+    VMX_REG_RAX,
+    VMX_REG_RCX,
+    VMX_REG_RDX,
+    VMX_REG_RBX,
+    VMX_REG_RSP,
+    VMX_REG_RBP,
+    VMX_REG_RSI,
+    VMX_REG_RDI,
+#ifdef CONFIG_X86_64
+    VMX_REG_R8,
+    VMX_REG_R9,
+    VMX_REG_R10,
+    VMX_REG_R11,
+    VMX_REG_R12,
+    VMX_REG_R13,
+    VMX_REG_R14,
+    VMX_REG_R15,
+#endif
+};
+
+enum vmx_sregs_enc {
+    VMX_SREG_ES,
+    VMX_SREG_CS,
+    VMX_SREG_SS,
+    VMX_SREG_DS,
+    VMX_SREG_FS,
+    VMX_SREG_GS,
+};
+
+union vmx_inst_info {
+    struct {
+        unsigned int scaling           :2; /* bit 0-1 */
+        unsigned int __rsvd0           :1; /* bit 2 */
+        unsigned int reg1              :4; /* bit 3-6 */
+        unsigned int addr_size         :3; /* bit 7-9 */
+        unsigned int memreg            :1; /* bit 10 */
+        unsigned int __rsvd1           :4; /* bit 11-14 */
+        unsigned int segment           :3; /* bit 15-17 */
+        unsigned int index_reg         :4; /* bit 18-21 */
+        unsigned int index_reg_invalid :1; /* bit 22 */
+        unsigned int base_reg          :4; /* bit 23-26 */
+        unsigned int base_reg_invalid  :1; /* bit 27 */
+        unsigned int reg2              :4; /* bit 28-31 */
+    } fields;
+    u32 word;
+};
+
 int nvmx_vcpu_initialise(struct vcpu *v);
 void nvmx_vcpu_destroy(struct vcpu *v);
 int nvmx_vcpu_reset(struct vcpu *v);
@@ -42,5 +94,7 @@ uint64_t nvmx_vcpu_guestcr3(struct vcpu 
 uint64_t nvmx_vcpu_hostcr3(struct vcpu *v);
 uint32_t nvmx_vcpu_asid(struct vcpu *v);
 
+int nvmx_handle_vmxon(struct cpu_user_regs *regs);
+int nvmx_handle_vmxoff(struct cpu_user_regs *regs);
 #endif /* __ASM_X86_HVM_VVMX_H__ */

Re: [PATCH 05 of 20] Emulation of guest VMXON/OFF instruction

[Tim Deegan]

At 16:57 +0800 on 02 Jun (1307033838), Eddie Dong wrote:
> diff -r 4e094881883f -r c8812151acfd xen/arch/x86/hvm/vmx/Makefile
> --- a/xen/arch/x86/hvm/vmx/Makefile	Thu Jun 02 16:33:20 2011 +0800
> +++ b/xen/arch/x86/hvm/vmx/Makefile	Thu Jun 02 16:33:20 2011 +0800
> @@ -5,3 +5,4 @@ obj-y += vmcs.o
>  obj-y += vmx.o
>  obj-y += vpmu_core2.o
>  obj-y += vvmx.o
> +obj-y += vvmx.o

Harmless, but wrong. :)

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

RE: [PATCH 05 of 20] Emulation of guest VMXON/OFF instruction

[Dong, Eddie]

> >  obj-y += vpmu_core2.o
> >  obj-y += vvmx.o
> > +obj-y += vvmx.o
> 
> Harmless, but wrong. :)
> 

Thanks, a patch merge introduced error :)
Fixed.
Eddie

[PATCH 06 of 20] Define structure and access APIs for virtual VMCS

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003600 -28800
# Node ID 8264b01b476b1b695727f78d92ab0ce553aa7516
# Parent  c8812151acfd6d9468f3407bc6a1a278cd764567
Define structure and access APIs for virtual VMCS.


Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r c8812151acfd -r 8264b01b476b xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:20 2011 +0800
@@ -124,6 +124,84 @@ enum vmx_ops_result {
 #define CASE_GET_REG(REG, reg)      \
     case VMX_REG_ ## REG: value = regs->reg; break
 
+static int vvmcs_offset(u32 width, u32 type, u32 index)
+{
+    int offset;
+
+    offset = (index & 0x1f) | type << 5 | width << 7;
+
+    if ( offset == 0 )    /* vpid */
+        offset = 0x3f;
+
+    return offset;
+}
+
+u64 __get_vvmcs(void *vvmcs, u32 vmcs_encoding)
+{
+    union vmcs_encoding enc;
+    u64 *content = (u64 *) vvmcs;
+    int offset;
+    u64 res;
+
+    enc.word = vmcs_encoding;
+    offset = vvmcs_offset(enc.width, enc.type, enc.index);
+    res = content[offset];
+
+    switch ( enc.width ) {
+    case VVMCS_WIDTH_16:
+        res &= 0xffff;
+        break;
+   case VVMCS_WIDTH_64:
+        if ( enc.access_type )
+            res >>= 32;
+        break;
+    case VVMCS_WIDTH_32:
+        res &= 0xffffffff;
+        break;
+    case VVMCS_WIDTH_NATURAL:
+    default:
+        break;
+    }
+
+    return res;
+}
+
+void __set_vvmcs(void *vvmcs, u32 vmcs_encoding, u64 val)
+{
+    union vmcs_encoding enc;
+    u64 *content = (u64 *) vvmcs;
+    int offset;
+    u64 res;
+
+    enc.word = vmcs_encoding;
+    offset = vvmcs_offset(enc.width, enc.type, enc.index);
+    res = content[offset];
+
+    switch ( enc.width ) {
+    case VVMCS_WIDTH_16:
+        res = val & 0xffff;
+        break;
+    case VVMCS_WIDTH_64:
+        if ( enc.access_type )
+        {
+            res &= 0xffffffff;
+            res |= val << 32;
+        }
+        else
+            res = val;
+        break;
+    case VVMCS_WIDTH_32:
+        res = val & 0xffffffff;
+        break;
+    case VVMCS_WIDTH_NATURAL:
+    default:
+        res = val;
+        break;
+    }
+
+    content[offset] = res;
+}
+
 static unsigned long reg_read(struct cpu_user_regs *regs,
                               enum vmx_regs_enc index)
 {
diff -r c8812151acfd -r 8264b01b476b xen/include/asm-x86/hvm/vmx/vvmx.h
--- a/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:20 2011 +0800
@@ -96,5 +96,61 @@ uint32_t nvmx_vcpu_asid(struct vcpu *v);
 
 int nvmx_handle_vmxon(struct cpu_user_regs *regs);
 int nvmx_handle_vmxoff(struct cpu_user_regs *regs);
+/*
+ * Virtual VMCS layout
+ *
+ * Since physical VMCS layout is unknown, a custom layout is used
+ * for virtual VMCS seen by guest. It occupies a 4k page, and the
+ * field is offset by an 9-bit offset into u64[], The offset is as
+ * follow, which means every <width, type> pair has a max of 32
+ * fields available.
+ *
+ *             9       7      5               0
+ *             --------------------------------
+ *     offset: | width | type |     index     |
+ *             --------------------------------
+ *
+ * Also, since the lower range <width=0, type={0,1}> has only one
+ * field: VPID, it is moved to a higher offset (63), and leaves the
+ * lower range to non-indexed field like VMCS revision.
+ *
+ */
+
+#define VVMCS_REVISION 0x40000001u
+
+struct vvmcs_header {
+    u32 revision;
+    u32 abort;
+};
+
+union vmcs_encoding {
+    struct {
+        u32 access_type : 1;
+        u32 index : 9;
+        u32 type : 2;
+        u32 rsv1 : 1;
+        u32 width : 2;
+        u32 rsv2 : 17;
+    };
+    u32 word;
+};
+
+enum vvmcs_encoding_width {
+    VVMCS_WIDTH_16 = 0,
+    VVMCS_WIDTH_64,
+    VVMCS_WIDTH_32,
+    VVMCS_WIDTH_NATURAL,
+};
+
+enum vvmcs_encoding_type {
+    VVMCS_TYPE_CONTROL = 0,
+    VVMCS_TYPE_RO,
+    VVMCS_TYPE_GSTATE,
+    VVMCS_TYPE_HSTATE,
+};
+
+u64 __get_vvmcs(void *vvmcs, u32 vmcs_encoding);
+void __set_vvmcs(void *vvmcs, u32 vmcs_encoding, u64 val);
+
 #endif /* __ASM_X86_HVM_VVMX_H__ */

[PATCH 07 of 20] Emulation of guest vmptrld

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003600 -28800
# Node ID 4dad232d7fc3bd62979a1b442d989fe0ca4baafe
# Parent  8264b01b476b1b695727f78d92ab0ce553aa7516
Emulation of guest vmptrld

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r 8264b01b476b -r 4dad232d7fc3 xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:20 2011 +0800
@@ -2444,11 +2444,15 @@ asmlinkage void vmx_vmexit_handler(struc
             update_guest_eip();
         break;
 
+    case EXIT_REASON_VMPTRLD:
+        if ( nvmx_handle_vmptrld(regs) == X86EMUL_OKAY )
+            update_guest_eip();
+        break;
+
     case EXIT_REASON_MWAIT_INSTRUCTION:
     case EXIT_REASON_MONITOR_INSTRUCTION:
     case EXIT_REASON_VMCLEAR:
     case EXIT_REASON_VMLAUNCH:
-    case EXIT_REASON_VMPTRLD:
     case EXIT_REASON_VMPTRST:
     case EXIT_REASON_VMREAD:
     case EXIT_REASON_VMRESUME:
diff -r 8264b01b476b -r 4dad232d7fc3 xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:20 2011 +0800
@@ -356,6 +356,41 @@ static void vmreturn(struct cpu_user_reg
     regs->eflags = eflags;
 }
 
+static void __map_io_bitmap(struct vcpu *v, u64 vmcs_reg)
+{
+    struct nestedvmx *nvmx = &vcpu_2_nvmx(v);
+    unsigned long gpa;
+    unsigned long mfn;
+    p2m_type_t p2mt;
+
+    if ( vmcs_reg == IO_BITMAP_A )
+    {
+        if (nvmx->iobitmap[0]) {
+            unmap_domain_page_global(nvmx->iobitmap[0]);
+        }
+        gpa = __get_vvmcs(vcpu_nestedhvm(v).nv_vvmcx, IO_BITMAP_A);
+        mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(v->domain),
+                              gpa >> PAGE_SHIFT, &p2mt));
+        nvmx->iobitmap[0] = map_domain_page_global(mfn);
+    }
+    else if ( vmcs_reg == IO_BITMAP_B )
+    {
+        if (nvmx->iobitmap[1]) {
+            unmap_domain_page_global(nvmx->iobitmap[1]);
+        }
+        gpa = __get_vvmcs(vcpu_nestedhvm(v).nv_vvmcx, IO_BITMAP_B);
+        mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(v->domain),
+                               gpa >> PAGE_SHIFT, &p2mt));
+        nvmx->iobitmap[1] = map_domain_page_global(mfn);
+    }
+}
+
+static inline void map_io_bitmap_all(struct vcpu *v)
+{
+   __map_io_bitmap (v, IO_BITMAP_A);
+   __map_io_bitmap (v, IO_BITMAP_B);
+}
+
 /*
  * VMX instructions handling
  */
@@ -364,6 +399,7 @@ int nvmx_handle_vmxon(struct cpu_user_re
 {
     struct vcpu *v=current;
     struct nestedvmx *nvmx = &vcpu_2_nvmx(v);
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
     struct vmx_inst_decoded decode;
     unsigned long gpa = 0;
     int rc;
@@ -372,7 +408,22 @@ int nvmx_handle_vmxon(struct cpu_user_re
     if ( rc != X86EMUL_OKAY )
         return rc;
 
+    if ( nvmx->vmxon_region_pa )
+        gdprintk(XENLOG_WARNING, 
+                 "vmxon again: orig %lx new %lx\n",
+                 nvmx->vmxon_region_pa, gpa);
+
     nvmx->vmxon_region_pa = gpa;
+
+    /*
+     * `fork' the host vmcs to shadow_vmcs
+     * vmcs_lock is not needed since we are on current
+     */
+    nvcpu->nv_n1vmcx = v->arch.hvm_vmx.vmcs;
+    __vmpclear(virt_to_maddr(v->arch.hvm_vmx.vmcs));
+    memcpy(nvcpu->nv_n2vmcx, v->arch.hvm_vmx.vmcs, PAGE_SIZE);
+    __vmptrld(virt_to_maddr(v->arch.hvm_vmx.vmcs));
+    v->arch.hvm_vmx.launched = 0;
     vmreturn(regs, VMSUCCEED);
 
     return X86EMUL_OKAY;
@@ -394,3 +445,38 @@ int nvmx_handle_vmxoff(struct cpu_user_r
     return X86EMUL_OKAY;
 }
 
+int nvmx_handle_vmptrld(struct cpu_user_regs *regs)
+{
+    struct vcpu *v = current;
+    struct vmx_inst_decoded decode;
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+    unsigned long gpa = 0;
+    unsigned long mfn;
+    p2m_type_t p2mt;
+    int rc;
+
+    rc = decode_vmx_inst(regs, &decode, &gpa, 0);
+    if ( rc != X86EMUL_OKAY )
+        return rc;
+
+    if ( gpa == vcpu_2_nvmx(v).vmxon_region_pa || gpa & 0xfff )
+    {
+        vmreturn(regs, VMFAIL_INVALID);
+        goto out;
+    }
+
+    if ( nvcpu->nv_vvmcxaddr == VMCX_EADDR )
+    {
+        mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(v->domain),
+                               gpa >> PAGE_SHIFT, &p2mt));
+        nvcpu->nv_vvmcx = map_domain_page_global(mfn);
+        nvcpu->nv_vvmcxaddr = gpa;
+        map_io_bitmap_all (v);
+    }
+
+    vmreturn(regs, VMSUCCEED);
+
+out:
+    return X86EMUL_OKAY;
+}
+
diff -r 8264b01b476b -r 4dad232d7fc3 xen/include/asm-x86/hvm/vmx/vvmx.h
--- a/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:20 2011 +0800
@@ -152,5 +152,8 @@ enum vvmcs_encoding_type {
 u64 __get_vvmcs(void *vvmcs, u32 vmcs_encoding);
 void __set_vvmcs(void *vvmcs, u32 vmcs_encoding, u64 val);
 
+void nvmx_destroy_vmcs(struct vcpu *v);
+int nvmx_handle_vmptrld(struct cpu_user_regs *regs);
+
 #endif /* __ASM_X86_HVM_VVMX_H__ */

Re: [PATCH 07 of 20] Emulation of guest vmptrld

[Tim Deegan]

At 16:57 +0800 on 02 Jun (1307033840), Eddie Dong wrote:
> diff -r 8264b01b476b -r 4dad232d7fc3 xen/arch/x86/hvm/vmx/vvmx.c
> --- a/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:20 2011 +0800
> +++ b/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:20 2011 +0800
> @@ -356,6 +356,41 @@ static void vmreturn(struct cpu_user_reg
>      regs->eflags = eflags;
>  }
>  
> +static void __map_io_bitmap(struct vcpu *v, u64 vmcs_reg)
> +{
> +    struct nestedvmx *nvmx = &vcpu_2_nvmx(v);
> +    unsigned long gpa;
> +    unsigned long mfn;
> +    p2m_type_t p2mt;
> +
> +    if ( vmcs_reg == IO_BITMAP_A )
> +    {
> +        if (nvmx->iobitmap[0]) {
> +            unmap_domain_page_global(nvmx->iobitmap[0]);
> +        }
> +        gpa = __get_vvmcs(vcpu_nestedhvm(v).nv_vvmcx, IO_BITMAP_A);
> +        mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(v->domain),
> +                              gpa >> PAGE_SHIFT, &p2mt));
> +        nvmx->iobitmap[0] = map_domain_page_global(mfn);

Why are these maps _global?  It might be OK to use 2 more global
mappings per VCPU but the reason should probably go in a comment beside
the call.

Also, I don't see where these mappings get torn down on domain
destruction. 

(While I'm looking at this code, this function is quite ugly.  Why have
a single function if you're going to duplicate its contents anyway?)

> +    }
> +    else if ( vmcs_reg == IO_BITMAP_B )
> +    {
> +        if (nvmx->iobitmap[1]) {
> +            unmap_domain_page_global(nvmx->iobitmap[1]);
> +        }
> +        gpa = __get_vvmcs(vcpu_nestedhvm(v).nv_vvmcx, IO_BITMAP_B);
> +        mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(v->domain),
> +                               gpa >> PAGE_SHIFT, &p2mt));
> +        nvmx->iobitmap[1] = map_domain_page_global(mfn);
> +    }
> +}
> +
> +static inline void map_io_bitmap_all(struct vcpu *v)
> +{
> +   __map_io_bitmap (v, IO_BITMAP_A);
> +   __map_io_bitmap (v, IO_BITMAP_B);
> +}
> +
>  /*
>   * VMX instructions handling
>   */
> @@ -364,6 +399,7 @@ int nvmx_handle_vmxon(struct cpu_user_re
>  {
>      struct vcpu *v=current;
>      struct nestedvmx *nvmx = &vcpu_2_nvmx(v);
> +    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
>      struct vmx_inst_decoded decode;
>      unsigned long gpa = 0;
>      int rc;
> @@ -372,7 +408,22 @@ int nvmx_handle_vmxon(struct cpu_user_re
>      if ( rc != X86EMUL_OKAY )
>          return rc;
>  
> +    if ( nvmx->vmxon_region_pa )
> +        gdprintk(XENLOG_WARNING, 
> +                 "vmxon again: orig %lx new %lx\n",
> +                 nvmx->vmxon_region_pa, gpa);
> +
>      nvmx->vmxon_region_pa = gpa;
> +
> +    /*
> +     * `fork' the host vmcs to shadow_vmcs
> +     * vmcs_lock is not needed since we are on current
> +     */
> +    nvcpu->nv_n1vmcx = v->arch.hvm_vmx.vmcs;
> +    __vmpclear(virt_to_maddr(v->arch.hvm_vmx.vmcs));
> +    memcpy(nvcpu->nv_n2vmcx, v->arch.hvm_vmx.vmcs, PAGE_SIZE);
> +    __vmptrld(virt_to_maddr(v->arch.hvm_vmx.vmcs));
> +    v->arch.hvm_vmx.launched = 0;
>      vmreturn(regs, VMSUCCEED);
>  
>      return X86EMUL_OKAY;
> @@ -394,3 +445,38 @@ int nvmx_handle_vmxoff(struct cpu_user_r
>      return X86EMUL_OKAY;
>  }
>  
> +int nvmx_handle_vmptrld(struct cpu_user_regs *regs)
> +{
> +    struct vcpu *v = current;
> +    struct vmx_inst_decoded decode;
> +    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
> +    unsigned long gpa = 0;
> +    unsigned long mfn;
> +    p2m_type_t p2mt;
> +    int rc;
> +
> +    rc = decode_vmx_inst(regs, &decode, &gpa, 0);
> +    if ( rc != X86EMUL_OKAY )
> +        return rc;
> +
> +    if ( gpa == vcpu_2_nvmx(v).vmxon_region_pa || gpa & 0xfff )
> +    {
> +        vmreturn(regs, VMFAIL_INVALID);
> +        goto out;
> +    }
> +
> +    if ( nvcpu->nv_vvmcxaddr == VMCX_EADDR )
> +    {
> +        mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(v->domain),
> +                               gpa >> PAGE_SHIFT, &p2mt));
> +        nvcpu->nv_vvmcx = map_domain_page_global(mfn);

Again, why _global?

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

RE: [PATCH 07 of 20] Emulation of guest vmptrld

[Dong, Eddie]

> > +    if ( vmcs_reg == IO_BITMAP_A )
> > +    {
> > +        if (nvmx->iobitmap[0]) {
> > +            unmap_domain_page_global(nvmx->iobitmap[0]);
> > +        }
> > +        gpa = __get_vvmcs(vcpu_nestedhvm(v).nv_vvmcx,
> IO_BITMAP_A);
> > +        mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(v->domain),
> > +                              gpa >> PAGE_SHIFT, &p2mt));
> > +        nvmx->iobitmap[0] = map_domain_page_global(mfn);
> 
> Why are these maps _global?  It might be OK to use 2 more global
> mappings per VCPU but the reason should probably go in a comment beside
> the call.

Do you mean to use hvm_map_guest_frame_ro? Fine to me.
> 
> Also, I don't see where these mappings get torn down on domain
> destruction.
> 
Yes. Fixed in nvmx_vcpu_destroy.

> (While I'm looking at this code, this function is quite ugly.  Why have
> a single function if you're going to duplicate its contents anyway?)

??? We don't know fi guest changed the bitmap, so we have to check each time.

> 
> +
> > +    if ( nvcpu->nv_vvmcxaddr == VMCX_EADDR )
> > +    {
> > +        mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(v->domain),
> > +                               gpa >> PAGE_SHIFT, &p2mt));
> > +        nvcpu->nv_vvmcx = map_domain_page_global(mfn);
> 
> Again, why _global?

Will fix with hvm_map_guest_frame.

Thx, Eddie

Re: [PATCH 07 of 20] Emulation of guest vmptrld

[Tim Deegan]

At 14:07 +0800 on 03 Jun (1307110060), Dong, Eddie wrote:
> > > +    if ( vmcs_reg == IO_BITMAP_A )
> > > +    {
> > > +        if (nvmx->iobitmap[0]) {
> > > +            unmap_domain_page_global(nvmx->iobitmap[0]);
> > > +        }
> > > +        gpa = __get_vvmcs(vcpu_nestedhvm(v).nv_vvmcx,
> > IO_BITMAP_A);
> > > +        mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(v->domain),
> > > +                              gpa >> PAGE_SHIFT, &p2mt));
> > > +        nvmx->iobitmap[0] = map_domain_page_global(mfn);
> > 
> > Why are these maps _global?  It might be OK to use 2 more global
> > mappings per VCPU but the reason should probably go in a comment beside
> > the call.
> 
> Do you mean to use hvm_map_guest_frame_ro? Fine to me.

Yes, I think that would be better unless you know there's a point where
the bitmaps are accessed on a vcpu other than current.  (On 64-bit it
makes no difference but on 32-bit map_domain_page_global() uses up a
global shared resource).

> > 
> > Also, I don't see where these mappings get torn down on domain
> > destruction.
> > 
> Yes. Fixed in nvmx_vcpu_destroy.
> 
> > (While I'm looking at this code, this function is quite ugly.  Why have
> > a single function if you're going to duplicate its contents anyway?)
> 
> ??? We don't know fi guest changed the bitmap, so we have to check each time.

I think I wasn't clear.  The logic is fine, I was just cavilling about
coding style.  You have some code that's basically

f1() { BUNCH_O_CODE(1) }

f2() { BUNCH_O_CODE(2) }

and places that need to call f1(), f2() or both.  Merging those into a
single function is a good idea, but the function should look like

f(x) { 
  int i = (x ? 1 : 2)
  BUNCH_O_CODE(i)
}

and what you have is

f(x) {
  if (x) 
     BUNCH_O_CODE(1)
  else
     BUNCH_O_CODE(2)
}

which keeps the duplication. 

Cheers,

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

RE: [PATCH 07 of 20] Emulation of guest vmptrld

[Dong, Eddie]

> > > (While I'm looking at this code, this function is quite ugly.  Why have
> > > a single function if you're going to duplicate its contents anyway?)
> >
> > ??? We don't know fi guest changed the bitmap, so we have to check each
> time.
> 
> I think I wasn't clear.  The logic is fine, I was just cavilling about
> coding style.  You have some code that's basically
> 
I see, yes it is better and fixed.

Thx, Eddie

[PATCH 08 of 20] Emulation of guest VMPTRST

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003600 -28800
# Node ID 54332433d873777e57e6ac47ee841a2a96c2f543
# Parent  4dad232d7fc3bd62979a1b442d989fe0ca4baafe
Emulation of guest VMPTRST

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r 4dad232d7fc3 -r 54332433d873 xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:20 2011 +0800
@@ -2449,11 +2449,15 @@ asmlinkage void vmx_vmexit_handler(struc
             update_guest_eip();
         break;
 
+    case EXIT_REASON_VMPTRST:
+        if ( nvmx_handle_vmptrst(regs) == X86EMUL_OKAY )
+            update_guest_eip();
+        break;
+
     case EXIT_REASON_MWAIT_INSTRUCTION:
     case EXIT_REASON_MONITOR_INSTRUCTION:
     case EXIT_REASON_VMCLEAR:
     case EXIT_REASON_VMLAUNCH:
-    case EXIT_REASON_VMPTRST:
     case EXIT_REASON_VMREAD:
     case EXIT_REASON_VMRESUME:
     case EXIT_REASON_VMWRITE:
diff -r 4dad232d7fc3 -r 54332433d873 xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:20 2011 +0800
@@ -480,3 +480,25 @@ out:
     return X86EMUL_OKAY;
 }
 
+int nvmx_handle_vmptrst(struct cpu_user_regs *regs)
+{
+    struct vcpu *v = current;
+    struct vmx_inst_decoded decode;
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+    unsigned long gpa = 0;
+    int rc;
+
+    rc = decode_vmx_inst(regs, &decode, &gpa, 0);
+    if ( rc != X86EMUL_OKAY )
+        return rc;
+
+    gpa = nvcpu->nv_vvmcxaddr;
+
+    rc = hvm_copy_to_guest_virt(decode.mem, &gpa, decode.len, 0);
+    if ( rc != HVMCOPY_okay )
+        return X86EMUL_EXCEPTION;
+
+    vmreturn(regs, VMSUCCEED);
+    return X86EMUL_OKAY;
+}
+
diff -r 4dad232d7fc3 -r 54332433d873 xen/include/asm-x86/hvm/vmx/vvmx.h
--- a/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:20 2011 +0800
@@ -154,6 +154,7 @@ void __set_vvmcs(void *vvmcs, u32 vmcs_e
 
 void nvmx_destroy_vmcs(struct vcpu *v);
 int nvmx_handle_vmptrld(struct cpu_user_regs *regs);
+int nvmx_handle_vmptrst(struct cpu_user_regs *regs);
 
 #endif /* __ASM_X86_HVM_VVMX_H__ */

[PATCH 09 of 20] Emulation of guest VMCLEAR

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003600 -28800
# Node ID 35cc736e8a75a0a349790871232f8761ceae41be
# Parent  54332433d873777e57e6ac47ee841a2a96c2f543
Emulation of guest VMCLEAR

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r 54332433d873 -r 35cc736e8a75 xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:20 2011 +0800
@@ -2444,6 +2444,11 @@ asmlinkage void vmx_vmexit_handler(struc
             update_guest_eip();
         break;
 
+    case EXIT_REASON_VMCLEAR:
+        if ( nvmx_handle_vmclear(regs) == X86EMUL_OKAY )
+            update_guest_eip();
+        break;
+ 
     case EXIT_REASON_VMPTRLD:
         if ( nvmx_handle_vmptrld(regs) == X86EMUL_OKAY )
             update_guest_eip();
@@ -2456,7 +2461,6 @@ asmlinkage void vmx_vmexit_handler(struc
 
     case EXIT_REASON_MWAIT_INSTRUCTION:
     case EXIT_REASON_MONITOR_INSTRUCTION:
-    case EXIT_REASON_VMCLEAR:
     case EXIT_REASON_VMLAUNCH:
     case EXIT_REASON_VMREAD:
     case EXIT_REASON_VMRESUME:
diff -r 54332433d873 -r 35cc736e8a75 xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:20 2011 +0800
@@ -356,6 +356,14 @@ static void vmreturn(struct cpu_user_reg
     regs->eflags = eflags;
 }
 
+static void __clear_current_vvmcs(struct vcpu *v)
+{
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+    
+    if ( nvcpu->nv_n2vmcx )
+        __vmpclear(virt_to_maddr(nvcpu->nv_n2vmcx));
+}
+
 static void __map_io_bitmap(struct vcpu *v, u64 vmcs_reg)
 {
     struct nestedvmx *nvmx = &vcpu_2_nvmx(v);
@@ -391,6 +399,26 @@ static inline void map_io_bitmap_all(str
    __map_io_bitmap (v, IO_BITMAP_B);
 }
 
+static void nvmx_purge_vvmcs(struct vcpu *v)
+{
+    struct nestedvmx *nvmx = &vcpu_2_nvmx(v);
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+
+    __clear_current_vvmcs(v);
+    if ( nvcpu->nv_vvmcxaddr != VMCX_EADDR )
+        unmap_domain_page_global(nvcpu->nv_vvmcx);
+    nvcpu->nv_vvmcx == NULL;
+    nvcpu->nv_vvmcxaddr = VMCX_EADDR;
+    if ( nvmx->iobitmap[0] ) {
+        unmap_domain_page_global(nvmx->iobitmap[0]);
+        nvmx->iobitmap[0] = NULL;
+    }
+    if ( nvmx->iobitmap[1] ) {
+        unmap_domain_page_global(nvmx->iobitmap[1]);
+        nvmx->iobitmap[1] = NULL;
+    }
+}
+
 /*
  * VMX instructions handling
  */
@@ -439,6 +467,7 @@ int nvmx_handle_vmxoff(struct cpu_user_r
     if ( rc != X86EMUL_OKAY )
         return rc;
 
+    nvmx_purge_vvmcs(v);
     nvmx->vmxon_region_pa = 0;
 
     vmreturn(regs, VMSUCCEED);
@@ -465,6 +494,9 @@ int nvmx_handle_vmptrld(struct cpu_user_
         goto out;
     }
 
+    if ( nvcpu->nv_vvmcxaddr != gpa )
+        nvmx_purge_vvmcs(v);
+
     if ( nvcpu->nv_vvmcxaddr == VMCX_EADDR )
     {
         mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(v->domain),
@@ -502,3 +534,37 @@ int nvmx_handle_vmptrst(struct cpu_user_
     return X86EMUL_OKAY;
 }
 
+int nvmx_handle_vmclear(struct cpu_user_regs *regs)
+{
+    struct vcpu *v = current;
+    struct vmx_inst_decoded decode;
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+    unsigned long gpa = 0;
+    int rc;
+
+    rc = decode_vmx_inst(regs, &decode, &gpa, 0);
+    if ( rc != X86EMUL_OKAY )
+        return rc;
+
+    if ( gpa & 0xfff )
+    {
+        vmreturn(regs, VMFAIL_INVALID);
+        goto out;
+    }
+
+    if ( gpa != nvcpu->nv_vvmcxaddr && nvcpu->nv_vvmcxaddr != VMCX_EADDR )
+    {
+        gdprintk(XENLOG_WARNING, 
+                 "vmclear gpa %lx not the same with current vmcs %lx\n",
+                 gpa, nvcpu->nv_vvmcxaddr);
+        vmreturn(regs, VMSUCCEED);
+        goto out;
+    }
+    nvmx_purge_vvmcs(v);
+
+    vmreturn(regs, VMSUCCEED);
+
+out:
+    return X86EMUL_OKAY;
+}
+
diff -r 54332433d873 -r 35cc736e8a75 xen/include/asm-x86/hvm/vmx/vvmx.h
--- a/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:20 2011 +0800
@@ -155,6 +155,7 @@ void __set_vvmcs(void *vvmcs, u32 vmcs_e
 void nvmx_destroy_vmcs(struct vcpu *v);
 int nvmx_handle_vmptrld(struct cpu_user_regs *regs);
 int nvmx_handle_vmptrst(struct cpu_user_regs *regs);
+int nvmx_handle_vmclear(struct cpu_user_regs *regs);
 
 #endif /* __ASM_X86_HVM_VVMX_H__ */

[PATCH 10 of 20] Emulation of guest VMWRITE

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003600 -28800
# Node ID 16e0e95f457e9b3f8ff0528c8f2b0f88b1c41109
# Parent  35cc736e8a75a0a349790871232f8761ceae41be
Emulation of guest VMWRITE

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r 35cc736e8a75 -r 16e0e95f457e xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:20 2011 +0800
@@ -2459,12 +2459,16 @@ asmlinkage void vmx_vmexit_handler(struc
             update_guest_eip();
         break;
 
+    case EXIT_REASON_VMWRITE:
+        if ( nvmx_handle_vmwrite(regs) == X86EMUL_OKAY )
+            update_guest_eip();
+        break;
+
     case EXIT_REASON_MWAIT_INSTRUCTION:
     case EXIT_REASON_MONITOR_INSTRUCTION:
     case EXIT_REASON_VMLAUNCH:
     case EXIT_REASON_VMREAD:
     case EXIT_REASON_VMRESUME:
-    case EXIT_REASON_VMWRITE:
     case EXIT_REASON_GETSEC:
     case EXIT_REASON_INVEPT:
     case EXIT_REASON_INVVPID:
diff -r 35cc736e8a75 -r 16e0e95f457e xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:20 2011 +0800
@@ -568,3 +568,27 @@ out:
     return X86EMUL_OKAY;
 }
 
+int nvmx_handle_vmwrite(struct cpu_user_regs *regs)
+{
+    struct vcpu *v = current;
+    struct vmx_inst_decoded decode;
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+    u64 operandS, vmcs_encoding;
+
+    if ( decode_vmx_inst(regs, &decode, &operandS, 0)
+             != X86EMUL_OKAY )
+        return X86EMUL_EXCEPTION;
+
+    vmcs_encoding = reg_read(regs, decode.reg2);
+    __set_vvmcs(nvcpu->nv_vvmcx, vmcs_encoding, operandS);
+
+    if ( vmcs_encoding == IO_BITMAP_A || vmcs_encoding == IO_BITMAP_A_HIGH )
+        __map_io_bitmap (v, IO_BITMAP_A);
+    else if ( vmcs_encoding == IO_BITMAP_B || 
+              vmcs_encoding == IO_BITMAP_B_HIGH )
+        __map_io_bitmap (v, IO_BITMAP_B);
+
+    vmreturn(regs, VMSUCCEED);
+    return X86EMUL_OKAY;
+}
+
diff -r 35cc736e8a75 -r 16e0e95f457e xen/include/asm-x86/hvm/vmx/vvmx.h
--- a/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:20 2011 +0800
@@ -156,6 +156,7 @@ void nvmx_destroy_vmcs(struct vcpu *v);
 int nvmx_handle_vmptrld(struct cpu_user_regs *regs);
 int nvmx_handle_vmptrst(struct cpu_user_regs *regs);
 int nvmx_handle_vmclear(struct cpu_user_regs *regs);
+int nvmx_handle_vmwrite(struct cpu_user_regs *regs);
 
 #endif /* __ASM_X86_HVM_VVMX_H__ */

[PATCH 11 of 20] Emulation of guest VMREAD

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003600 -28800
# Node ID 4631a951120093ade781c4f4542741266b615576
# Parent  16e0e95f457e9b3f8ff0528c8f2b0f88b1c41109
Emulation of guest VMREAD

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r 16e0e95f457e -r 4631a9511200 xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:20 2011 +0800
@@ -2459,6 +2459,11 @@ asmlinkage void vmx_vmexit_handler(struc
             update_guest_eip();
         break;
 
+    case EXIT_REASON_VMREAD:
+        if ( nvmx_handle_vmread(regs) == X86EMUL_OKAY )
+            update_guest_eip();
+        break;
+ 
     case EXIT_REASON_VMWRITE:
         if ( nvmx_handle_vmwrite(regs) == X86EMUL_OKAY )
             update_guest_eip();
@@ -2467,7 +2472,6 @@ asmlinkage void vmx_vmexit_handler(struc
     case EXIT_REASON_MWAIT_INSTRUCTION:
     case EXIT_REASON_MONITOR_INSTRUCTION:
     case EXIT_REASON_VMLAUNCH:
-    case EXIT_REASON_VMREAD:
     case EXIT_REASON_VMRESUME:
     case EXIT_REASON_GETSEC:
     case EXIT_REASON_INVEPT:
diff -r 16e0e95f457e -r 4631a9511200 xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:20 2011 +0800
@@ -121,6 +121,8 @@ enum vmx_ops_result {
     VMFAIL_INVALID,
 };
 
+#define CASE_SET_REG(REG, reg)      \
+    case VMX_REG_ ## REG: regs->reg = value; break
 #define CASE_GET_REG(REG, reg)      \
     case VMX_REG_ ## REG: value = regs->reg; break
 
@@ -233,6 +235,32 @@ static unsigned long reg_read(struct cpu
     return value;
 }
 
+static void reg_write(struct cpu_user_regs *regs,
+                      enum vmx_regs_enc index,
+                      unsigned long value)
+{
+    switch ( index ) {
+    CASE_SET_REG(RAX, eax);
+    CASE_SET_REG(RCX, ecx);
+    CASE_SET_REG(RDX, edx);
+    CASE_SET_REG(RBX, ebx);
+    CASE_SET_REG(RBP, ebp);
+    CASE_SET_REG(RSI, esi);
+    CASE_SET_REG(RDI, edi);
+    CASE_SET_REG(RSP, esp);
+    CASE_SET_REG(R8, r8);
+    CASE_SET_REG(R9, r9);
+    CASE_SET_REG(R10, r10);
+    CASE_SET_REG(R11, r11);
+    CASE_SET_REG(R12, r12);
+    CASE_SET_REG(R13, r13);
+    CASE_SET_REG(R14, r14);
+    CASE_SET_REG(R15, r15);
+    default:
+        break;
+    }
+}
+
 static int vmx_inst_check_privilege(struct cpu_user_regs *regs, int vmxop_check)
 {
     struct vcpu *v = current;
@@ -568,6 +596,35 @@ out:
     return X86EMUL_OKAY;
 }
 
+int nvmx_handle_vmread(struct cpu_user_regs *regs)
+{
+    struct vcpu *v = current;
+    struct vmx_inst_decoded decode;
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+    u64 value = 0;
+    int rc;
+
+    rc = decode_vmx_inst(regs, &decode, NULL, 0);
+    if ( rc != X86EMUL_OKAY )
+        return rc;
+
+    value = __get_vvmcs(nvcpu->nv_vvmcx, reg_read(regs, decode.reg2));
+
+    switch ( decode.type ) {
+    case VMX_INST_MEMREG_TYPE_MEMORY:
+        rc = hvm_copy_to_guest_virt(decode.mem, &value, decode.len, 0);
+        if ( rc != HVMCOPY_okay )
+            return X86EMUL_EXCEPTION;
+        break;
+    case VMX_INST_MEMREG_TYPE_REG:
+        reg_write(regs, decode.reg1, value);
+        break;
+    }
+
+    vmreturn(regs, VMSUCCEED);
+    return X86EMUL_OKAY;
+}
+
 int nvmx_handle_vmwrite(struct cpu_user_regs *regs)
 {
     struct vcpu *v = current;
diff -r 16e0e95f457e -r 4631a9511200 xen/include/asm-x86/hvm/vmx/vvmx.h
--- a/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:20 2011 +0800
@@ -156,6 +156,7 @@ void nvmx_destroy_vmcs(struct vcpu *v);
 int nvmx_handle_vmptrld(struct cpu_user_regs *regs);
 int nvmx_handle_vmptrst(struct cpu_user_regs *regs);
 int nvmx_handle_vmclear(struct cpu_user_regs *regs);
+int nvmx_handle_vmread(struct cpu_user_regs *regs);
 int nvmx_handle_vmwrite(struct cpu_user_regs *regs);
 
 #endif /* __ASM_X86_HVM_VVMX_H__ */

[PATCH 12 of 20] Add APIs to switch n1/n2 VMCS

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003601 -28800
# Node ID 62cc6c7516e010ef673c75bba83f901785b063d5
# Parent  4631a951120093ade781c4f4542741266b615576
Add APIs to switch n1/n2 VMCS.

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r 4631a9511200 -r 62cc6c7516e0 xen/arch/x86/hvm/vmx/vmcs.c
--- a/xen/arch/x86/hvm/vmx/vmcs.c	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vmcs.c	Thu Jun 02 16:33:21 2011 +0800
@@ -669,6 +669,38 @@ void vmx_disable_intercept_for_msr(struc
     }
 }
 
+/*
+ * Switch VMCS between layer 1 & 2 guest
+ */
+void vmx_vmcs_switch(struct vcpu *v,
+                             struct vmcs_struct *from,
+                             struct vmcs_struct *to)
+{
+    /* no foreign access */
+    if ( unlikely(v != current) )
+        return;
+
+    if ( unlikely(current->arch.hvm_vmx.vmcs != from) )
+        return;
+
+    spin_lock(&v->arch.hvm_vmx.vmcs_lock);
+
+    __vmpclear(virt_to_maddr(from));
+    __vmptrld(virt_to_maddr(to));
+
+    v->arch.hvm_vmx.vmcs = to;
+    v->arch.hvm_vmx.launched = 0;
+    this_cpu(current_vmcs) = to;
+
+    if ( v->arch.hvm_vmx.hostenv_migrated )
+    {
+        v->arch.hvm_vmx.hostenv_migrated = 0;
+        vmx_set_host_env(v);
+    }
+
+    spin_unlock(&v->arch.hvm_vmx.vmcs_lock);
+}
+
 static int construct_vmcs(struct vcpu *v)
 {
     struct domain *d = v->domain;
@@ -1078,6 +1110,13 @@ void vmx_do_resume(struct vcpu *v)
         hvm_migrate_timers(v);
         hvm_migrate_pirqs(v);
         vmx_set_host_env(v);
+        /*
+         * Both n1 VMCS and n2 VMCS need to update the host environment after 
+         * VCPU migration. The environment of current VMCS is updated in place,
+         * but the action of another VMCS is deferred till it is switched in.
+         */
+        v->arch.hvm_vmx.hostenv_migrated = 1;
+
         hvm_asid_flush_vcpu(v);
     }
 
diff -r 4631a9511200 -r 62cc6c7516e0 xen/include/asm-x86/hvm/vmx/vmcs.h
--- a/xen/include/asm-x86/hvm/vmx/vmcs.h	Thu Jun 02 16:33:20 2011 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vmcs.h	Thu Jun 02 16:33:21 2011 +0800
@@ -123,6 +123,7 @@ struct arch_vmx_struct {
     struct segment_register vm86_saved_seg[x86_seg_tr + 1];
     /* Remember EFLAGS while in virtual 8086 mode */
     uint32_t             vm86_saved_eflags;
+    int                  hostenv_migrated;
 };
 
 int vmx_create_vmcs(struct vcpu *v);
@@ -390,6 +391,9 @@ int vmx_read_guest_msr(u32 msr, u64 *val
 int vmx_write_guest_msr(u32 msr, u64 val);
 int vmx_add_guest_msr(u32 msr);
 int vmx_add_host_load_msr(u32 msr);
+void vmx_vmcs_switch(struct vcpu *v,
+                      struct vmcs_struct *from,
+                      struct vmcs_struct *to);
 
 #endif /* ASM_X86_HVM_VMX_VMCS_H__ */

Re: [PATCH 12 of 20] Add APIs to switch n1/n2 VMCS

[Tim Deegan]

At 16:57 +0800 on 02 Jun (1307033845), Eddie Dong wrote:
> diff -r 4631a9511200 -r 62cc6c7516e0 xen/arch/x86/hvm/vmx/vmcs.c
> --- a/xen/arch/x86/hvm/vmx/vmcs.c	Thu Jun 02 16:33:20 2011 +0800
> +++ b/xen/arch/x86/hvm/vmx/vmcs.c	Thu Jun 02 16:33:21 2011 +0800
> @@ -669,6 +669,38 @@ void vmx_disable_intercept_for_msr(struc
>      }
>  }
>  
> +/*
> + * Switch VMCS between layer 1 & 2 guest
> + */
> +void vmx_vmcs_switch(struct vcpu *v,
> +                             struct vmcs_struct *from,
> +                             struct vmcs_struct *to)
> +{
> +    /* no foreign access */
> +    if ( unlikely(v != current) )
> +        return;
> +
> +    if ( unlikely(current->arch.hvm_vmx.vmcs != from) )
> +        return;

Do you really want this function to fail silently if called with v !=
current?  Use ASSERT(), or, even better, remove the first argument
entirely.

Cheers,

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

RE: [PATCH 12 of 20] Add APIs to switch n1/n2 VMCS

[Dong, Eddie]

> > +    /* no foreign access */
> > +    if ( unlikely(v != current) )
> > +        return;
> > +
> > +    if ( unlikely(current->arch.hvm_vmx.vmcs != from) )
> > +        return;
> 
> Do you really want this function to fail silently if called with v !=
> current?  Use ASSERT(), or, even better, remove the first argument
> entirely.
> 
Deleted.

Thx, Eddie

[PATCH 13 of 20] Emulation of VMRESUME/VMLAUNCH

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003601 -28800
# Node ID 279a27a3b1a90380c8fa579e87835cb58a8f4aac
# Parent  62cc6c7516e010ef673c75bba83f901785b063d5
Emulation of VMRESUME/VMLAUNCH

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r 62cc6c7516e0 -r 279a27a3b1a9 xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:21 2011 +0800
@@ -2175,6 +2175,11 @@ asmlinkage void vmx_vmexit_handler(struc
     /* Now enable interrupts so it's safe to take locks. */
     local_irq_enable();
 
+    /* XXX: This looks ugly, but we need a mechanism to ensure
+     * any pending vmresume has really happened
+     */
+    vcpu_nestedhvm(v).nv_vmswitch_in_progress = 0;
+
     if ( unlikely(exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) )
         return vmx_failed_vmentry(exit_reason, regs);
 
@@ -2469,10 +2474,18 @@ asmlinkage void vmx_vmexit_handler(struc
             update_guest_eip();
         break;
 
+    case EXIT_REASON_VMLAUNCH:
+        if ( nvmx_handle_vmlaunch(regs) == X86EMUL_OKAY )
+            update_guest_eip();
+        break;
+
+    case EXIT_REASON_VMRESUME:
+        if ( nvmx_handle_vmresume(regs) == X86EMUL_OKAY )
+            update_guest_eip();
+        break;
+
     case EXIT_REASON_MWAIT_INSTRUCTION:
     case EXIT_REASON_MONITOR_INSTRUCTION:
-    case EXIT_REASON_VMLAUNCH:
-    case EXIT_REASON_VMRESUME:
     case EXIT_REASON_GETSEC:
     case EXIT_REASON_INVEPT:
     case EXIT_REASON_INVVPID:
diff -r 62cc6c7516e0 -r 279a27a3b1a9 xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:21 2011 +0800
@@ -261,6 +261,13 @@ static void reg_write(struct cpu_user_re
     }
 }
 
+static inline u32 __n2_exec_control(struct vcpu *v)
+{
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+
+    return __get_vvmcs(nvcpu->nv_vvmcx, CPU_BASED_VM_EXEC_CONTROL);
+}
+
 static int vmx_inst_check_privilege(struct cpu_user_regs *regs, int vmxop_check)
 {
     struct vcpu *v = current;
@@ -502,6 +509,34 @@ int nvmx_handle_vmxoff(struct cpu_user_r
     return X86EMUL_OKAY;
 }
 
+int nvmx_handle_vmresume(struct cpu_user_regs *regs)
+{
+    struct vcpu *v = current;
+    struct nestedvmx *nvmx = &vcpu_2_nvmx(v);
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+    int rc;
+
+    rc = vmx_inst_check_privilege(regs, 0);
+    if ( rc != X86EMUL_OKAY )
+        return rc;
+
+    /* check VMCS is valid and IO BITMAP is set */
+    if ( (nvcpu->nv_vvmcxaddr != VMCX_EADDR) &&
+            ((nvmx->iobitmap[0] && nvmx->iobitmap[1]) ||
+            !(__n2_exec_control(v) & CPU_BASED_ACTIVATE_IO_BITMAP) ) )
+        nvcpu->nv_vmentry_pending = 1;
+    else
+        vmreturn(regs, VMFAIL_INVALID);
+
+    return X86EMUL_OKAY;
+}
+
+int nvmx_handle_vmlaunch(struct cpu_user_regs *regs)
+{
+    /* TODO: check for initial launch/resume */
+    return nvmx_handle_vmresume(regs);
+}
+
 int nvmx_handle_vmptrld(struct cpu_user_regs *regs)
 {
     struct vcpu *v = current;
diff -r 62cc6c7516e0 -r 279a27a3b1a9 xen/include/asm-x86/hvm/vmx/vvmx.h
--- a/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:21 2011 +0800
@@ -158,6 +158,8 @@ int nvmx_handle_vmptrst(struct cpu_user_
 int nvmx_handle_vmclear(struct cpu_user_regs *regs);
 int nvmx_handle_vmread(struct cpu_user_regs *regs);
 int nvmx_handle_vmwrite(struct cpu_user_regs *regs);
+int nvmx_handle_vmresume(struct cpu_user_regs *regs);
+int nvmx_handle_vmlaunch(struct cpu_user_regs *regs);
 
 #endif /* __ASM_X86_HVM_VVMX_H__ */

[PATCH 14 of 20] Extend VMCS control fields for n2 guest

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003601 -28800
# Node ID aacbe98da103be572c9f96d6c85788f74f574117
# Parent  279a27a3b1a90380c8fa579e87835cb58a8f4aac
Extend VMCS control fields for n2 guest

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r 279a27a3b1a9 -r aacbe98da103 xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:21 2011 +0800
@@ -54,6 +54,7 @@
 #include <asm/xenoprof.h>
 #include <asm/debugger.h>
 #include <asm/apic.h>
+#include <asm/hvm/nestedhvm.h>
 
 enum handler_return { HNDL_done, HNDL_unhandled, HNDL_exception_raised };
 
@@ -361,18 +362,28 @@ long_mode_do_msr_write(unsigned int msr,
 
 void vmx_update_cpu_exec_control(struct vcpu *v)
 {
-    __vmwrite(CPU_BASED_VM_EXEC_CONTROL, v->arch.hvm_vmx.exec_control);
+    if ( nestedhvm_vcpu_in_guestmode(v) )
+        nvmx_update_exec_control(v, v->arch.hvm_vmx.exec_control);
+    else
+        __vmwrite(CPU_BASED_VM_EXEC_CONTROL, v->arch.hvm_vmx.exec_control);
 }
 
 static void vmx_update_secondary_exec_control(struct vcpu *v)
 {
-    __vmwrite(SECONDARY_VM_EXEC_CONTROL,
-              v->arch.hvm_vmx.secondary_exec_control);
+    if ( nestedhvm_vcpu_in_guestmode(v) )
+        nvmx_update_secondary_exec_control(v,
+            v->arch.hvm_vmx.secondary_exec_control);
+    else
+        __vmwrite(SECONDARY_VM_EXEC_CONTROL,
+                  v->arch.hvm_vmx.secondary_exec_control);
 }
 
 void vmx_update_exception_bitmap(struct vcpu *v)
 {
-    __vmwrite(EXCEPTION_BITMAP, v->arch.hvm_vmx.exception_bitmap);
+    if ( nestedhvm_vcpu_in_guestmode(v) )
+        nvmx_update_exception_bitmap(v, v->arch.hvm_vmx.exception_bitmap);
+    else
+        __vmwrite(EXCEPTION_BITMAP, v->arch.hvm_vmx.exception_bitmap);
 }
 
 static int vmx_guest_x86_mode(struct vcpu *v)
diff -r 279a27a3b1a9 -r aacbe98da103 xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:21 2011 +0800
@@ -25,6 +25,7 @@
 #include <asm/p2m.h>
 #include <asm/hvm/vmx/vmx.h>
 #include <asm/hvm/vmx/vvmx.h>
+#include <asm/hvm/nestedhvm.h>
 
 int nvmx_vcpu_initialise(struct vcpu *v)
 {
@@ -391,6 +392,93 @@ static void vmreturn(struct cpu_user_reg
     regs->eflags = eflags;
 }
 
+/*
+ * Nested VMX uses "strict" condition to exit from 
+ * L2 guest if either L1 VMM or L0 VMM expect to exit.
+ */
+static inline u32 __shadow_control(struct vcpu *v,
+                                 unsigned int field,
+                                 u32 host_value)
+{
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+
+    return (u32) __get_vvmcs(nvcpu->nv_vvmcx, field) | host_value;
+}
+
+static void set_shadow_control(struct vcpu *v,
+                               unsigned int field,
+                               u32 host_value)
+{
+    __vmwrite(field, __shadow_control(v, field, host_value));
+}
+
+unsigned long *_shadow_io_bitmap(struct vcpu *v)
+{
+    struct nestedvmx *nvmx = &vcpu_2_nvmx(v);
+    int port80, portED;
+    u8 *bitmap;
+
+    bitmap = nvmx->iobitmap[0];
+    port80 = bitmap[0x80 >> 3] & (1 << (0x80 & 0x7)) ? 1 : 0;
+    portED = bitmap[0xed >> 3] & (1 << (0xed & 0x7)) ? 1 : 0;
+
+    return nestedhvm_vcpu_iomap_get(port80, portED);
+}
+
+void nvmx_update_exec_control(struct vcpu *v, unsigned long host_cntrl)
+{
+#define PIO_CNTRL_BITS    ( CPU_BASED_ACTIVATE_IO_BITMAP         \
+             | CPU_BASED_UNCOND_IO_EXITING)
+    u32 pio_cntrl = PIO_CNTRL_BITS;
+    unsigned long *bitmap; 
+    u32 shadow_cntrl;
+ 
+    shadow_cntrl = __n2_exec_control(v);
+    pio_cntrl &= shadow_cntrl;
+    /* Enforce the removed features */
+#define REMOVED_EXEC_CONTROL_BITS (CPU_BASED_TPR_SHADOW          \
+             | CPU_BASED_ACTIVATE_MSR_BITMAP                     \
+             | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS             \
+             | CPU_BASED_ACTIVATE_IO_BITMAP                      \
+             | CPU_BASED_UNCOND_IO_EXITING)
+    shadow_cntrl &= ~REMOVED_EXEC_CONTROL_BITS;
+    shadow_cntrl |= host_cntrl;
+    if ( pio_cntrl == CPU_BASED_UNCOND_IO_EXITING ) {
+        /* L1 VMM intercepts all I/O instructions */
+        shadow_cntrl |= CPU_BASED_UNCOND_IO_EXITING;
+        shadow_cntrl &= ~CPU_BASED_ACTIVATE_IO_BITMAP;
+    }
+    else {
+        /* Use IO_BITMAP in shadow */
+        if ( pio_cntrl == 0 ) {
+            /* 
+             * L1 VMM doesn't intercept IO instruction.
+             * Use host configuration and reset IO_BITMAP
+             */
+            bitmap = hvm_io_bitmap;
+        }
+        else {
+            /* use IO bitmap */
+            bitmap = _shadow_io_bitmap(v);
+        }
+        __vmwrite(IO_BITMAP_A, virt_to_maddr(bitmap));
+        __vmwrite(IO_BITMAP_B, virt_to_maddr(bitmap) + PAGE_SIZE);
+    }
+
+    __vmwrite(CPU_BASED_VM_EXEC_CONTROL, shadow_cntrl);
+}
+
+void nvmx_update_secondary_exec_control(struct vcpu *v,
+                                            unsigned long value)
+{
+    set_shadow_control(v, SECONDARY_VM_EXEC_CONTROL, value);
+}
+
+void nvmx_update_exception_bitmap(struct vcpu *v, unsigned long value)
+{
+    set_shadow_control(v, EXCEPTION_BITMAP, value);
+}
+
 static void __clear_current_vvmcs(struct vcpu *v)
 {
     struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
diff -r 279a27a3b1a9 -r aacbe98da103 xen/include/asm-x86/hvm/vmx/vvmx.h
--- a/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:21 2011 +0800
@@ -161,5 +161,10 @@ int nvmx_handle_vmwrite(struct cpu_user_
 int nvmx_handle_vmresume(struct cpu_user_regs *regs);
 int nvmx_handle_vmlaunch(struct cpu_user_regs *regs);
 
+void nvmx_update_exec_control(struct vcpu *v, unsigned long value);
+void nvmx_update_secondary_exec_control(struct vcpu *v,
+                                        unsigned long value);
+void nvmx_update_exception_bitmap(struct vcpu *v, unsigned long value);
+
 #endif /* __ASM_X86_HVM_VVMX_H__ */

[PATCH 15 of 20] Switch shadow/virtual VMCS between n1/n2 guests

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003601 -28800
# Node ID bd15acfc9b822ccf27b5c7603e600e5e11733907
# Parent  aacbe98da103be572c9f96d6c85788f74f574117
Switch shadow/virtual VMCS between n1/n2 guests.

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r aacbe98da103 -r bd15acfc9b82 xen/arch/x86/hvm/vmx/entry.S
--- a/xen/arch/x86/hvm/vmx/entry.S	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/entry.S	Thu Jun 02 16:33:21 2011 +0800
@@ -119,6 +119,7 @@ vmx_asm_vmexit_handler:
 .globl vmx_asm_do_vmentry
 vmx_asm_do_vmentry:
         call vmx_intr_assist
+        call nvmx_switch_guest
 
         get_current(bx)
         cli
diff -r aacbe98da103 -r bd15acfc9b82 xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:21 2011 +0800
@@ -474,6 +474,48 @@ void nvmx_update_secondary_exec_control(
     set_shadow_control(v, SECONDARY_VM_EXEC_CONTROL, value);
 }
 
+static void nvmx_update_pin_control(struct vcpu *v,
+					unsigned long host_cntrl)
+{
+    u32 shadow_cntrl;
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+
+#define REMOVED_PIN_CONTROL_BITS (PIN_BASED_PREEMPT_TIMER)
+    shadow_cntrl = __get_vvmcs(nvcpu->nv_vvmcx, PIN_BASED_VM_EXEC_CONTROL);
+    shadow_cntrl &= ~REMOVED_PIN_CONTROL_BITS;
+    shadow_cntrl |= host_cntrl;
+    __vmwrite(PIN_BASED_VM_EXEC_CONTROL, shadow_cntrl);
+}
+
+static void nvmx_update_exit_control(struct vcpu *v,
+					unsigned long host_cntrl)
+{
+    u32 shadow_cntrl;
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+
+#define REMOVED_EXIT_CONTROL_BITS    ((1<<2) |           \
+                (VM_EXIT_SAVE_GUEST_PAT) |               \
+                (VM_EXIT_SAVE_GUEST_EFER) |              \
+                (VM_EXIT_SAVE_PREEMPT_TIMER))
+    shadow_cntrl = __get_vvmcs(nvcpu->nv_vvmcx, VM_EXIT_CONTROLS);
+    shadow_cntrl &= ~REMOVED_EXIT_CONTROL_BITS;
+    shadow_cntrl |= host_cntrl;
+    __vmwrite(VM_EXIT_CONTROLS, shadow_cntrl);
+}
+
+static void nvmx_update_entry_control(struct vcpu *v)
+{
+    u32 shadow_cntrl;
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+
+    /* VM_ENTRY_CONTROLS: enforce removed features */
+#define REMOVED_ENTRY_CONTROL_BITS (VM_ENTRY_LOAD_GUEST_PAT \
+            | VM_ENTRY_LOAD_GUEST_EFER)
+    shadow_cntrl = __get_vvmcs(nvcpu->nv_vvmcx, VM_ENTRY_CONTROLS);
+    shadow_cntrl &= ~REMOVED_ENTRY_CONTROL_BITS;
+    __vmwrite(VM_ENTRY_CONTROLS, shadow_cntrl);
+}
+
 void nvmx_update_exception_bitmap(struct vcpu *v, unsigned long value)
 {
     set_shadow_control(v, EXCEPTION_BITMAP, value);
@@ -543,6 +585,361 @@ static void nvmx_purge_vvmcs(struct vcpu
 }
 
 /*
+ * Context synchronized between shadow and virtual VMCS.
+ */
+static unsigned long vmcs_gstate_field[] = {
+    /* 16 BITS */
+    GUEST_ES_SELECTOR,
+    GUEST_CS_SELECTOR,
+    GUEST_SS_SELECTOR,
+    GUEST_DS_SELECTOR,
+    GUEST_FS_SELECTOR,
+    GUEST_GS_SELECTOR,
+    GUEST_LDTR_SELECTOR,
+    GUEST_TR_SELECTOR,
+    /* 64 BITS */
+    VMCS_LINK_POINTER,
+    GUEST_IA32_DEBUGCTL,
+#ifndef CONFIG_X86_64
+    VMCS_LINK_POINTER_HIGH,
+    GUEST_IA32_DEBUGCTL_HIGH,
+#endif
+    /* 32 BITS */
+    GUEST_ES_LIMIT,
+    GUEST_CS_LIMIT,
+    GUEST_SS_LIMIT,
+    GUEST_DS_LIMIT,
+    GUEST_FS_LIMIT,
+    GUEST_GS_LIMIT,
+    GUEST_LDTR_LIMIT,
+    GUEST_TR_LIMIT,
+    GUEST_GDTR_LIMIT,
+    GUEST_IDTR_LIMIT,
+    GUEST_ES_AR_BYTES,
+    GUEST_CS_AR_BYTES,
+    GUEST_SS_AR_BYTES,
+    GUEST_DS_AR_BYTES,
+    GUEST_FS_AR_BYTES,
+    GUEST_GS_AR_BYTES,
+    GUEST_LDTR_AR_BYTES,
+    GUEST_TR_AR_BYTES,
+    GUEST_INTERRUPTIBILITY_INFO,
+    GUEST_ACTIVITY_STATE,
+    GUEST_SYSENTER_CS,
+    /* natural */
+    GUEST_ES_BASE,
+    GUEST_CS_BASE,
+    GUEST_SS_BASE,
+    GUEST_DS_BASE,
+    GUEST_FS_BASE,
+    GUEST_GS_BASE,
+    GUEST_LDTR_BASE,
+    GUEST_TR_BASE,
+    GUEST_GDTR_BASE,
+    GUEST_IDTR_BASE,
+    GUEST_DR7,
+    /*
+     * Following guest states are in local cache (cpu_user_regs)
+     GUEST_RSP,
+     GUEST_RIP,
+     */
+    GUEST_RFLAGS,
+    GUEST_PENDING_DBG_EXCEPTIONS,
+    GUEST_SYSENTER_ESP,
+    GUEST_SYSENTER_EIP,
+};
+
+/*
+ * Context: shadow -> virtual VMCS
+ */
+static unsigned long vmcs_ro_field[] = {
+    GUEST_PHYSICAL_ADDRESS,
+    VM_INSTRUCTION_ERROR,
+    VM_EXIT_REASON,
+    VM_EXIT_INTR_INFO,
+    VM_EXIT_INTR_ERROR_CODE,
+    IDT_VECTORING_INFO,
+    IDT_VECTORING_ERROR_CODE,
+    VM_EXIT_INSTRUCTION_LEN,
+    VMX_INSTRUCTION_INFO,
+    EXIT_QUALIFICATION,
+    GUEST_LINEAR_ADDRESS
+};
+
+static struct vmcs_host_to_guest {
+    unsigned long host_field;
+    unsigned long guest_field;
+} vmcs_h2g_field[] = {
+    {HOST_ES_SELECTOR, GUEST_ES_SELECTOR},
+    {HOST_CS_SELECTOR, GUEST_CS_SELECTOR},
+    {HOST_SS_SELECTOR, GUEST_SS_SELECTOR},
+    {HOST_DS_SELECTOR, GUEST_DS_SELECTOR},
+    {HOST_FS_SELECTOR, GUEST_FS_SELECTOR},
+    {HOST_GS_SELECTOR, GUEST_GS_SELECTOR},
+    {HOST_TR_SELECTOR, GUEST_TR_SELECTOR},
+    {HOST_SYSENTER_CS, GUEST_SYSENTER_CS},
+    {HOST_FS_BASE, GUEST_FS_BASE},
+    {HOST_GS_BASE, GUEST_GS_BASE},
+    {HOST_TR_BASE, GUEST_TR_BASE},
+    {HOST_GDTR_BASE, GUEST_GDTR_BASE},
+    {HOST_IDTR_BASE, GUEST_IDTR_BASE},
+    {HOST_SYSENTER_ESP, GUEST_SYSENTER_ESP},
+    {HOST_SYSENTER_EIP, GUEST_SYSENTER_EIP},
+};
+
+static void vvmcs_to_shadow(void *vvmcs, unsigned int field)
+{
+    u64 value;
+
+    value = __get_vvmcs(vvmcs, field);
+    __vmwrite(field, value);
+}
+
+static void shadow_to_vvmcs(void *vvmcs, unsigned int field)
+{
+    u64 value;
+    int rc;
+
+    value = __vmread_safe(field, &rc);
+    if ( !rc )
+        __set_vvmcs(vvmcs, field, value);
+}
+
+static void load_shadow_control(struct vcpu *v)
+{
+    /* TODO: Make sure the shadow control doesn't set the bits 
+     * L0 VMM doesn't handle.
+     */
+
+    /*
+     * Set shadow controls:  PIN_BASED, CPU_BASED, EXIT, ENTRY
+     * and EXCEPTION
+     * Enforce the removed features
+     */
+    nvmx_update_pin_control(v, vmx_pin_based_exec_control);
+    vmx_update_cpu_exec_control(v);
+    nvmx_update_exit_control(v, vmx_vmexit_control);
+    nvmx_update_entry_control(v);
+    vmx_update_exception_bitmap(v);
+}
+
+static void load_shadow_guest_state(struct vcpu *v)
+{
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+    void *vvmcs = nvcpu->nv_vvmcx;
+    int i;
+
+    /* vvmcs.gstate to shadow vmcs.gstate */
+    for ( i = 0; i < ARRAY_SIZE(vmcs_gstate_field); i++ )
+        vvmcs_to_shadow(vvmcs, vmcs_gstate_field[i]);
+
+    hvm_set_cr0(__get_vvmcs(vvmcs, GUEST_CR0));
+    hvm_set_cr4(__get_vvmcs(vvmcs, GUEST_CR4));
+    hvm_set_cr3(__get_vvmcs(vvmcs, GUEST_CR3));
+
+    vvmcs_to_shadow(vvmcs, VM_ENTRY_INTR_INFO);
+    vvmcs_to_shadow(vvmcs, VM_ENTRY_EXCEPTION_ERROR_CODE);
+    vvmcs_to_shadow(vvmcs, VM_ENTRY_INSTRUCTION_LEN);
+
+    /* XXX: should refer to GUEST_HOST_MASK of both L0 and L1 */
+    vvmcs_to_shadow(vvmcs, CR0_READ_SHADOW);
+    vvmcs_to_shadow(vvmcs, CR4_READ_SHADOW);
+    vvmcs_to_shadow(vvmcs, CR0_GUEST_HOST_MASK);
+    vvmcs_to_shadow(vvmcs, CR4_GUEST_HOST_MASK);
+
+    /* TODO: PDPTRs for nested ept */
+    /* TODO: CR3 target control */
+}
+
+static void virtual_vmentry(struct cpu_user_regs *regs)
+{
+    struct vcpu *v = current;
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+    void *vvmcs = nvcpu->nv_vvmcx;
+#ifdef __x86_64__
+    unsigned long lm_l1, lm_l2;
+#endif
+
+    vmx_vmcs_switch(v, v->arch.hvm_vmx.vmcs, nvcpu->nv_n2vmcx);
+
+    nestedhvm_vcpu_enter_guestmode(v);
+    nvcpu->nv_vmentry_pending = 0;
+    nvcpu->nv_vmswitch_in_progress = 1;
+
+#ifdef __x86_64__
+    /*
+     * EFER handling:
+     * hvm_set_efer won't work if CR0.PG = 1, so we change the value
+     * directly to make hvm_long_mode_enabled(v) work in L2.
+     * An additional update_paging_modes is also needed if
+     * there is 32/64 switch. v->arch.hvm_vcpu.guest_efer doesn't
+     * need to be saved, since its value on vmexit is determined by
+     * L1 exit_controls
+     */
+    lm_l1 = !!hvm_long_mode_enabled(v);
+    lm_l2 = !!(__get_vvmcs(vvmcs, VM_ENTRY_CONTROLS) &
+                           VM_ENTRY_IA32E_MODE);
+
+    if ( lm_l2 )
+        v->arch.hvm_vcpu.guest_efer |= EFER_LMA | EFER_LME;
+    else
+        v->arch.hvm_vcpu.guest_efer &= ~(EFER_LMA | EFER_LME);
+#endif
+
+    load_shadow_control(v);
+    load_shadow_guest_state(v);
+
+#ifdef __x86_64__
+    if ( lm_l1 != lm_l2 )
+    {
+        paging_update_paging_modes(v);
+    }
+#endif
+
+    regs->rip = __get_vvmcs(vvmcs, GUEST_RIP);
+    regs->rsp = __get_vvmcs(vvmcs, GUEST_RSP);
+    regs->rflags = __get_vvmcs(vvmcs, GUEST_RFLAGS);
+
+    /* TODO: EPT_POINTER */
+}
+
+static void sync_vvmcs_guest_state(struct vcpu *v, struct cpu_user_regs *regs)
+{
+    int i;
+    unsigned long mask;
+    unsigned long cr;
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+    void *vvmcs = nvcpu->nv_vvmcx;
+
+    /* copy shadow vmcs.gstate back to vvmcs.gstate */
+    for ( i = 0; i < ARRAY_SIZE(vmcs_gstate_field); i++ )
+        shadow_to_vvmcs(vvmcs, vmcs_gstate_field[i]);
+    /* RIP, RSP are in user regs */
+    __set_vvmcs(vvmcs, GUEST_RIP, regs->rip);
+    __set_vvmcs(vvmcs, GUEST_RSP, regs->rsp);
+
+    /* SDM 20.6.6: L2 guest execution may change GUEST CR0/CR4 */
+    mask = __get_vvmcs(vvmcs, CR0_GUEST_HOST_MASK);
+    if ( ~mask )
+    {
+        cr = __get_vvmcs(vvmcs, GUEST_CR0);
+        cr = (cr & mask) | (__vmread(GUEST_CR4) & ~mask);
+        __set_vvmcs(vvmcs, GUEST_CR0, cr);
+    }
+
+    mask = __get_vvmcs(vvmcs, CR4_GUEST_HOST_MASK);
+    if ( ~mask )
+    {
+        cr = __get_vvmcs(vvmcs, GUEST_CR4);
+        cr = (cr & mask) | (__vmread(GUEST_CR4) & ~mask);
+        __set_vvmcs(vvmcs, GUEST_CR4, cr);
+    }
+
+    /* CR3 sync if exec doesn't want cr3 load exiting: i.e. nested EPT */
+    if ( !(__n2_exec_control(v) & CPU_BASED_CR3_LOAD_EXITING) )
+        shadow_to_vvmcs(vvmcs, GUEST_CR3);
+}
+
+static void sync_vvmcs_ro(struct vcpu *v)
+{
+    int i;
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+
+    for ( i = 0; i < ARRAY_SIZE(vmcs_ro_field); i++ )
+        shadow_to_vvmcs(nvcpu->nv_vvmcx, vmcs_ro_field[i]);
+}
+
+static void load_vvmcs_host_state(struct vcpu *v)
+{
+    int i;
+    u64 r;
+    void *vvmcs = vcpu_nestedhvm(v).nv_vvmcx;
+
+    for ( i = 0; i < ARRAY_SIZE(vmcs_h2g_field); i++ )
+    {
+        r = __get_vvmcs(vvmcs, vmcs_h2g_field[i].host_field);
+        __vmwrite(vmcs_h2g_field[i].guest_field, r);
+    }
+
+    hvm_set_cr0(__get_vvmcs(vvmcs, HOST_CR0));
+    hvm_set_cr4(__get_vvmcs(vvmcs, HOST_CR4));
+    hvm_set_cr3(__get_vvmcs(vvmcs, HOST_CR3));
+
+    __set_vvmcs(vvmcs, VM_ENTRY_INTR_INFO, 0);
+}
+
+static void virtual_vmexit(struct cpu_user_regs *regs)
+{
+    struct vcpu *v = current;
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+#ifdef __x86_64__
+    unsigned long lm_l1, lm_l2;
+#endif
+
+    sync_vvmcs_ro(v);
+    sync_vvmcs_guest_state(v, regs);
+
+    vmx_vmcs_switch(v, v->arch.hvm_vmx.vmcs, nvcpu->nv_n1vmcx);
+
+    nestedhvm_vcpu_exit_guestmode(v);
+    nvcpu->nv_vmexit_pending = 0;
+
+#ifdef __x86_64__
+    lm_l2 = !!hvm_long_mode_enabled(v);
+    lm_l1 = !!(__get_vvmcs(nvcpu->nv_vvmcx, VM_EXIT_CONTROLS) &
+                           VM_EXIT_IA32E_MODE);
+
+    if ( lm_l1 )
+        v->arch.hvm_vcpu.guest_efer |= EFER_LMA | EFER_LME;
+    else
+        v->arch.hvm_vcpu.guest_efer &= ~(EFER_LMA | EFER_LME);
+#endif
+
+    vmx_update_cpu_exec_control(v);
+    vmx_update_exception_bitmap(v);
+
+    load_vvmcs_host_state(v);
+
+#ifdef __x86_64__
+    if ( lm_l1 != lm_l2 )
+        paging_update_paging_modes(v);
+#endif
+
+    regs->rip = __get_vvmcs(nvcpu->nv_vvmcx, HOST_RIP);
+    regs->rsp = __get_vvmcs(nvcpu->nv_vvmcx, HOST_RSP);
+    regs->rflags = __vmread(GUEST_RFLAGS);
+
+    vmreturn(regs, VMSUCCEED);
+}
+
+asmlinkage void nvmx_switch_guest(void)
+{
+    struct vcpu *v = current;
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+    struct cpu_user_regs *regs = guest_cpu_user_regs();
+
+    /*
+     * a softirq may interrupt us between a virtual vmentry is
+     * just handled and the true vmentry. If during this window,
+     * a L1 virtual interrupt causes another virtual vmexit, we
+     * cannot let that happen or VM_ENTRY_INTR_INFO will be lost.
+     */
+    if ( unlikely(nvcpu->nv_vmswitch_in_progress) )
+        return;
+
+    if ( nestedhvm_vcpu_in_guestmode(v) && nvcpu->nv_vmexit_pending )
+    {
+        local_irq_enable();
+        virtual_vmexit(regs);
+    }
+    else if ( !nestedhvm_vcpu_in_guestmode(v) && nvcpu->nv_vmentry_pending )
+    {
+        local_irq_enable();
+        virtual_vmentry(regs);
+    }
+}
+
+/*
  * VMX instructions handling
  */
 
diff -r aacbe98da103 -r bd15acfc9b82 xen/include/asm-x86/hvm/vmx/vvmx.h
--- a/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:21 2011 +0800
@@ -165,6 +165,7 @@ void nvmx_update_exec_control(struct vcp
 void nvmx_update_secondary_exec_control(struct vcpu *v,
                                         unsigned long value);
 void nvmx_update_exception_bitmap(struct vcpu *v, unsigned long value);
+asmlinkage void nvmx_switch_guest(void);
 
 #endif /* __ASM_X86_HVM_VVMX_H__ */

Re: [PATCH 15 of 20] Switch shadow/virtual VMCS between n1/n2 guests

[Tim Deegan]

At 16:57 +0800 on 02 Jun (1307033848), Eddie Dong wrote:
> +static void nvmx_update_exit_control(struct vcpu *v,
> +					unsigned long host_cntrl)
> +{
> +    u32 shadow_cntrl;
> +    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
> +
> +#define REMOVED_EXIT_CONTROL_BITS    ((1<<2) |           \

Define a macro for whatever 1<<2 means here, please. 

> +                (VM_EXIT_SAVE_GUEST_PAT) |               \
> +                (VM_EXIT_SAVE_GUEST_EFER) |              \
> +                (VM_EXIT_SAVE_PREEMPT_TIMER))
> +    shadow_cntrl = __get_vvmcs(nvcpu->nv_vvmcx, VM_EXIT_CONTROLS);
> +    shadow_cntrl &= ~REMOVED_EXIT_CONTROL_BITS;
> +    shadow_cntrl |= host_cntrl;
> +    __vmwrite(VM_EXIT_CONTROLS, shadow_cntrl);
> +}
[...]
> +static void sync_vvmcs_guest_state(struct vcpu *v, struct cpu_user_regs *regs)
> +{
> +    int i;
> +    unsigned long mask;
> +    unsigned long cr;
> +    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
> +    void *vvmcs = nvcpu->nv_vvmcx;
> +
> +    /* copy shadow vmcs.gstate back to vvmcs.gstate */
> +    for ( i = 0; i < ARRAY_SIZE(vmcs_gstate_field); i++ )
> +        shadow_to_vvmcs(vvmcs, vmcs_gstate_field[i]);
> +    /* RIP, RSP are in user regs */
> +    __set_vvmcs(vvmcs, GUEST_RIP, regs->rip);
> +    __set_vvmcs(vvmcs, GUEST_RSP, regs->rsp);
> +
> +    /* SDM 20.6.6: L2 guest execution may change GUEST CR0/CR4 */
> +    mask = __get_vvmcs(vvmcs, CR0_GUEST_HOST_MASK);
> +    if ( ~mask )
> +    {
> +        cr = __get_vvmcs(vvmcs, GUEST_CR0);
> +        cr = (cr & mask) | (__vmread(GUEST_CR4) & ~mask);

Cut-n-paste error?                      ^^^^^^^^^

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

RE: [PATCH 15 of 20] Switch shadow/virtual VMCS between n1/n2 guests

[Dong, Eddie]

> > +    u32 shadow_cntrl;
> > +    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
> > +
> > +#define REMOVED_EXIT_CONTROL_BITS    ((1<<2) |           \
> 
> Define a macro for whatever 1<<2 means here, please.
> 

Done.

> > +                (VM_EXIT_SAVE_GUEST_PAT) |               \
> > +                (VM_EXIT_SAVE_GUEST_EFER) |              \
> > +                (VM_EXIT_SAVE_PREEMPT_TIMER))
> > +    shadow_cntrl = __get_vvmcs(nvcpu->nv_vvmcx,
> VM_EXIT_CONTROLS);
> > +    shadow_cntrl &= ~REMOVED_EXIT_CONTROL_BITS;
> > +    shadow_cntrl |= host_cntrl;
> > +    __vmwrite(VM_EXIT_CONTROLS, shadow_cntrl);
> > +}



> > +    /* SDM 20.6.6: L2 guest execution may change GUEST CR0/CR4 */
> > +    mask = __get_vvmcs(vvmcs, CR0_GUEST_HOST_MASK);
> > +    if ( ~mask )
> > +    {
> > +        cr = __get_vvmcs(vvmcs, GUEST_CR0);
> > +        cr = (cr & mask) | (__vmread(GUEST_CR4) & ~mask);
> 
> Cut-n-paste error?                      ^^^^^^^^^
> 
Oh, Yes, Thanks.
Eddie

Re: [PATCH 15 of 20] Switch shadow/virtual VMCS between n1/n2 guests

[Tim Deegan]

Hi,

> +asmlinkage void nvmx_switch_guest(void)
> +{
> +    struct vcpu *v = current;
> +    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
> +    struct cpu_user_regs *regs = guest_cpu_user_regs();
> +
> +    /*
> +     * a softirq may interrupt us between a virtual vmentry is
> +     * just handled and the true vmentry. If during this window,
> +     * a L1 virtual interrupt causes another virtual vmexit, we
> +     * cannot let that happen or VM_ENTRY_INTR_INFO will be lost.
> +     */
> +    if ( unlikely(nvcpu->nv_vmswitch_in_progress) )
> +        return;
> +
> +    if ( nestedhvm_vcpu_in_guestmode(v) && nvcpu->nv_vmexit_pending )
> +    {
> +        local_irq_enable();

Why?  Is this function every called with interrupts disabled?  And if
so, will its caller deal with having them enabled when it exits?

> +        virtual_vmexit(regs);
> +    }
> +    else if ( !nestedhvm_vcpu_in_guestmode(v) && nvcpu->nv_vmentry_pending )
> +    {
> +        local_irq_enable();

ditto.

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

[PATCH 16 of 20] interrupt/exception handling for n2 guest

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003601 -28800
# Node ID f14f451a780e60e920c057e44fa1bc3ee40495a7
# Parent  bd15acfc9b822ccf27b5c7603e600e5e11733907
interrupt/exception handling for n2 guest

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r bd15acfc9b82 -r f14f451a780e xen/arch/x86/hvm/vmx/intr.c
--- a/xen/arch/x86/hvm/vmx/intr.c	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/intr.c	Thu Jun 02 16:33:21 2011 +0800
@@ -35,6 +35,7 @@
 #include <asm/hvm/vmx/vmcs.h>
 #include <asm/hvm/vpic.h>
 #include <asm/hvm/vlapic.h>
+#include <asm/hvm/nestedhvm.h>
 #include <public/hvm/ioreq.h>
 #include <asm/hvm/trace.h>
 
@@ -109,6 +110,102 @@ static void enable_intr_window(struct vc
     }
 }
 
+/*
+ * Injecting interrupts for nested virtualization
+ *
+ *  When injecting virtual interrupts (originated from L0), there are
+ *  two major possibilities, within L1 context and within L2 context
+ *   1. L1 context (in_nesting == 0)
+ *     Everything is the same as without nested, check RFLAGS.IF to
+ *     see if the injection can be done, using VMCS to inject the
+ *     interrupt
+ *
+ *   2. L2 context (in_nesting == 1)
+ *     Causes a virtual VMExit, RFLAGS.IF is ignored, whether to ack
+ *     irq according to intr_ack_on_exit, shouldn't block normally,
+ *     except for:
+ *    a. context transition
+ *     interrupt needs to be blocked at virtual VMEntry time
+ *    b. L2 idtv reinjection
+ *     if L2 idtv is handled within L0 (e.g. L0 shadow page fault),
+ *     it needs to be reinjected without exiting to L1, interrupt
+ *     injection should be blocked as well at this point.
+ *
+ *  Unfortunately, interrupt blocking in L2 won't work with simple
+ *  intr_window_open (which depends on L2's IF). To solve this,
+ *  the following algorithm can be used:
+ *   v->arch.hvm_vmx.exec_control.VIRTUAL_INTR_PENDING now denotes
+ *   only L0 control, physical control may be different from it.
+ *       - if in L1, it behaves normally, intr window is written
+ *         to physical control as it is
+ *       - if in L2, replace it to MTF (or NMI window) if possible
+ *       - if MTF/NMI window is not used, intr window can still be
+ *         used but may have negative impact on interrupt performance.
+ */
+
+enum hvm_intblk nvmx_intr_blocked(struct vcpu *v)
+{
+    int r = hvm_intblk_none;
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+
+    if ( nestedhvm_vcpu_in_guestmode(v) )
+    {
+        if ( nvcpu->nv_vmexit_pending ||
+             nvcpu->nv_vmswitch_in_progress ||
+             (__vmread(VM_ENTRY_INTR_INFO) & INTR_INFO_VALID_MASK) )
+            r = hvm_intblk_rflags_ie;
+    }
+    else if ( nvcpu->nv_vmentry_pending )
+        r = hvm_intblk_rflags_ie;
+
+    return r;
+}
+
+static int nvmx_intr_intercept(struct vcpu *v, struct hvm_intack intack)
+{
+    u32 exit_ctrl;
+
+    /*
+     * TODO:
+     *   - if L1 intr-window exiting == 0
+     *   - vNMI
+     */
+
+    if ( nvmx_intr_blocked(v) != hvm_intblk_none )
+    {
+        enable_intr_window(v, intack);
+        return 1;
+    }
+
+    if ( nestedhvm_vcpu_in_guestmode(v) )
+    {
+        if ( intack.source == hvm_intsrc_pic ||
+                 intack.source == hvm_intsrc_lapic )
+        {
+            vmx_inject_extint(intack.vector);
+
+            exit_ctrl = __get_vvmcs(vcpu_nestedhvm(v).nv_vvmcx,
+                            VM_EXIT_CONTROLS);
+            if ( exit_ctrl & VM_EXIT_ACK_INTR_ON_EXIT )
+            {
+                /* for now, duplicate the ack path in vmx_intr_assist */
+                hvm_vcpu_ack_pending_irq(v, intack);
+                pt_intr_post(v, intack);
+
+                intack = hvm_vcpu_has_pending_irq(v);
+                if ( unlikely(intack.source != hvm_intsrc_none) )
+                    enable_intr_window(v, intack);
+            }
+            else
+                enable_intr_window(v, intack);
+
+            return 1;
+        }
+    }
+
+    return 0;
+}
+
 asmlinkage void vmx_intr_assist(void)
 {
     struct hvm_intack intack;
@@ -132,6 +229,9 @@ asmlinkage void vmx_intr_assist(void)
         if ( likely(intack.source == hvm_intsrc_none) )
             goto out;
 
+        if ( unlikely(nvmx_intr_intercept(v, intack)) )
+            goto out;
+
         intblk = hvm_interrupt_blocked(v, intack);
         if ( intblk == hvm_intblk_tpr )
         {
diff -r bd15acfc9b82 -r f14f451a780e xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:21 2011 +0800
@@ -1243,6 +1243,31 @@ void ept_sync_domain(struct domain *d)
                      __ept_sync_domain, d, 1);
 }
 
+void nvmx_enqueue_n2_exceptions(struct vcpu *v, 
+            unsigned long intr_fields, int error_code)
+{
+    struct nestedvmx *nvmx = &vcpu_2_nvmx(v);
+
+    if ( !(nvmx->intr.intr_info & INTR_INFO_VALID_MASK) ) {
+        /* enqueue the exception till the VMCS switch back to L1 */
+        nvmx->intr.intr_info = intr_fields;
+        nvmx->intr.error_code = error_code;
+        vcpu_nestedhvm(v).nv_vmexit_pending = 1;
+        return;
+    }
+    else
+        gdprintk(XENLOG_ERR, "Double Fault on Nested Guest: exception %lx %x"
+                 "on %lx %x\n", intr_fields, error_code,
+                 nvmx->intr.intr_info, nvmx->intr.error_code);
+}
+
+static int nvmx_vmexit_exceptions(struct vcpu *v, unsigned int trapnr,
+                      int errcode, unsigned long cr2)
+{
+    nvmx_enqueue_n2_exceptions(v, trapnr, errcode);
+    return NESTEDHVM_VMEXIT_DONE;
+}
+
 static void __vmx_inject_exception(int trap, int type, int error_code)
 {
     unsigned long intr_fields;
@@ -1272,11 +1297,16 @@ static void __vmx_inject_exception(int t
 
 void vmx_inject_hw_exception(int trap, int error_code)
 {
-    unsigned long intr_info = __vmread(VM_ENTRY_INTR_INFO);
+    unsigned long intr_info;
     struct vcpu *curr = current;
 
     int type = X86_EVENTTYPE_HW_EXCEPTION;
 
+    if ( nestedhvm_vcpu_in_guestmode(curr) )
+        intr_info = vcpu_2_nvmx(curr).intr.intr_info;
+    else
+        intr_info = __vmread(VM_ENTRY_INTR_INFO);
+
     switch ( trap )
     {
     case TRAP_debug:
@@ -1308,7 +1338,16 @@ void vmx_inject_hw_exception(int trap, i
             error_code = 0;
     }
 
-    __vmx_inject_exception(trap, type, error_code);
+    if ( nestedhvm_vcpu_in_guestmode(curr) &&
+         nvmx_intercepts_exception(curr, trap, error_code) )
+    {
+        nvmx_enqueue_n2_exceptions (curr, 
+            INTR_INFO_VALID_MASK | (type<<8) | trap,
+            error_code); 
+        return;
+    }
+    else
+        __vmx_inject_exception(trap, type, error_code);
 
     if ( trap == TRAP_page_fault )
         HVMTRACE_LONG_2D(PF_INJECT, error_code,
@@ -1319,12 +1358,38 @@ void vmx_inject_hw_exception(int trap, i
 
 void vmx_inject_extint(int trap)
 {
+    struct vcpu *v = current;
+    u32    pin_based_cntrl;
+
+    if ( nestedhvm_vcpu_in_guestmode(v) ) {
+        pin_based_cntrl = __get_vvmcs(vcpu_nestedhvm(v).nv_vvmcx, 
+                                     PIN_BASED_VM_EXEC_CONTROL);
+        if ( pin_based_cntrl && PIN_BASED_EXT_INTR_MASK ) {
+            nvmx_enqueue_n2_exceptions (v, 
+               INTR_INFO_VALID_MASK | (X86_EVENTTYPE_EXT_INTR<<8) | trap,
+               HVM_DELIVER_NO_ERROR_CODE);
+            return;
+        }
+    }
     __vmx_inject_exception(trap, X86_EVENTTYPE_EXT_INTR,
                            HVM_DELIVER_NO_ERROR_CODE);
 }
 
 void vmx_inject_nmi(void)
 {
+    struct vcpu *v = current;
+    u32    pin_based_cntrl;
+
+    if ( nestedhvm_vcpu_in_guestmode(v) ) {
+        pin_based_cntrl = __get_vvmcs(vcpu_nestedhvm(v).nv_vvmcx, 
+                                     PIN_BASED_VM_EXEC_CONTROL);
+        if ( pin_based_cntrl && PIN_BASED_NMI_EXITING ) {
+            nvmx_enqueue_n2_exceptions (v, 
+               INTR_INFO_VALID_MASK | (X86_EVENTTYPE_NMI<<8) | TRAP_nmi,
+               HVM_DELIVER_NO_ERROR_CODE);
+            return;
+        }
+    }
     __vmx_inject_exception(2, X86_EVENTTYPE_NMI,
                            HVM_DELIVER_NO_ERROR_CODE);
 }
@@ -1424,7 +1489,10 @@ static struct hvm_function_table __read_
     .nhvm_vcpu_reset      = nvmx_vcpu_reset,
     .nhvm_vcpu_guestcr3   = nvmx_vcpu_guestcr3,
     .nhvm_vcpu_hostcr3    = nvmx_vcpu_hostcr3,
-    .nhvm_vcpu_asid       = nvmx_vcpu_asid
+    .nhvm_vcpu_asid       = nvmx_vcpu_asid,
+    .nhvm_vmcx_guest_intercepts_trap = nvmx_intercepts_exception,
+    .nhvm_vcpu_vmexit_trap = nvmx_vmexit_exceptions,
+    .nhvm_intr_blocked    = nvmx_intr_blocked
 };
 
 struct hvm_function_table * __init start_vmx(void)
@@ -2237,7 +2305,8 @@ asmlinkage void vmx_vmexit_handler(struc
     hvm_maybe_deassert_evtchn_irq();
 
     idtv_info = __vmread(IDT_VECTORING_INFO);
-    if ( exit_reason != EXIT_REASON_TASK_SWITCH )
+    if ( !nestedhvm_vcpu_in_guestmode(v) && 
+         exit_reason != EXIT_REASON_TASK_SWITCH )
         vmx_idtv_reinject(idtv_info);
 
     switch ( exit_reason )
@@ -2585,6 +2654,9 @@ asmlinkage void vmx_vmexit_handler(struc
         domain_crash(v->domain);
         break;
     }
+
+    if ( nestedhvm_vcpu_in_guestmode(v) )
+        nvmx_idtv_handling();
 }
 
 asmlinkage void vmx_vmenter_helper(void)
diff -r bd15acfc9b82 -r f14f451a780e xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:21 2011 +0800
@@ -392,6 +392,27 @@ static void vmreturn(struct cpu_user_reg
     regs->eflags = eflags;
 }
 
+int nvmx_intercepts_exception(struct vcpu *v, unsigned int trap,
+                               int error_code)
+{
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+    u32 exception_bitmap, pfec_match=0, pfec_mask=0;
+    int r;
+
+    ASSERT ( trap < 32 );
+
+    exception_bitmap = __get_vvmcs(nvcpu->nv_vvmcx, EXCEPTION_BITMAP);
+    r = exception_bitmap & (1 << trap) ? 1: 0;
+
+    if ( trap == TRAP_page_fault ) {
+        pfec_match = __get_vvmcs(nvcpu->nv_vvmcx, PAGE_FAULT_ERROR_CODE_MATCH);
+        pfec_mask  = __get_vvmcs(nvcpu->nv_vvmcx, PAGE_FAULT_ERROR_CODE_MASK);
+        if ( (error_code & pfec_mask) != pfec_match )
+            r = !r;
+    }
+    return r;
+}
+
 /*
  * Nested VMX uses "strict" condition to exit from 
  * L2 guest if either L1 VMM or L0 VMM expect to exit.
@@ -465,6 +486,7 @@ void nvmx_update_exec_control(struct vcp
         __vmwrite(IO_BITMAP_B, virt_to_maddr(bitmap) + PAGE_SIZE);
     }
 
+    /* TODO: change L0 intr window to MTF or NMI window */
     __vmwrite(CPU_BASED_VM_EXEC_CONTROL, shadow_cntrl);
 }
 
@@ -868,6 +890,42 @@ static void load_vvmcs_host_state(struct
     __set_vvmcs(vvmcs, VM_ENTRY_INTR_INFO, 0);
 }
 
+static void sync_exception_state(struct vcpu *v)
+{
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+    struct nestedvmx *nvmx = &vcpu_2_nvmx(v);
+
+    if ( !(nvmx->intr.intr_info & INTR_INFO_VALID_MASK) )
+        return;
+
+    switch ( nvmx->intr.intr_info & INTR_INFO_INTR_TYPE_MASK )
+    {
+    case X86_EVENTTYPE_EXT_INTR:
+        /* rename exit_reason to EXTERNAL_INTERRUPT */
+        __set_vvmcs(nvcpu->nv_vvmcx, VM_EXIT_REASON,
+                    EXIT_REASON_EXTERNAL_INTERRUPT);
+        __set_vvmcs(nvcpu->nv_vvmcx, EXIT_QUALIFICATION, 0);
+        __set_vvmcs(nvcpu->nv_vvmcx, VM_EXIT_INTR_INFO,
+                    nvmx->intr.intr_info);
+        break;
+
+    case X86_EVENTTYPE_HW_EXCEPTION:
+    case X86_EVENTTYPE_SW_INTERRUPT:
+    case X86_EVENTTYPE_SW_EXCEPTION:
+        /* throw to L1 */
+        __set_vvmcs(nvcpu->nv_vvmcx, VM_EXIT_INTR_INFO,
+                    nvmx->intr.intr_info);
+        __set_vvmcs(nvcpu->nv_vvmcx, VM_EXIT_INTR_ERROR_CODE,
+                    nvmx->intr.error_code);
+        break;
+    case X86_EVENTTYPE_NMI:
+    default:
+        gdprintk(XENLOG_ERR, "Exception state %lx not handled\n",
+               nvmx->intr.intr_info); 
+        break;
+    }
+}
+
 static void virtual_vmexit(struct cpu_user_regs *regs)
 {
     struct vcpu *v = current;
@@ -878,6 +936,7 @@ static void virtual_vmexit(struct cpu_us
 
     sync_vvmcs_ro(v);
     sync_vvmcs_guest_state(v, regs);
+    sync_exception_state(v);
 
     vmx_vmcs_switch(v, v->arch.hvm_vmx.vmcs, nvcpu->nv_n1vmcx);
 
@@ -1169,3 +1228,40 @@ int nvmx_handle_vmwrite(struct cpu_user_
     return X86EMUL_OKAY;
 }
 
+void nvmx_idtv_handling(void)
+{
+    struct vcpu *v = current;
+    struct nestedvmx *nvmx = &vcpu_2_nvmx(v);
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+    unsigned int idtv_info = __vmread(IDT_VECTORING_INFO);
+
+    if ( likely(!(idtv_info & INTR_INFO_VALID_MASK)) )
+        return;
+
+    /*
+     * If L0 can solve the fault that causes idt vectoring, it should
+     * be reinjected, otherwise, pass to L1.
+     */
+    if ( (__vmread(VM_EXIT_REASON) != EXIT_REASON_EPT_VIOLATION &&
+          !(nvmx->intr.intr_info & INTR_INFO_VALID_MASK)) ||
+         (__vmread(VM_EXIT_REASON) == EXIT_REASON_EPT_VIOLATION &&
+          !nvcpu->nv_vmexit_pending) )
+    {
+        __vmwrite(VM_ENTRY_INTR_INFO, idtv_info & ~INTR_INFO_RESVD_BITS_MASK);
+        if ( idtv_info & INTR_INFO_DELIVER_CODE_MASK )
+           __vmwrite(VM_ENTRY_EXCEPTION_ERROR_CODE,
+                        __vmread(IDT_VECTORING_ERROR_CODE));
+        /*
+         * SDM 23.2.4, if L1 tries to inject a software interrupt
+         * and the delivery fails, VM_EXIT_INSTRUCTION_LEN receives
+         * the value of previous VM_ENTRY_INSTRUCTION_LEN.
+         *
+         * This means EXIT_INSTRUCTION_LEN is always valid here, for
+         * software interrupts both injected by L1, and generated in L2.
+         */
+        __vmwrite(VM_ENTRY_INSTRUCTION_LEN, __vmread(VM_EXIT_INSTRUCTION_LEN));
+   }
+
+    /* TODO: NMI */
+}
+
diff -r bd15acfc9b82 -r f14f451a780e xen/include/asm-x86/hvm/vmx/vvmx.h
--- a/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:21 2011 +0800
@@ -93,6 +93,9 @@ int nvmx_vcpu_reset(struct vcpu *v);
 uint64_t nvmx_vcpu_guestcr3(struct vcpu *v);
 uint64_t nvmx_vcpu_hostcr3(struct vcpu *v);
 uint32_t nvmx_vcpu_asid(struct vcpu *v);
+enum hvm_intblk nvmx_intr_blocked(struct vcpu *v);
+int nvmx_intercepts_exception(struct vcpu *v, 
+                              unsigned int trap, int error_code);
 
 int nvmx_handle_vmxon(struct cpu_user_regs *regs);
 int nvmx_handle_vmxoff(struct cpu_user_regs *regs);
@@ -166,6 +169,7 @@ void nvmx_update_secondary_exec_control(
                                         unsigned long value);
 void nvmx_update_exception_bitmap(struct vcpu *v, unsigned long value);
 asmlinkage void nvmx_switch_guest(void);
+void nvmx_idtv_handling(void);
 
 #endif /* __ASM_X86_HVM_VVMX_H__ */

[PATCH 17 of 20] VM exit handler of n2-guest

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003601 -28800
# Node ID 24d4d7d3e4c44c8dc61f464bca9aae57480dfe75
# Parent  f14f451a780e60e920c057e44fa1bc3ee40495a7
VM exit handler of n2-guest

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r f14f451a780e -r 24d4d7d3e4c4 xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:21 2011 +0800
@@ -943,6 +943,10 @@ static void vmx_set_segment_register(str
 static void vmx_set_tsc_offset(struct vcpu *v, u64 offset)
 {
     vmx_vmcs_enter(v);
+
+    if ( nestedhvm_vcpu_in_guestmode(v) )
+        offset += nvmx_get_tsc_offset(v);
+
     __vmwrite(TSC_OFFSET, offset);
 #if defined (__i386__)
     __vmwrite(TSC_OFFSET_HIGH, offset >> 32);
@@ -2258,6 +2262,11 @@ asmlinkage void vmx_vmexit_handler(struc
      * any pending vmresume has really happened
      */
     vcpu_nestedhvm(v).nv_vmswitch_in_progress = 0;
+    if ( nestedhvm_vcpu_in_guestmode(v) )
+    {
+        if ( nvmx_n2_vmexit_handler(regs, exit_reason) )
+            goto out;
+    }
 
     if ( unlikely(exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY) )
         return vmx_failed_vmentry(exit_reason, regs);
@@ -2655,6 +2664,7 @@ asmlinkage void vmx_vmexit_handler(struc
         break;
     }
 
+out:
     if ( nestedhvm_vcpu_in_guestmode(v) )
         nvmx_idtv_handling();
 }
diff -r f14f451a780e -r 24d4d7d3e4c4 xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:21 2011 +0800
@@ -288,13 +288,19 @@ static int vmx_inst_check_privilege(stru
     if ( (regs->eflags & X86_EFLAGS_VM) ||
          (hvm_long_mode_enabled(v) && cs.attr.fields.l == 0) )
         goto invalid_op;
-    /* TODO: check vmx operation mode */
+    else if ( nestedhvm_vcpu_in_guestmode(v) )
+        goto vmexit;
 
     if ( (cs.sel & 3) > 0 )
         goto gp_fault;
 
     return X86EMUL_OKAY;
 
+vmexit:
+    gdprintk(XENLOG_ERR, "vmx_inst_check_privilege: vmexit\n");
+    vcpu_nestedhvm(v).nv_vmexit_pending = 1;
+    return X86EMUL_EXCEPTION;
+    
 invalid_op:
     gdprintk(XENLOG_ERR, "vmx_inst_check_privilege: invalid_op\n");
     hvm_inject_exception(TRAP_invalid_op, 0, 0);
@@ -606,6 +612,18 @@ static void nvmx_purge_vvmcs(struct vcpu
     }
 }
 
+u64 nvmx_get_tsc_offset(struct vcpu *v)
+{
+    u64 offset = 0;
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+
+    if ( __get_vvmcs(nvcpu->nv_vvmcx, CPU_BASED_VM_EXEC_CONTROL) &
+         CPU_BASED_USE_TSC_OFFSETING )
+        offset = __get_vvmcs(nvcpu->nv_vvmcx, TSC_OFFSET);
+
+    return offset;
+}
+
 /*
  * Context synchronized between shadow and virtual VMCS.
  */
@@ -759,6 +777,8 @@ static void load_shadow_guest_state(stru
     hvm_set_cr4(__get_vvmcs(vvmcs, GUEST_CR4));
     hvm_set_cr3(__get_vvmcs(vvmcs, GUEST_CR3));
 
+    hvm_funcs.set_tsc_offset(v, v->arch.hvm_vcpu.cache_tsc_offset);
+
     vvmcs_to_shadow(vvmcs, VM_ENTRY_INTR_INFO);
     vvmcs_to_shadow(vvmcs, VM_ENTRY_EXCEPTION_ERROR_CODE);
     vvmcs_to_shadow(vvmcs, VM_ENTRY_INSTRUCTION_LEN);
@@ -887,6 +907,8 @@ static void load_vvmcs_host_state(struct
     hvm_set_cr4(__get_vvmcs(vvmcs, HOST_CR4));
     hvm_set_cr3(__get_vvmcs(vvmcs, HOST_CR3));
 
+    hvm_funcs.set_tsc_offset(v, v->arch.hvm_vcpu.cache_tsc_offset);
+
     __set_vvmcs(vvmcs, VM_ENTRY_INTR_INFO, 0);
 }
 
@@ -1265,3 +1287,252 @@ void nvmx_idtv_handling(void)
     /* TODO: NMI */
 }
 
+/*
+ * L2 VMExit handling
+ *    return 1: Done or skip the normal layer 0 hypervisor process.
+ *              Typically it requires layer 1 hypervisor processing
+ *              or it may be already processed here.
+ *           0: Require the normal layer 0 process.
+ */
+int nvmx_n2_vmexit_handler(struct cpu_user_regs *regs,
+                               unsigned int exit_reason)
+{
+    struct vcpu *v = current;
+    struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
+    struct nestedvmx *nvmx = &vcpu_2_nvmx(v);
+    u32 ctrl;
+    u16 port;
+    u8 *bitmap;
+
+    nvcpu->nv_vmexit_pending = 0;
+    nvmx->intr.intr_info = 0;
+    nvmx->intr.error_code = 0;
+
+    switch (exit_reason) {
+    case EXIT_REASON_EXCEPTION_NMI:
+    {
+        u32 intr_info = __vmread(VM_EXIT_INTR_INFO);
+        u32 valid_mask = (X86_EVENTTYPE_HW_EXCEPTION << 8) |
+                         INTR_INFO_VALID_MASK;
+        u64 exec_bitmap;
+        int vector = intr_info & INTR_INFO_VECTOR_MASK;
+
+        /*
+         * decided by L0 and L1 exception bitmap, if the vetor is set by
+         * both, L0 has priority on #PF, L1 has priority on others
+         */
+        if ( vector == TRAP_page_fault )
+        {
+            if ( paging_mode_hap(v->domain) )
+                nvcpu->nv_vmexit_pending = 1;
+        }
+        else if ( (intr_info & valid_mask) == valid_mask )
+        {
+            exec_bitmap =__get_vvmcs(nvcpu->nv_vvmcx, EXCEPTION_BITMAP);
+
+            if ( exec_bitmap & (1 << vector) )
+                nvcpu->nv_vmexit_pending = 1;
+        }
+        break;
+    }
+
+    case EXIT_REASON_WBINVD:
+    case EXIT_REASON_EPT_VIOLATION:
+    case EXIT_REASON_EPT_MISCONFIG:
+    case EXIT_REASON_EXTERNAL_INTERRUPT:
+        /* pass to L0 handler */
+        break;
+
+    case VMX_EXIT_REASONS_FAILED_VMENTRY:
+    case EXIT_REASON_TRIPLE_FAULT:
+    case EXIT_REASON_TASK_SWITCH:
+    case EXIT_REASON_CPUID:
+    case EXIT_REASON_MSR_READ:
+    case EXIT_REASON_MSR_WRITE:
+    case EXIT_REASON_VMCALL:
+    case EXIT_REASON_VMCLEAR:
+    case EXIT_REASON_VMLAUNCH:
+    case EXIT_REASON_VMPTRLD:
+    case EXIT_REASON_VMPTRST:
+    case EXIT_REASON_VMREAD:
+    case EXIT_REASON_VMRESUME:
+    case EXIT_REASON_VMWRITE:
+    case EXIT_REASON_VMXOFF:
+    case EXIT_REASON_VMXON:
+    case EXIT_REASON_INVEPT:
+        /* inject to L1 */
+        nvcpu->nv_vmexit_pending = 1;
+        break;
+    case EXIT_REASON_IO_INSTRUCTION:
+        ctrl = __n2_exec_control(v);
+        if ( ctrl & CPU_BASED_ACTIVATE_IO_BITMAP )
+        {
+            port = __vmread(EXIT_QUALIFICATION) >> 16;
+            bitmap = nvmx->iobitmap[port >> 15];
+            if ( bitmap[(port <<1) >> 4] & (1 << (port & 0x7)) )
+                nvcpu->nv_vmexit_pending = 1;
+        }
+        else if ( ctrl & CPU_BASED_UNCOND_IO_EXITING )
+            nvcpu->nv_vmexit_pending = 1;
+        break;
+
+    case EXIT_REASON_PENDING_VIRT_INTR:
+    {
+        ctrl = v->arch.hvm_vmx.exec_control;
+
+        /*
+         * if both open intr/nmi window, L0 has priority.
+         *
+         * Note that this is not strictly correct, in L2 context,
+         * L0's intr/nmi window flag should be replaced to MTF,
+         * causing an imediate VMExit, but MTF may not be available
+         * on all hardware.
+         */
+        if ( !(ctrl & CPU_BASED_VIRTUAL_INTR_PENDING) )
+            nvcpu->nv_vmexit_pending = 1;
+
+        break;
+    }
+    case EXIT_REASON_PENDING_VIRT_NMI:
+    {
+        ctrl = v->arch.hvm_vmx.exec_control;
+
+        if ( !(ctrl & CPU_BASED_VIRTUAL_NMI_PENDING) )
+            nvcpu->nv_vmexit_pending = 1;
+
+        break;
+    }
+
+    /* L1 has priority handling several other types of exits */
+    case EXIT_REASON_HLT:
+    {
+        ctrl = __n2_exec_control(v);
+
+        if ( ctrl & CPU_BASED_HLT_EXITING )
+            nvcpu->nv_vmexit_pending = 1;
+
+        break;
+    }
+
+    case EXIT_REASON_RDTSC:
+    {
+        ctrl = __n2_exec_control(v);
+
+        if ( ctrl & CPU_BASED_RDTSC_EXITING )
+            nvcpu->nv_vmexit_pending = 1;
+        else
+        {
+            uint64_t tsc;
+
+            /*
+             * special handler is needed if L1 doesn't intercept rdtsc,
+             * avoiding changing guest_tsc and messing up timekeeping in L1
+             */
+            tsc = hvm_get_guest_tsc(v);
+            tsc += __get_vvmcs(nvcpu->nv_vvmcx, TSC_OFFSET);
+            regs->eax = (uint32_t)tsc;
+            regs->edx = (uint32_t)(tsc >> 32);
+
+            return 1;
+        }
+
+        break;
+    }
+
+    case EXIT_REASON_RDPMC:
+    {
+        ctrl = __n2_exec_control(v);
+
+        if ( ctrl & CPU_BASED_RDPMC_EXITING )
+            nvcpu->nv_vmexit_pending = 1;
+
+        break;
+    }
+
+    case EXIT_REASON_MWAIT_INSTRUCTION:
+    {
+        ctrl = __n2_exec_control(v);
+
+        if ( ctrl & CPU_BASED_MWAIT_EXITING )
+            nvcpu->nv_vmexit_pending = 1;
+
+        break;
+    }
+
+    case EXIT_REASON_PAUSE_INSTRUCTION:
+    {
+        ctrl = __n2_exec_control(v);
+
+        if ( ctrl & CPU_BASED_PAUSE_EXITING )
+            nvcpu->nv_vmexit_pending = 1;
+
+        break;
+    }
+
+    case EXIT_REASON_MONITOR_INSTRUCTION:
+    {
+        ctrl = __n2_exec_control(v);
+
+        if ( ctrl & CPU_BASED_MONITOR_EXITING )
+            nvcpu->nv_vmexit_pending = 1;
+
+        break;
+    }
+
+    case EXIT_REASON_DR_ACCESS:
+    {
+        ctrl = __n2_exec_control(v);
+
+        if ( ctrl & CPU_BASED_MOV_DR_EXITING )
+            nvcpu->nv_vmexit_pending = 1;
+
+        break;
+    }
+
+    case EXIT_REASON_INVLPG:
+    {
+        ctrl = __n2_exec_control(v);
+
+        if ( ctrl & CPU_BASED_INVLPG_EXITING )
+            nvcpu->nv_vmexit_pending = 1;
+
+        break;
+    }
+
+    case EXIT_REASON_CR_ACCESS:
+    {
+        u64 exit_qualification = __vmread(EXIT_QUALIFICATION);
+        int cr = exit_qualification & 15;
+        int write = (exit_qualification >> 4) & 3;
+        u32 mask = 0;
+
+        /* also according to guest exec_control */
+        ctrl = __n2_exec_control(v);
+
+        if ( cr == 3 )
+        {
+            mask = write? CPU_BASED_CR3_STORE_EXITING:
+                          CPU_BASED_CR3_LOAD_EXITING;
+            if ( ctrl & mask )
+                nvcpu->nv_vmexit_pending = 1;
+        }
+        else if ( cr == 8 )
+        {
+            mask = write? CPU_BASED_CR8_STORE_EXITING:
+                          CPU_BASED_CR8_LOAD_EXITING;
+            if ( ctrl & mask )
+                nvcpu->nv_vmexit_pending = 1;
+        }
+        else  /* CR0, CR4, CLTS, LMSW */
+            nvcpu->nv_vmexit_pending = 1;
+
+        break;
+    }
+    default:
+        gdprintk(XENLOG_WARNING, "Unknown nested vmexit reason %x.\n",
+                 exit_reason);
+    }
+
+    return ( nvcpu->nv_vmexit_pending == 1 );
+}
+
diff -r f14f451a780e -r 24d4d7d3e4c4 xen/include/asm-x86/hvm/vmx/vvmx.h
--- a/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:21 2011 +0800
@@ -170,6 +170,9 @@ void nvmx_update_secondary_exec_control(
 void nvmx_update_exception_bitmap(struct vcpu *v, unsigned long value);
 asmlinkage void nvmx_switch_guest(void);
 void nvmx_idtv_handling(void);
+u64 nvmx_get_tsc_offset(struct vcpu *v);
+int nvmx_n2_vmexit_handler(struct cpu_user_regs *regs,
+                          unsigned int exit_reason);
 
 #endif /* __ASM_X86_HVM_VVMX_H__ */

Re: [PATCH 17 of 20] VM exit handler of n2-guest

[Tim Deegan]

At 16:57 +0800 on 02 Jun (1307033850), Eddie Dong wrote:
> +    case EXIT_REASON_WBINVD:
> +    case EXIT_REASON_EPT_VIOLATION:
> +    case EXIT_REASON_EPT_MISCONFIG:
> +    case EXIT_REASON_EXTERNAL_INTERRUPT:
> +        /* pass to L0 handler */
> +        break;

If the L1 guest asked to intercept WBINVD, will it ever get the VMEXIT?
I didn't see any code in the L0 WBINVD handler to pass it on.

Cheers,

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

RE: [PATCH 17 of 20] VM exit handler of n2-guest

[Dong, Eddie]

> At 16:57 +0800 on 02 Jun (1307033850), Eddie Dong wrote:
> > +    case EXIT_REASON_WBINVD:
> > +    case EXIT_REASON_EPT_VIOLATION:
> > +    case EXIT_REASON_EPT_MISCONFIG:
> > +    case EXIT_REASON_EXTERNAL_INTERRUPT:
> > +        /* pass to L0 handler */
> > +        break;
> 
> If the L1 guest asked to intercept WBINVD, will it ever get the VMEXIT?
> I didn't see any code in the L0 WBINVD handler to pass it on.
> 
Current patch doesn't expose Secondary Processor-Based VM-Execution Controls. So WBINVD exiting capability is removed in L1 guest.

Thx, Eddie

Re: [PATCH 17 of 20] VM exit handler of n2-guest

[Tim Deegan]

At 16:06 +0800 on 03 Jun (1307117213), Dong, Eddie wrote:
> > At 16:57 +0800 on 02 Jun (1307033850), Eddie Dong wrote:
> > > +    case EXIT_REASON_WBINVD:
> > > +    case EXIT_REASON_EPT_VIOLATION:
> > > +    case EXIT_REASON_EPT_MISCONFIG:
> > > +    case EXIT_REASON_EXTERNAL_INTERRUPT:
> > > +        /* pass to L0 handler */
> > > +        break;
> > 
> > If the L1 guest asked to intercept WBINVD, will it ever get the VMEXIT?
> > I didn't see any code in the L0 WBINVD handler to pass it on.
> > 

> Current patch doesn't expose Secondary Processor-Based VM-Execution
> Controls. So WBINVD exiting capability is removed in L1 guest.

Ah, OK, thanks. 

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

[PATCH 18 of 20] Lazy FPU for n2 guest

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003601 -28800
# Node ID 0cedbe9214c1632a0f1816d8b6d7442dc5f40065
# Parent  24d4d7d3e4c44c8dc61f464bca9aae57480dfe75
Lazy FPU for n2 guest

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r 24d4d7d3e4c4 -r 0cedbe9214c1 xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:21 2011 +0800
@@ -842,6 +842,9 @@ static void virtual_vmentry(struct cpu_u
     regs->rsp = __get_vvmcs(vvmcs, GUEST_RSP);
     regs->rflags = __get_vvmcs(vvmcs, GUEST_RFLAGS);
 
+    /* updating host cr0 to sync TS bit */
+    __vmwrite(HOST_CR0, v->arch.hvm_vmx.host_cr0);
+
     /* TODO: EPT_POINTER */
 }
 
@@ -990,6 +993,9 @@ static void virtual_vmexit(struct cpu_us
     regs->rsp = __get_vvmcs(nvcpu->nv_vvmcx, HOST_RSP);
     regs->rflags = __vmread(GUEST_RFLAGS);
 
+    /* updating host cr0 to sync TS bit */
+    __vmwrite(HOST_CR0, v->arch.hvm_vmx.host_cr0);
+
     vmreturn(regs, VMSUCCEED);
 }
 
@@ -1319,13 +1325,18 @@ int nvmx_n2_vmexit_handler(struct cpu_us
 
         /*
          * decided by L0 and L1 exception bitmap, if the vetor is set by
-         * both, L0 has priority on #PF, L1 has priority on others
+         * both, L0 has priority on #PF and #NM, L1 has priority on others
          */
         if ( vector == TRAP_page_fault )
         {
             if ( paging_mode_hap(v->domain) )
                 nvcpu->nv_vmexit_pending = 1;
         }
+        else if ( vector == TRAP_no_device )
+        {
+            if ( v->fpu_dirtied )
+                nvcpu->nv_vmexit_pending = 1;
+        }
         else if ( (intr_info & valid_mask) == valid_mask )
         {
             exec_bitmap =__get_vvmcs(nvcpu->nv_vvmcx, EXCEPTION_BITMAP);

[PATCH 19 of 20] Add VMXE bits in virtual CR4

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003601 -28800
# Node ID c046b25135205ff58c0b729c0b94cd920cdbb7e2
# Parent  0cedbe9214c1632a0f1816d8b6d7442dc5f40065
Add VMXE bits in virtual CR4

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r 0cedbe9214c1 -r c046b2513520 xen/include/asm-x86/cpufeature.h
--- a/xen/include/asm-x86/cpufeature.h	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/include/asm-x86/cpufeature.h	Thu Jun 02 16:33:21 2011 +0800
@@ -216,6 +216,8 @@
 
 #define cpu_has_svm		boot_cpu_has(X86_FEATURE_SVM)
 
+#define cpu_has_vmx		boot_cpu_has(X86_FEATURE_VMXE)
+
 #endif /* __ASM_I386_CPUFEATURE_H */
 
 /* 
diff -r 0cedbe9214c1 -r c046b2513520 xen/include/asm-x86/hvm/hvm.h
--- a/xen/include/asm-x86/hvm/hvm.h	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/include/asm-x86/hvm/hvm.h	Thu Jun 02 16:33:21 2011 +0800
@@ -313,6 +313,8 @@ static inline int hvm_do_pmu_interrupt(s
         X86_CR4_DE  | X86_CR4_PSE | X86_CR4_PAE |       \
         X86_CR4_MCE | X86_CR4_PGE | X86_CR4_PCE |       \
         X86_CR4_OSFXSR | X86_CR4_OSXMMEXCPT |           \
+	((nestedhvm_enabled((_v)->domain) &&            \
+          cpu_has_vmx) ? X86_CR4_VMXE : 0)  |       	\
         (xsave_enabled(_v) ? X86_CR4_OSXSAVE : 0))))
 
 /* These exceptions must always be intercepted. */

Re: [PATCH 19 of 20] Add VMXE bits in virtual CR4

[Tim Deegan]

At 16:57 +0800 on 02 Jun (1307033852), Eddie Dong wrote:
> diff -r 0cedbe9214c1 -r c046b2513520 xen/include/asm-x86/hvm/hvm.h
> --- a/xen/include/asm-x86/hvm/hvm.h	Thu Jun 02 16:33:21 2011 +0800
> +++ b/xen/include/asm-x86/hvm/hvm.h	Thu Jun 02 16:33:21 2011 +0800
> @@ -313,6 +313,8 @@ static inline int hvm_do_pmu_interrupt(s
>          X86_CR4_DE  | X86_CR4_PSE | X86_CR4_PAE |       \
>          X86_CR4_MCE | X86_CR4_PGE | X86_CR4_PCE |       \
>          X86_CR4_OSFXSR | X86_CR4_OSXMMEXCPT |           \
> +	((nestedhvm_enabled((_v)->domain) &&            \
> +          cpu_has_vmx) ? X86_CR4_VMXE : 0)  |       	\

Should we also add VMXE to this mask even if !nestedhvm_enabled()?

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

RE: [PATCH 19 of 20] Add VMXE bits in virtual CR4

[Dong, Eddie]

> > diff -r 0cedbe9214c1 -r c046b2513520 xen/include/asm-x86/hvm/hvm.h
> > --- a/xen/include/asm-x86/hvm/hvm.h	Thu Jun 02 16:33:21 2011 +0800
> > +++ b/xen/include/asm-x86/hvm/hvm.h	Thu Jun 02 16:33:21 2011 +0800
> > @@ -313,6 +313,8 @@ static inline int hvm_do_pmu_interrupt(s
> >          X86_CR4_DE  | X86_CR4_PSE | X86_CR4_PAE |       \
> >          X86_CR4_MCE | X86_CR4_PGE | X86_CR4_PCE |       \
> >          X86_CR4_OSFXSR | X86_CR4_OSXMMEXCPT |           \
> > +	((nestedhvm_enabled((_v)->domain) &&            \
> > +          cpu_has_vmx) ? X86_CR4_VMXE : 0)  |       	\
> 
> Should we also add VMXE to this mask even if !nestedhvm_enabled()?
> 
Fine.
Eddie

[PATCH 20 of 20] n2 MSR handling and capability exposure

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307003601 -28800
# Node ID ee55fa0471a6b72569b567286ae264bc1dcdbb4b
# Parent  c046b25135205ff58c0b729c0b94cd920cdbb7e2
n2 MSR handling and capability exposure

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r c046b2513520 -r ee55fa0471a6 xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 02 16:33:21 2011 +0800
@@ -1778,8 +1778,11 @@ static int vmx_msr_read_intercept(unsign
         *msr_content |= (u64)__vmread(GUEST_IA32_DEBUGCTL_HIGH) << 32;
 #endif
         break;
-    case MSR_IA32_VMX_BASIC...MSR_IA32_VMX_PROCBASED_CTLS2:
-        goto gp_fault;
+    case IA32_FEATURE_CONTROL_MSR:
+    case MSR_IA32_VMX_BASIC...MSR_IA32_VMX_TRUE_ENTRY_CTLS:
+        if ( !nvmx_msr_read_intercept(msr, msr_content) )
+            goto gp_fault;
+        break;
     case MSR_IA32_MISC_ENABLE:
         rdmsrl(MSR_IA32_MISC_ENABLE, *msr_content);
         /* Debug Trace Store is not supported. */
@@ -1940,8 +1943,11 @@ static int vmx_msr_write_intercept(unsig
 
         break;
     }
-    case MSR_IA32_VMX_BASIC...MSR_IA32_VMX_PROCBASED_CTLS2:
-        goto gp_fault;
+    case IA32_FEATURE_CONTROL_MSR:
+    case MSR_IA32_VMX_BASIC...MSR_IA32_VMX_TRUE_ENTRY_CTLS:
+        if ( !nvmx_msr_write_intercept(msr, msr_content) )
+            goto gp_fault;
+        break;
     default:
         if ( vpmu_do_wrmsr(msr, msr_content) )
             return X86EMUL_OKAY;
diff -r c046b2513520 -r ee55fa0471a6 xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 02 16:33:21 2011 +0800
@@ -1256,6 +1256,94 @@ int nvmx_handle_vmwrite(struct cpu_user_
     return X86EMUL_OKAY;
 }
 
+/*
+ * Capability reporting
+ */
+int nvmx_msr_read_intercept(unsigned int msr, u64 *msr_content)
+{
+    u32 eax, edx;
+    u64 data = 0;
+    int r = 1;
+    u32 mask = 0;
+
+    if ( !nestedhvm_enabled(current->domain) )
+        return 0;
+
+    switch (msr) {
+    case MSR_IA32_VMX_BASIC:
+        rdmsr(msr, eax, edx);
+        data = edx;
+        data = (data & ~0x1fff) | 0x1000;     /* request 4KB for guest VMCS */
+        data &= ~(1 << 23);                   /* disable TRUE_xxx_CTLS */
+        data = (data << 32) | VVMCS_REVISION; /* VVMCS revision */
+        break;
+    case MSR_IA32_VMX_PINBASED_CTLS:
+#define REMOVED_PIN_CONTROL_CAP (PIN_BASED_PREEMPT_TIMER)
+        rdmsr(msr, eax, edx);
+        data = edx;
+        data = (data << 32) | eax;
+        break;
+    case MSR_IA32_VMX_PROCBASED_CTLS:
+        rdmsr(msr, eax, edx);
+#define REMOVED_EXEC_CONTROL_CAP (CPU_BASED_TPR_SHADOW \
+            | CPU_BASED_ACTIVATE_MSR_BITMAP            \
+            | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)
+        data = edx & ~REMOVED_EXEC_CONTROL_CAP;
+        data = (data << 32) | eax;
+        break;
+    case MSR_IA32_VMX_EXIT_CTLS:
+        rdmsr(msr, eax, edx);
+#define REMOVED_EXIT_CONTROL_CAP (VM_EXIT_SAVE_GUEST_PAT \
+            | VM_EXIT_LOAD_HOST_PAT                      \
+            | VM_EXIT_SAVE_GUEST_EFER                    \
+            | VM_EXIT_LOAD_HOST_EFER                     \
+            | VM_EXIT_SAVE_PREEMPT_TIMER)
+        data = edx & ~REMOVED_EXIT_CONTROL_CAP;
+        data = (data << 32) | eax;
+        break;
+    case MSR_IA32_VMX_ENTRY_CTLS:
+        rdmsr(msr, eax, edx);
+#define REMOVED_ENTRY_CONTROL_CAP (VM_ENTRY_LOAD_GUEST_PAT \
+            | VM_ENTRY_LOAD_GUEST_EFER)
+        data = edx & ~REMOVED_ENTRY_CONTROL_CAP;
+        data = (data << 32) | eax;
+        break;
+    case MSR_IA32_VMX_PROCBASED_CTLS2:
+        mask = 0;
+
+        rdmsr(msr, eax, edx);
+        data = edx & mask;
+        data = (data << 32) | eax;
+        break;
+
+    /* pass through MSRs */
+    case IA32_FEATURE_CONTROL_MSR:
+    case MSR_IA32_VMX_MISC:
+    case MSR_IA32_VMX_CR0_FIXED0:
+    case MSR_IA32_VMX_CR0_FIXED1:
+    case MSR_IA32_VMX_CR4_FIXED0:
+    case MSR_IA32_VMX_CR4_FIXED1:
+    case MSR_IA32_VMX_VMCS_ENUM:
+        rdmsr(msr, eax, edx);
+        data = edx;
+        data = (data << 32) | eax;
+        break;
+
+    default:
+        r = 0;
+        break;
+    }
+
+    *msr_content = data;
+    return r;
+}
+
+int nvmx_msr_write_intercept(unsigned int msr, u64 msr_content)
+{
+    /* silently ignore for now */
+    return 1;
+}
+
 void nvmx_idtv_handling(void)
 {
     struct vcpu *v = current;
diff -r c046b2513520 -r ee55fa0471a6 xen/include/asm-x86/hvm/vmx/vvmx.h
--- a/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:21 2011 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 02 16:33:21 2011 +0800
@@ -163,6 +163,10 @@ int nvmx_handle_vmread(struct cpu_user_r
 int nvmx_handle_vmwrite(struct cpu_user_regs *regs);
 int nvmx_handle_vmresume(struct cpu_user_regs *regs);
 int nvmx_handle_vmlaunch(struct cpu_user_regs *regs);
+int nvmx_msr_read_intercept(unsigned int msr,
+                                u64 *msr_content);
+int nvmx_msr_write_intercept(unsigned int msr,
+                                 u64 msr_content);
 
 void nvmx_update_exec_control(struct vcpu *v, unsigned long value);
 void nvmx_update_secondary_exec_control(struct vcpu *v,

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Tim Deegan]

At 16:57 +0800 on 02 Jun (1307033853), Eddie Dong wrote:
> +    case MSR_IA32_VMX_PINBASED_CTLS:
> +#define REMOVED_PIN_CONTROL_CAP (PIN_BASED_PREEMPT_TIMER)
> +        rdmsr(msr, eax, edx);
> +        data = edx;
> +        data = (data << 32) | eax;
> +        break;

You don't actually mask the value here. 

BTW, I don't really like defining all these REMOVED_* macros, each
of which is used only once a few lines from the definition (here and
elsewhere in the series).  It just adds clutter for no benefit.

Tim.

> +    case MSR_IA32_VMX_PROCBASED_CTLS:
> +        rdmsr(msr, eax, edx);
> +#define REMOVED_EXEC_CONTROL_CAP (CPU_BASED_TPR_SHADOW \
> +            | CPU_BASED_ACTIVATE_MSR_BITMAP            \
> +            | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)
> +        data = edx & ~REMOVED_EXEC_CONTROL_CAP;
> +        data = (data << 32) | eax;
> +        break;
> +    case MSR_IA32_VMX_EXIT_CTLS:
> +        rdmsr(msr, eax, edx);
> +#define REMOVED_EXIT_CONTROL_CAP (VM_EXIT_SAVE_GUEST_PAT \
> +            | VM_EXIT_LOAD_HOST_PAT                      \
> +            | VM_EXIT_SAVE_GUEST_EFER                    \
> +            | VM_EXIT_LOAD_HOST_EFER                     \
> +            | VM_EXIT_SAVE_PREEMPT_TIMER)
> +        data = edx & ~REMOVED_EXIT_CONTROL_CAP;
> +        data = (data << 32) | eax;
> +        break;
> +    case MSR_IA32_VMX_ENTRY_CTLS:
> +        rdmsr(msr, eax, edx);
> +#define REMOVED_ENTRY_CONTROL_CAP (VM_ENTRY_LOAD_GUEST_PAT \
> +            | VM_ENTRY_LOAD_GUEST_EFER)
> +        data = edx & ~REMOVED_ENTRY_CONTROL_CAP;
> +        data = (data << 32) | eax;
> +        break;
> +    case MSR_IA32_VMX_PROCBASED_CTLS2:
> +        mask = 0;
> +
> +        rdmsr(msr, eax, edx);
> +        data = edx & mask;
> +        data = (data << 32) | eax;
> +        break;
> +

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Tim Deegan]

At 16:07 +0100 on 02 Jun (1307030872), Tim Deegan wrote:
> At 16:57 +0800 on 02 Jun (1307033853), Eddie Dong wrote:
> > +    case MSR_IA32_VMX_PINBASED_CTLS:
> > +#define REMOVED_PIN_CONTROL_CAP (PIN_BASED_PREEMPT_TIMER)
> > +        rdmsr(msr, eax, edx);
> > +        data = edx;
> > +        data = (data << 32) | eax;
> > +        break;
> 
> You don't actually mask the value here. 
> 
> BTW, I don't really like defining all these REMOVED_* macros, each
> of which is used only once a few lines from the definition (here and
> elsewhere in the series).  It just adds clutter for no benefit.
> 

Oh, I forgot to say: will this feature-blacklisting work over live
migration to a machine with a different CPU?  There isn't an equivalnet
of the CPUID masking feature to make all the machines in a cluster seem
to have the same VMX features. 

Elsewhere we use whitelisting for passsing hardware capability flags to
HVM guests; I think we should use whitelists here too. 

Cheers,

Tim.

 
> > +    case MSR_IA32_VMX_PROCBASED_CTLS:
> > +        rdmsr(msr, eax, edx);
> > +#define REMOVED_EXEC_CONTROL_CAP (CPU_BASED_TPR_SHADOW \
> > +            | CPU_BASED_ACTIVATE_MSR_BITMAP            \
> > +            | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)
> > +        data = edx & ~REMOVED_EXEC_CONTROL_CAP;
> > +        data = (data << 32) | eax;
> > +        break;
> > +    case MSR_IA32_VMX_EXIT_CTLS:
> > +        rdmsr(msr, eax, edx);
> > +#define REMOVED_EXIT_CONTROL_CAP (VM_EXIT_SAVE_GUEST_PAT \
> > +            | VM_EXIT_LOAD_HOST_PAT                      \
> > +            | VM_EXIT_SAVE_GUEST_EFER                    \
> > +            | VM_EXIT_LOAD_HOST_EFER                     \
> > +            | VM_EXIT_SAVE_PREEMPT_TIMER)
> > +        data = edx & ~REMOVED_EXIT_CONTROL_CAP;
> > +        data = (data << 32) | eax;
> > +        break;
> > +    case MSR_IA32_VMX_ENTRY_CTLS:
> > +        rdmsr(msr, eax, edx);
> > +#define REMOVED_ENTRY_CONTROL_CAP (VM_ENTRY_LOAD_GUEST_PAT \
> > +            | VM_ENTRY_LOAD_GUEST_EFER)
> > +        data = edx & ~REMOVED_ENTRY_CONTROL_CAP;
> > +        data = (data << 32) | eax;
> > +        break;
> > +    case MSR_IA32_VMX_PROCBASED_CTLS2:
> > +        mask = 0;
> > +
> > +        rdmsr(msr, eax, edx);
> > +        data = edx & mask;
> > +        data = (data << 32) | eax;
> > +        break;
> > +
> 

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Keir Fraser]

On 02/06/2011 16:11, "Tim Deegan" <Tim.Deegan@citrix.com> wrote:

>> BTW, I don't really like defining all these REMOVED_* macros, each
>> of which is used only once a few lines from the definition (here and
>> elsewhere in the series).  It just adds clutter for no benefit.
>> 
> 
> Oh, I forgot to say: will this feature-blacklisting work over live
> migration to a machine with a different CPU?  There isn't an equivalnet
> of the CPUID masking feature to make all the machines in a cluster seem
> to have the same VMX features.
> 
> Elsewhere we use whitelisting for passsing hardware capability flags to
> HVM guests; I think we should use whitelists here too.

Blacklists create a total mess of doom. We should absolutely disallow the
creation of any new ones. I think HVM guests are currently clean in this
regard and should stay that way.

 -- Keir

RE: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Dong, Eddie]

> 
> Oh, I forgot to say: will this feature-blacklisting work over live
> migration to a machine with a different CPU?  There isn't an equivalnet
> of the CPUID masking feature to make all the machines in a cluster seem
> to have the same VMX features.

That seems to be an issue neutral to nested virtualization. We should be able to migrate among same processors. But it is difficult to migrate a L2 guest to other machine as L1 guest. It may be OK evnetually, but not addressed right now.

My understanding is that same CPUID doesn't mean exactly same capability. 

> 
> Elsewhere we use whitelisting for passsing hardware capability flags to
> HVM guests; I think we should use whitelists here too.
> 

Thx, Eddie

RE: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Dong, Eddie]

> 
> At 16:57 +0800 on 02 Jun (1307033853), Eddie Dong wrote:
> > +    case MSR_IA32_VMX_PINBASED_CTLS:
> > +#define REMOVED_PIN_CONTROL_CAP (PIN_BASED_PREEMPT_TIMER)
> > +        rdmsr(msr, eax, edx);
> > +        data = edx;
> > +        data = (data << 32) | eax;
> > +        break;
> 
> You don't actually mask the value here.

Fixed. 

> 
> BTW, I don't really like defining all these REMOVED_* macros, each
> of which is used only once a few lines from the definition (here and
> elsewhere in the series).  It just adds clutter for no benefit.

OK, removed to be in code itself.

Thx, Eddie

Re: [PATCH 00 of 20] NestedVMX support

[Tim Deegan]

Hi, 

Thanks for these patches.  They look pretty good; I have a few comments
on the individual patches that I'll post separately.

Overall the only worry I have is the number of TODOs left at the end of
the series.  Some of them are obvioulsy ony important when you come to
do the nested EPT work.  I'd appreciate a comment on whether you think
any of these is important:

+static int nvmx_intr_intercept(struct vcpu *v, struct hvm_intack
intack)
+{
+    u32 exit_ctrl;
+
+    /*
+     * TODO:
+     *   - if L1 intr-window exiting == 0
+     *   - vNMI
+     */


+static int decode_vmx_inst(struct cpu_user_regs *regs,
+                           struct vmx_inst_decoded *decode,
+                           unsigned long *poperandS, int vmxon_check)
+{
[...]
+        /* TODO: segment type check */

This one, at least, I think does need to be fixed!


+static void load_shadow_control(struct vcpu *v)
+{
+    /* TODO: Make sure the shadow control doesn't set the bits 
+     * L0 VMM doesn't handle.
+     */


+int nvmx_handle_vmlaunch(struct cpu_user_regs *regs)
+{
+    /* TODO: check for initial launch/resume */
+    return nvmx_handle_vmresume(regs);
+}


+void nvmx_idtv_handling(void)
+{
[...]
+    /* TODO: NMI */
+}


+static void load_shadow_guest_state(struct vcpu *v)
+{
[...]
+    /* XXX: should refer to GUEST_HOST_MASK of both L0 and L1 */


Cheers,

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

RE: [PATCH 00 of 20] NestedVMX support

[Dong, Eddie]

> -----Original Message-----
> From: Tim Deegan [mailto:Tim.Deegan@citrix.com]
> Sent: Thursday, June 02, 2011 10:34 PM
> To: Dong, Eddie
> Cc: xen-devel@lists.xensource.com
> Subject: Re: [Xen-devel] [PATCH 00 of 20] NestedVMX support> 
> Hi,
> 
> Thanks for these patches.  They look pretty good; I have a few comments
> on the individual patches that I'll post separately.
> 
> Overall the only worry I have is the number of TODOs left at the end of
> the series.  Some of them are obvioulsy ony important when you come to
> do the nested EPT work.  I'd appreciate a comment on whether you think
> any of these is important:
> 
> +static int nvmx_intr_intercept(struct vcpu *v, struct hvm_intack
> intack)
> +{
> +    u32 exit_ctrl;
> +
> +    /*
> +     * TODO:
> +     *   - if L1 intr-window exiting == 0
> +     *   - vNMI
> +     */
> 

Deleted.

> 
> +static int decode_vmx_inst(struct cpu_user_regs *regs,
> +                           struct vmx_inst_decoded *decode,
> +                           unsigned long *poperandS, int
> vmxon_check)
> +{
> [...]
> +        /* TODO: segment type check */
> 

Fixed.

> This one, at least, I think does need to be fixed!
> 
> 
> +static void load_shadow_control(struct vcpu *v)
> +{
> +    /* TODO: Make sure the shadow control doesn't set the bits
> +     * L0 VMM doesn't handle.
> +     */
> 
deleted
> 
> +int nvmx_handle_vmlaunch(struct cpu_user_regs *regs)
> +{
> +    /* TODO: check for initial launch/resume */
> +    return nvmx_handle_vmresume(regs);
> +}
> 

Handled w/ correct launch state.
> 
> +void nvmx_idtv_handling(void)
> +{
> [...]
> +    /* TODO: NMI */
> +}
> 
deleted
> 
> +static void load_shadow_guest_state(struct vcpu *v)
> +{
> [...]
> +    /* XXX: should refer to GUEST_HOST_MASK of both L0 and L1 */
> 
Deleted and will revisit later.

> 
> Cheers,
> 
> Tim.
> 
> --
> Tim Deegan <Tim.Deegan@citrix.com>
> Principal Software Engineer, Xen Platform Team
> Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

[PATCH 00 of 20] Rebased Nested VMX v2

[Eddie Dong]

Tim/Keir:

This patch series enable the nestedvmx support with all fixes to comments.

patch 1-2 is a pre-cleanup.
patch 3 is for nested vmx data structure patch 4 for nestedhvm ops patch 5,
7,8,9, 10, 11 is for VMX instruction emulation patch 6 for virtual VMCS data
structure and access APIs.
patch 12 for VMCS switching
Patch 13 for vmreseume/launch emulation
patch 14 for shadow control VMCS fields
patch 15 for n1/n2 guest VMCS switch
patch 16 for interrupt/exceptions
patch 17 for nested vm exit
patch 18 for lazy FPU and patch 19 VMXE bits in CR4 patch 20 for MSR handling
and capability


Thanks, Eddie

[PATCH 20 of 20] n2 MSR handling and capability exposure

[Eddie Dong]

# HG changeset patch
# User Eddie Dong <eddie.dong@intel.com>
# Date 1307607849 -28800
# Node ID 480cfcea095bddb2e7c495a092632e80d552332d
# Parent  3189a3e6f05ecc7e14bc55e944c48d7e50613a96
n2 MSR handling and capability exposure

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r 3189a3e6f05e -r 480cfcea095b xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 09 16:24:09 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vmx.c	Thu Jun 09 16:24:09 2011 +0800
@@ -1778,8 +1778,11 @@ static int vmx_msr_read_intercept(unsign
         *msr_content |= (u64)__vmread(GUEST_IA32_DEBUGCTL_HIGH) << 32;
 #endif
         break;
-    case MSR_IA32_VMX_BASIC...MSR_IA32_VMX_PROCBASED_CTLS2:
-        goto gp_fault;
+    case IA32_FEATURE_CONTROL_MSR:
+    case MSR_IA32_VMX_BASIC...MSR_IA32_VMX_TRUE_ENTRY_CTLS:
+        if ( !nvmx_msr_read_intercept(msr, msr_content) )
+            goto gp_fault;
+        break;
     case MSR_IA32_MISC_ENABLE:
         rdmsrl(MSR_IA32_MISC_ENABLE, *msr_content);
         /* Debug Trace Store is not supported. */
@@ -1940,8 +1943,11 @@ static int vmx_msr_write_intercept(unsig
 
         break;
     }
-    case MSR_IA32_VMX_BASIC...MSR_IA32_VMX_PROCBASED_CTLS2:
-        goto gp_fault;
+    case IA32_FEATURE_CONTROL_MSR:
+    case MSR_IA32_VMX_BASIC...MSR_IA32_VMX_TRUE_ENTRY_CTLS:
+        if ( !nvmx_msr_write_intercept(msr, msr_content) )
+            goto gp_fault;
+        break;
     default:
         if ( vpmu_do_wrmsr(msr, msr_content) )
             return X86EMUL_OKAY;
diff -r 3189a3e6f05e -r 480cfcea095b xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 09 16:24:09 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Thu Jun 09 16:24:09 2011 +0800
@@ -22,6 +22,7 @@
 
 #include <xen/config.h>
 #include <asm/types.h>
+#include <asm/mtrr.h>
 #include <asm/p2m.h>
 #include <asm/hvm/vmx/vmx.h>
 #include <asm/hvm/vmx/vvmx.h>
@@ -1254,6 +1255,100 @@ int nvmx_handle_vmwrite(struct cpu_user_
     return X86EMUL_OKAY;
 }
 
+/*
+ * Capability reporting
+ */
+int nvmx_msr_read_intercept(unsigned int msr, u64 *msr_content)
+{
+    u64 data = 0, tmp;
+    int r = 1;
+
+    if ( !nestedhvm_enabled(current->domain) )
+        return 0;
+
+    /*
+     * Remove unsupport features from n1 guest capability MSR
+     */
+    switch (msr) {
+    case MSR_IA32_VMX_BASIC:
+        data = VVMCS_REVISION | (PAGE_SIZE) << 32 | 
+               ((u64)MTRR_TYPE_WRBACK) << 50 | (1L << 55);
+        break;
+    case MSR_IA32_VMX_PINBASED_CTLS:
+        /* 1-seetings */
+        data = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
+        data <<= 32;
+	/* 0-settings */
+        data |= 0;
+        break;
+    case MSR_IA32_VMX_PROCBASED_CTLS:
+        /* 1-seetings */
+        data = (CPU_BASED_HLT_EXITING |
+               CPU_BASED_VIRTUAL_INTR_PENDING |
+               CPU_BASED_CR8_LOAD_EXITING |
+               CPU_BASED_CR8_STORE_EXITING |
+               CPU_BASED_INVLPG_EXITING |
+               CPU_BASED_CR3_LOAD_EXITING |
+               CPU_BASED_CR3_STORE_EXITING |
+               CPU_BASED_MONITOR_EXITING |
+               CPU_BASED_MWAIT_EXITING |
+               CPU_BASED_MOV_DR_EXITING |
+               CPU_BASED_ACTIVATE_IO_BITMAP |
+               CPU_BASED_USE_TSC_OFFSETING |
+               CPU_BASED_UNCOND_IO_EXITING |
+               CPU_BASED_RDTSC_EXITING);
+        /* bit 1, 4-6,8,13-16,26 must be 1 (refer G4 of SDM) */
+        tmp = ( (1<<26) | (0xf << 13) | 0x100 | (0x7 << 4) | 0x2);
+        /* 0-settings */
+        data = ((data | tmp) << 32) | (tmp);
+        break;
+    case MSR_IA32_VMX_EXIT_CTLS:
+        /* 1-seetings */
+        /* bit 0-8, 10,11,13,14,16,17 must be 1 (refer G4 of SDM) */
+        tmp = 0x36dff;
+        data = VM_EXIT_ACK_INTR_ON_EXIT;
+#ifdef __x86_64__
+        data |= VM_EXIT_IA32E_MODE;
+#endif
+	/* 0-settings */
+        data = ((data | tmp) << 32) | tmp;
+        break;
+    case MSR_IA32_VMX_ENTRY_CTLS:
+        /* bit 0-8, and 12 must be 1 (refer G5 of SDM) */
+        data = 0x11ff;
+        data |= VM_ENTRY_IA32E_MODE;
+        data = (data << 32) | data;
+        break;
+
+    case IA32_FEATURE_CONTROL_MSR:
+        data = IA32_FEATURE_CONTROL_MSR_LOCK | 
+               IA32_FEATURE_CONTROL_MSR_ENABLE_VMXON_OUTSIDE_SMX;
+         break;
+    /* pass through MSRs */
+    case MSR_IA32_VMX_MISC:
+    case MSR_IA32_VMX_CR0_FIXED0:
+    case MSR_IA32_VMX_CR0_FIXED1:
+    case MSR_IA32_VMX_CR4_FIXED0:
+    case MSR_IA32_VMX_CR4_FIXED1:
+    case MSR_IA32_VMX_VMCS_ENUM:
+        gdprintk(XENLOG_WARNING, "VMX MSR %x not fully supported yet.\n", msr);
+        break;
+
+    default:
+        r = 0;
+        break;
+    }
+
+    *msr_content = data;
+    return r;
+}
+
+int nvmx_msr_write_intercept(unsigned int msr, u64 msr_content)
+{
+    /* silently ignore for now */
+    return 1;
+}
+
 void nvmx_idtv_handling(void)
 {
     struct vcpu *v = current;
diff -r 3189a3e6f05e -r 480cfcea095b xen/include/asm-x86/hvm/vmx/vvmx.h
--- a/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 09 16:24:09 2011 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 09 16:24:09 2011 +0800
@@ -163,6 +163,10 @@ int nvmx_handle_vmread(struct cpu_user_r
 int nvmx_handle_vmwrite(struct cpu_user_regs *regs);
 int nvmx_handle_vmresume(struct cpu_user_regs *regs);
 int nvmx_handle_vmlaunch(struct cpu_user_regs *regs);
+int nvmx_msr_read_intercept(unsigned int msr,
+                                u64 *msr_content);
+int nvmx_msr_write_intercept(unsigned int msr,
+                                 u64 msr_content);
 
 void nvmx_update_exec_control(struct vcpu *v, u32 value);
 void nvmx_update_secondary_exec_control(struct vcpu *v,

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Tim Deegan]

At 16:25 +0800 on 09 Jun (1307636725), Eddie Dong wrote:
> +    /* pass through MSRs */
> +    case MSR_IA32_VMX_MISC:
> +    case MSR_IA32_VMX_CR0_FIXED0:
> +    case MSR_IA32_VMX_CR0_FIXED1:
> +    case MSR_IA32_VMX_CR4_FIXED0:
> +    case MSR_IA32_VMX_CR4_FIXED1:
> +    case MSR_IA32_VMX_VMCS_ENUM:
> +        gdprintk(XENLOG_WARNING, "VMX MSR %x not fully supported yet.\n", msr);

Are you planning to fix this before checking in this series?  I'm pretty
sure that at least MSR_IA32_VMX_VMCS_ENUM should be trivial, since you
define your own VMCS format. 

Tim.

> +        break;
> +
> +    default:
> +        r = 0;
> +        break;
> +    }
> +
> +    *msr_content = data;
> +    return r;
> +}
> +
> +int nvmx_msr_write_intercept(unsigned int msr, u64 msr_content)
> +{
> +    /* silently ignore for now */
> +    return 1;
> +}
> +
>  void nvmx_idtv_handling(void)
>  {
>      struct vcpu *v = current;
> diff -r 3189a3e6f05e -r 480cfcea095b xen/include/asm-x86/hvm/vmx/vvmx.h
> --- a/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 09 16:24:09 2011 +0800
> +++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Thu Jun 09 16:24:09 2011 +0800
> @@ -163,6 +163,10 @@ int nvmx_handle_vmread(struct cpu_user_r
>  int nvmx_handle_vmwrite(struct cpu_user_regs *regs);
>  int nvmx_handle_vmresume(struct cpu_user_regs *regs);
>  int nvmx_handle_vmlaunch(struct cpu_user_regs *regs);
> +int nvmx_msr_read_intercept(unsigned int msr,
> +                                u64 *msr_content);
> +int nvmx_msr_write_intercept(unsigned int msr,
> +                                 u64 msr_content);
>  
>  void nvmx_update_exec_control(struct vcpu *v, u32 value);
>  void nvmx_update_secondary_exec_control(struct vcpu *v,
> 
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

RE: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Dong, Eddie]

[-- Attachment #1: Type: text/plain, Size: 566 bytes --]

> > +    case MSR_IA32_VMX_MISC:
> > +    case MSR_IA32_VMX_CR0_FIXED0:
> > +    case MSR_IA32_VMX_CR0_FIXED1:
> > +    case MSR_IA32_VMX_CR4_FIXED0:
> > +    case MSR_IA32_VMX_CR4_FIXED1:
> > +    case MSR_IA32_VMX_VMCS_ENUM:
> > +        gdprintk(XENLOG_WARNING, "VMX MSR %x not fully supported
> yet.\n", msr);
> 
> Are you planning to fix this before checking in this series?  I'm pretty
> sure that at least MSR_IA32_VMX_VMCS_ENUM should be trivial, since you
> define your own VMCS format.
> 

Sure, the updated patch is attached.

Thx, Eddie

[-- Attachment #2: msr_capability --]
[-- Type: application/octet-stream, Size: 5950 bytes --]


n2 MSR handling and capability exposure

Signed-off-by: Qing He <qing.he@intel.com>
Signed-off-by: Eddie Dong <eddie.dong@intel.com>

diff -r 4242db588ea6 xen/arch/x86/hvm/vmx/vmx.c
--- a/xen/arch/x86/hvm/vmx/vmx.c	Sun Jun 12 13:50:43 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vmx.c	Tue Jun 14 10:22:00 2011 +0800
@@ -1774,8 +1774,11 @@ static int vmx_msr_read_intercept(unsign
         *msr_content |= (u64)__vmread(GUEST_IA32_DEBUGCTL_HIGH) << 32;
 #endif
         break;
-    case MSR_IA32_VMX_BASIC...MSR_IA32_VMX_PROCBASED_CTLS2:
-        goto gp_fault;
+    case IA32_FEATURE_CONTROL_MSR:
+    case MSR_IA32_VMX_BASIC...MSR_IA32_VMX_TRUE_ENTRY_CTLS:
+        if ( !nvmx_msr_read_intercept(msr, msr_content) )
+            goto gp_fault;
+        break;
     case MSR_IA32_MISC_ENABLE:
         rdmsrl(MSR_IA32_MISC_ENABLE, *msr_content);
         /* Debug Trace Store is not supported. */
@@ -1935,8 +1938,11 @@ static int vmx_msr_write_intercept(unsig
 
         break;
     }
-    case MSR_IA32_VMX_BASIC...MSR_IA32_VMX_PROCBASED_CTLS2:
-        goto gp_fault;
+    case IA32_FEATURE_CONTROL_MSR:
+    case MSR_IA32_VMX_BASIC...MSR_IA32_VMX_TRUE_ENTRY_CTLS:
+        if ( !nvmx_msr_write_intercept(msr, msr_content) )
+            goto gp_fault;
+        break;
     default:
         if ( vpmu_do_wrmsr(msr, msr_content) )
             return X86EMUL_OKAY;
diff -r 4242db588ea6 xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Sun Jun 12 13:50:43 2011 +0800
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Tue Jun 14 10:22:00 2011 +0800
@@ -22,6 +22,7 @@
 
 #include <xen/config.h>
 #include <asm/types.h>
+#include <asm/mtrr.h>
 #include <asm/p2m.h>
 #include <asm/hvm/vmx/vmx.h>
 #include <asm/hvm/vmx/vvmx.h>
@@ -1230,6 +1231,113 @@ int nvmx_handle_vmwrite(struct cpu_user_
     return X86EMUL_OKAY;
 }
 
+/*
+ * Capability reporting
+ */
+int nvmx_msr_read_intercept(unsigned int msr, u64 *msr_content)
+{
+    u64 data = 0, tmp;
+    int r = 1;
+
+    if ( !nestedhvm_enabled(current->domain) )
+        return 0;
+
+    /*
+     * Remove unsupport features from n1 guest capability MSR
+     */
+    switch (msr) {
+    case MSR_IA32_VMX_BASIC:
+        data = VVMCS_REVISION | (PAGE_SIZE) << 32 | 
+               ((u64)MTRR_TYPE_WRBACK) << 50 | (1L << 55);
+        break;
+    case MSR_IA32_VMX_PINBASED_CTLS:
+        /* 1-seetings */
+        data = PIN_BASED_EXT_INTR_MASK | PIN_BASED_NMI_EXITING;
+        data <<= 32;
+	/* 0-settings */
+        data |= 0;
+        break;
+    case MSR_IA32_VMX_PROCBASED_CTLS:
+        /* 1-seetings */
+        data = (CPU_BASED_HLT_EXITING |
+               CPU_BASED_VIRTUAL_INTR_PENDING |
+               CPU_BASED_CR8_LOAD_EXITING |
+               CPU_BASED_CR8_STORE_EXITING |
+               CPU_BASED_INVLPG_EXITING |
+               CPU_BASED_CR3_LOAD_EXITING |
+               CPU_BASED_CR3_STORE_EXITING |
+               CPU_BASED_MONITOR_EXITING |
+               CPU_BASED_MWAIT_EXITING |
+               CPU_BASED_MOV_DR_EXITING |
+               CPU_BASED_ACTIVATE_IO_BITMAP |
+               CPU_BASED_USE_TSC_OFFSETING |
+               CPU_BASED_UNCOND_IO_EXITING |
+               CPU_BASED_RDTSC_EXITING);
+        /* bit 1, 4-6,8,13-16,26 must be 1 (refer G4 of SDM) */
+        tmp = ( (1<<26) | (0xf << 13) | 0x100 | (0x7 << 4) | 0x2);
+        /* 0-settings */
+        data = ((data | tmp) << 32) | (tmp);
+        break;
+    case MSR_IA32_VMX_EXIT_CTLS:
+        /* 1-seetings */
+        /* bit 0-8, 10,11,13,14,16,17 must be 1 (refer G4 of SDM) */
+        tmp = 0x36dff;
+        data = VM_EXIT_ACK_INTR_ON_EXIT;
+#ifdef __x86_64__
+        data |= VM_EXIT_IA32E_MODE;
+#endif
+	/* 0-settings */
+        data = ((data | tmp) << 32) | tmp;
+        break;
+    case MSR_IA32_VMX_ENTRY_CTLS:
+        /* bit 0-8, and 12 must be 1 (refer G5 of SDM) */
+        data = 0x11ff;
+        data |= VM_ENTRY_IA32E_MODE;
+        data = (data << 32) | data;
+        break;
+
+    case IA32_FEATURE_CONTROL_MSR:
+        data = IA32_FEATURE_CONTROL_MSR_LOCK | 
+               IA32_FEATURE_CONTROL_MSR_ENABLE_VMXON_OUTSIDE_SMX;
+        break;
+    case MSR_IA32_VMX_VMCS_ENUM:
+        /* The max index of VVMCS encoding is 0x1f. */
+        data = 0x1f << 1;
+        break;
+    case MSR_IA32_VMX_CR0_FIXED0:
+        /* PG, PE bits must be 1 in VMX operation */
+        data = X86_CR0_PE | X86_CR0_PG;
+        break;
+    case MSR_IA32_VMX_CR0_FIXED1:
+        /* allow 0-settings for all bits */
+        data = 0xffffffff;
+        break;
+    case MSR_IA32_VMX_CR4_FIXED0:
+        /* VMXE bit must be 1 in VMX operation */
+        data = X86_CR4_VMXE;
+        break;
+    case MSR_IA32_VMX_CR4_FIXED1:
+        /* allow 0-settings except SMXE */
+        data = 0x267ff & ~X86_CR4_SMXE;
+        break;
+    case MSR_IA32_VMX_MISC:
+        gdprintk(XENLOG_WARNING, "VMX MSR %x not fully supported yet.\n", msr);
+        break;
+    default:
+        r = 0;
+        break;
+    }
+
+    *msr_content = data;
+    return r;
+}
+
+int nvmx_msr_write_intercept(unsigned int msr, u64 msr_content)
+{
+    /* silently ignore for now */
+    return 1;
+}
+
 void nvmx_idtv_handling(void)
 {
     struct vcpu *v = current;
diff -r 4242db588ea6 xen/include/asm-x86/hvm/vmx/vvmx.h
--- a/xen/include/asm-x86/hvm/vmx/vvmx.h	Sun Jun 12 13:50:43 2011 +0800
+++ b/xen/include/asm-x86/hvm/vmx/vvmx.h	Tue Jun 14 10:22:00 2011 +0800
@@ -164,6 +164,10 @@ int nvmx_handle_vmread(struct cpu_user_r
 int nvmx_handle_vmwrite(struct cpu_user_regs *regs);
 int nvmx_handle_vmresume(struct cpu_user_regs *regs);
 int nvmx_handle_vmlaunch(struct cpu_user_regs *regs);
+int nvmx_msr_read_intercept(unsigned int msr,
+                                u64 *msr_content);
+int nvmx_msr_write_intercept(unsigned int msr,
+                                 u64 msr_content);
 
 void nvmx_update_exec_control(struct vcpu *v, u32 value);
 void nvmx_update_secondary_exec_control(struct vcpu *v,

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Tim Deegan]

At 10:29 +0800 on 14 Jun (1308047377), Dong, Eddie wrote:
> > > +    case MSR_IA32_VMX_MISC:
> > > +    case MSR_IA32_VMX_CR0_FIXED0:
> > > +    case MSR_IA32_VMX_CR0_FIXED1:
> > > +    case MSR_IA32_VMX_CR4_FIXED0:
> > > +    case MSR_IA32_VMX_CR4_FIXED1:
> > > +    case MSR_IA32_VMX_VMCS_ENUM:
> > > +        gdprintk(XENLOG_WARNING, "VMX MSR %x not fully supported
> > yet.\n", msr);
> > 
> > Are you planning to fix this before checking in this series?  I'm pretty
> > sure that at least MSR_IA32_VMX_VMCS_ENUM should be trivial, since you
> > define your own VMCS format.
> > 
> 
> Sure, the updated patch is attached.

Thanks.  I've applied the full series.  I cleaned up a few things for
the 32-bit build as well.

Cheers,

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Jeroen Groenewegen van der Weyden]

Hi,

Based on these patch series, I gave it go.
I noticed until now that the system becomes in-responsive a few second 
after activating the second nested guest.
steps:
1) dom0 creates domu (pass)
2) domU creates first kvm guest (pass)
3) domU creates second kvm guest (fail)
after the third step the domu becomes in-responsive. eg no network / vnc 
console response. in dom0 is shows domu is stil running.

I'm able to provide any input needed, just tell me what you need (and how).

mfg,
Jeroen

My HW setup
mobo asus z8nr-d12
proc: 2 times e5645
mem: 96 GB

My goal/test
in short: HW -> dom0 -> domU -> 4 time kvm guest

dom0
opensuse 11.4 x86
xen: unstable cs 23553

domu (HVM)
sles11sp1
mem: 8 GB
vcpu: 4

domu guests
1) qemu-kvm: propetary OS
2) qemu-kvm: propetary OS
3) qemu-kvm: SLES10SP3
4) qemu-kvm: SLES10SP3



Op 15-6-2011 14:45, Tim Deegan schreef:
> At 10:29 +0800 on 14 Jun (1308047377), Dong, Eddie wrote:
>>>> +    case MSR_IA32_VMX_MISC:
>>>> +    case MSR_IA32_VMX_CR0_FIXED0:
>>>> +    case MSR_IA32_VMX_CR0_FIXED1:
>>>> +    case MSR_IA32_VMX_CR4_FIXED0:
>>>> +    case MSR_IA32_VMX_CR4_FIXED1:
>>>> +    case MSR_IA32_VMX_VMCS_ENUM:
>>>> +        gdprintk(XENLOG_WARNING, "VMX MSR %x not fully supported
>>> yet.\n", msr);
>>>
>>> Are you planning to fix this before checking in this series?  I'm pretty
>>> sure that at least MSR_IA32_VMX_VMCS_ENUM should be trivial, since you
>>> define your own VMCS format.
>>>
>> Sure, the updated patch is attached.
> Thanks.  I've applied the full series.  I cleaned up a few things for
> the 32-bit build as well.
>
> Cheers,
>
> Tim.
>

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Jeroen Groenewegen van der Weyden]

Hi Tim/Eddi/Christoph,

I just gave it a go again with cs23640. But the behaviour is still the 
same. so L1 becomes inresponsive after starting the second nested quest.
I can deliver any trace input you want/need.

mfg,
Jeroen.

Op 20-6-2011 8:31, Jeroen Groenewegen van der Weyden schreef:
> Hi,
>
> Based on these patch series, I gave it go.
> I noticed until now that the system becomes in-responsive a few second 
> after activating the second nested guest.
> steps:
> 1) dom0 creates domu (pass)
> 2) domU creates first kvm guest (pass)
> 3) domU creates second kvm guest (fail)
> after the third step the domu becomes in-responsive. eg no network / 
> vnc console response. in dom0 is shows domu is stil running.
>
> I'm able to provide any input needed, just tell me what you need (and 
> how).
>
> mfg,
> Jeroen
>
> My HW setup
> mobo asus z8nr-d12
> proc: 2 times e5645
> mem: 96 GB
>
> My goal/test
> in short: HW -> dom0 -> domU -> 4 time kvm guest
>
> dom0
> opensuse 11.4 x86
> xen: unstable cs 23553
>
> domu (HVM)
> sles11sp1
> mem: 8 GB
> vcpu: 4
>
> domu guests
> 1) qemu-kvm: propetary OS
> 2) qemu-kvm: propetary OS
> 3) qemu-kvm: SLES10SP3
> 4) qemu-kvm: SLES10SP3
>
>
>
> Op 15-6-2011 14:45, Tim Deegan schreef:
>> At 10:29 +0800 on 14 Jun (1308047377), Dong, Eddie wrote:
>>>>> +    case MSR_IA32_VMX_MISC:
>>>>> +    case MSR_IA32_VMX_CR0_FIXED0:
>>>>> +    case MSR_IA32_VMX_CR0_FIXED1:
>>>>> +    case MSR_IA32_VMX_CR4_FIXED0:
>>>>> +    case MSR_IA32_VMX_CR4_FIXED1:
>>>>> +    case MSR_IA32_VMX_VMCS_ENUM:
>>>>> +        gdprintk(XENLOG_WARNING, "VMX MSR %x not fully supported
>>>> yet.\n", msr);
>>>>
>>>> Are you planning to fix this before checking in this series?  I'm 
>>>> pretty
>>>> sure that at least MSR_IA32_VMX_VMCS_ENUM should be trivial, since you
>>>> define your own VMCS format.
>>>>
>>> Sure, the updated patch is attached.
>> Thanks.  I've applied the full series.  I cleaned up a few things for
>> the 32-bit build as well.
>>
>> Cheers,
>>
>> Tim.
>>
>
>
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel
>

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Tim Deegan]

Eddie?  Did you test with KVM guests?  Is this expected to work?

Tim.

At 15:14 +0100 on 30 Jun (1309446872), Jeroen Groenewegen van der Weyden wrote:
> Hi Tim/Eddi/Christoph,
> 
> I just gave it a go again with cs23640. But the behaviour is still the 
> same. so L1 becomes inresponsive after starting the second nested quest.
> I can deliver any trace input you want/need.
> 
> mfg,
> Jeroen.
> 
> Op 20-6-2011 8:31, Jeroen Groenewegen van der Weyden schreef:
> > Hi,
> >
> > Based on these patch series, I gave it go.
> > I noticed until now that the system becomes in-responsive a few second 
> > after activating the second nested guest.
> > steps:
> > 1) dom0 creates domu (pass)
> > 2) domU creates first kvm guest (pass)
> > 3) domU creates second kvm guest (fail)
> > after the third step the domu becomes in-responsive. eg no network / 
> > vnc console response. in dom0 is shows domu is stil running.
> >
> > I'm able to provide any input needed, just tell me what you need (and 
> > how).
> >
> > mfg,
> > Jeroen
> >
> > My HW setup
> > mobo asus z8nr-d12
> > proc: 2 times e5645
> > mem: 96 GB
> >
> > My goal/test
> > in short: HW -> dom0 -> domU -> 4 time kvm guest
> >
> > dom0
> > opensuse 11.4 x86
> > xen: unstable cs 23553
> >
> > domu (HVM)
> > sles11sp1
> > mem: 8 GB
> > vcpu: 4
> >
> > domu guests
> > 1) qemu-kvm: propetary OS
> > 2) qemu-kvm: propetary OS
> > 3) qemu-kvm: SLES10SP3
> > 4) qemu-kvm: SLES10SP3
> >
> >
> >
> > Op 15-6-2011 14:45, Tim Deegan schreef:
> >> At 10:29 +0800 on 14 Jun (1308047377), Dong, Eddie wrote:
> >>>>> +    case MSR_IA32_VMX_MISC:
> >>>>> +    case MSR_IA32_VMX_CR0_FIXED0:
> >>>>> +    case MSR_IA32_VMX_CR0_FIXED1:
> >>>>> +    case MSR_IA32_VMX_CR4_FIXED0:
> >>>>> +    case MSR_IA32_VMX_CR4_FIXED1:
> >>>>> +    case MSR_IA32_VMX_VMCS_ENUM:
> >>>>> +        gdprintk(XENLOG_WARNING, "VMX MSR %x not fully supported
> >>>> yet.\n", msr);
> >>>>
> >>>> Are you planning to fix this before checking in this series?  I'm 
> >>>> pretty
> >>>> sure that at least MSR_IA32_VMX_VMCS_ENUM should be trivial, since you
> >>>> define your own VMCS format.
> >>>>
> >>> Sure, the updated patch is attached.
> >> Thanks.  I've applied the full series.  I cleaned up a few things for
> >> the 32-bit build as well.
> >>
> >> Cheers,
> >>
> >> Tim.
> >>
> >
> >
> > _______________________________________________
> > Xen-devel mailing list
> > Xen-devel@lists.xensource.com
> > http://lists.xensource.com/xen-devel
> >
> 

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Christoph Egger]

KVM on Xen works for me. Also KVM on Xen on Xen.

Christoph


On 07/01/11 11:01, Tim Deegan wrote:
> Eddie?  Did you test with KVM guests?  Is this expected to work?
>
> Tim.
>
> At 15:14 +0100 on 30 Jun (1309446872), Jeroen Groenewegen van der Weyden wrote:
>> Hi Tim/Eddi/Christoph,
>>
>> I just gave it a go again with cs23640. But the behaviour is still the
>> same. so L1 becomes inresponsive after starting the second nested quest.
>> I can deliver any trace input you want/need.
>>
>> mfg,
>> Jeroen.
>>
>> Op 20-6-2011 8:31, Jeroen Groenewegen van der Weyden schreef:
>>> Hi,
>>>
>>> Based on these patch series, I gave it go.
>>> I noticed until now that the system becomes in-responsive a few second
>>> after activating the second nested guest.
>>> steps:
>>> 1) dom0 creates domu (pass)
>>> 2) domU creates first kvm guest (pass)
>>> 3) domU creates second kvm guest (fail)
>>> after the third step the domu becomes in-responsive. eg no network /
>>> vnc console response. in dom0 is shows domu is stil running.
>>>
>>> I'm able to provide any input needed, just tell me what you need (and
>>> how).
>>>
>>> mfg,
>>> Jeroen
>>>
>>> My HW setup
>>> mobo asus z8nr-d12
>>> proc: 2 times e5645
>>> mem: 96 GB
>>>
>>> My goal/test
>>> in short: HW ->  dom0 ->  domU ->  4 time kvm guest
>>>
>>> dom0
>>> opensuse 11.4 x86
>>> xen: unstable cs 23553
>>>
>>> domu (HVM)
>>> sles11sp1
>>> mem: 8 GB
>>> vcpu: 4
>>>
>>> domu guests
>>> 1) qemu-kvm: propetary OS
>>> 2) qemu-kvm: propetary OS
>>> 3) qemu-kvm: SLES10SP3
>>> 4) qemu-kvm: SLES10SP3
>>>
>>>
>>>
>>> Op 15-6-2011 14:45, Tim Deegan schreef:
>>>> At 10:29 +0800 on 14 Jun (1308047377), Dong, Eddie wrote:
>>>>>>> +    case MSR_IA32_VMX_MISC:
>>>>>>> +    case MSR_IA32_VMX_CR0_FIXED0:
>>>>>>> +    case MSR_IA32_VMX_CR0_FIXED1:
>>>>>>> +    case MSR_IA32_VMX_CR4_FIXED0:
>>>>>>> +    case MSR_IA32_VMX_CR4_FIXED1:
>>>>>>> +    case MSR_IA32_VMX_VMCS_ENUM:
>>>>>>> +        gdprintk(XENLOG_WARNING, "VMX MSR %x not fully supported
>>>>>> yet.\n", msr);
>>>>>>
>>>>>> Are you planning to fix this before checking in this series?  I'm
>>>>>> pretty
>>>>>> sure that at least MSR_IA32_VMX_VMCS_ENUM should be trivial, since you
>>>>>> define your own VMCS format.
>>>>>>
>>>>> Sure, the updated patch is attached.
>>>> Thanks.  I've applied the full series.  I cleaned up a few things for
>>>> the 32-bit build as well.
>>>>
>>>> Cheers,
>>>>
>>>> Tim.


-- 
---to satisfy European Law for business letters:
Advanced Micro Devices GmbH
Einsteinring 24, 85689 Dornach b. Muenchen
Geschaeftsfuehrer: Alberto Bozzo, Andrew Bowd
Sitz: Dornach, Gemeinde Aschheim, Landkreis Muenchen
Registergericht Muenchen, HRB Nr. 43632

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Jeroen Groenewegen van der Weyden]

Just a remark to avoid misunderstanding

The first nested  KVM guest is running ok, for as long as i keep it running
When starting a second nested KVM guest (in the same Xen DOMU) that is 
when the Xen DOMU becomes in-responsive. In DOM0 it it shown that it is 
still running and consuming time.


The complete test is like this

dom0
opensuse 11.4 x86
xen: unstable cs 23640

domu1 (HVM)
sles11sp1
mem: 8 GB
vcpu: 4

domu1 guests
1) qemu-kvm: propetary OS
2) qemu-kvm: propetary OS
3) qemu-kvm: SLES10SP3
4) qemu-kvm: SLES10SP3

mfg,
Jeroen

Op 1-7-2011 11:54, Christoph Egger schreef:
>
> KVM on Xen works for me. Also KVM on Xen on Xen.
>
> Christoph
>
>
> On 07/01/11 11:01, Tim Deegan wrote:
>> Eddie?  Did you test with KVM guests?  Is this expected to work?
>>
>> Tim.
>>
>> At 15:14 +0100 on 30 Jun (1309446872), Jeroen Groenewegen van der 
>> Weyden wrote:
>>> Hi Tim/Eddi/Christoph,
>>>
>>> I just gave it a go again with cs23640. But the behaviour is still the
>>> same. so L1 becomes inresponsive after starting the second nested 
>>> quest.
>>> I can deliver any trace input you want/need.
>>>
>>> mfg,
>>> Jeroen.
>>>
>>> Op 20-6-2011 8:31, Jeroen Groenewegen van der Weyden schreef:
>>>> Hi,
>>>>
>>>> Based on these patch series, I gave it go.
>>>> I noticed until now that the system becomes in-responsive a few second
>>>> after activating the second nested guest.
>>>> steps:
>>>> 1) dom0 creates domu (pass)
>>>> 2) domU creates first kvm guest (pass)
>>>> 3) domU creates second kvm guest (fail)
>>>> after the third step the domu becomes in-responsive. eg no network /
>>>> vnc console response. in dom0 is shows domu is stil running.
>>>>
>>>> I'm able to provide any input needed, just tell me what you need (and
>>>> how).
>>>>
>>>> mfg,
>>>> Jeroen
>>>>
>>>> My HW setup
>>>> mobo asus z8nr-d12
>>>> proc: 2 times e5645
>>>> mem: 96 GB
>>>>
>>>> My goal/test
>>>> in short: HW ->  dom0 ->  domU ->  4 time kvm guest
>>>>
>>>> dom0
>>>> opensuse 11.4 x86
>>>> xen: unstable cs 23553
>>>>
>>>> domu (HVM)
>>>> sles11sp1
>>>> mem: 8 GB
>>>> vcpu: 4
>>>>
>>>> domu guests
>>>> 1) qemu-kvm: propetary OS
>>>> 2) qemu-kvm: propetary OS
>>>> 3) qemu-kvm: SLES10SP3
>>>> 4) qemu-kvm: SLES10SP3
>>>>
>>>>
>>>>
>>>> Op 15-6-2011 14:45, Tim Deegan schreef:
>>>>> At 10:29 +0800 on 14 Jun (1308047377), Dong, Eddie wrote:
>>>>>>>> +    case MSR_IA32_VMX_MISC:
>>>>>>>> +    case MSR_IA32_VMX_CR0_FIXED0:
>>>>>>>> +    case MSR_IA32_VMX_CR0_FIXED1:
>>>>>>>> +    case MSR_IA32_VMX_CR4_FIXED0:
>>>>>>>> +    case MSR_IA32_VMX_CR4_FIXED1:
>>>>>>>> +    case MSR_IA32_VMX_VMCS_ENUM:
>>>>>>>> +        gdprintk(XENLOG_WARNING, "VMX MSR %x not fully supported
>>>>>>> yet.\n", msr);
>>>>>>>
>>>>>>> Are you planning to fix this before checking in this series?  I'm
>>>>>>> pretty
>>>>>>> sure that at least MSR_IA32_VMX_VMCS_ENUM should be trivial, 
>>>>>>> since you
>>>>>>> define your own VMCS format.
>>>>>>>
>>>>>> Sure, the updated patch is attached.
>>>>> Thanks.  I've applied the full series.  I cleaned up a few things for
>>>>> the 32-bit build as well.
>>>>>
>>>>> Cheers,
>>>>>
>>>>> Tim.
>
>

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Tim Deegan]

At 12:11 +0200 on 01 Jul (1309522280), Jeroen Groenewegen van der Weyden wrote:
> Just a remark to avoid misunderstanding
> 
> The first nested  KVM guest is running ok, for as long as i keep it running
> When starting a second nested KVM guest (in the same Xen DOMU) that
> is when the Xen DOMU becomes in-responsive. In DOM0 it it shown that
> it is still running and consuming time.
> 
> 
> The complete test is like this
> 
> dom0
> opensuse 11.4 x86
> xen: unstable cs 23640

You're testing on Intel hardware, right?

Eddie, any comment?

Tim.

> domu1 (HVM)
> sles11sp1
> mem: 8 GB
> vcpu: 4
> 
> domu1 guests
> 1) qemu-kvm: propetary OS
> 2) qemu-kvm: propetary OS
> 3) qemu-kvm: SLES10SP3
> 4) qemu-kvm: SLES10SP3
> 
> mfg,
> Jeroen
> 
> Op 1-7-2011 11:54, Christoph Egger schreef:
> >
> >KVM on Xen works for me. Also KVM on Xen on Xen.
> >
> >Christoph
> >
> >
> >On 07/01/11 11:01, Tim Deegan wrote:
> >>Eddie?  Did you test with KVM guests?  Is this expected to work?
> >>
> >>Tim.
> >>
> >>At 15:14 +0100 on 30 Jun (1309446872), Jeroen Groenewegen van
> >>der Weyden wrote:
> >>>Hi Tim/Eddi/Christoph,
> >>>
> >>>I just gave it a go again with cs23640. But the behaviour is still the
> >>>same. so L1 becomes inresponsive after starting the second
> >>>nested quest.
> >>>I can deliver any trace input you want/need.
> >>>
> >>>mfg,
> >>>Jeroen.
> >>>
> >>>Op 20-6-2011 8:31, Jeroen Groenewegen van der Weyden schreef:
> >>>>Hi,
> >>>>
> >>>>Based on these patch series, I gave it go.
> >>>>I noticed until now that the system becomes in-responsive a few second
> >>>>after activating the second nested guest.
> >>>>steps:
> >>>>1) dom0 creates domu (pass)
> >>>>2) domU creates first kvm guest (pass)
> >>>>3) domU creates second kvm guest (fail)
> >>>>after the third step the domu becomes in-responsive. eg no network /
> >>>>vnc console response. in dom0 is shows domu is stil running.
> >>>>
> >>>>I'm able to provide any input needed, just tell me what you need (and
> >>>>how).
> >>>>
> >>>>mfg,
> >>>>Jeroen
> >>>>
> >>>>My HW setup
> >>>>mobo asus z8nr-d12
> >>>>proc: 2 times e5645
> >>>>mem: 96 GB
> >>>>
> >>>>My goal/test
> >>>>in short: HW ->  dom0 ->  domU ->  4 time kvm guest
> >>>>
> >>>>dom0
> >>>>opensuse 11.4 x86
> >>>>xen: unstable cs 23553
> >>>>
> >>>>domu (HVM)
> >>>>sles11sp1
> >>>>mem: 8 GB
> >>>>vcpu: 4
> >>>>
> >>>>domu guests
> >>>>1) qemu-kvm: propetary OS
> >>>>2) qemu-kvm: propetary OS
> >>>>3) qemu-kvm: SLES10SP3
> >>>>4) qemu-kvm: SLES10SP3
> >>>>
> >>>>
> >>>>
> >>>>Op 15-6-2011 14:45, Tim Deegan schreef:
> >>>>>At 10:29 +0800 on 14 Jun (1308047377), Dong, Eddie wrote:
> >>>>>>>>+    case MSR_IA32_VMX_MISC:
> >>>>>>>>+    case MSR_IA32_VMX_CR0_FIXED0:
> >>>>>>>>+    case MSR_IA32_VMX_CR0_FIXED1:
> >>>>>>>>+    case MSR_IA32_VMX_CR4_FIXED0:
> >>>>>>>>+    case MSR_IA32_VMX_CR4_FIXED1:
> >>>>>>>>+    case MSR_IA32_VMX_VMCS_ENUM:
> >>>>>>>>+        gdprintk(XENLOG_WARNING, "VMX MSR %x not fully supported
> >>>>>>>yet.\n", msr);
> >>>>>>>
> >>>>>>>Are you planning to fix this before checking in this series?  I'm
> >>>>>>>pretty
> >>>>>>>sure that at least MSR_IA32_VMX_VMCS_ENUM should be
> >>>>>>>trivial, since you
> >>>>>>>define your own VMCS format.
> >>>>>>>
> >>>>>>Sure, the updated patch is attached.
> >>>>>Thanks.  I've applied the full series.  I cleaned up a few things for
> >>>>>the 32-bit build as well.
> >>>>>
> >>>>>Cheers,
> >>>>>
> >>>>>Tim.
> >
> >
> 
> 
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Jeroen Groenewegen van der Weyden]

You're testing on Intel hardware, right?

- Yes, the proc is a Xeon e5645. Two of them on a asus z8nr-d12 mobo.

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Jeroen Groenewegen van der Weyden]

Eddie, did I mention I'm eager and capable of testing anything you need.

mfg,
jeroen

Op 4-7-2011 10:58, Tim Deegan schreef:
> At 12:11 +0200 on 01 Jul (1309522280), Jeroen Groenewegen van der Weyden wrote:
>> Just a remark to avoid misunderstanding
>>
>> The first nested  KVM guest is running ok, for as long as i keep it running
>> When starting a second nested KVM guest (in the same Xen DOMU) that
>> is when the Xen DOMU becomes in-responsive. In DOM0 it it shown that
>> it is still running and consuming time.
>>
>>
>> The complete test is like this
>>
>> dom0
>> opensuse 11.4 x86
>> xen: unstable cs 23640
> You're testing on Intel hardware, right?
>
> Eddie, any comment?
>
> Tim.
>
>> domu1 (HVM)
>> sles11sp1
>> mem: 8 GB
>> vcpu: 4
>>
>> domu1 guests
>> 1) qemu-kvm: propetary OS
>> 2) qemu-kvm: propetary OS
>> 3) qemu-kvm: SLES10SP3
>> 4) qemu-kvm: SLES10SP3
>>
>> mfg,
>> Jeroen
>>
>> Op 1-7-2011 11:54, Christoph Egger schreef:
>>> KVM on Xen works for me. Also KVM on Xen on Xen.
>>>
>>> Christoph
>>>
>>>
>>> On 07/01/11 11:01, Tim Deegan wrote:
>>>> Eddie?  Did you test with KVM guests?  Is this expected to work?
>>>>
>>>> Tim.
>>>>
>>>> At 15:14 +0100 on 30 Jun (1309446872), Jeroen Groenewegen van
>>>> der Weyden wrote:
>>>>> Hi Tim/Eddi/Christoph,
>>>>>
>>>>> I just gave it a go again with cs23640. But the behaviour is still the
>>>>> same. so L1 becomes inresponsive after starting the second
>>>>> nested quest.
>>>>> I can deliver any trace input you want/need.
>>>>>
>>>>> mfg,
>>>>> Jeroen.
>>>>>
>>>>> Op 20-6-2011 8:31, Jeroen Groenewegen van der Weyden schreef:
>>>>>> Hi,
>>>>>>
>>>>>> Based on these patch series, I gave it go.
>>>>>> I noticed until now that the system becomes in-responsive a few second
>>>>>> after activating the second nested guest.
>>>>>> steps:
>>>>>> 1) dom0 creates domu (pass)
>>>>>> 2) domU creates first kvm guest (pass)
>>>>>> 3) domU creates second kvm guest (fail)
>>>>>> after the third step the domu becomes in-responsive. eg no network /
>>>>>> vnc console response. in dom0 is shows domu is stil running.
>>>>>>
>>>>>> I'm able to provide any input needed, just tell me what you need (and
>>>>>> how).
>>>>>>
>>>>>> mfg,
>>>>>> Jeroen
>>>>>>
>>>>>> My HW setup
>>>>>> mobo asus z8nr-d12
>>>>>> proc: 2 times e5645
>>>>>> mem: 96 GB
>>>>>>
>>>>>> My goal/test
>>>>>> in short: HW ->   dom0 ->   domU ->   4 time kvm guest
>>>>>>
>>>>>> dom0
>>>>>> opensuse 11.4 x86
>>>>>> xen: unstable cs 23553
>>>>>>
>>>>>> domu (HVM)
>>>>>> sles11sp1
>>>>>> mem: 8 GB
>>>>>> vcpu: 4
>>>>>>
>>>>>> domu guests
>>>>>> 1) qemu-kvm: propetary OS
>>>>>> 2) qemu-kvm: propetary OS
>>>>>> 3) qemu-kvm: SLES10SP3
>>>>>> 4) qemu-kvm: SLES10SP3
>>>>>>
>>>>>>
>>>>>>
>>>>>> Op 15-6-2011 14:45, Tim Deegan schreef:
>>>>>>> At 10:29 +0800 on 14 Jun (1308047377), Dong, Eddie wrote:
>>>>>>>>>> +    case MSR_IA32_VMX_MISC:
>>>>>>>>>> +    case MSR_IA32_VMX_CR0_FIXED0:
>>>>>>>>>> +    case MSR_IA32_VMX_CR0_FIXED1:
>>>>>>>>>> +    case MSR_IA32_VMX_CR4_FIXED0:
>>>>>>>>>> +    case MSR_IA32_VMX_CR4_FIXED1:
>>>>>>>>>> +    case MSR_IA32_VMX_VMCS_ENUM:
>>>>>>>>>> +        gdprintk(XENLOG_WARNING, "VMX MSR %x not fully supported
>>>>>>>>> yet.\n", msr);
>>>>>>>>>
>>>>>>>>> Are you planning to fix this before checking in this series?  I'm
>>>>>>>>> pretty
>>>>>>>>> sure that at least MSR_IA32_VMX_VMCS_ENUM should be
>>>>>>>>> trivial, since you
>>>>>>>>> define your own VMCS format.
>>>>>>>>>
>>>>>>>> Sure, the updated patch is attached.
>>>>>>> Thanks.  I've applied the full series.  I cleaned up a few things for
>>>>>>> the 32-bit build as well.
>>>>>>>
>>>>>>> Cheers,
>>>>>>>
>>>>>>> Tim.
>>>
>>
>> _______________________________________________
>> Xen-devel mailing list
>> Xen-devel@lists.xensource.com
>> http://lists.xensource.com/xen-devel

RE: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Dong, Eddie]

Jeroen:
	Sorry for later response.
	I did a double check locally, I was able to create 2 layer 2 guests. My environment is L1 guest: 64 bits Linux 2.6.25 + KVM-76. One L2 guest is 32 bits Linux, another one is 64 bits guest.
	For layer 1 guest, I am setting: 

hap=1
nestedhvm = 1

Thx, Eddie

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Jeroen Groenewegen van der Weyden]

Hi eddie, I use the same settings. however my setup becomes in-response 
after start of the second nested kvm quest. were does that leave us? 
Should I make some trace and sent it to you. I can give you access to my 
system as well if you want. just give me your thoughts on how to move 
forward.

mfg,
Jeroen.

Op 18-7-2011 17:41, Dong, Eddie schreef:
> Jeroen:
> 	Sorry for later response.
> 	I did a double check locally, I was able to create 2 layer 2 guests. My environment is L1 guest: 64 bits Linux 2.6.25 + KVM-76. One L2 guest is 32 bits Linux, another one is 64 bits guest.
> 	For layer 1 guest, I am setting:
>
> hap=1
> nestedhvm = 1
>
> Thx, Eddie
>
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel
>

RE: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Dong, Eddie]

Hi Jeroen:
	What L1 guest do u use? Xen or KVM? 
Thx, Eddie

> -----Original Message-----
> From: Jeroen Groenewegen van der Weyden [mailto:groen692@grosc.com]
> Sent: Tuesday, July 19, 2011 10:00 PM
> To: Dong, Eddie
> Cc: xen-devel@lists.xensource.com; Tim Deegan;
> Christoph.Egger@amd.com
> Subject: Re: [Xen-devel] [PATCH 20 of 20] n2 MSR handling and capability
> exposure
> 
> Hi eddie, I use the same settings. however my setup becomes in-response
> after start of the second nested kvm quest. were does that leave us?
> Should I make some trace and sent it to you. I can give you access to my
> system as well if you want. just give me your thoughts on how to move
> forward.
> 
> mfg,
> Jeroen.
> 
> Op 18-7-2011 17:41, Dong, Eddie schreef:
> > Jeroen:
> > 	Sorry for later response.
> > 	I did a double check locally, I was able to create 2 layer 2 guests. My
> environment is L1 guest: 64 bits Linux 2.6.25 + KVM-76. One L2 guest is 32
> bits Linux, another one is 64 bits guest.
> > 	For layer 1 guest, I am setting:
> >
> > hap=1
> > nestedhvm = 1
> >
> > Thx, Eddie
> >
> > _______________________________________________
> > Xen-devel mailing list
> > Xen-devel@lists.xensource.com
> > http://lists.xensource.com/xen-devel
> >

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Jeroen Groenewegen van der Weyden]

Eddie, Here my setup,

in short: HW ->    dom0 opensuse 11.4(xen-cs23728) ->   domU (HVM-sles11sp1) KVM ->    4 time kvm guest

dom0
opensuse 11.4 x86
kernel 2.6.37.6-0.5-xen
xen: unstable cs 23728

- domu (HVM)
- sles11sp1
- mem: 8 GB
- vcpu: 4
- kvm

--domu guests
--1) qemu-kvm: propetary OS
--2) qemu-kvm: propetary OS
--3) qemu-kvm: SLES10SP3
--4) qemu-kvm: SLES10SP3


mfg,
Jeroen

Op 20-7-2011 15:44, Dong, Eddie schreef:
> Hi Jeroen:
> 	What L1 guest do u use? Xen or KVM?
> Thx, Eddie
>
>> -----Original Message-----
>> From: Jeroen Groenewegen van der Weyden [mailto:groen692@grosc.com]
>> Sent: Tuesday, July 19, 2011 10:00 PM
>> To: Dong, Eddie
>> Cc: xen-devel@lists.xensource.com; Tim Deegan;
>> Christoph.Egger@amd.com
>> Subject: Re: [Xen-devel] [PATCH 20 of 20] n2 MSR handling and capability
>> exposure
>>
>> Hi eddie, I use the same settings. however my setup becomes in-response
>> after start of the second nested kvm quest. were does that leave us?
>> Should I make some trace and sent it to you. I can give you access to my
>> system as well if you want. just give me your thoughts on how to move
>> forward.
>>
>> mfg,
>> Jeroen.
>>
>> Op 18-7-2011 17:41, Dong, Eddie schreef:
>>> Jeroen:
>>> 	Sorry for later response.
>>> 	I did a double check locally, I was able to create 2 layer 2 guests. My
>> environment is L1 guest: 64 bits Linux 2.6.25 + KVM-76. One L2 guest is 32
>> bits Linux, another one is 64 bits guest.
>>> 	For layer 1 guest, I am setting:
>>>
>>> hap=1
>>> nestedhvm = 1
>>>
>>> Thx, Eddie
>>>
>>> _______________________________________________
>>> Xen-devel mailing list
>>> Xen-devel@lists.xensource.com
>>> http://lists.xensource.com/xen-devel
>>>
>

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Tim Deegan]

Hi, 


At 16:12 +0200 on 20 Jul (1311178333), Jeroen Groenewegen van der Weyden wrote:
> in short: HW ->    dom0 opensuse 11.4(xen-cs23728) ->   domU (HVM-sles11sp1) KVM ->    4 time kvm guest
> 
> dom0
> opensuse 11.4 x86
> kernel 2.6.37.6-0.5-xen
> xen: unstable cs 23728
> 
> - domu (HVM)
> - sles11sp1
> - mem: 8 GB
> - vcpu: 4
> - kvm

FWIW, I can reproduce this with a Debian 2.6.32-5-686 n1 guest on
current unstable tip.  Running two copies of 'kvm' inside that
(i.e. simple n2s without any disks) I see this on the n0 console:

(XEN) vvmx.c:1181:d1 vmclear gpa 3661d000 not the same as current vmcs 0000000036459000
(XEN) vvmx.c:1181:d1 vmclear gpa 36459000 not the same as current vmcs 000000003661d000

and the n1 guest locks up using 100% cpu on one of its vcpus. 

Reducing the n1 guest to only 1 VCPU works around the issue, though I
still see one of those complaints about vmclear, so that might be a red
herring.

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Tim Deegan]

[-- Attachment #1: Type: text/plain, Size: 1290 bytes --]

Hi, 

At 15:08 +0100 on 25 Jul (1311606523), Tim Deegan wrote:
> FWIW, I can reproduce this with a Debian 2.6.32-5-686 n1 guest on
> current unstable tip.  Running two copies of 'kvm' inside that
> (i.e. simple n2s without any disks) I see this on the n0 console:
> 
> (XEN) vvmx.c:1181:d1 vmclear gpa 3661d000 not the same as current vmcs 0000000036459000
> (XEN) vvmx.c:1181:d1 vmclear gpa 36459000 not the same as current vmcs 000000003661d000
> 
> and the n1 guest locks up using 100% cpu on one of its vcpus. 

AFAICS when KVM has two VMs sharing a CPU, it just switches between them
with VMPTRLD, rather than VMCLEARing the current one on every context
switch.  When it migrates one of them away, it VMCLEARs it, even if it's
not the most recently loaded VMCS.

Xen's emulation of VMCLEAR doesn't clear the 'launched' bit in this
case, though the SDM says it should.  The attached patch fixes the hang
for me, but has had only very light testing (i.e. I haven't checked that
proper OSes running inside the KVM VMs behave correctly).

Eddie, does this look right to you?

Jeroen, can you try it and see if it fixes your problems?

Cheers,

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

[-- Attachment #2: vmclear --]
[-- Type: text/plain, Size: 1232 bytes --]

Nested VMX: always mark VVMCS as not-launched on VMCLEAR.

The SDM says to flush changes and clear the launch state even if this
isn't the "current VMCS", and KVM seems to rely on this behaviour.

Signed-off-by: Tim Deegan <Tim.Deegan@citrix.com>

diff -r 9dbbf1631193 xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Mon Jul 25 14:21:13 2011 +0100
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Mon Jul 25 17:16:01 2011 +0100
@@ -1162,6 +1162,7 @@ int nvmx_handle_vmclear(struct cpu_user_
     struct vmx_inst_decoded decode;
     struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
     unsigned long gpa = 0;
+    void *vvmcs;
     int rc;
 
     rc = decode_vmx_inst(regs, &decode, &gpa, 0);
@@ -1179,6 +1180,13 @@ int nvmx_handle_vmclear(struct cpu_user_
         gdprintk(XENLOG_WARNING, 
                  "vmclear gpa %lx not the same as current vmcs %"PRIpaddr"\n",
                  gpa, nvcpu->nv_vvmcxaddr);
+
+        /* Even if this VMCS isn't the current one, we must clear it. */
+        vvmcs = hvm_map_guest_frame_rw(gpa >> PAGE_SHIFT);
+        if ( vvmcs ) 
+            __set_vvmcs(vvmcs, NVMX_LAUNCH_STATE, 0);
+        hvm_unmap_guest_frame(vvmcs);
+
         vmreturn(regs, VMSUCCEED);
         goto out;
     }

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Jeroen Groenewegen van der Weyden]

Hi Tim,

I think the behaviour is still the same,

1) cs23726
2) vvmc.c patched with attachment.
3) new compile

after a little while the domu becomes ir-responsive.

with xm dmesg I see a lot of these:
(XEN) vvmx.c:1182:d3 vmclear gpa 1f5a89000 not the same as current vmcs 
00000001f448f000
(XEN) vvmx.c:1182:d3 vmclear gpa 1f5a89000 not the same as current vmcs 
00000001f448f000


Note: I have to say, patching this on this level is not common practice 
for me. although I think I did it the right way. please keep in mind I 
can make mistakes on this level.

mfg,
Jeroen.

Op 25-7-2011 18:16, Tim Deegan schreef:
> Hi,
>
> At 15:08 +0100 on 25 Jul (1311606523), Tim Deegan wrote:
>> FWIW, I can reproduce this with a Debian 2.6.32-5-686 n1 guest on
>> current unstable tip.  Running two copies of 'kvm' inside that
>> (i.e. simple n2s without any disks) I see this on the n0 console:
>>
>> (XEN) vvmx.c:1181:d1 vmclear gpa 3661d000 not the same as current vmcs 0000000036459000
>> (XEN) vvmx.c:1181:d1 vmclear gpa 36459000 not the same as current vmcs 000000003661d000
>>
>> and the n1 guest locks up using 100% cpu on one of its vcpus.
> AFAICS when KVM has two VMs sharing a CPU, it just switches between them
> with VMPTRLD, rather than VMCLEARing the current one on every context
> switch.  When it migrates one of them away, it VMCLEARs it, even if it's
> not the most recently loaded VMCS.
>
> Xen's emulation of VMCLEAR doesn't clear the 'launched' bit in this
> case, though the SDM says it should.  The attached patch fixes the hang
> for me, but has had only very light testing (i.e. I haven't checked that
> proper OSes running inside the KVM VMs behave correctly).
>
> Eddie, does this look right to you?
>
> Jeroen, can you try it and see if it fixes your problems?
>
> Cheers,
>
> Tim.
>

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Tim Deegan]

Hi, 

At 09:15 +0200 on 26 Jul (1311671730), Jeroen Groenewegen van der Weyden wrote:
> I think the behaviour is still the same,
> 
> 1) cs23726
> 2) vvmc.c patched with attachment.
> 3) new compile
> 
> after a little while the domu becomes ir-responsive.

Dang. :(

> with xm dmesg I see a lot of these:
> (XEN) vvmx.c:1182:d3 vmclear gpa 1f5a89000 not the same as current
> vmcs 00000001f448f000
> (XEN) vvmx.c:1182:d3 vmclear gpa 1f5a89000 not the same as current
> vmcs 00000001f448f000

Yeah; with the patch applied, those should be harmlesss.

If you give your first-level guest only one vcpu, does the problem go
away?

> Note: I have to say, patching this on this level is not common
> practice for me. although I think I did it the right way. please
> keep in mind I can make mistakes on this level.

If you want to double-check that you've done the patch right,
edit xen/arch/x86/hvm/vmx/vvmx.c, and at line 1185, just under the line
` /* Even if this VMCS isn't the current one, we must clear it. */ '
add a line ` printk("boo!\n"); '.  Then when you recompile and test you
should see "boo!" printed just after each "vvmx.c:1182:d3" line on the
console.

Cheers,

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Tim Deegan]

At 11:00 +0100 on 26 Jul (1311678018), Tim Deegan wrote:
> If you want to double-check that you've done the patch right,
> edit xen/arch/x86/hvm/vmx/vvmx.c, and at line 1185, just under the line
> ` /* Even if this VMCS isn't the current one, we must clear it. */ '
> add a line ` printk("boo!\n"); '.  Then when you recompile and test you
> should see "boo!" printed just after each "vvmx.c:1182:d3" line on the
> console.

Oh, and when the domU is hung, can you run the command 
/usr/lib/xen/bin/xenctx -aCS 1 
(replacing '1' with the domid of your domU) three or four times 
and post the output here?  On a 64-bit dom0 I think the correct path
will be /usr/lib64/xen/bin/xenctx.

Cheers,

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Jeroen Groenewegen van der Weyden]

[-- Attachment #1: Type: text/plain, Size: 789 bytes --]

Here my input


mfg,
jeroen

Op 26-7-2011 12:11, Tim Deegan schreef:
> At 11:00 +0100 on 26 Jul (1311678018), Tim Deegan wrote:
>> If you want to double-check that you've done the patch right,
>> edit xen/arch/x86/hvm/vmx/vvmx.c, and at line 1185, just under the line
>> ` /* Even if this VMCS isn't the current one, we must clear it. */ '
>> add a line ` printk("boo!\n"); '.  Then when you recompile and test you
>> should see "boo!" printed just after each "vvmx.c:1182:d3" line on the
>> console.
> Oh, and when the domU is hung, can you run the command
> /usr/lib/xen/bin/xenctx -aCS 1
> (replacing '1' with the domid of your domU) three or four times
> and post the output here?  On a 64-bit dom0 I think the correct path
> will be /usr/lib64/xen/bin/xenctx.
>
> Cheers,
>
> Tim.
>


[-- Attachment #2: 1.xenctx --]
[-- Type: text/plain, Size: 11856 bytes --]

rip: ffffffff810784d0 
flags: 00000202 i nz
rsp: ffff8801f17c9ce8
rax: 000000000000000f	rcx: 000000000000000f	rdx: 0000000000000000
rbx: ffff88001146ffc0	rsi: 000000000000000f	rdi: 0000000000000000
rbp: ffff8801f17c9d28	 r8: 000000000000000f	 r9: ffffffff8140bba0
r10: 0000000000000000	r11: ffffffff8101f2b0	r12: ffff88001146ffd0
r13: ffff88001146ffc0	r14: 0000000000000003	r15: 0000000000000001
 cs: 0010	 ss: 0018	 ds: 002b	 es: 002b
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 00000000
cr3: 1f572c000
cr4: 000026f0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffff810784d0)
39 eb 74 41 45 85 ff 75 0e 0f 1f 40 00 eb 0e 66 0f 1f 44 00 00 <f3> 90 f6 45 20 01 75 f8 48 8b 5c 


Stack:
 0000000000013700 0000000000000000 0000000000000003 ffffffff81927680
 ffff8801f5698100 ffffffffa019e920 0000000000000001 ffffffff810786f3
 ffff88001146ffc0 ffff88001146ffc0 ffffffffa019e920 ffff8801f5698100
 0000000000000001 ffff8801f15e2280 ffff8801f17c9dd8 ffff8801f5698100

Stack Trace:
* [<ffffffff810784d0>]  <--
    0000000000013700
    0000000000000000
    0000000000000003
  [<ffffffff81927680>] 
    ffff8801f5698100
  [<ffffffffa019e920>] 
    0000000000000001
  [<ffffffff810786f3>] 
    ffff88001146ffc0
    ffff88001146ffc0
  [<ffffffffa019e920>] 
    ffff8801f5698100
    0000000000000001
    ffff8801f15e2280
    ffff8801f17c9dd8
    ffff8801f5698100
    0000000000000000
    00007fc30a713c90
    ffff8801f5698100
    000000004004ae86
    0000000000c76ad0
  [<ffffffffa019fcf4>] 
    0000000000000286
    ffff8801f17c9e10
    ffff8801f17c9de0
    00000001f5713000
    ffff8801f5698100
    ffff8801f5698100
    ffff8801f5698100
  [<ffffffffa0377e7e>] 
  [<fffffffffffffffb>] 
  [<ffffffffa037cdb5>] 
    ffff88010000007b
    ffff8801f17c9df0
    ffff8801f17c9df0
    ffff8801f15e2280
    0000000000000000
    000000000086b000
    ffff880203941040
    0000000000000b00
    0000000000000000
    0000000000000000
    0000000000000000
  [<fffffffffffffffb>] 
    ffff8801f5698100
    00007fc30a713c90
    000000000000000f
  [<ffffffffa03736a6>] 
    ffff8801f17c9ea8
  [<ffffffff8105aacc>] 
    ffff8801f15e2280
    00007fc30a713ce0
    00007fc30a713c20
  [<ffffffff8105a0bd>] 
    00000000000000fa
  [<ffffffff8105ad7d>] 
    0000000000000026
  [<ffffffff81ad9a28>] 
    0000000000003320
    0000000000000000
    ffff88020377d900
    00007fc30a713c90
    000000004004ae86
    000000000000000f
    000000004004ae86
  [<ffffffff8110c63f>] 
    ffff88020377d900
    00007fc30a713c90
    00007fc30a713c90
  [<ffffffff8110cacb>] 
    0000000200c77c74
    000000000086bb00
    0000000000000000
    ffff88020377d900
    00007fc30a713c90
  [<ffffffff8110cbe1>] 
    00007fc310ccc000
    00000001810595c9
    0000000000ca1ec0
    000000000000007b
    0000000000ca1ec0
    00007fc30a713fa0
    00007fc310ccb000
  [<ffffffff81002f7b>] 
    0000000000000246
    0000000000000000
    0000000000000000
    00000000ffffff80
    0000000000000010
    00007fc30e85abd7
    00007fc30a713c90
    000000004004ae86
    000000000000000f
    0000000000000010
    00007fc30e85abd7
    0000000000000033
    0000000000000246
    00007fc30a713ca0
    000000000000002b
rip: ffffffffa01a1c9d 
flags: 00000042 z
rsp: ffff8801f20ddc90
rax: 0000000000000000	rcx: 00000000c0332008	rdx: 00000000c0332000
rbx: 00000000ffffffff	rsi: 00000000c0352433	rdi: 00000000c1602593
rbp: 00000000c02a53ec	 r8: 0000000000000000	 r9: 0000000000000000
r10: 0000000000000000	r11: 0000000000000000	r12: 0000000000000000
r13: 0000000000000000	r14: 0000000000000000	r15: 0000000000000000
 cs: 0010	 ss: 0018	 ds: 002b	 es: 002b
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 080b0028
cr3: 1f2030000
cr4: 000026e0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffffa01a1c9d)
b1 d8 01 00 00 4c 8b b9 e0 01 00 00 48 8b 89 70 01 00 00 75 05 <0f> 01 c2 eb 03 0f 01 c3 48 87 0c 


Stack:
 ffff8801f14bc0c0 ffff8801f14bc0c0 0000000000006c14 0000000000000000
 ffff8801f14bc2b4 0000000000013700 ffffffff811e2fd0 ffff8802056853d0
 ffffffff811e2fd0 ffff880203ad0870 0000000000000000 0000000000000001
 ffff880203ad0870 0000000000000001 ffffffff81066f81 fffffffffffffffb

Stack Trace:
* [<ffffffffa01a1c9d>]  <--
    ffff8801f14bc0c0
    ffff8801f14bc0c0
    0000000000006c14
    0000000000000000
    ffff8801f14bc2b4
    0000000000013700
  [<ffffffff811e2fd0>] 
    ffff8802056853d0
  [<ffffffff811e2fd0>] 
    ffff880203ad0870
    0000000000000000
    0000000000000001
    ffff880203ad0870
    0000000000000001
  [<ffffffff81066f81>] 
  [<fffffffffffffffb>] 
    ffff88001142fe00
    0000000000000282
  [<ffffffff81067cdf>] 
    0000000000000001
    000000000000fdc0
    000000000000fdc0
    ffff8802056853c0
  [<ffffffffa038e898>] 
    0000000000000070
  [<ffffffffa038ed00>] 
  [<fffffffffffffffb>] 
    ffff8801f14bc0c0
  [<fffffffffffffffb>] 
    ffff8801f14bc0c0
    ffff8801f14bc118
    0000000000000000
    ffff8801f147a000
    ffff8801f14bc119
  [<ffffffffa0380866>] 
    ffff8801f21281c0
    ffff8801f20dc000
    0000000000014310
    ffff8801f20ddfd8
    ffff8801f5748080
    ffff8801f5748080
    ffff8801f5748080
    ffff8801f5748080
    ffff8801f20ddfd8
    ffff8801f14bd750
    ffff8801f14bd748
    00ffffff8107454f
    ffff8801dea9ddd8
  [<ffffffff81ad7370>] 
    fffffffe7ffbfeff
  [<ffffffff81074a05>] 
    ffff8801f20dc000
  [<fffffffffffffffb>] 
    ffff8801f14bc0c0
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000c77950
  [<ffffffffa037385a>] 
    0000000000000000
    ffff8801f54a2538
    ffff880011473700
    ffff8801f20ddec0
  [<ffffffff8103fe03>] 
    ffff8801f7970ac0
    ffff8801f7970ac0
    ffff880011473700
    ffff8801f7970ac0
    ffff880011242f78
  [<ffffffff811245c0>] 
    ffff8801f20ddf70
    ffff880202d95bc0
    0000000000000000
    000000000000ae80
    0000000000000010
    000000000000ae80
  [<ffffffff8110c63f>] 
    ffff880202d95bc0
    0000000000000000
    0000000000000000
  [<ffffffff8110cacb>] 
    0000000000013700
    0000000000013700
    0000000000000000
    ffff880202d95bc0
    0000000000000000
  [<ffffffff8110cbe1>] 
    0000000001123d20
    0000000181003d75
    00007fd0b975a000
    0000000000000000
    0000000000cb2bb0
    0000000000000001
    00007fd0b9759000
  [<ffffffff81002f7b>] 
    0000000000000246
    0000000000000000
    0000000000100000
    000000000086bb00
    0000000000000010
    0000000000000000
    0000000000000000
    000000000000ae80
    0000000000000010
    0000000000000010
    00007fd0b731dbd7
    0000000000000033
    0000000000010202
    00007fd0b462a678
    000000000000002b
rip: ffffffff8100aef2 
flags: 00000246 i z p
rsp: ffff8802073dbf30
rax: ffff8802073dbfd8	rcx: 00000000ffffffff	rdx: 0000000000000000
rbx: ffff8802073dbfd8	rsi: 0000000000000001	rdi: ffffffff81a2e308
rbp: ffffffff81927680	 r8: 0000000000000000	 r9: 0000000000000000
r10: 0000000000000000	r11: 00000000000003ff	r12: 0000000000000000
r13: 0000000000000000	r14: 0000000000000000	r15: 0000000000000000
 cs: 0010	 ss: 0018	 ds: 0018	 es: 0018
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 00000000
cr3: 1f5751000
cr4: 000026f0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffff8100aef2)
65 48 8b 04 25 88 b5 00 00 83 88 3c e0 ff ff 04 c3 66 90 fb f4 <eb> e9 66 66 66 2e 0f 1f 84 00 00 


Stack:
 ffffffff8100205a 0000000000000018 0000000000000000 0000000000000000
 0000000000000000 0000000000000000 0000000000000000 0000000000000000
 0000000000000000 0000000000000000 0000000000000000 0000000000000000
 0000000000000000 0000000000000000 0000000000000000 0000000000000000

Stack Trace:
* [<ffffffff8100aef2>]  <--
  [<ffffffff8100205a>] 
    0000000000000018
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
rip: ffffffffa01a1c9d 
flags: 00000042 z
rsp: ffff8801f2151c90
rax: 0000000000000000	rcx: 00000000c0332008	rdx: 00000000c0332000
rbx: 00000000ffffffff	rsi: 00000000c035245a	rdi: 00000000c14015ba
rbp: 00000000c02a53ec	 r8: 0000000000000000	 r9: 0000000000000000
r10: 0000000000000000	r11: 0000000000000000	r12: 0000000000000000
r13: 0000000000000000	r14: 0000000000000000	r15: 0000000000000000
 cs: 0010	 ss: 0018	 ds: 002b	 es: 002b
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 0807f000
cr3: 1f23fa000
cr4: 000026e0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffffa01a1c9d)
b1 d8 01 00 00 4c 8b b9 e0 01 00 00 48 8b 89 70 01 00 00 75 05 <0f> 01 c2 eb 03 0f 01 c3 48 87 0c 


Stack:
 ffff8801f1600080 ffff8801f1600080 0000000000006c14 0000000000000003
 ffff8801f1600274 00000000ffffffff ffffffffa0378937 ffff8802056854d0
 0000000000000000 ffff88020513d670 0000000000000000 0000000000000003
 ffff88020513d670 0000000000000003 ffffffff81066f81 ffff8801f2151e08

Stack Trace:
* [<ffffffffa01a1c9d>]  <--
    ffff8801f1600080
    ffff8801f1600080
    0000000000006c14
    0000000000000003
    ffff8801f1600274
    00000000ffffffff
  [<ffffffffa0378937>] 
    ffff8802056854d0
    0000000000000000
    ffff88020513d670
    0000000000000000
    0000000000000003
    ffff88020513d670
    0000000000000003
  [<ffffffff81066f81>] 
    ffff8801f2151e08
    ffff88001146fe00
    0000000000000282
  [<ffffffff81067cdf>] 
    0000000000000003
    000000000000fdc0
    000000000000fdc0
    ffff8802056854c0
  [<ffffffffa038e898>] 
    0000000000000010
  [<ffffffffa038ed00>] 
    ffff8801f2151e08
    ffff8801f1600080
    ffff8801f2151e08
    ffff8801f1600080
    ffff8801f16000d8
    0000000000000000
    ffff8801f15f0000
    ffff8801f16000d9
  [<ffffffffa0380866>] 
    ffff8801f154c3c0
    ffff8801f2150000
    0000000000014310
    ffff8801f2151fd8
    ffff8801f154c1c0
    ffff8801f154c1c0
    ffff8801f154c1c0
    ffff8801f154c1c0
    ffff8801f2151fd8
    ffff8801f1601710
    ffff8801f1601708
    00ff8801f154c1c0
    00000000000000ab
    0000000007e5df6d
    fffffffe7ffbfeff
    0000000000000b00
    0000000000000000
  [<fffffffffffffffb>] 
    ffff8801f1600080
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000c77910
  [<ffffffffa037385a>] 
    0000000000000000
    ffff8801f54ae5f8
    ffff880011453700
    ffff8801f2151ec0
  [<ffffffff8103fe03>] 
    ffff880205753040
    ffff880205753040
    ffff880011453700
    ffff880203941040
    0000000000000304
    ffff8801de86fac8
    ffff8801f2151f70
    ffff8801f7a6d140
    0000000000000000
    000000000000ae80
    0000000000000010
    000000000000ae80
  [<ffffffff8110c63f>] 
    ffff8801f7a6d140
    0000000000000000
    0000000000000000
  [<ffffffff8110cacb>] 
    0000000000013700
    0000000000013700
    0000000000000000
    ffff8801f7a6d140
    0000000000000000
  [<ffffffff8110cbe1>] 
    00007f4c669d2000
    0000000100cb2b70
    00007f4c669d2000
    0000000000000000
    0000000000cb2b70
    0000000000000001
    00007f4c669d1000
  [<ffffffff81002f7b>] 
    0000000000000246
    0000000000000001
    0000000000002cf3
    000000000086bb00
    0000000000000010
    00007f4c64595bd7
    0000000000000000
    000000000000ae80
    0000000000000010
    0000000000000010
    00007f4c64595bd7
    0000000000000033
    0000000000000202
    00007f4c618a2cb0
    000000000000002b

[-- Attachment #3: 2.xenctx --]
[-- Type: text/plain, Size: 11731 bytes --]

rip: ffffffff8101f0db 
flags: 00000083 s nz c
rsp: ffff880011403ee0
rax: ffffffff818120e0	rcx: 0000000000000020	rdx: 00000000003ce63b
rbx: 0000000000000000	rsi: 0000000000006170	rdi: ffffffffff5fb380
rbp: 0000004130d2de00	 r8: ffff88001140cdc0	 r9: 0000000000015ab9
r10: ffff88001140fe18	r11: ffffffff8101a450	r12: ffff88001140cdc0
r13: 0000004130d2de01	r14: ffffffff8155dca5	r15: ffff880011403f48
 cs: 0010	 ss: 0018	 ds: 002b	 es: 002b
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 00000000
cr3: 1f572c000
cr4: 000026f0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffff8101f0db)
90 90 90 90 90 90 90 90 90 90 89 ff 48 81 ef 00 50 a0 00 89 37 <c3> 0f 1f 40 00 89 ff 8b 87 00 b0 


Stack:
 ffffffff8101a468 ffffffff8155dca5 ffffffff810725a6 0000000000000080
 ffff88001140fdc8 ffff88001140fdc0 0000004130d2de00 0000004130d2de00
 ffffffff810673eb ffff88001146ffd0 000000413095f057 000000413095f057
 0000000000000001 000000413095f057 ffff880011403f58 0000000000000046

Stack Trace:
* [<ffffffff8101f0db>]  <--
  [<ffffffff8101a468>] 
  [<ffffffff8155dca5>] 
  [<ffffffff810725a6>] 
    0000000000000080
    ffff88001140fdc8
    ffff88001140fdc0
    0000004130d2de00
    0000004130d2de00
  [<ffffffff810673eb>] 
    ffff88001146ffd0
    000000413095f057
    000000413095f057
    0000000000000001
    000000413095f057
    ffff880011403f58
    0000000000000046
    ffff88001140cdc0
    0000000000000000
    ffff88001146ffd0
    ffff88001146ffc0
    0000000000000003
    0000000000000001
  [<ffffffff8101b077>] 
    ffff8801f17c9c60
    ffff88001146ffc0
    ffff8801f17c9c60
  [<ffffffff81003a93>] 
    ffff8801f17c9c60
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
rip: ffffffffa01a1c9d 
flags: 00000042 z
rsp: ffff8801f20ddc90
rax: 0000000000000000	rcx: 00000000c0332008	rdx: 00000000c0332000
rbx: 00000000ffffffff	rsi: 00000000c0352433	rdi: 00000000c1602593
rbp: 00000000c02a53ec	 r8: 0000000000000000	 r9: 0000000000000000
r10: 0000000000000000	r11: 0000000000000000	r12: 0000000000000000
r13: 0000000000000000	r14: 0000000000000000	r15: 0000000000000000
 cs: 0010	 ss: 0018	 ds: 002b	 es: 002b
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 080b0028
cr3: 1f2030000
cr4: 000026e0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffffa01a1c9d)
b1 d8 01 00 00 4c 8b b9 e0 01 00 00 48 8b 89 70 01 00 00 75 05 <0f> 01 c2 eb 03 0f 01 c3 48 87 0c 


Stack:
 ffff8801f14bc0c0 ffff8801f14bc0c0 0000000000006c14 0000000000000000
 ffff8801f14bc2b4 0000000000013700 ffffffff811e2fd0 ffff8802056853d0
 ffffffff811e2fd0 ffff880203ad0870 0000000000000000 0000000000000001
 ffff880203ad0870 0000000000000001 ffffffff81066f81 fffffffffffffffb

Stack Trace:
* [<ffffffffa01a1c9d>]  <--
    ffff8801f14bc0c0
    ffff8801f14bc0c0
    0000000000006c14
    0000000000000000
    ffff8801f14bc2b4
    0000000000013700
  [<ffffffff811e2fd0>] 
    ffff8802056853d0
  [<ffffffff811e2fd0>] 
    ffff880203ad0870
    0000000000000000
    0000000000000001
    ffff880203ad0870
    0000000000000001
  [<ffffffff81066f81>] 
  [<fffffffffffffffb>] 
    ffff88001142fe00
    0000000000000282
  [<ffffffff81067cdf>] 
    0000000000000001
    000000000000fdc0
    000000000000fdc0
    ffff8802056853c0
  [<ffffffffa038e898>] 
    0000000000000070
  [<ffffffffa038ed00>] 
  [<fffffffffffffffb>] 
    ffff8801f14bc0c0
  [<fffffffffffffffb>] 
    ffff8801f14bc0c0
    ffff8801f14bc118
    0000000000000000
    ffff8801f147a000
    ffff8801f14bc119
  [<ffffffffa0380866>] 
    ffff8801f21281c0
    ffff8801f20dc000
    0000000000014310
    ffff8801f20ddfd8
    ffff8801f5748080
    ffff8801f5748080
    ffff8801f5748080
    ffff8801f5748080
    ffff8801f20ddfd8
    ffff8801f14bd750
    ffff8801f14bd748
    00ffffff8107454f
    ffff8801dea9ddd8
  [<ffffffff81ad7370>] 
    fffffffe7ffbfeff
  [<ffffffff81074a05>] 
    ffff8801f20dc000
  [<fffffffffffffffb>] 
    ffff8801f14bc0c0
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000c77950
  [<ffffffffa037385a>] 
    0000000000000000
    ffff8801f54a2538
    ffff880011473700
    ffff8801f20ddec0
  [<ffffffff8103fe03>] 
    ffff8801f7970ac0
    ffff8801f7970ac0
    ffff880011473700
    ffff8801f7970ac0
    ffff880011242f78
  [<ffffffff811245c0>] 
    ffff8801f20ddf70
    ffff880202d95bc0
    0000000000000000
    000000000000ae80
    0000000000000010
    000000000000ae80
  [<ffffffff8110c63f>] 
    ffff880202d95bc0
    0000000000000000
    0000000000000000
  [<ffffffff8110cacb>] 
    0000000000013700
    0000000000013700
    0000000000000000
    ffff880202d95bc0
    0000000000000000
  [<ffffffff8110cbe1>] 
    0000000001123d20
    0000000181003d75
    00007fd0b975a000
    0000000000000000
    0000000000cb2bb0
    0000000000000001
    00007fd0b9759000
  [<ffffffff81002f7b>] 
    0000000000000246
    0000000000000000
    0000000000100000
    000000000086bb00
    0000000000000010
    0000000000000000
    0000000000000000
    000000000000ae80
    0000000000000010
    0000000000000010
    00007fd0b731dbd7
    0000000000000033
    0000000000010202
    00007fd0b462a678
    000000000000002b
rip: ffffffffa01a0b02 
flags: 00000202 i nz
rsp: ffff8801f5657d68
rax: 0000000000000001	rcx: 00000000000004a1	rdx: 000000000000440c
rbx: ffff8801f57c0040	rsi: ffff8801f207f000	rdi: ffff8801f57c0040
rbp: ffff8801f57c0230	 r8: 0000000000000000	 r9: 0000000000000000
r10: 0000000000000000	r11: ffffffffa01a0b60	r12: ffff8801f207f000
r13: 0000000000000001	r14: ffff8801f207f000	r15: ffff8801f57c0099
 cs: 0010	 ss: 0018	 ds: 002b	 es: 002b
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 80050033
cr2: 00000000
cr3: 1f5751000
cr4: 000026f0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffffa01a0b02)
ff 90 08 01 00 00 48 8b 8b e8 01 00 00 ba 0c 44 00 00 0f 78 d0 <89> c0 48 01 c8 48 89 83 e8 01 00 


Stack:
 880011444000007f 0000000011000008 ffff8801f57c0040 ffffffffa01a0bc5
 0000000000000000 ffff8801f57c0040 ffff8801f57c0098 ffffffffa03808d3
 0000000000013700 ffff8801f5656000 0000000000014310 ffff8801f5657fd8
 ffff8801f550c600 ffff8801f550c600 ffff8801f550c600 ffff8801f550c600

Stack Trace:
* [<ffffffffa01a0b02>]  <--
    880011444000007f
    0000000011000008
    ffff8801f57c0040
  [<ffffffffa01a0bc5>] 
    0000000000000000
    ffff8801f57c0040
    ffff8801f57c0098
  [<ffffffffa03808d3>] 
    0000000000013700
    ffff8801f5656000
    0000000000014310
    ffff8801f5657fd8
    ffff8801f550c600
    ffff8801f550c600
    ffff8801f550c600
    ffff8801f550c600
    ffff8801f5657fd8
    ffff8801f57c16d0
    ffff8801f57c16c8
    0000000000000096
    0000000000000000
  [<ffffffff81817860>] 
    fffffffe7ffbfeff
    ffff8801f550cd90
    0000000000000026
  [<fffffffffffffffb>] 
    ffff8801f57c0040
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000c76840
  [<ffffffffa037385a>] 
    ffff8801f5657ea8
  [<ffffffff8105aacc>] 
    ffff8801f550c600
    00007ff18675bce0
    00007ff18675bc20
  [<ffffffff8105a0bd>] 
    0000000000000000
  [<ffffffff8105ad7d>] 
    0000000000000026
    00000000fffffffa
    0000000000002b8d
    0000000000000200
    ffff8801f91a5cc0
    0000000000000000
    000000000000ae80
    000000000000000f
    000000000000ae80
  [<ffffffff8110c63f>] 
    ffff8801f91a5cc0
    0000000000000000
    0000000000000000
  [<ffffffff8110cacb>] 
    00007fff874c4380
  [<ffffffff810fd6d0>] 
    0000000000000000
    ffff8801f91a5cc0
    0000000000000000
  [<ffffffff8110cbe1>] 
    0000000000000000
    00000001810595c9
    00007ff18c513000
    0000000000000000
    0000000000c904f0
    0000000000000001
    00007ff18c512000
  [<ffffffff81002f7b>] 
    0000000000000246
    0000000000000000
    000000002158fe01
    000000000086bb00
    0000000000000010
    00000000000000aa
    0000000000000000
    000000000000ae80
    000000000000000f
    0000000000000010
    00007ff18a0a1bd7
    0000000000000033
    0000000000000207
    00007ff18675bb20
    000000000000002b
rip: ffffffffa01a1c9d 
flags: 00000042 z
rsp: ffff8801f2151c90
rax: 0000000000000000	rcx: 00000000c0332008	rdx: 00000000c0332000
rbx: 00000000ffffffff	rsi: 00000000c035245a	rdi: 00000000c14015ba
rbp: 00000000c02a53ec	 r8: 0000000000000000	 r9: 0000000000000000
r10: 0000000000000000	r11: 0000000000000000	r12: 0000000000000000
r13: 0000000000000000	r14: 0000000000000000	r15: 0000000000000000
 cs: 0010	 ss: 0018	 ds: 002b	 es: 002b
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 0807f000
cr3: 1f23fa000
cr4: 000026e0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffffa01a1c9d)
b1 d8 01 00 00 4c 8b b9 e0 01 00 00 48 8b 89 70 01 00 00 75 05 <0f> 01 c2 eb 03 0f 01 c3 48 87 0c 


Stack:
 ffff8801f1600080 ffff8801f1600080 0000000000006c14 0000000000000003
 ffff8801f1600274 00000000ffffffff ffffffffa0378937 ffff8802056854d0
 0000000000000000 ffff88020513d670 0000000000000000 0000000000000003
 ffff88020513d670 0000000000000003 ffffffff81066f81 ffff8801f2151e08

Stack Trace:
* [<ffffffffa01a1c9d>]  <--
    ffff8801f1600080
    ffff8801f1600080
    0000000000006c14
    0000000000000003
    ffff8801f1600274
    00000000ffffffff
  [<ffffffffa0378937>] 
    ffff8802056854d0
    0000000000000000
    ffff88020513d670
    0000000000000000
    0000000000000003
    ffff88020513d670
    0000000000000003
  [<ffffffff81066f81>] 
    ffff8801f2151e08
    ffff88001146fe00
    0000000000000282
  [<ffffffff81067cdf>] 
    0000000000000003
    000000000000fdc0
    000000000000fdc0
    ffff8802056854c0
  [<ffffffffa038e898>] 
    0000000000000010
  [<ffffffffa038ed00>] 
    ffff8801f2151e08
    ffff8801f1600080
    ffff8801f2151e08
    ffff8801f1600080
    ffff8801f16000d8
    0000000000000000
    ffff8801f15f0000
    ffff8801f16000d9
  [<ffffffffa0380866>] 
    ffff8801f154c3c0
    ffff8801f2150000
    0000000000014310
    ffff8801f2151fd8
    ffff8801f154c1c0
    ffff8801f154c1c0
    ffff8801f154c1c0
    ffff8801f154c1c0
    ffff8801f2151fd8
    ffff8801f1601710
    ffff8801f1601708
    00ff8801f154c1c0
    00000000000000ab
    0000000007e5df6d
    fffffffe7ffbfeff
    0000000000000b00
    0000000000000000
  [<fffffffffffffffb>] 
    ffff8801f1600080
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000c77910
  [<ffffffffa037385a>] 
    0000000000000000
    ffff8801f54ae5f8
    ffff880011453700
    ffff8801f2151ec0
  [<ffffffff8103fe03>] 
    ffff880205753040
    ffff880205753040
    ffff880011453700
    ffff880203941040
    0000000000000304
    ffff8801de86fac8
    ffff8801f2151f70
    ffff8801f7a6d140
    0000000000000000
    000000000000ae80
    0000000000000010
    000000000000ae80
  [<ffffffff8110c63f>] 
    ffff8801f7a6d140
    0000000000000000
    0000000000000000
  [<ffffffff8110cacb>] 
    0000000000013700
    0000000000013700
    0000000000000000
    ffff8801f7a6d140
    0000000000000000
  [<ffffffff8110cbe1>] 
    00007f4c669d2000
    0000000100cb2b70
    00007f4c669d2000
    0000000000000000
    0000000000cb2b70
    0000000000000001
    00007f4c669d1000
  [<ffffffff81002f7b>] 
    0000000000000246
    0000000000000001
    0000000000002cf3
    000000000086bb00
    0000000000000010
    00007f4c64595bd7
    0000000000000000
    000000000000ae80
    0000000000000010
    0000000000000010
    00007f4c64595bd7
    0000000000000033
    0000000000000202
    00007f4c618a2cb0
    000000000000002b

[-- Attachment #4: 3.xenctx --]
[-- Type: text/plain, Size: 11856 bytes --]

rip: ffffffff810784d2 
flags: 00000202 i nz
rsp: ffff8801f17c9ce8
rax: 000000000000000f	rcx: 000000000000000f	rdx: 0000000000000000
rbx: ffff88001146ffc0	rsi: 000000000000000f	rdi: 0000000000000000
rbp: ffff8801f17c9d28	 r8: 000000000000000f	 r9: ffffffff8140bba0
r10: 0000000000000000	r11: ffffffff8101f2b0	r12: ffff88001146ffd0
r13: ffff88001146ffc0	r14: 0000000000000003	r15: 0000000000000001
 cs: 0010	 ss: 0018	 ds: 002b	 es: 002b
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 00000000
cr3: 1f572c000
cr4: 000026f0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffff810784d2)
74 41 45 85 ff 75 0e 0f 1f 40 00 eb 0e 66 0f 1f 44 00 00 f3 90 <f6> 45 20 01 75 f8 48 8b 5c 24 08 


Stack:
 0000000000013700 0000000000000000 0000000000000003 ffffffff81927680
 ffff8801f5698100 ffffffffa019e920 0000000000000001 ffffffff810786f3
 ffff88001146ffc0 ffff88001146ffc0 ffffffffa019e920 ffff8801f5698100
 0000000000000001 ffff8801f15e2280 ffff8801f17c9dd8 ffff8801f5698100

Stack Trace:
* [<ffffffff810784d2>]  <--
    0000000000013700
    0000000000000000
    0000000000000003
  [<ffffffff81927680>] 
    ffff8801f5698100
  [<ffffffffa019e920>] 
    0000000000000001
  [<ffffffff810786f3>] 
    ffff88001146ffc0
    ffff88001146ffc0
  [<ffffffffa019e920>] 
    ffff8801f5698100
    0000000000000001
    ffff8801f15e2280
    ffff8801f17c9dd8
    ffff8801f5698100
    0000000000000000
    00007fc30a713c90
    ffff8801f5698100
    000000004004ae86
    0000000000c76ad0
  [<ffffffffa019fcf4>] 
    0000000000000286
    ffff8801f17c9e10
    ffff8801f17c9de0
    00000001f5713000
    ffff8801f5698100
    ffff8801f5698100
    ffff8801f5698100
  [<ffffffffa0377e7e>] 
  [<fffffffffffffffb>] 
  [<ffffffffa037cdb5>] 
    ffff88010000007b
    ffff8801f17c9df0
    ffff8801f17c9df0
    ffff8801f15e2280
    0000000000000000
    000000000086b000
    ffff880203941040
    0000000000000b00
    0000000000000000
    0000000000000000
    0000000000000000
  [<fffffffffffffffb>] 
    ffff8801f5698100
    00007fc30a713c90
    000000000000000f
  [<ffffffffa03736a6>] 
    ffff8801f17c9ea8
  [<ffffffff8105aacc>] 
    ffff8801f15e2280
    00007fc30a713ce0
    00007fc30a713c20
  [<ffffffff8105a0bd>] 
    00000000000000fa
  [<ffffffff8105ad7d>] 
    0000000000000026
  [<ffffffff81ad9a28>] 
    0000000000003320
    0000000000000000
    ffff88020377d900
    00007fc30a713c90
    000000004004ae86
    000000000000000f
    000000004004ae86
  [<ffffffff8110c63f>] 
    ffff88020377d900
    00007fc30a713c90
    00007fc30a713c90
  [<ffffffff8110cacb>] 
    0000000200c77c74
    000000000086bb00
    0000000000000000
    ffff88020377d900
    00007fc30a713c90
  [<ffffffff8110cbe1>] 
    00007fc310ccc000
    00000001810595c9
    0000000000ca1ec0
    000000000000007b
    0000000000ca1ec0
    00007fc30a713fa0
    00007fc310ccb000
  [<ffffffff81002f7b>] 
    0000000000000246
    0000000000000000
    0000000000000000
    00000000ffffff80
    0000000000000010
    00007fc30e85abd7
    00007fc30a713c90
    000000004004ae86
    000000000000000f
    0000000000000010
    00007fc30e85abd7
    0000000000000033
    0000000000000246
    00007fc30a713ca0
    000000000000002b
rip: ffffffffa01a1c9d 
flags: 00000042 z
rsp: ffff8801f20ddc90
rax: 0000000000000000	rcx: 00000000c0332008	rdx: 00000000c0332000
rbx: 00000000ffffffff	rsi: 00000000c0352433	rdi: 00000000c1602593
rbp: 00000000c02a53ec	 r8: 0000000000000000	 r9: 0000000000000000
r10: 0000000000000000	r11: 0000000000000000	r12: 0000000000000000
r13: 0000000000000000	r14: 0000000000000000	r15: 0000000000000000
 cs: 0010	 ss: 0018	 ds: 002b	 es: 002b
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 080b0028
cr3: 1f2030000
cr4: 000026e0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffffa01a1c9d)
b1 d8 01 00 00 4c 8b b9 e0 01 00 00 48 8b 89 70 01 00 00 75 05 <0f> 01 c2 eb 03 0f 01 c3 48 87 0c 


Stack:
 ffff8801f14bc0c0 ffff8801f14bc0c0 0000000000006c14 0000000000000000
 ffff8801f14bc2b4 0000000000013700 ffffffff811e2fd0 ffff8802056853d0
 ffffffff811e2fd0 ffff880203ad0870 0000000000000000 0000000000000001
 ffff880203ad0870 0000000000000001 ffffffff81066f81 fffffffffffffffb

Stack Trace:
* [<ffffffffa01a1c9d>]  <--
    ffff8801f14bc0c0
    ffff8801f14bc0c0
    0000000000006c14
    0000000000000000
    ffff8801f14bc2b4
    0000000000013700
  [<ffffffff811e2fd0>] 
    ffff8802056853d0
  [<ffffffff811e2fd0>] 
    ffff880203ad0870
    0000000000000000
    0000000000000001
    ffff880203ad0870
    0000000000000001
  [<ffffffff81066f81>] 
  [<fffffffffffffffb>] 
    ffff88001142fe00
    0000000000000282
  [<ffffffff81067cdf>] 
    0000000000000001
    000000000000fdc0
    000000000000fdc0
    ffff8802056853c0
  [<ffffffffa038e898>] 
    0000000000000070
  [<ffffffffa038ed00>] 
  [<fffffffffffffffb>] 
    ffff8801f14bc0c0
  [<fffffffffffffffb>] 
    ffff8801f14bc0c0
    ffff8801f14bc118
    0000000000000000
    ffff8801f147a000
    ffff8801f14bc119
  [<ffffffffa0380866>] 
    ffff8801f21281c0
    ffff8801f20dc000
    0000000000014310
    ffff8801f20ddfd8
    ffff8801f5748080
    ffff8801f5748080
    ffff8801f5748080
    ffff8801f5748080
    ffff8801f20ddfd8
    ffff8801f14bd750
    ffff8801f14bd748
    00ffffff8107454f
    ffff8801dea9ddd8
  [<ffffffff81ad7370>] 
    fffffffe7ffbfeff
  [<ffffffff81074a05>] 
    ffff8801f20dc000
  [<fffffffffffffffb>] 
    ffff8801f14bc0c0
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000c77950
  [<ffffffffa037385a>] 
    0000000000000000
    ffff8801f54a2538
    ffff880011473700
    ffff8801f20ddec0
  [<ffffffff8103fe03>] 
    ffff8801f7970ac0
    ffff8801f7970ac0
    ffff880011473700
    ffff8801f7970ac0
    ffff880011242f78
  [<ffffffff811245c0>] 
    ffff8801f20ddf70
    ffff880202d95bc0
    0000000000000000
    000000000000ae80
    0000000000000010
    000000000000ae80
  [<ffffffff8110c63f>] 
    ffff880202d95bc0
    0000000000000000
    0000000000000000
  [<ffffffff8110cacb>] 
    0000000000013700
    0000000000013700
    0000000000000000
    ffff880202d95bc0
    0000000000000000
  [<ffffffff8110cbe1>] 
    0000000001123d20
    0000000181003d75
    00007fd0b975a000
    0000000000000000
    0000000000cb2bb0
    0000000000000001
    00007fd0b9759000
  [<ffffffff81002f7b>] 
    0000000000000246
    0000000000000000
    0000000000100000
    000000000086bb00
    0000000000000010
    0000000000000000
    0000000000000000
    000000000000ae80
    0000000000000010
    0000000000000010
    00007fd0b731dbd7
    0000000000000033
    0000000000010202
    00007fd0b462a678
    000000000000002b
rip: ffffffff8100aef2 
flags: 00000246 i z p
rsp: ffff8802073dbf30
rax: ffff8802073dbfd8	rcx: 00000000ffffffff	rdx: 0000000000000000
rbx: ffff8802073dbfd8	rsi: 0000000000000001	rdi: ffffffff81a2e308
rbp: ffffffff81927680	 r8: 0000000000000000	 r9: 0000000000000000
r10: 0000000000000000	r11: 00000000000003ff	r12: 0000000000000000
r13: 0000000000000000	r14: 0000000000000000	r15: 0000000000000000
 cs: 0010	 ss: 0018	 ds: 0018	 es: 0018
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 00000000
cr3: 1f5751000
cr4: 000026f0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffff8100aef2)
65 48 8b 04 25 88 b5 00 00 83 88 3c e0 ff ff 04 c3 66 90 fb f4 <eb> e9 66 66 66 2e 0f 1f 84 00 00 


Stack:
 ffffffff8100205a 0000000000000018 0000000000000000 0000000000000000
 0000000000000000 0000000000000000 0000000000000000 0000000000000000
 0000000000000000 0000000000000000 0000000000000000 0000000000000000
 0000000000000000 0000000000000000 0000000000000000 0000000000000000

Stack Trace:
* [<ffffffff8100aef2>]  <--
  [<ffffffff8100205a>] 
    0000000000000018
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
rip: ffffffffa01a1c9d 
flags: 00000042 z
rsp: ffff8801f2151c90
rax: 0000000000000000	rcx: 00000000c0332008	rdx: 00000000c0332000
rbx: 00000000ffffffff	rsi: 00000000c035245a	rdi: 00000000c14015ba
rbp: 00000000c02a53ec	 r8: 0000000000000000	 r9: 0000000000000000
r10: 0000000000000000	r11: 0000000000000000	r12: 0000000000000000
r13: 0000000000000000	r14: 0000000000000000	r15: 0000000000000000
 cs: 0010	 ss: 0018	 ds: 002b	 es: 002b
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 0807f000
cr3: 1f23fa000
cr4: 000026e0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffffa01a1c9d)
b1 d8 01 00 00 4c 8b b9 e0 01 00 00 48 8b 89 70 01 00 00 75 05 <0f> 01 c2 eb 03 0f 01 c3 48 87 0c 


Stack:
 ffff8801f1600080 ffff8801f1600080 0000000000006c14 0000000000000003
 ffff8801f1600274 00000000ffffffff ffffffffa0378937 ffff8802056854d0
 0000000000000000 ffff88020513d670 0000000000000000 0000000000000003
 ffff88020513d670 0000000000000003 ffffffff81066f81 ffff8801f2151e08

Stack Trace:
* [<ffffffffa01a1c9d>]  <--
    ffff8801f1600080
    ffff8801f1600080
    0000000000006c14
    0000000000000003
    ffff8801f1600274
    00000000ffffffff
  [<ffffffffa0378937>] 
    ffff8802056854d0
    0000000000000000
    ffff88020513d670
    0000000000000000
    0000000000000003
    ffff88020513d670
    0000000000000003
  [<ffffffff81066f81>] 
    ffff8801f2151e08
    ffff88001146fe00
    0000000000000282
  [<ffffffff81067cdf>] 
    0000000000000003
    000000000000fdc0
    000000000000fdc0
    ffff8802056854c0
  [<ffffffffa038e898>] 
    0000000000000010
  [<ffffffffa038ed00>] 
    ffff8801f2151e08
    ffff8801f1600080
    ffff8801f2151e08
    ffff8801f1600080
    ffff8801f16000d8
    0000000000000000
    ffff8801f15f0000
    ffff8801f16000d9
  [<ffffffffa0380866>] 
    ffff8801f154c3c0
    ffff8801f2150000
    0000000000014310
    ffff8801f2151fd8
    ffff8801f154c1c0
    ffff8801f154c1c0
    ffff8801f154c1c0
    ffff8801f154c1c0
    ffff8801f2151fd8
    ffff8801f1601710
    ffff8801f1601708
    00ff8801f154c1c0
    00000000000000ab
    0000000007e5df6d
    fffffffe7ffbfeff
    0000000000000b00
    0000000000000000
  [<fffffffffffffffb>] 
    ffff8801f1600080
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000c77910
  [<ffffffffa037385a>] 
    0000000000000000
    ffff8801f54ae5f8
    ffff880011453700
    ffff8801f2151ec0
  [<ffffffff8103fe03>] 
    ffff880205753040
    ffff880205753040
    ffff880011453700
    ffff880203941040
    0000000000000304
    ffff8801de86fac8
    ffff8801f2151f70
    ffff8801f7a6d140
    0000000000000000
    000000000000ae80
    0000000000000010
    000000000000ae80
  [<ffffffff8110c63f>] 
    ffff8801f7a6d140
    0000000000000000
    0000000000000000
  [<ffffffff8110cacb>] 
    0000000000013700
    0000000000013700
    0000000000000000
    ffff8801f7a6d140
    0000000000000000
  [<ffffffff8110cbe1>] 
    00007f4c669d2000
    0000000100cb2b70
    00007f4c669d2000
    0000000000000000
    0000000000cb2b70
    0000000000000001
    00007f4c669d1000
  [<ffffffff81002f7b>] 
    0000000000000246
    0000000000000001
    0000000000002cf3
    000000000086bb00
    0000000000000010
    00007f4c64595bd7
    0000000000000000
    000000000000ae80
    0000000000000010
    0000000000000010
    00007f4c64595bd7
    0000000000000033
    0000000000000202
    00007f4c618a2cb0
    000000000000002b

[-- Attachment #5: 4.xenctx --]
[-- Type: text/plain, Size: 11856 bytes --]

rip: ffffffff810784d0 
flags: 00000202 i nz
rsp: ffff8801f17c9ce8
rax: 000000000000000f	rcx: 000000000000000f	rdx: 0000000000000000
rbx: ffff88001146ffc0	rsi: 000000000000000f	rdi: 0000000000000000
rbp: ffff8801f17c9d28	 r8: 000000000000000f	 r9: ffffffff8140bba0
r10: 0000000000000000	r11: ffffffff8101f2b0	r12: ffff88001146ffd0
r13: ffff88001146ffc0	r14: 0000000000000003	r15: 0000000000000001
 cs: 0010	 ss: 0018	 ds: 002b	 es: 002b
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 00000000
cr3: 1f572c000
cr4: 000026f0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffff810784d0)
39 eb 74 41 45 85 ff 75 0e 0f 1f 40 00 eb 0e 66 0f 1f 44 00 00 <f3> 90 f6 45 20 01 75 f8 48 8b 5c 


Stack:
 0000000000013700 0000000000000000 0000000000000003 ffffffff81927680
 ffff8801f5698100 ffffffffa019e920 0000000000000001 ffffffff810786f3
 ffff88001146ffc0 ffff88001146ffc0 ffffffffa019e920 ffff8801f5698100
 0000000000000001 ffff8801f15e2280 ffff8801f17c9dd8 ffff8801f5698100

Stack Trace:
* [<ffffffff810784d0>]  <--
    0000000000013700
    0000000000000000
    0000000000000003
  [<ffffffff81927680>] 
    ffff8801f5698100
  [<ffffffffa019e920>] 
    0000000000000001
  [<ffffffff810786f3>] 
    ffff88001146ffc0
    ffff88001146ffc0
  [<ffffffffa019e920>] 
    ffff8801f5698100
    0000000000000001
    ffff8801f15e2280
    ffff8801f17c9dd8
    ffff8801f5698100
    0000000000000000
    00007fc30a713c90
    ffff8801f5698100
    000000004004ae86
    0000000000c76ad0
  [<ffffffffa019fcf4>] 
    0000000000000286
    ffff8801f17c9e10
    ffff8801f17c9de0
    00000001f5713000
    ffff8801f5698100
    ffff8801f5698100
    ffff8801f5698100
  [<ffffffffa0377e7e>] 
  [<fffffffffffffffb>] 
  [<ffffffffa037cdb5>] 
    ffff88010000007b
    ffff8801f17c9df0
    ffff8801f17c9df0
    ffff8801f15e2280
    0000000000000000
    000000000086b000
    ffff880203941040
    0000000000000b00
    0000000000000000
    0000000000000000
    0000000000000000
  [<fffffffffffffffb>] 
    ffff8801f5698100
    00007fc30a713c90
    000000000000000f
  [<ffffffffa03736a6>] 
    ffff8801f17c9ea8
  [<ffffffff8105aacc>] 
    ffff8801f15e2280
    00007fc30a713ce0
    00007fc30a713c20
  [<ffffffff8105a0bd>] 
    00000000000000fa
  [<ffffffff8105ad7d>] 
    0000000000000026
  [<ffffffff81ad9a28>] 
    0000000000003320
    0000000000000000
    ffff88020377d900
    00007fc30a713c90
    000000004004ae86
    000000000000000f
    000000004004ae86
  [<ffffffff8110c63f>] 
    ffff88020377d900
    00007fc30a713c90
    00007fc30a713c90
  [<ffffffff8110cacb>] 
    0000000200c77c74
    000000000086bb00
    0000000000000000
    ffff88020377d900
    00007fc30a713c90
  [<ffffffff8110cbe1>] 
    00007fc310ccc000
    00000001810595c9
    0000000000ca1ec0
    000000000000007b
    0000000000ca1ec0
    00007fc30a713fa0
    00007fc310ccb000
  [<ffffffff81002f7b>] 
    0000000000000246
    0000000000000000
    0000000000000000
    00000000ffffff80
    0000000000000010
    00007fc30e85abd7
    00007fc30a713c90
    000000004004ae86
    000000000000000f
    0000000000000010
    00007fc30e85abd7
    0000000000000033
    0000000000000246
    00007fc30a713ca0
    000000000000002b
rip: ffffffffa01a1c9d 
flags: 00000042 z
rsp: ffff8801f20ddc90
rax: 0000000000000000	rcx: 00000000c0332008	rdx: 00000000c0332000
rbx: 00000000ffffffff	rsi: 00000000c0352433	rdi: 00000000c1602593
rbp: 00000000c02a53ec	 r8: 0000000000000000	 r9: 0000000000000000
r10: 0000000000000000	r11: 0000000000000000	r12: 0000000000000000
r13: 0000000000000000	r14: 0000000000000000	r15: 0000000000000000
 cs: 0010	 ss: 0018	 ds: 002b	 es: 002b
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 080b0028
cr3: 1f2030000
cr4: 000026e0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffffa01a1c9d)
b1 d8 01 00 00 4c 8b b9 e0 01 00 00 48 8b 89 70 01 00 00 75 05 <0f> 01 c2 eb 03 0f 01 c3 48 87 0c 


Stack:
 ffff8801f14bc0c0 ffff8801f14bc0c0 0000000000006c14 0000000000000000
 ffff8801f14bc2b4 0000000000013700 ffffffff811e2fd0 ffff8802056853d0
 ffffffff811e2fd0 ffff880203ad0870 0000000000000000 0000000000000001
 ffff880203ad0870 0000000000000001 ffffffff81066f81 fffffffffffffffb

Stack Trace:
* [<ffffffffa01a1c9d>]  <--
    ffff8801f14bc0c0
    ffff8801f14bc0c0
    0000000000006c14
    0000000000000000
    ffff8801f14bc2b4
    0000000000013700
  [<ffffffff811e2fd0>] 
    ffff8802056853d0
  [<ffffffff811e2fd0>] 
    ffff880203ad0870
    0000000000000000
    0000000000000001
    ffff880203ad0870
    0000000000000001
  [<ffffffff81066f81>] 
  [<fffffffffffffffb>] 
    ffff88001142fe00
    0000000000000282
  [<ffffffff81067cdf>] 
    0000000000000001
    000000000000fdc0
    000000000000fdc0
    ffff8802056853c0
  [<ffffffffa038e898>] 
    0000000000000070
  [<ffffffffa038ed00>] 
  [<fffffffffffffffb>] 
    ffff8801f14bc0c0
  [<fffffffffffffffb>] 
    ffff8801f14bc0c0
    ffff8801f14bc118
    0000000000000000
    ffff8801f147a000
    ffff8801f14bc119
  [<ffffffffa0380866>] 
    ffff8801f21281c0
    ffff8801f20dc000
    0000000000014310
    ffff8801f20ddfd8
    ffff8801f5748080
    ffff8801f5748080
    ffff8801f5748080
    ffff8801f5748080
    ffff8801f20ddfd8
    ffff8801f14bd750
    ffff8801f14bd748
    00ffffff8107454f
    ffff8801dea9ddd8
  [<ffffffff81ad7370>] 
    fffffffe7ffbfeff
  [<ffffffff81074a05>] 
    ffff8801f20dc000
  [<fffffffffffffffb>] 
    ffff8801f14bc0c0
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000c77950
  [<ffffffffa037385a>] 
    0000000000000000
    ffff8801f54a2538
    ffff880011473700
    ffff8801f20ddec0
  [<ffffffff8103fe03>] 
    ffff8801f7970ac0
    ffff8801f7970ac0
    ffff880011473700
    ffff8801f7970ac0
    ffff880011242f78
  [<ffffffff811245c0>] 
    ffff8801f20ddf70
    ffff880202d95bc0
    0000000000000000
    000000000000ae80
    0000000000000010
    000000000000ae80
  [<ffffffff8110c63f>] 
    ffff880202d95bc0
    0000000000000000
    0000000000000000
  [<ffffffff8110cacb>] 
    0000000000013700
    0000000000013700
    0000000000000000
    ffff880202d95bc0
    0000000000000000
  [<ffffffff8110cbe1>] 
    0000000001123d20
    0000000181003d75
    00007fd0b975a000
    0000000000000000
    0000000000cb2bb0
    0000000000000001
    00007fd0b9759000
  [<ffffffff81002f7b>] 
    0000000000000246
    0000000000000000
    0000000000100000
    000000000086bb00
    0000000000000010
    0000000000000000
    0000000000000000
    000000000000ae80
    0000000000000010
    0000000000000010
    00007fd0b731dbd7
    0000000000000033
    0000000000010202
    00007fd0b462a678
    000000000000002b
rip: ffffffff8100aef2 
flags: 00000246 i z p
rsp: ffff8802073dbf30
rax: ffff8802073dbfd8	rcx: 00000000ffffffff	rdx: 0000000000000000
rbx: ffff8802073dbfd8	rsi: 0000000000000001	rdi: ffffffff81a2e308
rbp: ffffffff81927680	 r8: 0000000000000000	 r9: 0000000000000000
r10: 0000000000000000	r11: 00000000000003ff	r12: 0000000000000000
r13: 0000000000000000	r14: 0000000000000000	r15: 0000000000000000
 cs: 0010	 ss: 0018	 ds: 0018	 es: 0018
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 00000000
cr3: 1f5751000
cr4: 000026f0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffff8100aef2)
65 48 8b 04 25 88 b5 00 00 83 88 3c e0 ff ff 04 c3 66 90 fb f4 <eb> e9 66 66 66 2e 0f 1f 84 00 00 


Stack:
 ffffffff8100205a 0000000000000018 0000000000000000 0000000000000000
 0000000000000000 0000000000000000 0000000000000000 0000000000000000
 0000000000000000 0000000000000000 0000000000000000 0000000000000000
 0000000000000000 0000000000000000 0000000000000000 0000000000000000

Stack Trace:
* [<ffffffff8100aef2>]  <--
  [<ffffffff8100205a>] 
    0000000000000018
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
rip: ffffffffa01a1c9d 
flags: 00000042 z
rsp: ffff8801f2151c90
rax: 0000000000000000	rcx: 00000000c0332008	rdx: 00000000c0332000
rbx: 00000000ffffffff	rsi: 00000000c035245a	rdi: 00000000c14015ba
rbp: 00000000c02a53ec	 r8: 0000000000000000	 r9: 0000000000000000
r10: 0000000000000000	r11: 0000000000000000	r12: 0000000000000000
r13: 0000000000000000	r14: 0000000000000000	r15: 0000000000000000
 cs: 0010	 ss: 0018	 ds: 002b	 es: 002b
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 0807f000
cr3: 1f23fa000
cr4: 000026e0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffffa01a1c9d)
b1 d8 01 00 00 4c 8b b9 e0 01 00 00 48 8b 89 70 01 00 00 75 05 <0f> 01 c2 eb 03 0f 01 c3 48 87 0c 


Stack:
 ffff8801f1600080 ffff8801f1600080 0000000000006c14 0000000000000003
 ffff8801f1600274 00000000ffffffff ffffffffa0378937 ffff8802056854d0
 0000000000000000 ffff88020513d670 0000000000000000 0000000000000003
 ffff88020513d670 0000000000000003 ffffffff81066f81 ffff8801f2151e08

Stack Trace:
* [<ffffffffa01a1c9d>]  <--
    ffff8801f1600080
    ffff8801f1600080
    0000000000006c14
    0000000000000003
    ffff8801f1600274
    00000000ffffffff
  [<ffffffffa0378937>] 
    ffff8802056854d0
    0000000000000000
    ffff88020513d670
    0000000000000000
    0000000000000003
    ffff88020513d670
    0000000000000003
  [<ffffffff81066f81>] 
    ffff8801f2151e08
    ffff88001146fe00
    0000000000000282
  [<ffffffff81067cdf>] 
    0000000000000003
    000000000000fdc0
    000000000000fdc0
    ffff8802056854c0
  [<ffffffffa038e898>] 
    0000000000000010
  [<ffffffffa038ed00>] 
    ffff8801f2151e08
    ffff8801f1600080
    ffff8801f2151e08
    ffff8801f1600080
    ffff8801f16000d8
    0000000000000000
    ffff8801f15f0000
    ffff8801f16000d9
  [<ffffffffa0380866>] 
    ffff8801f154c3c0
    ffff8801f2150000
    0000000000014310
    ffff8801f2151fd8
    ffff8801f154c1c0
    ffff8801f154c1c0
    ffff8801f154c1c0
    ffff8801f154c1c0
    ffff8801f2151fd8
    ffff8801f1601710
    ffff8801f1601708
    00ff8801f154c1c0
    00000000000000ab
    0000000007e5df6d
    fffffffe7ffbfeff
    0000000000000b00
    0000000000000000
  [<fffffffffffffffb>] 
    ffff8801f1600080
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000c77910
  [<ffffffffa037385a>] 
    0000000000000000
    ffff8801f54ae5f8
    ffff880011453700
    ffff8801f2151ec0
  [<ffffffff8103fe03>] 
    ffff880205753040
    ffff880205753040
    ffff880011453700
    ffff880203941040
    0000000000000304
    ffff8801de86fac8
    ffff8801f2151f70
    ffff8801f7a6d140
    0000000000000000
    000000000000ae80
    0000000000000010
    000000000000ae80
  [<ffffffff8110c63f>] 
    ffff8801f7a6d140
    0000000000000000
    0000000000000000
  [<ffffffff8110cacb>] 
    0000000000013700
    0000000000013700
    0000000000000000
    ffff8801f7a6d140
    0000000000000000
  [<ffffffff8110cbe1>] 
    00007f4c669d2000
    0000000100cb2b70
    00007f4c669d2000
    0000000000000000
    0000000000cb2b70
    0000000000000001
    00007f4c669d1000
  [<ffffffff81002f7b>] 
    0000000000000246
    0000000000000001
    0000000000002cf3
    000000000086bb00
    0000000000000010
    00007f4c64595bd7
    0000000000000000
    000000000000ae80
    0000000000000010
    0000000000000010
    00007f4c64595bd7
    0000000000000033
    0000000000000202
    00007f4c618a2cb0
    000000000000002b

[-- Attachment #6: 5.xenctx --]
[-- Type: text/plain, Size: 10354 bytes --]

rip: ffffffffa03fedf8 
flags: 00010286 rf i s nz p
rsp: ffff880011403f18
rax: ffffc900030000c0	rcx: ffff880011400000	rdx: ffff8802055b7000
rbx: ffff8802055b7780	rsi: ffff8802055b7000	rdi: 0000000000000020
rbp: 0000000000000000	 r8: 000000000000000f	 r9: ffffffff8140bba0
r10: 0000000000000000	r11: ffffffff8101f2b0	r12: 0000000000000000
r13: 0000000000000020	r14: 0000000000000003	r15: 0000000000000001
 cs: 0010	 ss: 0018	 ds: 002b	 es: 002b
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 00000000
cr3: 1f572c000
cr4: 000026f0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffffa03fedf8)
9e 80 07 00 00 48 89 f2 48 8b 83 90 03 00 00 48 05 c0 00 00 00 <8b> 00 85 c0 0f 84 ce 00 00 00 f6 


Stack:
 ffff88020573a9c0 ffffffff8109f7c9 ffff88001146ffd0 ffff88020372f2c0
 ffff88020372f32c 0000000000000020 0000000000000000 ffffffff810a1f04
 0000000000000020 0000000000000020 0000000000000051 ffffffff81005d47
 ffff8801f17c9c38 ffffffff81005255 0000000000000001 ffff88001146ffc0

Stack Trace:
* [<ffffffffa03fedf8>]  <--
    ffff88020573a9c0
  [<ffffffff8109f7c9>] 
    ffff88001146ffd0
    ffff88020372f2c0
    ffff88020372f32c
    0000000000000020
    0000000000000000
  [<ffffffff810a1f04>] 
    0000000000000020
    0000000000000020
    0000000000000051
  [<ffffffff81005d47>] 
    ffff8801f17c9c38
  [<ffffffff81005255>] 
    0000000000000001
    ffff88001146ffc0
    ffff8801f17c9c60
    ffff88001146ffd0
    ffff88001146ffc0
  [<ffffffff81003913>] 
    ffff8801f17c9c60
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
rip: ffffffffa01a1c9d 
flags: 00000042 z
rsp: ffff8801f20ddc90
rax: 0000000000000000	rcx: 00000000c0332008	rdx: 00000000c0332000
rbx: 00000000ffffffff	rsi: 00000000c0352433	rdi: 00000000c1602593
rbp: 00000000c02a53ec	 r8: 0000000000000000	 r9: 0000000000000000
r10: 0000000000000000	r11: 0000000000000000	r12: 0000000000000000
r13: 0000000000000000	r14: 0000000000000000	r15: 0000000000000000
 cs: 0010	 ss: 0018	 ds: 002b	 es: 002b
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 080b0028
cr3: 1f2030000
cr4: 000026e0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffffa01a1c9d)
b1 d8 01 00 00 4c 8b b9 e0 01 00 00 48 8b 89 70 01 00 00 75 05 <0f> 01 c2 eb 03 0f 01 c3 48 87 0c 


Stack:
 ffff8801f14bc0c0 ffff8801f14bc0c0 0000000000006c14 0000000000000000
 ffff8801f14bc2b4 0000000000013700 ffffffff811e2fd0 ffff8802056853d0
 ffffffff811e2fd0 ffff880203ad0870 0000000000000000 0000000000000001
 ffff880203ad0870 0000000000000001 ffffffff81066f81 fffffffffffffffb

Stack Trace:
* [<ffffffffa01a1c9d>]  <--
    ffff8801f14bc0c0
    ffff8801f14bc0c0
    0000000000006c14
    0000000000000000
    ffff8801f14bc2b4
    0000000000013700
  [<ffffffff811e2fd0>] 
    ffff8802056853d0
  [<ffffffff811e2fd0>] 
    ffff880203ad0870
    0000000000000000
    0000000000000001
    ffff880203ad0870
    0000000000000001
  [<ffffffff81066f81>] 
  [<fffffffffffffffb>] 
    ffff88001142fe00
    0000000000000282
  [<ffffffff81067cdf>] 
    0000000000000001
    000000000000fdc0
    000000000000fdc0
    ffff8802056853c0
  [<ffffffffa038e898>] 
    0000000000000070
  [<ffffffffa038ed00>] 
  [<fffffffffffffffb>] 
    ffff8801f14bc0c0
  [<fffffffffffffffb>] 
    ffff8801f14bc0c0
    ffff8801f14bc118
    0000000000000000
    ffff8801f147a000
    ffff8801f14bc119
  [<ffffffffa0380866>] 
    ffff8801f21281c0
    ffff8801f20dc000
    0000000000014310
    ffff8801f20ddfd8
    ffff8801f5748080
    ffff8801f5748080
    ffff8801f5748080
    ffff8801f5748080
    ffff8801f20ddfd8
    ffff8801f14bd750
    ffff8801f14bd748
    00ffffff8107454f
    ffff8801dea9ddd8
  [<ffffffff81ad7370>] 
    fffffffe7ffbfeff
  [<ffffffff81074a05>] 
    ffff8801f20dc000
  [<fffffffffffffffb>] 
    ffff8801f14bc0c0
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000c77950
  [<ffffffffa037385a>] 
    0000000000000000
    ffff8801f54a2538
    ffff880011473700
    ffff8801f20ddec0
  [<ffffffff8103fe03>] 
    ffff8801f7970ac0
    ffff8801f7970ac0
    ffff880011473700
    ffff8801f7970ac0
    ffff880011242f78
  [<ffffffff811245c0>] 
    ffff8801f20ddf70
    ffff880202d95bc0
    0000000000000000
    000000000000ae80
    0000000000000010
    000000000000ae80
  [<ffffffff8110c63f>] 
    ffff880202d95bc0
    0000000000000000
    0000000000000000
  [<ffffffff8110cacb>] 
    0000000000013700
    0000000000013700
    0000000000000000
    ffff880202d95bc0
    0000000000000000
  [<ffffffff8110cbe1>] 
    0000000001123d20
    0000000181003d75
    00007fd0b975a000
    0000000000000000
    0000000000cb2bb0
    0000000000000001
    00007fd0b9759000
  [<ffffffff81002f7b>] 
    0000000000000246
    0000000000000000
    0000000000100000
    000000000086bb00
    0000000000000010
    0000000000000000
    0000000000000000
    000000000000ae80
    0000000000000010
    0000000000000010
    00007fd0b731dbd7
    0000000000000033
    0000000000010202
    00007fd0b462a678
    000000000000002b
rip: ffffffff8100aef2 
flags: 00000246 i z p
rsp: ffff8802073dbf30
rax: ffff8802073dbfd8	rcx: 00000000ffffffff	rdx: 0000000000000000
rbx: ffff8802073dbfd8	rsi: 0000000000000001	rdi: ffffffff81a2e308
rbp: ffffffff81927680	 r8: 0000000000000000	 r9: 0000000000000000
r10: 0000000000000000	r11: 00000000000003ff	r12: 0000000000000000
r13: 0000000000000000	r14: 0000000000000000	r15: 0000000000000000
 cs: 0010	 ss: 0018	 ds: 0018	 es: 0018
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 00000000
cr3: 1f5751000
cr4: 000026f0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffff8100aef2)
65 48 8b 04 25 88 b5 00 00 83 88 3c e0 ff ff 04 c3 66 90 fb f4 <eb> e9 66 66 66 2e 0f 1f 84 00 00 


Stack:
 ffffffff8100205a 0000000000000018 0000000000000000 0000000000000000
 0000000000000000 0000000000000000 0000000000000000 0000000000000000
 0000000000000000 0000000000000000 0000000000000000 0000000000000000
 0000000000000000 0000000000000000 0000000000000000 0000000000000000

Stack Trace:
* [<ffffffff8100aef2>]  <--
  [<ffffffff8100205a>] 
    0000000000000018
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000000000
rip: ffffffffa01a1c9d 
flags: 00000042 z
rsp: ffff8801f2151c90
rax: 0000000000000000	rcx: 00000000c0332008	rdx: 00000000c0332000
rbx: 00000000ffffffff	rsi: 00000000c035245a	rdi: 00000000c14015ba
rbp: 00000000c02a53ec	 r8: 0000000000000000	 r9: 0000000000000000
r10: 0000000000000000	r11: 0000000000000000	r12: 0000000000000000
r13: 0000000000000000	r14: 0000000000000000	r15: 0000000000000000
 cs: 0010	 ss: 0018	 ds: 002b	 es: 002b
 fs: 0000 @ 0000000000000000
 gs: 0000 @ 0000000000000000/0000000000000000

cr0: 8005003b
cr2: 0807f000
cr3: 1f23fa000
cr4: 000026e0

dr0: 00000000
dr1: 00000000
dr2: 00000000
dr3: 00000000
dr6: ffff0ff0
dr7: 00000400
Code (instr addr ffffffffa01a1c9d)
b1 d8 01 00 00 4c 8b b9 e0 01 00 00 48 8b 89 70 01 00 00 75 05 <0f> 01 c2 eb 03 0f 01 c3 48 87 0c 


Stack:
 ffff8801f1600080 ffff8801f1600080 0000000000006c14 0000000000000003
 ffff8801f1600274 00000000ffffffff ffffffffa0378937 ffff8802056854d0
 0000000000000000 ffff88020513d670 0000000000000000 0000000000000003
 ffff88020513d670 0000000000000003 ffffffff81066f81 ffff8801f2151e08

Stack Trace:
* [<ffffffffa01a1c9d>]  <--
    ffff8801f1600080
    ffff8801f1600080
    0000000000006c14
    0000000000000003
    ffff8801f1600274
    00000000ffffffff
  [<ffffffffa0378937>] 
    ffff8802056854d0
    0000000000000000
    ffff88020513d670
    0000000000000000
    0000000000000003
    ffff88020513d670
    0000000000000003
  [<ffffffff81066f81>] 
    ffff8801f2151e08
    ffff88001146fe00
    0000000000000282
  [<ffffffff81067cdf>] 
    0000000000000003
    000000000000fdc0
    000000000000fdc0
    ffff8802056854c0
  [<ffffffffa038e898>] 
    0000000000000010
  [<ffffffffa038ed00>] 
    ffff8801f2151e08
    ffff8801f1600080
    ffff8801f2151e08
    ffff8801f1600080
    ffff8801f16000d8
    0000000000000000
    ffff8801f15f0000
    ffff8801f16000d9
  [<ffffffffa0380866>] 
    ffff8801f154c3c0
    ffff8801f2150000
    0000000000014310
    ffff8801f2151fd8
    ffff8801f154c1c0
    ffff8801f154c1c0
    ffff8801f154c1c0
    ffff8801f154c1c0
    ffff8801f2151fd8
    ffff8801f1601710
    ffff8801f1601708
    00ff8801f154c1c0
    00000000000000ab
    0000000007e5df6d
    fffffffe7ffbfeff
    0000000000000b00
    0000000000000000
  [<fffffffffffffffb>] 
    ffff8801f1600080
    0000000000000000
    0000000000000000
    0000000000000000
    0000000000c77910
  [<ffffffffa037385a>] 
    0000000000000000
    ffff8801f54ae5f8
    ffff880011453700
    ffff8801f2151ec0
  [<ffffffff8103fe03>] 
    ffff880205753040
    ffff880205753040
    ffff880011453700
    ffff880203941040
    0000000000000304
    ffff8801de86fac8
    ffff8801f2151f70
    ffff8801f7a6d140
    0000000000000000
    000000000000ae80
    0000000000000010
    000000000000ae80
  [<ffffffff8110c63f>] 
    ffff8801f7a6d140
    0000000000000000
    0000000000000000
  [<ffffffff8110cacb>] 
    0000000000013700
    0000000000013700
    0000000000000000
    ffff8801f7a6d140
    0000000000000000
  [<ffffffff8110cbe1>] 
    00007f4c669d2000
    0000000100cb2b70
    00007f4c669d2000
    0000000000000000
    0000000000cb2b70
    0000000000000001
    00007f4c669d1000
  [<ffffffff81002f7b>] 
    0000000000000246
    0000000000000001
    0000000000002cf3
    000000000086bb00
    0000000000000010
    00007f4c64595bd7
    0000000000000000
    000000000000ae80
    0000000000000010
    0000000000000010
    00007f4c64595bd7
    0000000000000033
    0000000000000202
    00007f4c618a2cb0
    000000000000002b

[-- Attachment #7: dmesg --]
[-- Type: text/plain, Size: 79999 bytes --]

 __  __            _  _    ____                     _        _     _      
 \ \/ /___ _ __   | || |  |___ \    _   _ _ __  ___| |_ __ _| |__ | | ___ 
  \  // _ \ \047_ \  | || |_   __) |__| | | | \047_ \/ __| __/ _` | \047_ \| |/ _ \
  /  \  __/ | | | |__   _| / __/|__| |_| | | | \__ \ || (_| | |_) | |  __/
 /_/\_\___|_| |_|    |_|(_)_____|   \__,_|_| |_|___/\__\__,_|_.__/|_|\___|
                                                                          
(XEN) Xen version 4.2-unstable (root@site) (gcc version 4.5.1 20101208 [gcc-4_5-branch revision 167585] (SUSE Linux) ) Tue Jul 26 12:41:19 CEST 2011
(XEN) Latest ChangeSet: Tue Jul 19 16:02:36 2011 +0100 23728:548b2826293e
(XEN) Bootloader: GNU GRUB 0.97
(XEN) Command line: vga=mode-0x31a
(XEN) Video information:
(XEN)  VGA is graphics mode 1280x1024, 16 bpp
(XEN)  VBE/DDC methods: V2; EDID transfer time: 1 seconds
(XEN) Disc information:
(XEN)  Found 5 MBR signatures
(XEN)  Found 5 EDD information structures
(XEN) Xen-e820 RAM map:
(XEN)  0000000000000000 - 000000000009fc00 (usable)
(XEN)  000000000009fc00 - 00000000000a0000 (reserved)
(XEN)  00000000000e4000 - 0000000000100000 (reserved)
(XEN)  0000000000100000 - 00000000bf780000 (usable)
(XEN)  00000000bf78e000 - 00000000bf790000 type 9
(XEN)  00000000bf790000 - 00000000bf79e000 (ACPI data)
(XEN)  00000000bf79e000 - 00000000bf7d0000 (ACPI NVS)
(XEN)  00000000bf7d0000 - 00000000bf7e0000 (reserved)
(XEN)  00000000bf7ec000 - 00000000c0000000 (reserved)
(XEN)  00000000e0000000 - 00000000f0000000 (reserved)
(XEN)  00000000fee00000 - 00000000fee01000 (reserved)
(XEN)  00000000ffe00000 - 0000000100000000 (reserved)
(XEN)  0000000100000000 - 0000001840000000 (usable)
(XEN) ACPI: RSDP 000FB410, 0024 (r2 ACPIAM)
(XEN) ACPI: XSDT BF790100, 008C (r1 120210 XSDT1403 20101202 MSFT       97)
(XEN) ACPI: FACP BF790290, 00F4 (r3 120210 FACP1403 20101202 MSFT       97)
(XEN) ACPI: DSDT BF790540, 5AE1 (r1  S0075 S0075001        1 INTL 20060113)
(XEN) ACPI: FACS BF79E000, 0040
(XEN) ACPI: APIC BF790390, 011E (r1 120210 APIC1403 20101202 MSFT       97)
(XEN) ACPI: MCFG BF7904B0, 003C (r1 120210 OEMMCFG  20101202 MSFT       97)
(XEN) ACPI: SPMI BF7904F0, 0041 (r5 120210 OEMSPMI  20101202 MSFT       97)
(XEN) ACPI: OEMB BF79E040, 0072 (r1 120210 OEMB1403 20101202 MSFT       97)
(XEN) ACPI: HPET BF798540, 0038 (r1 120210 OEMHPET  20101202 MSFT       97)
(XEN) ACPI: SRAT BF798580, 0228 (r1 120210 OEMSRAT         1 INTL        1)
(XEN) ACPI: DMAR BF79E0C0, 0140 (r1    AMI  OEMDMAR        1 MSFT       97)
(XEN) ACPI: SSDT BF7A4000, 0363 (r1 DpgPmm    CpuPm       12 INTL 20060113)
(XEN) ACPI: EINJ BF7987B0, 0130 (r1  AMIER AMI_EINJ 20101202 MSFT       97)
(XEN) ACPI: BERT BF798940, 0030 (r1  AMIER AMI_BERT 20101202 MSFT       97)
(XEN) ACPI: ERST BF798970, 01B0 (r1  AMIER AMI_ERST 20101202 MSFT       97)
(XEN) ACPI: HEST BF798B20, 00A8 (r1  AMIER ABC_HEST 20101202 MSFT       97)
(XEN) System RAM: 98295MB (100654204kB)
(XEN) SRAT: PXM 0 -> APIC 0 -> Node 0
(XEN) SRAT: PXM 1 -> APIC 32 -> Node 1
(XEN) SRAT: PXM 0 -> APIC 2 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 4 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 16 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 18 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 20 -> Node 0
(XEN) SRAT: PXM 1 -> APIC 34 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 36 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 48 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 50 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 52 -> Node 1
(XEN) SRAT: PXM 0 -> APIC 1 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 3 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 5 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 17 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 19 -> Node 0
(XEN) SRAT: PXM 0 -> APIC 21 -> Node 0
(XEN) SRAT: PXM 1 -> APIC 33 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 35 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 37 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 49 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 51 -> Node 1
(XEN) SRAT: PXM 1 -> APIC 53 -> Node 1
(XEN) SRAT: Node 0 PXM 0 0-c0000000
(XEN) SRAT: Node 0 PXM 0 100000000-c40000000
(XEN) SRAT: Node 1 PXM 1 c40000000-1840000000
(XEN) NUMA: Allocated memnodemap from 183db0f000 - 183db10000
(XEN) NUMA: Using 18 for the hash shift.
(XEN) Domain heap initialised DMA width 32 bits
(XEN) vesafb: framebuffer at 0xfb000000, mapped to 0xffff82c000000000, using 4096k, total 8192k
(XEN) vesafb: mode is 1280x1024x16, linelength=2560, font 8x16
(XEN) vesafb: Truecolor: size=0:5:6:5, shift=0:11:5:0
(XEN) found SMP MP-table at 000ff780
(XEN) DMI present.
(XEN) Using APIC driver default
(XEN) ACPI: PM-Timer IO Port: 0x808
(XEN) ACPI: ACPI SLEEP INFO: pm1x_cnt[804,0], pm1x_evt[800,0]
(XEN) ACPI:                  wakeup_vec[bf79e00c], vec_size[20]
(XEN) ACPI: Local APIC address 0xfee00000
(XEN) ACPI: LAPIC (acpi_id[0x01] lapic_id[0x00] enabled)
(XEN) Processor #0 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x02] lapic_id[0x02] enabled)
(XEN) Processor #2 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x03] lapic_id[0x04] enabled)
(XEN) Processor #4 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x04] lapic_id[0x10] enabled)
(XEN) Processor #16 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x05] lapic_id[0x12] enabled)
(XEN) Processor #18 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x06] lapic_id[0x14] enabled)
(XEN) Processor #20 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x07] lapic_id[0x20] enabled)
(XEN) Processor #32 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x08] lapic_id[0x22] enabled)
(XEN) Processor #34 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x09] lapic_id[0x24] enabled)
(XEN) Processor #36 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0a] lapic_id[0x30] enabled)
(XEN) Processor #48 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0b] lapic_id[0x32] enabled)
(XEN) Processor #50 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0c] lapic_id[0x34] enabled)
(XEN) Processor #52 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0d] lapic_id[0x01] enabled)
(XEN) Processor #1 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0e] lapic_id[0x03] enabled)
(XEN) Processor #3 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x0f] lapic_id[0x05] enabled)
(XEN) Processor #5 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x10] lapic_id[0x11] enabled)
(XEN) Processor #17 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x11] lapic_id[0x13] enabled)
(XEN) Processor #19 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x12] lapic_id[0x15] enabled)
(XEN) Processor #21 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x13] lapic_id[0x21] enabled)
(XEN) Processor #33 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x14] lapic_id[0x23] enabled)
(XEN) Processor #35 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x15] lapic_id[0x25] enabled)
(XEN) Processor #37 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x16] lapic_id[0x31] enabled)
(XEN) Processor #49 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x17] lapic_id[0x33] enabled)
(XEN) Processor #51 6:12 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x18] lapic_id[0x35] enabled)
(XEN) Processor #53 6:12 APIC version 21
(XEN) ACPI: LAPIC_NMI (acpi_id[0xff] high level lint[0x1])
(XEN) Overriding APIC driver with bigsmp
(XEN) ACPI: IOAPIC (id[0x06] address[0xfec00000] gsi_base[0])
(XEN) IOAPIC[0]: apic_id 6, version 32, address 0xfec00000, GSI 0-23
(XEN) ACPI: IOAPIC (id[0x07] address[0xfec8a000] gsi_base[24])
(XEN) IOAPIC[1]: apic_id 7, version 32, address 0xfec8a000, GSI 24-47
(XEN) ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
(XEN) ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level)
(XEN) ACPI: IRQ0 used by override.
(XEN) ACPI: IRQ2 used by override.
(XEN) ACPI: IRQ9 used by override.
(XEN) Enabling APIC mode:  Phys.  Using 2 I/O APICs
(XEN) ACPI: HPET id: 0x8086a301 base: 0xfed00000
(XEN) PCI: MCFG configuration 0: base e0000000 segment 0 buses 0 - 255
(XEN) PCI: MCFG area at e0000000 reserved in E820
(XEN) ERST table is invalid
(XEN) Using ACPI (MADT) for SMP configuration information
(XEN) IRQ limits: 48 GSI, 4576 MSI/MSI-X
(XEN) Using scheduler: SMP Credit Scheduler (credit)
(XEN) Detected 2400.177 MHz processor.
(XEN) Initing memory sharing.
(XEN) mce_intel.c:1214: MCA Capability: BCAST 1 SER 0 CMCI 1 firstbank 0 extended MCE MSR 0
(XEN) Intel machine check reporting enabled
(XEN) Intel VT-d Snoop Control enabled.
(XEN) Intel VT-d Dom0 DMA Passthrough not enabled.
(XEN) Intel VT-d Queued Invalidation enabled.
(XEN) Intel VT-d Interrupt Remapping enabled.
(XEN) Intel VT-d Shared EPT tables not enabled.
(XEN) I/O virtualisation enabled
(XEN)  - Dom0 mode: Relaxed
(XEN) Enabled directed EOI with ioapic_ack_old on!
(XEN) ENABLING IO-APIC IRQs
(XEN)  -> Using old ACK method
(XEN) ..TIMER: vector=0xF0 apic1=0 pin1=2 apic2=-1 pin2=-1
(XEN) Platform timer is 14.318MHz HPET
(XEN) Defaulting to alternative key handling; send \047A\047 to switch to normal mode.
(XEN) Allocated console ring of 256 KiB.
(XEN) VMX: Supported advanced features:
(XEN)  - APIC MMIO access virtualisation
(XEN)  - APIC TPR shadow
(XEN)  - Extended Page Tables (EPT)
(XEN)  - Virtual-Processor Identifiers (VPID)
(XEN)  - Virtual NMI
(XEN)  - MSR direct-access bitmap
(XEN)  - Unrestricted Guest
(XEN) EPT supports 1GB super page.
(XEN) EPT supports 2MB super page.
(XEN) HVM: ASIDs enabled.
(XEN) HVM: VMX enabled
(XEN) HVM: Hardware Assisted Paging detected.
(XEN) Brought up 24 CPUs
(XEN) ACPI sleep modes: S3
(XEN) mcheck_poll: Machine check polling timer started.
(XEN) *** LOADING DOMAIN 0 ***
(XEN) elf_parse_binary: phdr: paddr=0x2000 memsz=0x7cc000
(XEN) elf_parse_binary: phdr: paddr=0x7ce000 memsz=0x634d0
(XEN) elf_parse_binary: phdr: paddr=0x832000 memsz=0x888
(XEN) elf_parse_binary: phdr: paddr=0x833000 memsz=0xaa80
(XEN) elf_parse_binary: phdr: paddr=0x83e000 memsz=0x157000
(XEN) elf_parse_binary: memory: 0x2000 -> 0x995000
(XEN) elf_xen_parse_note: GUEST_OS = "linux"
(XEN) elf_xen_parse_note: GUEST_VERSION = "2.6"
(XEN) elf_xen_parse_note: XEN_VERSION = "xen-3.0"
(XEN) elf_xen_parse_note: VIRT_BASE = 0xffffffff80000000
(XEN) elf_xen_parse_note: PADDR_OFFSET = 0x0
(XEN) elf_xen_parse_note: ENTRY = 0xffffffff80002000
(XEN) elf_xen_parse_note: HYPERCALL_PAGE = 0xffffffff80003000
(XEN) elf_xen_parse_note: unknown xen elf note (0xd)
(XEN) elf_xen_parse_note: MOD_START_PFN = 0x1
(XEN) elf_xen_parse_note: INIT_P2M = 0xffffea0000000000
(XEN) elf_xen_parse_note: FEATURES = "writable_page_tables|writable_descriptor_tables|auto_translated_physmap|supervisor_mode_kernel"
(XEN) elf_xen_parse_note: LOADER = "generic"
(XEN) elf_xen_parse_note: SUSPEND_CANCEL = 0x1
(XEN) elf_xen_addr_calc_check: addresses:
(XEN)     virt_base        = 0xffffffff80000000
(XEN)     elf_paddr_offset = 0x0
(XEN)     virt_offset      = 0xffffffff80000000
(XEN)     virt_kstart      = 0xffffffff80002000
(XEN)     virt_kend        = 0xffffffff80995000
(XEN)     virt_entry       = 0xffffffff80002000
(XEN)     p2m_base         = 0xffffea0000000000
(XEN)  Xen  kernel: 64-bit, lsb, compat32
(XEN)  Dom0 kernel: 64-bit, lsb, paddr 0x2000 -> 0x995000
(XEN) PHYSICAL MEMORY ARRANGEMENT:
(XEN)  Dom0 alloc.:   0000001803000000->0000001804000000 (24798061 pages to be allocated)
(XEN)  Init. ramdisk: 000000183e7b0000->000000183ffff800
(XEN) VIRTUAL MEMORY ARRANGEMENT:
(XEN)  Loaded kernel: ffffffff80002000->ffffffff80995000
(XEN)  Init. ramdisk: 0000000000000000->0000000000000000
(XEN)  Phys-Mach map: ffffea0000000000->ffffea000bd45de8
(XEN)  Start info:    ffffffff80995000->ffffffff809954b4
(XEN)  Page tables:   ffffffff80996000->ffffffff8099f000
(XEN)  Boot stack:    ffffffff8099f000->ffffffff809a0000
(XEN)  TOTAL:         ffffffff80000000->ffffffff80c00000
(XEN)  ENTRY ADDRESS: ffffffff80002000
(XEN) Dom0 has maximum 24 VCPUs
(XEN) elf_load_binary: phdr 0 at 0xffffffff80002000 -> 0xffffffff807ce000
(XEN) elf_load_binary: phdr 1 at 0xffffffff807ce000 -> 0xffffffff808314d0
(XEN) elf_load_binary: phdr 2 at 0xffffffff80832000 -> 0xffffffff80832888
(XEN) elf_load_binary: phdr 3 at 0xffffffff80833000 -> 0xffffffff8083da80
(XEN) elf_load_binary: phdr 4 at 0xffffffff8083e000 -> 0xffffffff80899000
(XEN) Scrubbing Free RAM: .done.
(XEN) Std. Loglevel: All
(XEN) Guest Loglevel: All
(XEN) Xen is relinquishing VGA console.
(XEN) *** Serial input -> DOM0 (type \047CTRL-a\047 three times to switch input to Xen)
(XEN) Freed 244kB init memory.
(XEN) PCI add device 00:01.0
(XEN) PCI add device 00:02.0
(XEN) PCI add device 00:03.0
(XEN) PCI add device 00:07.0
(XEN) PCI add device 00:08.0
(XEN) PCI add device 00:09.0
(XEN) PCI add device 00:0a.0
(XEN) PCI add device 00:1c.0
(XEN) PCI add device 00:1c.4
(XEN) PCI add device 00:1c.5
(XEN) PCI add device 00:1f.2
(XEN) PCI add device 00:1f.5
(XEN) PCI add device 00:1a.7
(XEN) PCI add device 00:1d.7
(XEN) PCI add device 00:1a.0
(XEN) PCI add device 00:1a.1
(XEN) PCI add device 00:1d.0
(XEN) PCI add device 00:1d.1
(XEN) PCI add device 00:1d.2
(XEN) PCI add device 00:1d.3
(XEN) PCI add device 00:16.0
(XEN) PCI add device 00:16.1
(XEN) PCI add device 00:16.2
(XEN) PCI add device 00:16.3
(XEN) PCI add device 00:16.4
(XEN) PCI add device 00:16.5
(XEN) PCI add device 00:16.6
(XEN) PCI add device 00:16.7
(XEN) PCI add device 00:01.0
(XEN) PCI add device 00:02.0
(XEN) PCI add device 00:03.0
(XEN) PCI add device 00:07.0
(XEN) PCI add device 00:08.0
(XEN) PCI add device 00:09.0
(XEN) PCI add device 00:0a.0
(XEN) PCI add device 00:1c.0
(XEN) PCI add device 00:1c.4
(XEN) PCI add device 00:1c.5
(XEN) PCI add device 00:14.0
(XEN) PCI add device 00:1f.3
(XEN) PCI add device 03:00.0
(XEN) PCI add device 02:00.0
(XEN) mtrr: type mismatch for fb000000,800000 old: write-back new: write-combining
(XEN) memory.c:133:d0 Could not allocate order=18 extent: id=2 memflags=0 (0 of 1)
(XEN) memory.c:133:d0 Could not allocate order=9 extent: id=2 memflags=0 (2 of 4)
(XEN) memory.c:133:d0 Could not allocate order=9 extent: id=2 memflags=0 (0 of 4)
(XEN) memory.c:133:d0 Could not allocate order=9 extent: id=2 memflags=0 (0 of 4)
(XEN) memory.c:133:d0 Could not allocate order=9 extent: id=2 memflags=0 (0 of 4)
(XEN) memory.c:133:d0 Could not allocate order=9 extent: id=2 memflags=0 (0 of 4)
(XEN) memory.c:133:d0 Could not allocate order=9 extent: id=2 memflags=0 (0 of 4)
(XEN) memory.c:133:d0 Could not allocate order=9 extent: id=2 memflags=0 (0 of 2)
(XEN) HVM2: HVM Loader
(XEN) HVM2: Detected Xen v4.2-unstable
(XEN) HVM2: Xenbus rings @0xfeffc000, event channel 5
(XEN) HVM2: System requested ROMBIOS
(XEN) HVM2: CPU speed is 2400 MHz
(XEN) irq.c:264: Dom2 PCI link 0 changed 0 -> 5
(XEN) HVM2: PCI-ISA link 0 routed to IRQ5
(XEN) irq.c:264: Dom2 PCI link 1 changed 0 -> 10
(XEN) HVM2: PCI-ISA link 1 routed to IRQ10
(XEN) irq.c:264: Dom2 PCI link 2 changed 0 -> 11
(XEN) HVM2: PCI-ISA link 2 routed to IRQ11
(XEN) irq.c:264: Dom2 PCI link 3 changed 0 -> 5
(XEN) HVM2: PCI-ISA link 3 routed to IRQ5
(XEN) HVM2: pci dev 01:2 INTD->IRQ5
(XEN) HVM2: pci dev 01:3 INTA->IRQ10
(XEN) HVM2: pci dev 03:0 INTA->IRQ5
(XEN) HVM2: pci dev 04:0 INTA->IRQ5
(XEN) HVM2: pci dev 05:0 INTA->IRQ10
(XEN) HVM2: pci dev 06:0 INTA->IRQ11
(XEN) HVM2: pci dev 07:0 INTA->IRQ5
(XEN) HVM2: pci dev 02:0 bar 10 size 02000000: f0000008
(XEN) HVM2: pci dev 03:0 bar 14 size 01000000: f2000008
(XEN) HVM2: pci dev 04:0 bar 10 size 00020000: f3000000
(XEN) HVM2: pci dev 05:0 bar 10 size 00020000: f3020000
(XEN) HVM2: pci dev 06:0 bar 10 size 00020000: f3040000
(XEN) HVM2: pci dev 07:0 bar 10 size 00020000: f3060000
(XEN) HVM2: pci dev 02:0 bar 14 size 00001000: f3080000
(XEN) HVM2: pci dev 03:0 bar 10 size 00000100: 0000c001
(XEN) HVM2: pci dev 04:0 bar 14 size 00000040: 0000c101
(XEN) HVM2: pci dev 05:0 bar 14 size 00000040: 0000c141
(XEN) HVM2: pci dev 06:0 bar 14 size 00000040: 0000c181
(XEN) HVM2: pci dev 07:0 bar 14 size 00000040: 0000c1c1
(XEN) HVM2: pci dev 01:2 bar 20 size 00000020: 0000c201
(XEN) HVM2: pci dev 01:1 bar 20 size 00000010: 0000c221
(XEN) HVM2: Multiprocessor initialisation:
(XEN) HVM2:  - CPU0 ... 40-bit phys ... fixed MTRRs ... var MTRRs [2/8] ... done.
(XEN) HVM2:  - CPU1 ... 40-bit phys ... fixed MTRRs ... var MTRRs [2/8] ... done.
(XEN) HVM2:  - CPU2 ... 40-bit phys ... fixed MTRRs ... var MTRRs [2/8] ... done.
(XEN) HVM2:  - CPU3 ... 40-bit phys ... fixed MTRRs ... var MTRRs [2/8] ... done.
(XEN) HVM2: Testing HVM environment:
(XEN) HVM2:  - REP INSB across page boundaries ... passed
(XEN) HVM2:  - GS base MSRs and SWAPGS ... passed
(XEN) HVM2: Passed 2 of 2 tests
(XEN) HVM2: Writing SMBIOS tables ...
(XEN) HVM2: Loading ROMBIOS ...
(XEN) HVM2: 9852 bytes of ROMBIOS high-memory extensions:
(XEN) HVM2:   Relocating to 0xfc000000-0xfc00267c ... done
(XEN) HVM2: Creating MP tables ...
(XEN) HVM2: Loading Cirrus VGABIOS ...
(XEN) HVM2: Loading PCI Option ROM ...
(XEN) HVM2:  - Manufacturer: http://etherboot.org
(XEN) HVM2:  - Product name: gPXE
(XEN) HVM2: Loading ACPI ...
(XEN) HVM2: vm86 TSS at fc012880
(XEN) HVM2: BIOS map:
(XEN) HVM2:  c0000-c8fff: VGA BIOS
(XEN) HVM2:  c9000-dafff: Etherboot ROM
(XEN) HVM2:  f0000-fffff: Main BIOS
(XEN) HVM2: E820 table:
(XEN) HVM2:  [00]: 00000000:00000000 - 00000000:0009e000: RAM
(XEN) HVM2:  [01]: 00000000:0009e000 - 00000000:000a0000: RESERVED
(XEN) HVM2:  HOLE: 00000000:000a0000 - 00000000:000e0000
(XEN) HVM2:  [02]: 00000000:000e0000 - 00000000:00100000: RESERVED
(XEN) HVM2:  [03]: 00000000:00100000 - 00000000:f0000000: RAM
(XEN) HVM2:  HOLE: 00000000:f0000000 - 00000000:fc000000
(XEN) HVM2:  [04]: 00000000:fc000000 - 00000001:00000000: RESERVED
(XEN) HVM2:  [05]: 00000001:00000000 - 00000002:10400000: RAM
(XEN) HVM2: Invoking ROMBIOS ...
(XEN) HVM2: $Revision: 1.221 $ $Date: 2008/12/07 17:32:29 $
(XEN) stdvga.c:147:d2 entering stdvga and caching modes
(XEN) HVM2: VGABios $Id: vgabios.c,v 1.67 2008/01/27 09:44:12 vruppert Exp $
(XEN) HVM2: Bochs BIOS - build: 06/23/99
(XEN) HVM2: $Revision: 1.221 $ $Date: 2008/12/07 17:32:29 $
(XEN) HVM2: Options: apmbios pcibios eltorito PMM 
(XEN) HVM2: 
(XEN) HVM2: ata0-0: PCHS=16383/16/63 translation=lba LCHS=1024/255/63
(XEN) HVM2: ata0 master: QEMU HARDDISK ATA-7 Hard-Disk ( 250 GBytes)
(XEN) HVM2: IDE time out
(XEN) HVM2: ata1 master: QEMU DVD-ROM ATAPI-4 CD-Rom/DVD-Rom
(XEN) HVM2: IDE time out
(XEN) HVM2: 
(XEN) HVM2: 
(XEN) HVM2: 
(XEN) HVM2: Press F12 for boot menu.
(XEN) HVM2: 
(XEN) HVM2: Booting from CD-Rom...
(XEN) HVM2: 2895MB medium detected
(XEN) HVM2: Booting from 0000:7c00
(XEN) stdvga.c:151:d2 leaving stdvga
(XEN) stdvga.c:147:d2 entering stdvga and caching modes
(XEN) HVM2: IDE time out
(XEN) HVM2: int13_harddisk: function 41, unmapped device for ELDL=81
(XEN) HVM2: int13_harddisk: function 08, unmapped device for ELDL=81
(XEN) HVM2: *** int 15h function AX=00c0, BX=0000 not yet supported!
(XEN) HVM2: *** int 15h function AX=ec00, BX=0002 not yet supported!
(XEN) HVM2: KBD: unsupported int 16h function 03
(XEN) HVM2: *** int 15h function AX=e980, BX=0000 not yet supported!
(XEN) HVM2: int13_harddisk: function 41, unmapped device for ELDL=81
(XEN) HVM2: int13_harddisk: function 02, unmapped device for ELDL=81
(XEN) HVM2: int13_harddisk: function 41, unmapped device for ELDL=82
(XEN) HVM2: int13_harddisk: function 02, unmapped device for ELDL=82
(XEN) HVM2: int13_harddisk: function 41, unmapped device for ELDL=83
(XEN) HVM2: int13_harddisk: function 02, unmapped device for ELDL=83
(XEN) HVM2: int13_harddisk: function 41, unmapped device for ELDL=84
(XEN) HVM2: int13_harddisk: function 02, unmapped device for ELDL=84
(XEN) HVM2: int13_harddisk: function 41, unmapped device for ELDL=85
(XEN) HVM2: int13_harddisk: function 02, unmapped device for ELDL=85
(XEN) HVM2: int13_harddisk: function 41, unmapped device for ELDL=86
(XEN) HVM2: int13_harddisk: function 02, unmapped device for ELDL=86
(XEN) HVM2: int13_harddisk: function 41, unmapped device for ELDL=87
(XEN) HVM2: int13_harddisk: function 02, unmapped device for ELDL=87
(XEN) HVM2: int13_harddisk: function 41, ELDL out of range 88
(XEN) HVM2: int13_harddisk: function 02, ELDL out of range 88
(XEN) HVM2: int13_harddisk: function 41, ELDL out of range 89
(XEN) HVM2: int13_harddisk: function 02, ELDL out of range 89
(XEN) HVM2: int13_harddisk: function 41, ELDL out of range 8a
(XEN) HVM2: int13_harddisk: function 02, ELDL out of range 8a
(XEN) HVM2: int13_harddisk: function 41, ELDL out of range 8b
(XEN) HVM2: int13_harddisk: function 02, ELDL out of range 8b
(XEN) HVM2: int13_harddisk: function 41, ELDL out of range 8c
(XEN) HVM2: int13_harddisk: function 02, ELDL out of range 8c
(XEN) HVM2: int13_harddisk: function 41, ELDL out of range 8d
(XEN) HVM2: int13_harddisk: function 02, ELDL out of range 8d
(XEN) HVM2: int13_harddisk: function 41, ELDL out of range 8e
(XEN) HVM2: int13_harddisk: function 02, ELDL out of range 8e
(XEN) HVM2: int13_harddisk: function 41, ELDL out of range 8f
(XEN) HVM2: int13_harddisk: function 02, ELDL out of range 8f
(XEN) stdvga.c:151:d2 leaving stdvga
(XEN) irq.c:264: Dom2 PCI link 0 changed 5 -> 0
(XEN) irq.c:264: Dom2 PCI link 1 changed 10 -> 0
(XEN) irq.c:264: Dom2 PCI link 2 changed 11 -> 0
(XEN) irq.c:264: Dom2 PCI link 3 changed 5 -> 0
(XEN) traps.c:3074: GPF (0000): ffff82c4801c6f34 -> ffff82c4802166c5
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001dfb26000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1dfb26000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5793000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5793000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f5713000 not the same as current vmcs 00000001f151a000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied
(XEN) vvmx.c:1182:d2 vmclear gpa 1f151a000 not the same as current vmcs 00000001f5713000
(XEN) appropiate debug message to indicate tim\047s patch is applied

[-- Attachment #8: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Tim Deegan]

[-- Attachment #1: Type: text/plain, Size: 1079 bytes --]

At 12:46 +0200 on 26 Jul (1311684389), Jeroen Groenewegen van der Weyden wrote:
> Here my input

Thanks.  Looks very similar to the bug I thought I fixed with my patch.

VCPUs 1 and 3 are always at 0xa01a1c9d: <0f> 01 c2 eb 03 0f 01 c3
which is probably this fragment of code from KVM's vmx_vcpu_run():

        /* Enter guest mode */
        "jne .Llaunched \n\t"
	__ex(ASM_VMX_VMLAUNCH) "\n\t"
        "jmp .Lkvm_vmx_return \n\t"
        ".Llaunched: " __ex(ASM_VMX_VMRESUME) "\n\t"
	".Lkvm_vmx_return: "

So, just like the case I saw, they're trying to VMLAUNCH a VMCS and
failing.  That should only fail if the VMCS is already launched. 

I think the reason they're _stuck_ is that error paths for VMLAUNCH and
VMRESUME emulation are wrong; I can fix them up a little but I suspect
that won't solve the problem; just change it from a hang to some other
failure mode.

Can you try the attached patch instead of the previous one? 

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

[-- Attachment #2: vmclear --]
[-- Type: text/plain, Size: 2231 bytes --]

diff -r 9dbbf1631193 xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Mon Jul 25 14:21:13 2011 +0100
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Tue Jul 26 12:42:00 2011 +0100
@@ -1070,11 +1070,17 @@ int nvmx_handle_vmresume(struct cpu_user
     int launched;
     struct vcpu *v = current;
 
+    if ( vcpu_nestedhvm(v).nv_vvmcxaddr == VMCX_EADDR )
+    {
+        vmreturn (regs, VMFAIL_INVALID);
+        return X86EMUL_OKAY;        
+    }
+
     launched = __get_vvmcs(vcpu_nestedhvm(v).nv_vvmcx,
                            NVMX_LAUNCH_STATE);
     if ( !launched ) {
        vmreturn (regs, VMFAIL_VALID);
-       return X86EMUL_EXCEPTION;
+       return X86EMUL_OKAY;
     }
     return nvmx_vmresume(v,regs);
 }
@@ -1085,11 +1091,17 @@ int nvmx_handle_vmlaunch(struct cpu_user
     int rc;
     struct vcpu *v = current;
 
+    if ( vcpu_nestedhvm(v).nv_vvmcxaddr == VMCX_EADDR )
+    {
+        vmreturn (regs, VMFAIL_INVALID);
+        return X86EMUL_OKAY;        
+    }
+
     launched = __get_vvmcs(vcpu_nestedhvm(v).nv_vvmcx,
                            NVMX_LAUNCH_STATE);
     if ( launched ) {
        vmreturn (regs, VMFAIL_VALID);
-       rc = X86EMUL_EXCEPTION;
+       return X86EMUL_OKAY;
     }
     else {
         rc = nvmx_vmresume(v,regs);
@@ -1162,6 +1174,7 @@ int nvmx_handle_vmclear(struct cpu_user_
     struct vmx_inst_decoded decode;
     struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
     unsigned long gpa = 0;
+    void *vvmcs;
     int rc;
 
     rc = decode_vmx_inst(regs, &decode, &gpa, 0);
@@ -1176,9 +1189,15 @@ int nvmx_handle_vmclear(struct cpu_user_
 
     if ( gpa != nvcpu->nv_vvmcxaddr && nvcpu->nv_vvmcxaddr != VMCX_EADDR )
     {
-        gdprintk(XENLOG_WARNING, 
-                 "vmclear gpa %lx not the same as current vmcs %"PRIpaddr"\n",
+        gdprintk(XENLOG_WARNING, "vmclear gpa %lx != %"PRIpaddr"\n",
                  gpa, nvcpu->nv_vvmcxaddr);
+
+        /* Even if this VMCS isn't the current one, we must clear it. */
+        vvmcs = hvm_map_guest_frame_rw(gpa >> PAGE_SHIFT);
+        if ( vvmcs ) 
+            __set_vvmcs(vvmcs, NVMX_LAUNCH_STATE, 0);
+        hvm_unmap_guest_frame(vvmcs);
+
         vmreturn(regs, VMSUCCEED);
         goto out;
     }

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Jeroen Groenewegen van der Weyden]

[-- Attachment #1.1: Type: text/plain, Size: 1592 bytes --]

Tim, This improved a lot. the domu does not become in-responsive 
anymore. However, all four l2 guest are started. but 2 out of 4 are 
hanging/in-responsive after 30 to 60 seconds.

L1 domu (sles11sp1)
  -> seems to be ok

L2-1 propetary OS, seems to be ok
L2-2 propetary OS, seems to be ok
l2-3 sles10sp3, hanging/in-responsive
l2-4 sles10sp3, hanging/in-responsive

no degug/error messages in dmesg.

mfg,
jeroen

Op 26-7-2011 13:42, Tim Deegan schreef:
> At 12:46 +0200 on 26 Jul (1311684389), Jeroen Groenewegen van der Weyden wrote:
>> Here my input
> Thanks.  Looks very similar to the bug I thought I fixed with my patch.
>
> VCPUs 1 and 3 are always at 0xa01a1c9d:<0f>  01 c2 eb 03 0f 01 c3
> which is probably this fragment of code from KVM's vmx_vcpu_run():
>
>          /* Enter guest mode */
>          "jne .Llaunched \n\t"
> 	__ex(ASM_VMX_VMLAUNCH) "\n\t"
>          "jmp .Lkvm_vmx_return \n\t"
>          ".Llaunched: " __ex(ASM_VMX_VMRESUME) "\n\t"
> 	".Lkvm_vmx_return: "
>
> So, just like the case I saw, they're trying to VMLAUNCH a VMCS and
> failing.  That should only fail if the VMCS is already launched.
>
> I think the reason they're _stuck_ is that error paths for VMLAUNCH and
> VMRESUME emulation are wrong; I can fix them up a little but I suspect
> that won't solve the problem; just change it from a hang to some other
> failure mode.
>
> Can you try the attached patch instead of the previous one?
>
> Tim.
>
>
>
> _______________________________________________
> Xen-devel mailing list
> Xen-devel@lists.xensource.com
> http://lists.xensource.com/xen-devel


[-- Attachment #1.2: Type: text/html, Size: 2365 bytes --]

[-- Attachment #2: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Tim Deegan]

[-- Attachment #1: Type: text/plain, Size: 956 bytes --]

At 15:33 +0200 on 26 Jul (1311694437), Jeroen Groenewegen van der Weyden wrote:
> Tim, This improved a lot. the domu does not become in-responsive
> anymore. However, all four l2 guest are started. but 2 out of 4 are
> hanging/in-responsive after 30 to 60 seconds.
> 
> L1 domu (sles11sp1)
>  -> seems to be ok
> 
> L2-1 propetary OS, seems to be ok
> L2-2 propetary OS, seems to be ok
> l2-3 sles10sp3, hanging/in-responsive
> l2-4 sles10sp3, hanging/in-responsive
> 
> no degug/error messages in dmesg.

Anything in the l1 dmesg?

I'm trying to repro with actual linux guests but I've found that
PXELINUX is hanging for me. :(  I'll have a look at that later if I can.

In the meantime I realised I didn't quite get the logic right in the
first patch; there's one case missing.  Can you please try this one?

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

[-- Attachment #2: vmclear --]
[-- Type: text/plain, Size: 2843 bytes --]

diff -r 9dbbf1631193 xen/arch/x86/hvm/vmx/vvmx.c
--- a/xen/arch/x86/hvm/vmx/vvmx.c	Mon Jul 25 14:21:13 2011 +0100
+++ b/xen/arch/x86/hvm/vmx/vvmx.c	Tue Jul 26 15:19:30 2011 +0100
@@ -1070,11 +1070,17 @@ int nvmx_handle_vmresume(struct cpu_user
     int launched;
     struct vcpu *v = current;
 
+    if ( vcpu_nestedhvm(v).nv_vvmcxaddr == VMCX_EADDR )
+    {
+        vmreturn (regs, VMFAIL_INVALID);
+        return X86EMUL_OKAY;        
+    }
+
     launched = __get_vvmcs(vcpu_nestedhvm(v).nv_vvmcx,
                            NVMX_LAUNCH_STATE);
     if ( !launched ) {
        vmreturn (regs, VMFAIL_VALID);
-       return X86EMUL_EXCEPTION;
+       return X86EMUL_OKAY;
     }
     return nvmx_vmresume(v,regs);
 }
@@ -1085,11 +1091,17 @@ int nvmx_handle_vmlaunch(struct cpu_user
     int rc;
     struct vcpu *v = current;
 
+    if ( vcpu_nestedhvm(v).nv_vvmcxaddr == VMCX_EADDR )
+    {
+        vmreturn (regs, VMFAIL_INVALID);
+        return X86EMUL_OKAY;        
+    }
+
     launched = __get_vvmcs(vcpu_nestedhvm(v).nv_vvmcx,
                            NVMX_LAUNCH_STATE);
     if ( launched ) {
        vmreturn (regs, VMFAIL_VALID);
-       rc = X86EMUL_EXCEPTION;
+       return X86EMUL_OKAY;
     }
     else {
         rc = nvmx_vmresume(v,regs);
@@ -1162,6 +1174,7 @@ int nvmx_handle_vmclear(struct cpu_user_
     struct vmx_inst_decoded decode;
     struct nestedvcpu *nvcpu = &vcpu_nestedhvm(v);
     unsigned long gpa = 0;
+    void *vvmcs;
     int rc;
 
     rc = decode_vmx_inst(regs, &decode, &gpa, 0);
@@ -1171,24 +1184,28 @@ int nvmx_handle_vmclear(struct cpu_user_
     if ( gpa & 0xfff )
     {
         vmreturn(regs, VMFAIL_INVALID);
-        goto out;
+        return X86EMUL_OKAY;
+    }
+    
+    if ( gpa == nvcpu->nv_vvmcxaddr ) 
+    {
+        __set_vvmcs(nvcpu->nv_vvmcx, NVMX_LAUNCH_STATE, 0);
+        nvmx_purge_vvmcs(v);
+    }
+    else 
+    {
+        /* Even if this VMCS isn't the current one, we must clear it. */
+        vvmcs = hvm_map_guest_frame_rw(gpa >> PAGE_SHIFT);
+        if ( vvmcs ) 
+            __set_vvmcs(vvmcs, NVMX_LAUNCH_STATE, 0);
+        hvm_unmap_guest_frame(vvmcs);
+
+        if ( nvcpu->nv_vvmcxaddr != VMCX_EADDR )
+            gdprintk(XENLOG_WARNING, "vmclear gpa %lx != %"PRIpaddr"\n",
+                     gpa, nvcpu->nv_vvmcxaddr);
     }
 
-    if ( gpa != nvcpu->nv_vvmcxaddr && nvcpu->nv_vvmcxaddr != VMCX_EADDR )
-    {
-        gdprintk(XENLOG_WARNING, 
-                 "vmclear gpa %lx not the same as current vmcs %"PRIpaddr"\n",
-                 gpa, nvcpu->nv_vvmcxaddr);
-        vmreturn(regs, VMSUCCEED);
-        goto out;
-    }
-    if ( nvcpu->nv_vvmcxaddr != VMCX_EADDR )
-        __set_vvmcs(nvcpu->nv_vvmcx, NVMX_LAUNCH_STATE, 0);
-    nvmx_purge_vvmcs(v);
-
     vmreturn(regs, VMSUCCEED);
-
-out:
     return X86EMUL_OKAY;
 }
 

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Jeroen Groenewegen van der Weyden]

Tim! everyhting seems to work now. domu is working ok, all 4 nested kvm 
guests are running.

only dmesg seems te complain about something
(XEN) vvmx.c:1205:d2 vmclear gpa 1920de000 != 000000018c829000
(XEN) vvmx.c:1205:d2 vmclear gpa 1920de000 != 000000018c829000
(XEN) vvmx.c:1205:d2 vmclear gpa 1920de000 != 00000001f9372000
(XEN) vvmx.c:1205:d2 vmclear gpa 1ec8f7000 != 00000001f9372000
(XEN) vvmx.c:1205:d2 vmclear gpa 1920de000 != 00000001ec8f7000
(XEN) vvmx.c:1205:d2 vmclear gpa 18c829000 != 00000001ec8f7000
(XEN) vvmx.c:1205:d2 vmclear gpa 1920de000 != 00000001ec8f7000


mfg,
Jeroen

Op 26-7-2011 16:21, Tim Deegan schreef:
> At 15:33 +0200 on 26 Jul (1311694437), Jeroen Groenewegen van der Weyden wrote:
>> Tim, This improved a lot. the domu does not become in-responsive
>> anymore. However, all four l2 guest are started. but 2 out of 4 are
>> hanging/in-responsive after 30 to 60 seconds.
>>
>> L1 domu (sles11sp1)
>>   ->  seems to be ok
>>
>> L2-1 propetary OS, seems to be ok
>> L2-2 propetary OS, seems to be ok
>> l2-3 sles10sp3, hanging/in-responsive
>> l2-4 sles10sp3, hanging/in-responsive
>>
>> no degug/error messages in dmesg.
> Anything in the l1 dmesg?
>
> I'm trying to repro with actual linux guests but I've found that
> PXELINUX is hanging for me. :(  I'll have a look at that later if I can.
>
> In the meantime I realised I didn't quite get the logic right in the
> first patch; there's one case missing.  Can you please try this one?
>
> Tim.
>

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Tim Deegan]

At 17:25 +0200 on 26 Jul (1311701111), Jeroen Groenewegen van der Weyden wrote:
> Tim! everyhting seems to work now. domu is working ok, all 4 nested
> kvm guests are running.

Excellent.  Thank you for the testing.  I'll check those changes in. 

> only dmesg seems te complain about something
> (XEN) vvmx.c:1205:d2 vmclear gpa 1920de000 != 000000018c829000

That's OK; those are benign and I'll remove them. 

Cheers,

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Jeroen Groenewegen van der Weyden]

Tim, just to be complete, I tested them against cs23728. the lastest 
change set cs23749 fails to boot at my system, it hangs at the hpet.

Thank you for cooperation. If you need any further testing done with 
this nestedhvm just let me know. I can image you want some performance 
testing and improvments are necessary.

mvg,
Jeroen

Op 26-7-2011 17:48, Tim Deegan schreef:
> At 17:25 +0200 on 26 Jul (1311701111), Jeroen Groenewegen van der Weyden wrote:
>> Tim! everyhting seems to work now. domu is working ok, all 4 nested
>> kvm guests are running.
> Excellent.  Thank you for the testing.  I'll check those changes in.
>
>> only dmesg seems te complain about something
>> (XEN) vvmx.c:1205:d2 vmclear gpa 1920de000 != 000000018c829000
> That's OK; those are benign and I'll remove them.
>
> Cheers,
>
> Tim.
>

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Tim Deegan]

Hi, 

At 18:00 +0200 on 26 Jul (1311703215), Jeroen Groenewegen van der Weyden wrote:
> Thank you for cooperation. If you need any further testing done with
> this nestedhvm just let me know. I can image you want some
> performance testing and improvments are necessary.

Yes, the performance is proabbly pretty poor right now.  I believe Eddie
has plans for a nested-EPT patch series, which should make a big
difference.

Tim.

-- 
Tim Deegan <Tim.Deegan@citrix.com>
Principal Software Engineer, Xen Platform Team
Citrix Systems UK Ltd.  (Company #02937203, SL9 0BG)

Re: [PATCH 20 of 20] n2 MSR handling and capability exposure

[Jeroen Groenewegen van der Weyden]

Op 26-7-2011 12:00, Tim Deegan schreef:
> If you give your first-level guest only one vcpu, does the problem go
> away?
I did. I can not tell for sure 100% but it seems different/better. eg 
from ir-responsive to dead snail performance. after this the DOMU reboot 
it self but this is/could be a mechanism of the installed application.  
I'do more investigation to be sure and keep you informed.