[PATCH][v2] Hybrid extension support in Xen

[Sheng Yang <sheng@linux.intel.com>]

[-- Attachment #1: Type: Text/Plain, Size: 1254 bytes --]

Hi Keir

Here is the second version of Hybrid extension support in Xen. Mostly the 
patch did:

1. Enable SMP support through VCPU_OP in arch_set_info_guest().
2. A new hypercall in hvm_ops to enable hybrid.
3. Mapping IRQ to VIRQ when deliver to the guest.
4. Inject a guest defined vector to deliver notification for events.
5. Use CPUID leaf 0x40000002 to support hybrid feature
6. Reserve some space at the end of MMIO hole for grant table use.

Please review. Thanks!

-- 
regards
Yang, Sheng

--
 tools/firmware/hvmloader/config.h    |    6 ++-
 tools/firmware/hvmloader/hvmloader.c |    6 +++
 xen/arch/x86/domain.c                |    9 ++++
 xen/arch/x86/hvm/hvm.c               |   62 +++++++++++++++++++++++++++++---
 xen/arch/x86/hvm/irq.c               |   66 
++++++++++++++++++++++++++++-------
 xen/arch/x86/hvm/vmx/intr.c          |    3 +
 xen/arch/x86/traps.c                 |    9 ++++
 xen/include/asm-x86/hvm/hvm.h        |    4 +-
 xen/include/asm-x86/hvm/irq.h        |    4 +-
 xen/include/public/arch-x86/cpuid.h  |    7 +++
 xen/include/public/hvm/hvm_op.h      |    6 +++
 xen/include/public/xen.h             |    7 +++
 xen/include/xen/sched.h              |    9 ++++
 13 files changed, 175 insertions(+), 23 deletions(-)

[-- Attachment #2: hybrid-xen.patch --]
[-- Type: text/x-patch, Size: 15850 bytes --]

diff --git a/tools/firmware/hvmloader/config.h b/tools/firmware/hvmloader/config.h
--- a/tools/firmware/hvmloader/config.h
+++ b/tools/firmware/hvmloader/config.h
@@ -16,8 +16,12 @@
 
 /* MMIO hole: Hardcoded defaults, which can be dynamically expanded. */
 #define PCI_MEM_START       0xf0000000
-#define PCI_MEM_END         0xfc000000
+#define PCI_MEM_END         0xfbfe0000
 extern unsigned long pci_mem_start, pci_mem_end;
+
+/* Reserve 128KB for grant table */
+#define GNTTAB_MEMBASE	    0xfbfe0000
+#define GNTTAB_MEMSIZE	    0x20000
 
 /* We reserve 16MB for special BIOS mappings, etc. */
 #define RESERVED_MEMBASE    0xfc000000
diff --git a/tools/firmware/hvmloader/hvmloader.c b/tools/firmware/hvmloader/hvmloader.c
--- a/tools/firmware/hvmloader/hvmloader.c
+++ b/tools/firmware/hvmloader/hvmloader.c
@@ -618,6 +618,12 @@ static void build_e820_table(void)
     e820[nr].type = E820_RAM;
     nr++;
 
+    /* Reserved for grant table */
+    e820[nr].addr = GNTTAB_MEMBASE;
+    e820[nr].size = GNTTAB_MEMSIZE;
+    e820[nr].type = E820_RESERVED;
+    nr++;
+
     /*
      * Explicitly reserve space for special pages.
      * This space starts at RESERVED_MEMBASE an extends to cover various
diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -682,7 +682,16 @@ int arch_set_info_guest(
 
     if ( is_hvm_vcpu(v) )
     {
+        unsigned long eip, cs;
+
         hvm_set_info_guest(v);
+
+        eip = c(user_regs.eip);
+        if (eip != 0) {
+            cs = eip >> 12 << 8;
+            hvm_vcpu_reset_state(v, cs, 0);
+            hvm_funcs.set_tsc_offset(v, 0);
+        }
         goto out;
     }
 
diff --git a/xen/arch/x86/hvm/hvm.c b/xen/arch/x86/hvm/hvm.c
--- a/xen/arch/x86/hvm/hvm.c
+++ b/xen/arch/x86/hvm/hvm.c
@@ -2325,6 +2325,17 @@ static hvm_hypercall_t *hvm_hypercall32_
     HYPERCALL(hvm_op)
 };
 
+static hvm_hypercall_t *hvm_hypercall_hybrid64_table[NR_hypercalls] = {
+    [ __HYPERVISOR_memory_op ] = (hvm_hypercall_t *)hvm_memory_op,
+    [ __HYPERVISOR_grant_table_op ] = (hvm_hypercall_t *)hvm_grant_table_op,
+    HYPERCALL(xen_version),
+    HYPERCALL(console_io),
+    HYPERCALL(vcpu_op),
+    HYPERCALL(sched_op),
+    HYPERCALL(event_channel_op),
+    HYPERCALL(hvm_op),
+};
+
 #endif /* defined(__x86_64__) */
 
 int hvm_do_hypercall(struct cpu_user_regs *regs)
@@ -2355,7 +2366,8 @@ int hvm_do_hypercall(struct cpu_user_reg
     if ( (eax & 0x80000000) && is_viridian_domain(curr->domain) )
         return viridian_hypercall(regs);
 
-    if ( (eax >= NR_hypercalls) || !hvm_hypercall32_table[eax] )
+    if ( (eax >= NR_hypercalls) ||
+         (!hvm_hypercall32_table[eax] && !is_hybrid_vcpu(curr)) )
     {
         regs->eax = -ENOSYS;
         return HVM_HCALL_completed;
@@ -2370,11 +2382,18 @@ int hvm_do_hypercall(struct cpu_user_reg
                     regs->rdi, regs->rsi, regs->rdx, regs->r10, regs->r8);
 
         this_cpu(hvm_64bit_hcall) = 1;
-        regs->rax = hvm_hypercall64_table[eax](regs->rdi,
-                                               regs->rsi,
-                                               regs->rdx,
-                                               regs->r10,
-                                               regs->r8); 
+        if (is_hybrid_vcpu(curr))
+            regs->rax = hvm_hypercall_hybrid64_table[eax](regs->rdi,
+                                                          regs->rsi,
+                                                          regs->rdx,
+                                                          regs->r10,
+                                                          regs->r8);
+        else
+            regs->rax = hvm_hypercall64_table[eax](regs->rdi,
+                                                   regs->rsi,
+                                                   regs->rdx,
+                                                   regs->r10,
+                                                   regs->r8);
         this_cpu(hvm_64bit_hcall) = 0;
     }
     else
@@ -3109,6 +3128,37 @@ long do_hvm_op(unsigned long op, XEN_GUE
         break;
     }
 
+    case HVMOP_enable_hybrid: {
+        struct xen_hvm_hybrid_type a;
+        struct domain *d;
+
+        if ( copy_from_guest(&a, arg, 1) )
+            return -EFAULT;
+
+        rc = rcu_lock_target_domain_by_id(a.domid, &d);
+        if ( rc != 0 )
+            return rc;
+
+        rc = -EINVAL;
+        if ( !is_hvm_domain(d) )
+            goto param_fail5;
+
+        rc = xsm_hvm_param(d, op);
+        if ( rc )
+            goto param_fail5;
+
+        d->hybrid_enabled = XEN_HYBRID_ENABLED;
+        printk("HVM: Hybrid domain enabled\n");
+        if (a.flags & HVM_HYBRID_EVTCHN) {
+            update_domain_wallclock_time(d);
+            hvm_funcs.set_tsc_offset(d->vcpu[0], 0);
+            d->hybrid_enabled |= XEN_HYBRID_EVTCHN_ENABLED;
+        }
+param_fail5:
+        rcu_unlock_domain(d);
+        break;
+    }
+
     default:
     {
         gdprintk(XENLOG_WARNING, "Bad HVM op %ld.\n", op);
diff --git a/xen/arch/x86/hvm/irq.c b/xen/arch/x86/hvm/irq.c
--- a/xen/arch/x86/hvm/irq.c
+++ b/xen/arch/x86/hvm/irq.c
@@ -46,8 +46,18 @@ static void __hvm_pci_intx_assert(
     if ( (hvm_irq->pci_link_assert_count[link]++ == 0) && isa_irq &&
          (hvm_irq->gsi_assert_count[isa_irq]++ == 0) )
     {
-        vioapic_irq_positive_edge(d, isa_irq);
-        vpic_irq_positive_edge(d, isa_irq);
+        if ( !is_hybrid_evtchn_enabled_domain(d) )
+        {
+            vioapic_irq_positive_edge(d, isa_irq);
+            vpic_irq_positive_edge(d, isa_irq);
+        }
+        else
+        {
+            /* TODO fix the critical region here */
+            spin_unlock(&d->arch.hvm_domain.irq_lock);
+            send_guest_global_virq(d, VIRQ_EMUL_PIN(isa_irq));
+            spin_lock(&d->arch.hvm_domain.irq_lock);
+	}
     }
 }
 
@@ -76,8 +86,10 @@ static void __hvm_pci_intx_deassert(
     link    = hvm_pci_intx_link(device, intx);
     isa_irq = hvm_irq->pci_link.route[link];
     if ( (--hvm_irq->pci_link_assert_count[link] == 0) && isa_irq &&
-         (--hvm_irq->gsi_assert_count[isa_irq] == 0) )
-        vpic_irq_negative_edge(d, isa_irq);
+         (--hvm_irq->gsi_assert_count[isa_irq] == 0) ) {
+        if ( !is_hybrid_evtchn_enabled_domain(d) )
+            vpic_irq_negative_edge(d, isa_irq);
+    }
 }
 
 void hvm_pci_intx_deassert(
@@ -93,6 +105,7 @@ void hvm_isa_irq_assert(
 {
     struct hvm_irq *hvm_irq = &d->arch.hvm_domain.irq;
     unsigned int gsi = hvm_isa_irq_to_gsi(isa_irq);
+    int send_virq = 0;
 
     ASSERT(isa_irq <= 15);
 
@@ -101,11 +114,21 @@ void hvm_isa_irq_assert(
     if ( !__test_and_set_bit(isa_irq, &hvm_irq->isa_irq.i) &&
          (hvm_irq->gsi_assert_count[gsi]++ == 0) )
     {
-        vioapic_irq_positive_edge(d, gsi);
-        vpic_irq_positive_edge(d, isa_irq);
+        if ( !is_hybrid_evtchn_enabled_domain(d) )
+        {
+            vioapic_irq_positive_edge(d, gsi);
+            vpic_irq_positive_edge(d, isa_irq);
+        }
+        else
+        {
+            send_virq = 1;
+        }
     }
 
     spin_unlock(&d->arch.hvm_domain.irq_lock);
+
+    if (send_virq)
+	    send_guest_global_virq(d, VIRQ_EMUL_PIN(isa_irq));
 }
 
 void hvm_isa_irq_deassert(
@@ -120,7 +143,10 @@ void hvm_isa_irq_deassert(
 
     if ( __test_and_clear_bit(isa_irq, &hvm_irq->isa_irq.i) &&
          (--hvm_irq->gsi_assert_count[gsi] == 0) )
-        vpic_irq_negative_edge(d, isa_irq);
+    {
+        if ( !is_hybrid_evtchn_enabled_domain(d) )
+            vpic_irq_negative_edge(d, isa_irq);
+    }
 
     spin_unlock(&d->arch.hvm_domain.irq_lock);
 }
@@ -165,6 +191,8 @@ static void hvm_set_callback_irq_level(s
             __hvm_pci_intx_assert(d, pdev, pintx);
         else
             __hvm_pci_intx_deassert(d, pdev, pintx);
+    case HVMIRQ_callback_vector:
+        vcpu_kick(v);
     default:
         break;
     }
@@ -185,16 +213,17 @@ void hvm_maybe_deassert_evtchn_irq(void)
 
 void hvm_assert_evtchn_irq(struct vcpu *v)
 {
-    if ( v->vcpu_id != 0 )
-        return;
-
     if ( unlikely(in_irq() || !local_irq_is_enabled()) )
     {
         tasklet_schedule(&v->arch.hvm_vcpu.assert_evtchn_irq_tasklet);
         return;
     }
 
-    hvm_set_callback_irq_level(v);
+    /* set_call_irq_level can't deal with vcpu other than 0 */
+    if (v->vcpu_id != 0)
+        vcpu_kick(v);
+    else
+        hvm_set_callback_irq_level(v);
 }
 
 void hvm_set_pci_link_route(struct domain *d, u8 link, u8 isa_irq)
@@ -251,7 +280,7 @@ void hvm_set_callback_via(struct domain 
 
     via_type = (uint8_t)(via >> 56) + 1;
     if ( ((via_type == HVMIRQ_callback_gsi) && (via == 0)) ||
-         (via_type > HVMIRQ_callback_pci_intx) )
+         (via_type > HVMIRQ_callback_vector) )
         via_type = HVMIRQ_callback_none;
 
     spin_lock(&d->arch.hvm_domain.irq_lock);
@@ -297,6 +326,9 @@ void hvm_set_callback_via(struct domain 
         if ( hvm_irq->callback_via_asserted )
              __hvm_pci_intx_assert(d, pdev, pintx);
         break;
+    case HVMIRQ_callback_vector:
+        hvm_irq->callback_via.vector = (uint8_t)via;
+        break;
     default:
         break;
     }
@@ -312,6 +344,10 @@ void hvm_set_callback_via(struct domain 
     case HVMIRQ_callback_pci_intx:
         printk("PCI INTx Dev 0x%02x Int%c\n", pdev, 'A' + pintx);
         break;
+    case HVMIRQ_callback_vector:
+        printk("Set HVMIRQ_callback_vector to %u\n",
+               hvm_irq->callback_via.vector);
+        break;
     default:
         printk("None\n");
         break;
@@ -322,6 +358,10 @@ struct hvm_intack hvm_vcpu_has_pending_i
 {
     struct hvm_domain *plat = &v->domain->arch.hvm_domain;
     int vector;
+
+    if (plat->irq.callback_via_type == HVMIRQ_callback_vector &&
+            vcpu_info(v, evtchn_upcall_pending))
+        return hvm_intack_vector(plat->irq.callback_via.vector);
 
     if ( unlikely(v->nmi_pending) )
         return hvm_intack_nmi;
@@ -363,6 +403,8 @@ struct hvm_intack hvm_vcpu_ack_pending_i
     case hvm_intsrc_lapic:
         if ( !vlapic_ack_pending_irq(v, intack.vector) )
             intack = hvm_intack_none;
+        break;
+    case hvm_intsrc_vector:
         break;
     default:
         intack = hvm_intack_none;
diff --git a/xen/arch/x86/hvm/vmx/intr.c b/xen/arch/x86/hvm/vmx/intr.c
--- a/xen/arch/x86/hvm/vmx/intr.c
+++ b/xen/arch/x86/hvm/vmx/intr.c
@@ -164,7 +164,8 @@ asmlinkage void vmx_intr_assist(void)
     {
         HVMTRACE_2D(INJ_VIRQ, intack.vector, /*fake=*/ 0);
         vmx_inject_extint(intack.vector);
-        pt_intr_post(v, intack);
+        if (intack.source != hvm_intsrc_vector)
+             pt_intr_post(v, intack);
     }
 
     /* Is there another IRQ to queue up behind this one? */
diff --git a/xen/arch/x86/traps.c b/xen/arch/x86/traps.c
--- a/xen/arch/x86/traps.c
+++ b/xen/arch/x86/traps.c
@@ -686,6 +686,7 @@ int cpuid_hypervisor_leaves( uint32_t id
     struct domain *d = current->domain;
     /* Optionally shift out of the way of Viridian architectural leaves. */
     uint32_t base = is_viridian_domain(d) ? 0x40000100 : 0x40000000;
+    unsigned int tmp_eax, tmp_ebx, tmp_ecx, tmp_edx;
 
     idx -= base;
     if ( idx > 3 ) 
@@ -716,6 +717,14 @@ int cpuid_hypervisor_leaves( uint32_t id
         *edx = 0;          /* Features 2 */
         if ( !is_hvm_vcpu(current) )
             *ecx |= XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD;
+
+        /* Check if additional feature specified, e.g. Hybrid */
+        if ( !is_viridian_domain(d) ) {
+            domain_cpuid(d, 0x40000002, 0,
+                         &tmp_eax, &tmp_ebx, &tmp_ecx, &tmp_edx);
+            if (tmp_edx != 0)
+                *edx = tmp_edx & XEN_CPUID_FEAT2_MASK;
+        }
         break;
 
     case 3:
diff --git a/xen/include/asm-x86/hvm/hvm.h b/xen/include/asm-x86/hvm/hvm.h
--- a/xen/include/asm-x86/hvm/hvm.h
+++ b/xen/include/asm-x86/hvm/hvm.h
@@ -33,7 +33,8 @@ enum hvm_intsrc {
     hvm_intsrc_pic,
     hvm_intsrc_lapic,
     hvm_intsrc_nmi,
-    hvm_intsrc_mce
+    hvm_intsrc_mce,
+    hvm_intsrc_vector,
 };
 struct hvm_intack {
     uint8_t source; /* enum hvm_intsrc */
@@ -44,6 +45,7 @@ struct hvm_intack {
 #define hvm_intack_lapic(vec) ( (struct hvm_intack) { hvm_intsrc_lapic, vec } )
 #define hvm_intack_nmi        ( (struct hvm_intack) { hvm_intsrc_nmi,   2 } )
 #define hvm_intack_mce        ( (struct hvm_intack) { hvm_intsrc_mce,   18 } )
+#define hvm_intack_vector(vec)( (struct hvm_intack) { hvm_intsrc_vector, vec } )
 enum hvm_intblk {
     hvm_intblk_none,      /* not blocked (deliverable) */
     hvm_intblk_shadow,    /* MOV-SS or STI shadow */
diff --git a/xen/include/asm-x86/hvm/irq.h b/xen/include/asm-x86/hvm/irq.h
--- a/xen/include/asm-x86/hvm/irq.h
+++ b/xen/include/asm-x86/hvm/irq.h
@@ -54,12 +54,14 @@ struct hvm_irq {
         enum {
             HVMIRQ_callback_none,
             HVMIRQ_callback_gsi,
-            HVMIRQ_callback_pci_intx
+            HVMIRQ_callback_pci_intx,
+            HVMIRQ_callback_vector,
         } callback_via_type;
     };
     union {
         uint32_t gsi;
         struct { uint8_t dev, intx; } pci;
+        uint32_t vector;
     } callback_via;
 
     /* Number of INTx wires asserting each PCI-ISA link. */
diff --git a/xen/include/public/arch-x86/cpuid.h b/xen/include/public/arch-x86/cpuid.h
--- a/xen/include/public/arch-x86/cpuid.h
+++ b/xen/include/public/arch-x86/cpuid.h
@@ -65,4 +65,11 @@
 #define _XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD 0
 #define XEN_CPUID_FEAT1_MMU_PT_UPDATE_PRESERVE_AD  (1u<<0)
 
+/* Mask unsupported CPUID specified by user */
+#define XEN_CPUID_FEAT2_MASK 0x3ul
+#define _XEN_CPUID_FEAT2_HYBRID 0
+#define XEN_CPUID_FEAT2_HYBRID (1u<<0)
+#define _XEN_CPUID_FEAT2_HYBRID_EVTCHN 1
+#define XEN_CPUID_FEAT2_HYBRID_EVTCHN (1u<<1)
+
 #endif /* __XEN_PUBLIC_ARCH_X86_CPUID_H__ */
diff --git a/xen/include/public/hvm/hvm_op.h b/xen/include/public/hvm/hvm_op.h
--- a/xen/include/public/hvm/hvm_op.h
+++ b/xen/include/public/hvm/hvm_op.h
@@ -127,6 +127,12 @@ typedef struct xen_hvm_set_mem_type xen_
 typedef struct xen_hvm_set_mem_type xen_hvm_set_mem_type_t;
 DEFINE_XEN_GUEST_HANDLE(xen_hvm_set_mem_type_t);
 
+#define HVMOP_enable_hybrid    9
+struct xen_hvm_hybrid_type {
+    domid_t domid;
+    uint32_t flags;
+#define HVM_HYBRID_EVTCHN (1ull<<1)
+};
 
 #endif /* defined(__XEN__) || defined(__XEN_TOOLS__) */
 
diff --git a/xen/include/public/xen.h b/xen/include/public/xen.h
--- a/xen/include/public/xen.h
+++ b/xen/include/public/xen.h
@@ -158,7 +158,12 @@ DEFINE_XEN_GUEST_HANDLE(xen_pfn_t);
 #define VIRQ_ARCH_6    22
 #define VIRQ_ARCH_7    23
 
-#define NR_VIRQS       24
+#define VIRQ_EMUL_PIN_START 24
+#define VIRQ_EMUL_PIN_END 39
+#define VIRQ_EMUL_PIN_NUM 16
+#define VIRQ_EMUL_PIN(x) (VIRQ_EMUL_PIN_START + x)
+
+#define NR_VIRQS       40
 
 /*
  * HYPERVISOR_mmu_update(reqs, count, pdone, foreigndom)
diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h
--- a/xen/include/xen/sched.h
+++ b/xen/include/xen/sched.h
@@ -309,6 +309,10 @@ struct domain
 
     /* Memory paging support */
     struct mem_event_domain mem_event;
+
+#define XEN_HYBRID_ENABLED          (1u << 0)
+#define XEN_HYBRID_EVTCHN_ENABLED   (1u << 1)
+    uint64_t hybrid_enabled;
 };
 
 struct domain_setup_info
@@ -595,6 +599,11 @@ uint64_t get_cpu_idle_time(unsigned int 
 #define is_hvm_vcpu(v)   (is_hvm_domain(v->domain))
 #define need_iommu(d)    ((d)->need_iommu)
 
+#define is_hybrid_domain(d) ((d)->hybrid_enabled & XEN_HYBRID_ENABLED)
+#define is_hybrid_vcpu(v)   (is_hybrid_domain(v->domain))
+#define is_hybrid_evtchn_enabled_domain(d) (is_hybrid_domain(d) && \
+		(d)->hybrid_enabled & XEN_HYBRID_EVTCHN_ENABLED)
+
 void set_vcpu_migration_delay(unsigned int delay);
 unsigned int get_vcpu_migration_delay(void);
 

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel