[PATCH v3 10/17] x86: allocate per-vcpu stacks for interrupt entries

[Juergen Gross <jgross@suse.com>]

In case of XPTI being active for a pv-domain allocate and initialize
per-vcpu stacks. The stacks are added to the per-domain mappings of
the pv-domain.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
V3:
- move xpti code to xpti.c
- directly modify page table entries as needed for stub and stack
  page (Jan Beulich)
- use one page for all stacks and TSS
- remap global stub instead allocating one for each vcpu
---
 xen/arch/x86/pv/domain.c      |   2 +
 xen/arch/x86/pv/xpti.c        | 117 +++++++++++++++++++++++++++++++++++++++---
 xen/include/asm-x86/config.h  |  13 ++++-
 xen/include/asm-x86/current.h |  49 +++++++++++++-----
 xen/include/asm-x86/domain.h  |   3 ++
 xen/include/asm-x86/pv/mm.h   |   2 +
 6 files changed, 166 insertions(+), 20 deletions(-)

diff --git a/xen/arch/x86/pv/domain.c b/xen/arch/x86/pv/domain.c
index a007af94dd..550fbbf0fe 100644
--- a/xen/arch/x86/pv/domain.c
+++ b/xen/arch/x86/pv/domain.c
@@ -120,6 +120,8 @@ void pv_vcpu_destroy(struct vcpu *v)
     pv_destroy_gdt_ldt_l1tab(v);
     xfree(v->arch.pv_vcpu.trap_ctxt);
     v->arch.pv_vcpu.trap_ctxt = NULL;
+
+    xpti_vcpu_destroy(v);
 }
 
 int pv_vcpu_initialise(struct vcpu *v)
diff --git a/xen/arch/x86/pv/xpti.c b/xen/arch/x86/pv/xpti.c
index 0b17d77d74..1356541804 100644
--- a/xen/arch/x86/pv/xpti.c
+++ b/xen/arch/x86/pv/xpti.c
@@ -19,13 +19,28 @@
  * along with this program; If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <xen/domain_page.h>
 #include <xen/errno.h>
 #include <xen/init.h>
 #include <xen/lib.h>
 #include <xen/sched.h>
 
+#define XPTI_STACK_SIZE 512
+#define XPTI_STACK_N (XPTI_STACK_SIZE / 8)
+
+struct xpti_stack {
+    struct tss_struct tss;
+    char pad[PAGE_SIZE - sizeof(struct cpu_info) - sizeof(struct tss_struct) -
+             XPTI_STACK_SIZE * 4];
+    uint64_t df_stack[XPTI_STACK_N];
+    uint64_t nmi_stack[XPTI_STACK_N];
+    uint64_t mce_stack[XPTI_STACK_N];
+    uint64_t primary_stack[XPTI_STACK_N];
+    struct cpu_info cpu_info;
+};
+
 struct xpti_domain {
-    int pad;
+    l1_pgentry_t **perdom_l1tab;
 };
 
 static __read_mostly enum {
@@ -64,14 +79,92 @@ custom_runtime_param("xpti", parse_xpti);
 
 void xpti_domain_destroy(struct domain *d)
 {
-    xfree(d->arch.pv_domain.xpti);
+    struct xpti_domain *xd = d->arch.pv_domain.xpti;
+
+    if ( !xd )
+        return;
+
+    xfree(xd->perdom_l1tab);
+    xfree(xd);
     d->arch.pv_domain.xpti = NULL;
 }
 
+void xpti_vcpu_destroy(struct vcpu *v)
+{
+    if ( v->domain->arch.pv_domain.xpti )
+    {
+        free_xenheap_page(v->arch.pv_vcpu.stack_regs);
+        v->arch.pv_vcpu.stack_regs = NULL;
+        destroy_perdomain_mapping(v->domain, XPTI_START(v), STACK_PAGES);
+    }
+}
+
+static int xpti_vcpu_init(struct vcpu *v)
+{
+    struct domain *d = v->domain;
+    struct xpti_domain *xd = d->arch.pv_domain.xpti;
+    void *ptr;
+    struct cpu_info *info;
+    struct xpti_stack *stack;
+    struct tss_struct *tss;
+    l1_pgentry_t *pl1e;
+    unsigned int i;
+    int rc;
+
+    /* Populate page tables. */
+    rc = create_perdomain_mapping(d, XPTI_START(v), STACK_PAGES,
+                                  xd->perdom_l1tab, NULL);
+    if ( rc )
+        goto done;
+    pl1e = xd->perdom_l1tab[l2_table_offset(XPTI_START(v))] +
+           l1_table_offset(XPTI_START(v));
+
+    /* Map stacks and TSS. */
+    rc = create_perdomain_mapping(d, XPTI_TSS(v), 1,
+                                  NULL, NIL(struct page_info *));
+    if ( rc )
+        goto done;
+
+    ptr = alloc_xenheap_page();
+    if ( !ptr )
+    {
+        rc = -ENOMEM;
+        goto done;
+    }
+    clear_page(ptr);
+    l1e_write(pl1e + STACK_PAGES - 1,
+              l1e_from_pfn(virt_to_mfn(ptr), __PAGE_HYPERVISOR_RW));
+    info = (struct cpu_info *)((unsigned long)ptr + PAGE_SIZE) - 1;
+    info->flags = ON_VCPUSTACK;
+    v->arch.pv_vcpu.stack_regs = &info->guest_cpu_user_regs;
+
+    /* stack just used for generating the correct addresses. */
+    stack = (struct xpti_stack *)XPTI_TSS(v);
+    tss = ptr;
+    tss->rsp0 = (unsigned long)&stack->cpu_info.guest_cpu_user_regs.es;
+    tss->rsp1 = 0x8600111111111111ul; /* poison */
+    tss->rsp2 = 0x8600111111111111ul; /* poison */
+    tss->ist[IST_MCE - 1] = (unsigned long)&stack->mce_stack[XPTI_STACK_N];
+    tss->ist[IST_DF  - 1] = (unsigned long)&stack->df_stack[XPTI_STACK_N];
+    tss->ist[IST_NMI - 1] = (unsigned long)&stack->nmi_stack[XPTI_STACK_N];
+    for ( i = IST_MAX; i < ARRAY_SIZE(tss->ist); i++ )
+        tss->ist[i] = 0x8600111111111111ul; /* poison */
+    tss->bitmap = IOBMP_INVALID_OFFSET;
+
+    /* Map stub trampolines. */
+    l1e_write(pl1e + STACK_PAGES - 2,
+              l1e_from_pfn(virt_to_mfn(xpti_lstar), __PAGE_HYPERVISOR_RX));
+
+ done:
+    return rc;
+}
+
 int xpti_domain_init(struct domain *d)
 {
     bool xpti = false;
-    int ret = 0;
+    int ret = -ENOMEM;
+    struct vcpu *v;
+    struct xpti_domain *xd;
 
     if ( !is_pv_domain(d) || is_pv_32bit_domain(d) )
         return 0;
@@ -96,11 +189,21 @@ int xpti_domain_init(struct domain *d)
     if ( !xpti )
         return 0;
 
-    d->arch.pv_domain.xpti = xmalloc(struct xpti_domain);
-    if ( !d->arch.pv_domain.xpti )
-    {
-        ret = -ENOMEM;
+    xd = xzalloc(struct xpti_domain);
+    if ( !xd )
         goto done;
+    d->arch.pv_domain.xpti = xd;
+
+    xd->perdom_l1tab = xzalloc_array(l1_pgentry_t *,
+                   l2_table_offset((d->max_vcpus - 1) << XPTI_VA_SHIFT) + 1);
+    if ( !xd->perdom_l1tab )
+        goto done;
+
+    for_each_vcpu( d, v )
+    {
+        ret = xpti_vcpu_init(v);
+        if ( ret )
+            goto done;
     }
 
     printk("Enabling Xen Pagetable protection (XPTI) for Domain %d\n",
diff --git a/xen/include/asm-x86/config.h b/xen/include/asm-x86/config.h
index 9ef9d03ca7..b563a2f85b 100644
--- a/xen/include/asm-x86/config.h
+++ b/xen/include/asm-x86/config.h
@@ -66,6 +66,7 @@
 #endif
 
 #define STACK_ORDER 3
+#define STACK_PAGES (1 << STACK_ORDER)
 #define STACK_SIZE  (PAGE_SIZE << STACK_ORDER)
 
 #define TRAMPOLINE_STACK_SPACE  PAGE_SIZE
@@ -202,7 +203,7 @@ extern unsigned char boot_edid_info[128];
 /* Slot 260: per-domain mappings (including map cache). */
 #define PERDOMAIN_VIRT_START    (PML4_ADDR(260))
 #define PERDOMAIN_SLOT_MBYTES   (PML4_ENTRY_BYTES >> (20 + PAGETABLE_ORDER))
-#define PERDOMAIN_SLOTS         3
+#define PERDOMAIN_SLOTS         4
 #define PERDOMAIN_VIRT_SLOT(s)  (PERDOMAIN_VIRT_START + (s) * \
                                  (PERDOMAIN_SLOT_MBYTES << 20))
 /* Slot 261: machine-to-phys conversion table (256GB). */
@@ -310,6 +311,16 @@ extern unsigned long xen_phys_start;
 #define ARG_XLAT_START(v)        \
     (ARG_XLAT_VIRT_START + ((v)->vcpu_id << ARG_XLAT_VA_SHIFT))
 
+/* Per-vcpu XPTI pages. The fourth per-domain-mapping sub-area. */
+#define XPTI_VIRT_START          PERDOMAIN_VIRT_SLOT(3)
+#define XPTI_VA_SHIFT            (PAGE_SHIFT + STACK_ORDER)
+#define XPTI_TRAMPOLINE_OFF      ((STACK_PAGES - 2) << PAGE_SHIFT)
+#define XPTI_TSS_OFF             ((STACK_PAGES - 1) << PAGE_SHIFT)
+#define XPTI_START(v)            (XPTI_VIRT_START + \
+                                  ((v)->vcpu_id << XPTI_VA_SHIFT))
+#define XPTI_TRAMPOLINE(v)       (XPTI_START(v) + XPTI_TRAMPOLINE_OFF)
+#define XPTI_TSS(v)              (XPTI_START(v) + XPTI_TSS_OFF)
+
 #define NATIVE_VM_ASSIST_VALID   ((1UL << VMASST_TYPE_4gb_segments)        | \
                                   (1UL << VMASST_TYPE_4gb_segments_notify) | \
                                   (1UL << VMASST_TYPE_writable_pagetables) | \
diff --git a/xen/include/asm-x86/current.h b/xen/include/asm-x86/current.h
index 83d226a1ba..5963114e08 100644
--- a/xen/include/asm-x86/current.h
+++ b/xen/include/asm-x86/current.h
@@ -12,7 +12,7 @@
 #include <asm/page.h>
 
 /*
- * Xen's cpu stacks are 8 pages (8-page aligned), arranged as:
+ * Xen's physical cpu stacks are 8 pages (8-page aligned), arranged as:
  *
  * 7 - Primary stack (with a struct cpu_info at the top)
  * 6 - Primary stack
@@ -25,6 +25,19 @@
  */
 
 /*
+ * The vcpu stacks used for XPTI are 8-page aligned in virtual address space
+ * like the physical cpu stacks, but most of that area is unpopulated.
+ * As each stack needs only space for the interrupted context and (in case
+ * of the primary stack) maybe a cpu_info structure, all stacks can be put
+ * into a single page. The Syscall trampolines are mapped directly below the
+ * stack page.
+ *
+ * 7 - Primary stack (with a struct cpu_info at the top), IST stacks and TSS
+ * 6 - Syscall trampolines
+ * 0 - 5 unused
+ */
+
+/*
  * Identify which stack page the stack pointer is on.  Returns an index
  * as per the comment above.
  */
@@ -37,17 +50,29 @@ struct vcpu;
 
 struct cpu_info {
     struct cpu_user_regs guest_cpu_user_regs;
-    unsigned int processor_id;
-    struct vcpu *current_vcpu;
-    unsigned long per_cpu_offset;
-    unsigned long cr4;
-
-    /* See asm-x86/spec_ctrl_asm.h for usage. */
-    unsigned int shadow_spec_ctrl;
-    bool         use_shadow_spec_ctrl;
-    uint8_t      bti_ist_info;
-
-    unsigned long __pad;
+    union {
+        /* per physical cpu mapping */
+        struct {
+            struct vcpu *current_vcpu;
+            unsigned long per_cpu_offset;
+            unsigned long cr4;
+
+            /* See asm-x86/spec_ctrl_asm.h for usage. */
+            unsigned int shadow_spec_ctrl;
+            bool         use_shadow_spec_ctrl;
+            uint8_t      bti_ist_info;
+            unsigned long p_pad;
+        };
+        /* per vcpu mapping (xpti) */
+        struct {
+            unsigned long v_pad[4];
+            unsigned long stack_bottom_cpu;
+        };
+    };
+    unsigned int processor_id;  /* per physical cpu mapping only */
+    unsigned int flags;
+#define ON_VCPUSTACK      0x00000001
+#define VCPUSTACK_ACTIVE  0x00000002
     /* get_stack_bottom() must be 16-byte aligned */
 };
 
diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h
index b33c286807..1a4e92481c 100644
--- a/xen/include/asm-x86/domain.h
+++ b/xen/include/asm-x86/domain.h
@@ -505,6 +505,9 @@ struct pv_vcpu
     /* Deferred VA-based update state. */
     bool_t need_update_runstate_area;
     struct vcpu_time_info pending_system_time;
+
+    /* If XPTI is active: pointer to user regs on stack. */
+    struct cpu_user_regs *stack_regs;
 };
 
 typedef enum __packed {
diff --git a/xen/include/asm-x86/pv/mm.h b/xen/include/asm-x86/pv/mm.h
index dfac89df0b..34c51bcfba 100644
--- a/xen/include/asm-x86/pv/mm.h
+++ b/xen/include/asm-x86/pv/mm.h
@@ -31,6 +31,7 @@ void pv_destroy_gdt(struct vcpu *v);
 bool pv_map_ldt_shadow_page(unsigned int off);
 bool pv_destroy_ldt(struct vcpu *v);
 
+void xpti_vcpu_destroy(struct vcpu *v);
 int xpti_domain_init(struct domain *d);
 void xpti_domain_destroy(struct domain *d);
 
@@ -65,6 +66,7 @@ static inline bool pv_map_ldt_shadow_page(unsigned int off) { return false; }
 static inline bool pv_destroy_ldt(struct vcpu *v)
 { ASSERT_UNREACHABLE(); return false; }
 
+static inline void xpti_vcpu_init(struct vcpu *v) { }
 static inline int xpti_domain_init(struct domain *d) { return 0; }
 static inline void xpti_domain_destroy(struct domain *d) { }
 
-- 
2.13.6


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel