[RFCv2 30/38] x86/hyperlaunch: introduce concept of core domains

[RFCv2 30/38] x86/hyperlaunch: introduce concept of core domains

[Daniel P. Smith]

When constructing a disaggregated Xen system, there are certain domains with
particular capabilities that must be present and running at start-of-day. The
hardware domain is absolutely required, while a xenstore domain is mostly
required.

The function build_core_domains is introduced to encapsulate the construction
of the core domains.

Signed-off-by: Daniel P. Smith <dpsmith@apertussolutions.com>

---

Changes in RFCv2:
- rewrote build_core_domains due address the reordering event channel creation
---
 xen/arch/x86/domain-builder/core.c     | 66 +++++++++++++++++++++++---
 xen/arch/x86/include/asm/boot-domain.h |  2 +
 2 files changed, 61 insertions(+), 7 deletions(-)

diff --git a/xen/arch/x86/domain-builder/core.c b/xen/arch/x86/domain-builder/core.c
index 4eaf3a111208..af79792b5316 100644
--- a/xen/arch/x86/domain-builder/core.c
+++ b/xen/arch/x86/domain-builder/core.c
@@ -3,24 +3,76 @@
  * Copyright (C) 2025, Apertus Solutions, LLC
  */
 
+#include <xen/bug.h>
 #include <xen/domain-builder.h>
 #include <xen/init.h>
 #include <xen/lib.h>
 
 #include <asm/bootinfo.h>
+#include <asm/pv/shim.h>
+
+static int  __init build_core_domains(struct boot_info *bi)
+{
+    int count = 0;
+    struct boot_domain *bd;
+    int hw, xs;
+
+    hw = first_boot_domain_index(bi, DOMAIN_CAPS_HARDWARE);
+    if ( hw > MAX_NR_BOOTDOMS )
+        panic("%s: hardware domain missing\n", __func__);
+    else
+    {
+        bd = &bi->domains[hw];
+
+        arch_create_dom(bi, bd);
+        if ( bd->d )
+        {
+            bd->constructed = true;
+            count++;
+        }
+    }
+
+    xs = first_boot_domain_index(bi, DOMAIN_CAPS_XENSTORE);
+    if ( xs > MAX_NR_BOOTDOMS )
+        printk(XENLOG_WARNING "No xenstore domain was defined\n");
+    else
+    {
+        if ( !bi->domains[xs].constructed )
+        {
+            bd = &bi->domains[xs];
+
+            arch_create_dom(bi, bd);
+            if ( bd->d )
+            {
+                bd->constructed = true;
+                count++;
+            }
+        }
+    }
+
+    ASSERT(count <= bi->nr_domains);
+
+    return count;
+}
 
 unsigned int __init builder_create_domains(struct boot_info *bi)
 {
     unsigned int build_count = 0;
-    struct boot_domain *bd = &bi->domains[0];
-
-    if ( bd->capabilities & DOMAIN_CAPS_HARDWARE && bd->kernel == NULL )
-        panic("%s: hardware domain missing kernel\n", __func__);
 
+    if ( bi->nr_domains == 0 )
+        panic("%s: no domains defined\n", __func__);
 
-    arch_create_dom(bi, bd);
-    if ( bd->d )
-        build_count++;
+    if ( pv_shim )
+    {
+        arch_create_dom(bi, &bi->domains[0]);
+        if ( bi->domains[0].d )
+        {
+            bi->domains[0].constructed = true;
+            build_count++;
+        }
+    }
+    else
+        build_count = build_core_domains(bi);
 
     arch_builder_finalize(bi);
 
diff --git a/xen/arch/x86/include/asm/boot-domain.h b/xen/arch/x86/include/asm/boot-domain.h
index 66f3a71fd597..41246f31acce 100644
--- a/xen/arch/x86/include/asm/boot-domain.h
+++ b/xen/arch/x86/include/asm/boot-domain.h
@@ -36,6 +36,8 @@ struct boot_domain {
     struct domain *d;
 
     xen_pfn_t xs_page, cons_page;
+
+    bool constructed;
 };
 
 static inline bool __init has_dom0_caps(const struct boot_domain *bd)
-- 
2.30.2

[RFCv2 31/38] common/gzip: add function to read isize field

[Daniel P. Smith]

The gzip specification dictates that the last four bytes of a gzip
file will contain the modulo 2^32 of the original image size. Since
this is a function of gzip, relocate the logic under a gzip function.

Signed-off-by: Daniel P. Smith <dpsmith@apertussolutions.com>
---
 xen/arch/x86/bzimage.c   | 10 +++-------
 xen/common/gzip/gunzip.c | 12 ++++++++++++
 xen/include/xen/gunzip.h |  3 +++
 3 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/xen/arch/x86/bzimage.c b/xen/arch/x86/bzimage.c
index 66f648f311e4..eaea64b9c014 100644
--- a/xen/arch/x86/bzimage.c
+++ b/xen/arch/x86/bzimage.c
@@ -8,11 +8,6 @@
 #include <xen/libelf.h>
 #include <asm/bzimage.h>
 
-static __init unsigned long output_length(void *image, unsigned long image_len)
-{
-    return *(uint32_t *)(image + image_len - 4);
-}
-
 struct __packed setup_header {
         uint8_t         _pad0[0x1f1];           /* skip uninteresting stuff */
         uint8_t         setup_sects;
@@ -91,7 +86,8 @@ unsigned long __init bzimage_headroom(void *image_start,
         return 0;
 
     orig_image_len = image_length;
-    headroom = output_length(image_start, image_length);
+    /* Linux build system mimics gzip isize field for bzip2/lzma algos */
+    headroom = gzip_isize(image_start, image_length);
     if (gzip_check(image_start, image_length))
     {
         headroom += headroom >> 12; /* Add 8 bytes for every 32K input block */
@@ -124,7 +120,7 @@ int __init bzimage_parse(void *image_base, void **image_start,
 
     BUG_ON(!(image_base < *image_start));
 
-    output_len = output_length(*image_start, orig_image_len);
+    output_len = gzip_isize(*image_start, orig_image_len);
 
     if ( (err = perform_gunzip(image_base, *image_start, orig_image_len)) > 0 )
         err = decompress(*image_start, orig_image_len, image_base);
diff --git a/xen/common/gzip/gunzip.c b/xen/common/gzip/gunzip.c
index 89f45d4050ba..0963fe7bbd17 100644
--- a/xen/common/gzip/gunzip.c
+++ b/xen/common/gzip/gunzip.c
@@ -3,6 +3,7 @@
 #include <xen/init.h>
 #include <xen/lib.h>
 #include <xen/mm.h>
+#include <xen/unaligned.h>
 
 #define WSIZE           0x80000000U
 
@@ -106,6 +107,17 @@ __init int gzip_check(char *image, unsigned long image_len)
     return (magic0 == 0x1f) && ((magic1 == 0x8b) || (magic1 == 0x9e));
 }
 
+/*
+ * RFC 1952 specifies the last four bytes as the isize field, the size of the
+ * original (uncompressed) input data modulo 2^32.
+ */
+__init uint32_t gzip_isize(char *image, unsigned long image_len)
+{
+    uint32_t *ptr = (uint32_t *)(image + image_len - 4);
+
+    return get_unaligned(ptr);
+}
+
 __init int perform_gunzip(char *output, char *image, unsigned long image_len)
 {
     struct gunzip_state *s;
diff --git a/xen/include/xen/gunzip.h b/xen/include/xen/gunzip.h
index 805833127aba..1b45350c195f 100644
--- a/xen/include/xen/gunzip.h
+++ b/xen/include/xen/gunzip.h
@@ -1,7 +1,10 @@
 #ifndef __XEN_GUNZIP_H
 #define __XEN_GUNZIP_H
 
+#include <xen/types.h>
+
 int gzip_check(char *image, unsigned long image_len);
+uint32_t gzip_isize(char *image, unsigned long image_len);
 int perform_gunzip(char *output, char *image, unsigned long image_len);
 
 #endif
-- 
2.30.2

[RFCv2 32/38] x86/hyperlaunch: move headroom under domain builder

[Daniel P. Smith]

The function bzimage_headroom attempted to determine the necessary headroom for
all supported kernel types, not just bzImage. The result was convoluted logic
to handle three kernel image types, and then within the bzImage type, also
handle three types of payloads.

This commit moves the generalized headroom determination for the three kernel
types to the domain builder and scopes bziamge_headroom to only doing headroom
calculations for bzimage payload types.

Signed-off-by: Daniel P. Smith <dpsmith@apertussolutions.com>
---
 xen/arch/x86/bzimage.c                    | 43 +++++++++++------------
 xen/arch/x86/domain-builder/domain.c      | 24 +++++++++++++
 xen/arch/x86/include/asm/bzimage.h        |  1 +
 xen/arch/x86/include/asm/domain-builder.h |  2 ++
 xen/arch/x86/setup.c                      |  6 ++--
 xen/include/xen/gunzip.h                  |  9 +++++
 6 files changed, 58 insertions(+), 27 deletions(-)

diff --git a/xen/arch/x86/bzimage.c b/xen/arch/x86/bzimage.c
index eaea64b9c014..ba090b3ef9d6 100644
--- a/xen/arch/x86/bzimage.c
+++ b/xen/arch/x86/bzimage.c
@@ -47,8 +47,10 @@ struct __packed setup_header {
         uint32_t        payload_length;
     };
 
-static __init int bzimage_check(struct setup_header *hdr, unsigned long len)
+int __init bzimage_check(void *image, unsigned long len)
 {
+    struct setup_header *hdr = (struct setup_header *)image;
+
     if ( len < sizeof(struct setup_header) )
         return 0;
 
@@ -56,9 +58,10 @@ static __init int bzimage_check(struct setup_header *hdr, unsigned long len)
         return 0;
 
     if ( hdr->version < VERSION(2,8) ) {
-        printk("Cannot load bzImage v%d.%02d at least v2.08 is required\n",
-           hdr->version >> 8, hdr->version & 0xff);
-        return -EINVAL;
+        printk(XENLOG_ERR
+               "Cannot load bzImage v%d.%02d at least v2.08 is required\n",
+               hdr->version >> 8, hdr->version & 0xff);
+        return 0;
     }
     return 1;
 }
@@ -69,34 +72,28 @@ unsigned long __init bzimage_headroom(void *image_start,
                                       unsigned long image_length)
 {
     struct setup_header *hdr = (struct setup_header *)image_start;
-    int err;
     unsigned long headroom;
 
-    err = bzimage_check(hdr, image_length);
-    if ( err < 0 )
-        return 0;
-
-    if ( err > 0 )
-    {
-        image_start += (hdr->setup_sects + 1) * 512 + hdr->payload_offset;
-        image_length = hdr->payload_length;
-    }
-
-    if ( elf_is_elfbinary(image_start, image_length) )
-        return 0;
+    image_start += (hdr->setup_sects + 1) * 512 + hdr->payload_offset;
+    image_length = hdr->payload_length;
 
     orig_image_len = image_length;
     /* Linux build system mimics gzip isize field for bzip2/lzma algos */
     headroom = gzip_isize(image_start, image_length);
+    /*
+     * Final headroom:
+     *  - gzip: original size + heuristically derived value
+     *  - uncompressed kernel: zero the headroom
+     *  - bzip2 & lzma: original size + payload size
+     */
     if (gzip_check(image_start, image_length))
-    {
-        headroom += headroom >> 12; /* Add 8 bytes for every 32K input block */
-        headroom += (32768 + 18); /* Add 32K + 18 bytes of extra headroom */
-    } else
+        headroom = gzip_headroom(headroom);
+    else if ( elf_is_elfbinary(image_start, image_length) )
+        headroom = 0;
+    else
         headroom += image_length;
-    headroom = (headroom + 4095) & ~4095;
 
-    return headroom;
+    return ROUNDUP(headroom, PAGE_SIZE);
 }
 
 int __init bzimage_parse(void *image_base, void **image_start,
diff --git a/xen/arch/x86/domain-builder/domain.c b/xen/arch/x86/domain-builder/domain.c
index 673fc5b16ed3..9c2f30eee8bd 100644
--- a/xen/arch/x86/domain-builder/domain.c
+++ b/xen/arch/x86/domain-builder/domain.c
@@ -9,6 +9,7 @@
 #include <xen/err.h>
 #include <xen/event.h>
 #include <xen/grant_table.h>
+#include <xen/gunzip.h>
 #include <xen/init.h>
 #include <xen/lib.h> /* get types.h for libefl.h */
 #include <xen/libelf.h>
@@ -20,6 +21,7 @@
 #include <public/hvm/params.h>
 
 #include <asm/bootinfo.h>
+#include <asm/bzimage.h>
 #include <asm/cpu-policy.h>
 #include <asm/dom0_build.h>
 #include <asm/domain-builder.h>
@@ -247,6 +249,28 @@ static size_t __init domain_cmdline_size(const struct boot_info *bi,
     return s;
 }
 
+/* Xen x86 supports two kernel types that requires headroom:
+ *  - Linux bzImage
+ *  - Linux vmlinuz aka vmlinux.gz
+ */
+void __init arch_builder_headroom(struct boot_domain *bd)
+{
+    void *image = bootstrap_map_bm(bd->kernel);
+    unsigned long length = bd->kernel->size;
+
+    if ( bzimage_check(image, length) )
+        bd->kernel->headroom = bzimage_headroom(image, length);
+    else if ( gzip_check(image, length) )
+    {
+        unsigned long headroom = gzip_isize(image, length);
+        bd->kernel->headroom = gzip_headroom(headroom);
+    }
+    else
+        bd->kernel->headroom = 0;
+
+    bootstrap_unmap();
+}
+
 struct domain *__init arch_create_dom(
     struct boot_info *bi, struct boot_domain *bd)
 {
diff --git a/xen/arch/x86/include/asm/bzimage.h b/xen/arch/x86/include/asm/bzimage.h
index 7ed69d39103d..c3a042f177ac 100644
--- a/xen/arch/x86/include/asm/bzimage.h
+++ b/xen/arch/x86/include/asm/bzimage.h
@@ -3,6 +3,7 @@
 
 #include <xen/init.h>
 
+int bzimage_check(void *image, unsigned long len);
 unsigned long bzimage_headroom(void *image_start, unsigned long image_length);
 
 int bzimage_parse(void *image_base, void **image_start,
diff --git a/xen/arch/x86/include/asm/domain-builder.h b/xen/arch/x86/include/asm/domain-builder.h
index 6ef4776faf9a..f1749d2786c6 100644
--- a/xen/arch/x86/include/asm/domain-builder.h
+++ b/xen/arch/x86/include/asm/domain-builder.h
@@ -20,6 +20,8 @@ int hvm_alloc_xenstore_page(struct boot_domain *bd, xen_pfn_t *xs_gfn);
 unsigned long dom_paging_pages(
     const struct boot_domain *d, unsigned long nr_pages);
 
+void arch_builder_headroom(struct boot_domain *bd);
+
 unsigned long dom_compute_nr_pages(
     struct boot_domain *bd, struct elf_dom_parms *parms);
 
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index b2c7846be18f..36e6ba11ddcd 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -38,6 +38,7 @@
 #include <asm/bzimage.h>
 #include <asm/cpu-policy.h>
 #include <asm/desc.h>
+#include <asm/domain-builder.h>
 #include <asm/e820.h>
 #include <asm/edd.h>
 #include <asm/genapic.h>
@@ -1330,10 +1331,7 @@ void asmlinkage __init noreturn __start_xen(void)
         xen->size  = __2M_rwdata_end - _stext;
     }
 
-    bi->domains[0].kernel->headroom =
-        bzimage_headroom(bootstrap_map_bm(bi->domains[0].kernel),
-                         bi->domains[0].kernel->size);
-    bootstrap_unmap();
+    arch_builder_headroom(&bi->domains[0]);
 
 #ifndef highmem_start
     /* Don't allow split below 4Gb. */
diff --git a/xen/include/xen/gunzip.h b/xen/include/xen/gunzip.h
index 1b45350c195f..c8a0d59eef9d 100644
--- a/xen/include/xen/gunzip.h
+++ b/xen/include/xen/gunzip.h
@@ -1,10 +1,19 @@
 #ifndef __XEN_GUNZIP_H
 #define __XEN_GUNZIP_H
 
+#include <xen/macros.h>
+#include <xen/page-size.h>
 #include <xen/types.h>
 
 int gzip_check(char *image, unsigned long image_len);
 uint32_t gzip_isize(char *image, unsigned long image_len);
 int perform_gunzip(char *output, char *image, unsigned long image_len);
 
+static inline unsigned long gzip_headroom(unsigned long headroom)
+{
+    headroom += headroom >> 12; /* Add 8 bytes for every 32K input block */
+    headroom += (32768 + 18); /* Add 32K + 18 bytes of extra headroom */
+
+    return ROUNDUP(headroom, PAGE_SIZE);
+}
 #endif
-- 
2.30.2

[RFCv2 33/38] x86/hyperlaunch: move kernel extraction under domain builder

[Daniel P. Smith]

The function bzimage_parse attempted to prepare the kernel image for copying
into the guest for all supported kernel types, not just bzImage. The result was
convoluted logic to handle three kernel image types, and then within the
bzImage type, also handle three types of payloads.

This commit moves the generalized kernel preparation for the three kernel types
to the domain builder. It descopes bziamge_parse to only handling bzImages.

Signed-off-by: Daniel P. Smith <dpsmith@apertussolutions.com>
---
 xen/arch/x86/bzimage.c                    | 34 ++++++++++-------------
 xen/arch/x86/domain-builder/domain.c      | 30 ++++++++++++++++++++
 xen/arch/x86/hvm/dom_build.c              |  3 +-
 xen/arch/x86/include/asm/bzimage.h        |  5 ++--
 xen/arch/x86/include/asm/domain-builder.h |  3 ++
 xen/arch/x86/pv/dom0_build.c              |  4 ++-
 6 files changed, 55 insertions(+), 24 deletions(-)

diff --git a/xen/arch/x86/bzimage.c b/xen/arch/x86/bzimage.c
index ba090b3ef9d6..65ee0eef029a 100644
--- a/xen/arch/x86/bzimage.c
+++ b/xen/arch/x86/bzimage.c
@@ -66,8 +66,6 @@ int __init bzimage_check(void *image, unsigned long len)
     return 1;
 }
 
-static unsigned long __initdata orig_image_len;
-
 unsigned long __init bzimage_headroom(void *image_start,
                                       unsigned long image_length)
 {
@@ -77,7 +75,6 @@ unsigned long __init bzimage_headroom(void *image_start,
     image_start += (hdr->setup_sects + 1) * 512 + hdr->payload_offset;
     image_length = hdr->payload_length;
 
-    orig_image_len = image_length;
     /* Linux build system mimics gzip isize field for bzip2/lzma algos */
     headroom = gzip_isize(image_start, image_length);
     /*
@@ -96,31 +93,28 @@ unsigned long __init bzimage_headroom(void *image_start,
     return ROUNDUP(headroom, PAGE_SIZE);
 }
 
-int __init bzimage_parse(void *image_base, void **image_start,
-                         unsigned long *image_len)
+int __init bzimage_parse(
+    void *image_base, void **image_start, unsigned long headroom,
+    unsigned long *image_len)
 {
     struct setup_header *hdr = (struct setup_header *)(*image_start);
-    int err = bzimage_check(hdr, *image_len);
+    int err = 0;
     unsigned long output_len;
 
-    if ( err < 0 )
-        return err;
-
-    if ( err > 0 )
-    {
-        *image_start += (hdr->setup_sects + 1) * 512 + hdr->payload_offset;
-        *image_len = hdr->payload_length;
-    }
+    *image_start += (hdr->setup_sects + 1) * 512 + hdr->payload_offset;
+    *image_len = hdr->payload_length;
 
     if ( elf_is_elfbinary(*image_start, *image_len) )
-        return 0;
+        return err;
 
-    BUG_ON(!(image_base < *image_start));
+    output_len = gzip_isize(*image_start, *image_len);
 
-    output_len = gzip_isize(*image_start, orig_image_len);
+    BUG_ON(!(image_base < *image_start));
 
-    if ( (err = perform_gunzip(image_base, *image_start, orig_image_len)) > 0 )
-        err = decompress(*image_start, orig_image_len, image_base);
+    if ( gzip_check(*image_start, *image_len) )
+        err = perform_gunzip(image_base, *image_start, *image_len);
+    else
+        err = decompress(*image_start, *image_len, image_base);
 
     if ( !err )
     {
@@ -128,7 +122,7 @@ int __init bzimage_parse(void *image_base, void **image_start,
         *image_len = output_len;
     }
 
-    return err > 0 ? 0 : err;
+    return err;
 }
 
 /*
diff --git a/xen/arch/x86/domain-builder/domain.c b/xen/arch/x86/domain-builder/domain.c
index 9c2f30eee8bd..deff6c8efaf1 100644
--- a/xen/arch/x86/domain-builder/domain.c
+++ b/xen/arch/x86/domain-builder/domain.c
@@ -271,6 +271,36 @@ void __init arch_builder_headroom(struct boot_domain *bd)
     bootstrap_unmap();
 }
 
+int __init arch_builder_prepare_kernel(
+    void *image_base, void **image_start, unsigned long headroom,
+    unsigned long *image_len)
+{
+    int rc = 0;
+
+    *image_start = image_base + headroom;
+    *image_len -= headroom;
+
+    if ( bzimage_check(*image_start, *image_len) )
+        rc = bzimage_parse(image_base, image_start, headroom, image_len);
+    else if ( gzip_check(*image_start, *image_len) )
+    {
+        unsigned long len = gzip_isize(*image_start, *image_len);
+        rc = perform_gunzip(image_base, *image_start, *image_len);
+        if ( !rc )
+        {
+            *image_start = image_base;
+            *image_len = len;
+        }
+    }
+    else if ( elf_is_elfbinary(*image_start, *image_len) )
+        rc = 0;
+    else
+        rc = -EINVAL;
+
+    return rc;
+}
+
+
 struct domain *__init arch_create_dom(
     struct boot_info *bi, struct boot_domain *bd)
 {
diff --git a/xen/arch/x86/hvm/dom_build.c b/xen/arch/x86/hvm/dom_build.c
index 3eab97b5288b..170caac6716e 100644
--- a/xen/arch/x86/hvm/dom_build.c
+++ b/xen/arch/x86/hvm/dom_build.c
@@ -745,7 +745,8 @@ static int __init pvh_load_kernel(
     struct vcpu *v = d->vcpu[0];
     int rc;
 
-    if ( (rc = bzimage_parse(image_base, &image_start, &image_len)) != 0 )
+    if ( (rc = arch_builder_prepare_kernel(image_base, &image_start,
+                                           image->headroom, &image_len)) != 0 )
     {
         printk("Error trying to detect bz compressed kernel\n");
         return rc;
diff --git a/xen/arch/x86/include/asm/bzimage.h b/xen/arch/x86/include/asm/bzimage.h
index c3a042f177ac..ebe4f9325c4f 100644
--- a/xen/arch/x86/include/asm/bzimage.h
+++ b/xen/arch/x86/include/asm/bzimage.h
@@ -6,7 +6,8 @@
 int bzimage_check(void *image, unsigned long len);
 unsigned long bzimage_headroom(void *image_start, unsigned long image_length);
 
-int bzimage_parse(void *image_base, void **image_start,
-                  unsigned long *image_len);
+int bzimage_parse(
+    void *image_base, void **image_start, unsigned long headroom,
+    unsigned long *image_len);
 
 #endif /* __X86_BZIMAGE_H__ */
diff --git a/xen/arch/x86/include/asm/domain-builder.h b/xen/arch/x86/include/asm/domain-builder.h
index f1749d2786c6..3adeadc15423 100644
--- a/xen/arch/x86/include/asm/domain-builder.h
+++ b/xen/arch/x86/include/asm/domain-builder.h
@@ -21,6 +21,9 @@ unsigned long dom_paging_pages(
     const struct boot_domain *d, unsigned long nr_pages);
 
 void arch_builder_headroom(struct boot_domain *bd);
+int arch_builder_prepare_kernel(
+    void *image_base, void **image_start, unsigned long headroom,
+    unsigned long *image_len);
 
 unsigned long dom_compute_nr_pages(
     struct boot_domain *bd, struct elf_dom_parms *parms);
diff --git a/xen/arch/x86/pv/dom0_build.c b/xen/arch/x86/pv/dom0_build.c
index 21cb0b11748e..72319c31fa0e 100644
--- a/xen/arch/x86/pv/dom0_build.c
+++ b/xen/arch/x86/pv/dom0_build.c
@@ -428,7 +428,9 @@ static int __init dom0_construct(struct boot_domain *bd)
 
     d->max_pages = ~0U;
 
-    if ( (rc = bzimage_parse(image_base, &image_start, &image_len)) != 0 )
+    if ( (rc = arch_builder_prepare_kernel(image_base, &image_start,
+                                           bd->kernel->headroom,
+                                           &image_len)) != 0 )
         return rc;
 
     if ( (rc = elf_init(&elf, image_start, image_len)) != 0 )
-- 
2.30.2

[RFCv2 34/38] x86/hyperlaunch: introduce multidomain kconfig option

[Daniel P. Smith]

This adds the MULTIDOMAIN_BUILDER kconfig option that will be used to enable
the domain construction path to be called multiple times. With the idea of
being able to construct multiple domains now introduced, rename construct_dom0()
to construct_dom().

Signed-off-by: Daniel P. Smith <dpsmith@apertussolutions.com>
---
 xen/arch/x86/dom0_build.c            |  9 +++++----
 xen/arch/x86/domain-builder/domain.c |  2 +-
 xen/arch/x86/include/asm/bootinfo.h  |  2 +-
 xen/arch/x86/include/asm/setup.h     |  2 +-
 xen/common/domain-builder/Kconfig    | 12 ++++++++++++
 5 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/xen/arch/x86/dom0_build.c b/xen/arch/x86/dom0_build.c
index 75969887b933..8f7615afb3f2 100644
--- a/xen/arch/x86/dom0_build.c
+++ b/xen/arch/x86/dom0_build.c
@@ -558,15 +558,16 @@ int __init dom0_setup_permissions(struct domain *d)
     return rc;
 }
 
-int __init construct_dom0(struct boot_domain *bd)
+int __init construct_dom(struct boot_domain *bd)
 {
     int rc;
     const struct domain *d = bd->d;
 
     /* Sanity! */
-    BUG_ON(!pv_shim && d->domain_id != 0);
-    BUG_ON(d->vcpu[0] == NULL);
-    BUG_ON(d->vcpu[0]->is_initialised);
+    if ( ! IS_ENABLED(CONFIG_MULTIDOMAIN_BUILDER) )
+        BUG_ON(!pv_shim && bd->d->domain_id != 0);
+    BUG_ON(bd->d->vcpu[0] == NULL);
+    BUG_ON(bd->d->vcpu[0]->is_initialised);
 
     process_pending_softirqs();
 
diff --git a/xen/arch/x86/domain-builder/domain.c b/xen/arch/x86/domain-builder/domain.c
index deff6c8efaf1..c453629700c1 100644
--- a/xen/arch/x86/domain-builder/domain.c
+++ b/xen/arch/x86/domain-builder/domain.c
@@ -397,7 +397,7 @@ struct domain *__init arch_create_dom(
         bd->cmdline = cmdline;
     }
 
-    if ( construct_dom0(bd) != 0 )
+    if ( construct_dom(bd) != 0 )
         panic("Could not construct domain 0\n");
 
     bd->cmdline = NULL;
diff --git a/xen/arch/x86/include/asm/bootinfo.h b/xen/arch/x86/include/asm/bootinfo.h
index 430ae08cf5ef..298cff303673 100644
--- a/xen/arch/x86/include/asm/bootinfo.h
+++ b/xen/arch/x86/include/asm/bootinfo.h
@@ -17,7 +17,7 @@
 #define MAX_NR_BOOTMODS 63
 
 /* Max number of boot domains that Xen can construct */
-#define MAX_NR_BOOTDOMS 1
+#define MAX_NR_BOOTDOMS 64
 
 /* Boot module binary type / purpose */
 enum bootmod_type {
diff --git a/xen/arch/x86/include/asm/setup.h b/xen/arch/x86/include/asm/setup.h
index 4b8fbdc67e05..3f6850d40d04 100644
--- a/xen/arch/x86/include/asm/setup.h
+++ b/xen/arch/x86/include/asm/setup.h
@@ -27,7 +27,7 @@ void subarch_init_memory(void);
 void init_IRQ(void);
 
 struct boot_domain;
-int construct_dom0(struct boot_domain *bd);
+int construct_dom(struct boot_domain *bd);
 
 const char *cmdline_cook(const char *p, const char *loader_name);
 
diff --git a/xen/common/domain-builder/Kconfig b/xen/common/domain-builder/Kconfig
index 44b8351af8ab..898a592a6340 100644
--- a/xen/common/domain-builder/Kconfig
+++ b/xen/common/domain-builder/Kconfig
@@ -12,4 +12,16 @@ config DOMAIN_BUILDER
 
 	  If unsure, say N.
 
+config MULTIDOMAIN_BUILDER
+	bool "Multiple domain building (UNSUPPORTED)" if UNSUPPORTED
+	depends on DOMAIN_BUILDER
+	default n
+	help
+      Enables the domain builder capability to build multiple domains
+	  using a flattened device tree.
+
+	  This feature is currently experimental.
+
+	  If unsure, say N.
+
 endmenu
-- 
2.30.2

[RFCv2 35/38] x86/hyperlaunch: add multidomain construction logic

[Daniel P. Smith]

Introduce the logic to loop over boot_info->domains and construct
each valid entry in the array.

Signed-off-by: Daniel P. Smith <dpsmith@apertussolutions.com>
---
 xen/arch/x86/domain-builder/core.c   | 30 ++++++++++++++++++++++++++++
 xen/arch/x86/domain-builder/domain.c |  7 +++++--
 xen/arch/x86/hvm/dom_build.c         |  5 ++++-
 xen/arch/x86/setup.c                 |  3 ++-
 4 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/xen/arch/x86/domain-builder/core.c b/xen/arch/x86/domain-builder/core.c
index af79792b5316..367c0de33cfb 100644
--- a/xen/arch/x86/domain-builder/core.c
+++ b/xen/arch/x86/domain-builder/core.c
@@ -58,6 +58,7 @@ static int  __init build_core_domains(struct boot_info *bi)
 unsigned int __init builder_create_domains(struct boot_info *bi)
 {
     unsigned int build_count = 0;
+    int i;
 
     if ( bi->nr_domains == 0 )
         panic("%s: no domains defined\n", __func__);
@@ -70,10 +71,39 @@ unsigned int __init builder_create_domains(struct boot_info *bi)
             bi->domains[0].constructed = true;
             build_count++;
         }
+
+        goto out;
     }
     else
         build_count = build_core_domains(bi);
 
+    if ( !IS_ENABLED(CONFIG_MULTIDOMAIN_BUILDER) )
+        goto out;
+
+    for ( i = 0; i < bi->nr_domains; i++ )
+    {
+        struct boot_domain *bd = &bi->domains[i];
+
+        if ( bd->constructed )
+            continue;
+
+        if ( bd->mode & BUILD_MODE_PARAVIRT )
+        {
+            printk(XENLOG_WARNING "don't support PV DomU, skipping %d\n", i);
+            continue;
+        }
+
+        arch_create_dom(bi, bd);
+        if ( bd->d )
+        {
+            bd->constructed = true;
+            build_count++;
+        }
+        else
+            printk(XENLOG_WARNING "failed to construct build domain %d\n", i);
+    }
+
+ out:
     arch_builder_finalize(bi);
 
     return build_count;
diff --git a/xen/arch/x86/domain-builder/domain.c b/xen/arch/x86/domain-builder/domain.c
index c453629700c1..9d9901fad505 100644
--- a/xen/arch/x86/domain-builder/domain.c
+++ b/xen/arch/x86/domain-builder/domain.c
@@ -326,8 +326,11 @@ struct domain *__init arch_create_dom(
                            ((hvm_hap_supported() && !opt_dom0_shadow) ?
                             XEN_DOMCTL_CDF_hap : 0));
 
-        dom_cfg.arch.emulation_flags |=
-            XEN_X86_EMU_LAPIC | XEN_X86_EMU_IOAPIC | XEN_X86_EMU_VPCI;
+        if ( bd->capabilities & DOMAIN_CAPS_HARDWARE )
+            dom_cfg.arch.emulation_flags |=
+                XEN_X86_EMU_LAPIC | XEN_X86_EMU_IOAPIC | XEN_X86_EMU_VPCI;
+        else
+            dom_cfg.arch.emulation_flags |= X86_EMU_LAPIC;
     }
 
     if ( iommu_enabled && (bd->capabilities & DOMAIN_CAPS_HARDWARE) )
diff --git a/xen/arch/x86/hvm/dom_build.c b/xen/arch/x86/hvm/dom_build.c
index 170caac6716e..3118e3483e46 100644
--- a/xen/arch/x86/hvm/dom_build.c
+++ b/xen/arch/x86/hvm/dom_build.c
@@ -885,7 +885,10 @@ static int __init pvh_load_kernel(
     }
 
     start_info.magic = XEN_HVM_START_MAGIC_VALUE;
-    start_info.flags = SIF_PRIVILEGED | SIF_INITDOMAIN;
+    if ( is_control_domain(d) )
+        start_info.flags = SIF_PRIVILEGED;
+    if ( is_hardware_domain(d) )
+        start_info.flags = SIF_INITDOMAIN;
     rc = hvm_copy_to_guest_phys(last_addr, &start_info, sizeof(start_info), v);
     if ( rc )
     {
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 36e6ba11ddcd..422fef7ce02a 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -1331,7 +1331,8 @@ void asmlinkage __init noreturn __start_xen(void)
         xen->size  = __2M_rwdata_end - _stext;
     }
 
-    arch_builder_headroom(&bi->domains[0]);
+    for ( i = 0; i < bi->nr_domains; i++ )
+        arch_builder_headroom(&bi->domains[i]);
 
 #ifndef highmem_start
     /* Don't allow split below 4Gb. */
-- 
2.30.2

[RFCv2 36/38] x86/hyperlaunch: enable unpausing mulitple domains

[Daniel P. Smith]

This commit enables the domain builder to unpause all domains
that have been flagged to start on boot.

Signed-off-by: Daniel P. Smith <dpsmith@apertussolutions.com>
---
 xen/arch/x86/domain-builder/core.c     | 20 ++++++++++++++++++++
 xen/arch/x86/include/asm/boot-domain.h |  8 +++++---
 xen/arch/x86/setup.c                   |  8 +++++++-
 xen/include/xen/domain-builder.h       |  1 +
 4 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/xen/arch/x86/domain-builder/core.c b/xen/arch/x86/domain-builder/core.c
index 367c0de33cfb..dbe547ff0a87 100644
--- a/xen/arch/x86/domain-builder/core.c
+++ b/xen/arch/x86/domain-builder/core.c
@@ -7,6 +7,7 @@
 #include <xen/domain-builder.h>
 #include <xen/init.h>
 #include <xen/lib.h>
+#include <xen/sched.h>
 
 #include <asm/bootinfo.h>
 #include <asm/pv/shim.h>
@@ -109,6 +110,25 @@ unsigned int __init builder_create_domains(struct boot_info *bi)
     return build_count;
 }
 
+int __init builder_unpause_domains(struct boot_info *bi)
+{
+    int i, count = 0;
+
+    for ( i = 0; i < bi->nr_domains; i++ )
+    {
+        struct boot_domain *bd = &bi->domains[i];
+
+        if ( bd->capabilities & DOMAIN_CAPS_HARDWARE ||
+             bd->mode & BUILD_MODE_START_ON_BOOT )
+        {
+            domain_unpause_by_systemcontroller(bd->d);
+            count++;
+        }
+    }
+
+    return count;
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/xen/arch/x86/include/asm/boot-domain.h b/xen/arch/x86/include/asm/boot-domain.h
index 41246f31acce..b04d48010799 100644
--- a/xen/arch/x86/include/asm/boot-domain.h
+++ b/xen/arch/x86/include/asm/boot-domain.h
@@ -18,9 +18,11 @@ struct boot_domain {
     /* Bitmap. See DOMAIN_CAPS_MASK for a list */
     uint32_t capabilities;
 
-                                          /* On     | Off    */
-#define BUILD_MODE_PARAVIRT      (1 << 0) /* PV     | PVH/HVM */
-#define BUILD_MODE_ENABLE_DM     (1 << 1) /* HVM    | PVH     */
+                                          /* On       | Off     */
+#define BUILD_MODE_PARAVIRT      (1 << 0) /* PV       | PVH/HVM */
+#define BUILD_MODE_ENABLE_DM     (1 << 1) /* HVM      | PVH     */
+#define BUILD_MODE_LONG          (1 << 2) /* 64 BIT   | 32 BIT  */
+#define BUILD_MODE_START_ON_BOOT (1 << 3) /* UNPAUSED | PAUSED  */
     uint32_t mode;
 
     unsigned long mem_pages;
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index 422fef7ce02a..b55a1da9db91 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -790,6 +790,7 @@ static inline bool using_2M_mapping(void)
 
 static void noreturn init_done(void)
 {
+    struct boot_info *bi = &xen_boot_info;
     void *va;
     unsigned long start, end;
     int err;
@@ -803,7 +804,12 @@ static void noreturn init_done(void)
     if ( IS_ENABLED(CONFIG_SELF_TESTS) && cpu_has_xen_shstk )
         stub_selftest();
 
-    domain_unpause_by_systemcontroller(dom0);
+    err = builder_unpause_domains(bi);
+    if ( err == 0 )
+        panic("domain builder: failed to schedule any domain to start\n");
+    else
+        printk("domain builder: unpaused %d of %d domains at boot\n", err,
+               bi->nr_domains);
 
     /* MUST be done prior to removing .init data. */
     unregister_init_virtual_region();
diff --git a/xen/include/xen/domain-builder.h b/xen/include/xen/domain-builder.h
index ca9d9032b35b..eb8f5773b17e 100644
--- a/xen/include/xen/domain-builder.h
+++ b/xen/include/xen/domain-builder.h
@@ -46,5 +46,6 @@ struct domain *arch_create_dom(
 int arch_builder_finalize(struct boot_info *bi);
 
 unsigned int builder_create_domains(struct boot_info *bi);
+int builder_unpause_domains(struct boot_info *bi);
 
 #endif /* __XEN_DOMAIN_BUILDER_H__ */
-- 
2.30.2

[RFCv2 37/38] x86/hyperlaunch: generalize domid assignment

[Daniel P. Smith]

Signed-off-by: Daniel P. Smith <dpsmith@apertussolutions.com>
---
 xen/common/domain-builder/fdt.c | 32 +++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/xen/common/domain-builder/fdt.c b/xen/common/domain-builder/fdt.c
index 1b3492571b15..414bbf5d9fb1 100644
--- a/xen/common/domain-builder/fdt.c
+++ b/xen/common/domain-builder/fdt.c
@@ -16,6 +16,21 @@
 
 #include "fdt.h"
 
+#define MAX_DOMID  DOMID_FIRST_RESERVED
+static __initdata DECLARE_BITMAP(domid_alloc, MAX_DOMID);
+
+static domid_t __init find_next_domid(void)
+{
+    unsigned long n = find_next_zero_bit(domid_alloc, MAX_DOMID, 1);
+
+    if ( n == MAX_DOMID )
+        return DOMID_INVALID;
+
+    set_bit(n, domid_alloc);
+
+    return (domid_t) n;
+}
+
 static int __init fdt_prop_as_u32(const struct fdt_property *prop,
                                   uint32_t *val)
 {
@@ -231,18 +246,17 @@ static int __init fdt_process_domain_node(
 
             if ( val >= DOMID_FIRST_RESERVED )
             {
-                printk(XENLOG_ERR "  invalid domain id for domain %s\n", name);
-                return -EINVAL;
-            }
-
-            for ( unsigned int i = 0; i < bi->nr_domains; i++ )
-            {
-                if ( bi->domains[i].domid == val )
+                if ( (val = find_next_domid()) == DOMID_INVALID )
                 {
-                    printk(XENLOG_ERR "  duplicate id for domain %s\n", name);
-                    return -EINVAL;
+                    printk("  unable to allocate domid for domain %s\n", name);
+                    return -EFAULT;
                 }
             }
+            else if ( test_and_set_bit(val, domid_alloc) )
+            {
+                printk(XENLOG_ERR "  duplicate id for domain %s\n", name);
+                return -EINVAL;
+            }
 
             bd->domid = val;
             printk(XENLOG_INFO "  domid: %d\n", bd->domid);
-- 
2.30.2

[RFCv2 38/38] tools: introduce hyperlaunch domain late init

[Daniel P. Smith]

The late domain init helper is a helper tool for late setup of Xenstore for a
domain that was created by the hypervisor using hyperlaunch.

Signed-off-by: Daniel P. Smith <dpsmith@apertussolutions.com>
---
 .gitignore                        |   1 +
 tools/helpers/Makefile            |  12 +
 tools/helpers/late-init-domains.c | 364 ++++++++++++++++++++++++++++++
 tools/helpers/late-init-domains.h |  18 ++
 tools/helpers/xs-helpers.c        | 117 ++++++++++
 tools/helpers/xs-helpers.h        |  26 +++
 6 files changed, 538 insertions(+)
 create mode 100644 tools/helpers/late-init-domains.c
 create mode 100644 tools/helpers/late-init-domains.h
 create mode 100644 tools/helpers/xs-helpers.c
 create mode 100644 tools/helpers/xs-helpers.h

diff --git a/.gitignore b/.gitignore
index 53f5df000383..7b0c390dbe0d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -122,6 +122,7 @@ tools/flask/utils/flask-label-pci
 tools/helpers/init-dom0less
 tools/helpers/init-xenstore-domain
 tools/helpers/xen-init-dom0
+tools/helpers/late-init-domains
 tools/hotplug/common/hotplugpath.sh
 tools/hotplug/FreeBSD/rc.d/xencommons
 tools/hotplug/FreeBSD/rc.d/xendriverdomain
diff --git a/tools/helpers/Makefile b/tools/helpers/Makefile
index 09590eb5b6f0..26fa079e8b1f 100644
--- a/tools/helpers/Makefile
+++ b/tools/helpers/Makefile
@@ -14,6 +14,7 @@ ifeq ($(CONFIG_ARM),y)
 TARGETS += init-dom0less
 endif
 endif
+TARGETS += late-init-domains
 
 XEN_INIT_DOM0_OBJS = xen-init-dom0.o init-dom-json.o
 $(XEN_INIT_DOM0_OBJS): CFLAGS += $(CFLAGS_libxentoollog)
@@ -39,6 +40,14 @@ $(INIT_DOM0LESS_OBJS): CFLAGS += $(CFLAGS_libxenctrl)
 $(INIT_DOM0LESS_OBJS): CFLAGS += $(CFLAGS_libxenevtchn)
 init-dom0less: LDLIBS += $(call xenlibs-ldlibs,ctrl evtchn toollog store light guest foreignmemory)
 
+LATE_INIT_DOMAINS_OBJS = late-init-domains.o xs-helpers.o init-dom-json.o
+$(LATE_INIT_DOMAINS_OBJS): CFLAGS += $(CFLAGS_libxentoollog)
+$(LATE_INIT_DOMAINS_OBJS): CFLAGS += $(CFLAGS_libxenguest)
+$(LATE_INIT_DOMAINS_OBJS): CFLAGS += $(CFLAGS_libxenlight)
+$(LATE_INIT_DOMAINS_OBJS): CFLAGS += $(CFLAGS_libxenctrl)
+$(LATE_INIT_DOMAINS_OBJS): CFLAGS += $(CFLAGS_libxenstore)
+late-init-domains: LDLIBS += $(call xenlibs-ldlibs,ctrl toollog store light guest)
+
 .PHONY: all
 all: $(TARGETS)
 
@@ -51,6 +60,9 @@ init-xenstore-domain: $(INIT_XENSTORE_DOMAIN_OBJS)
 init-dom0less: $(INIT_DOM0LESS_OBJS)
 	$(CC) $(LDFLAGS) -o $@ $(INIT_DOM0LESS_OBJS) $(LDLIBS) $(APPEND_LDFLAGS)
 
+late-init-domains: $(LATE_INIT_DOMAINS_OBJS)
+	$(CC) $(LDFLAGS) -o $@ $(LATE_INIT_DOMAINS_OBJS) $(LDLIBS)  $(APPEND_LDFLAGS)
+
 .PHONY: install
 install: all
 	$(INSTALL_DIR) $(DESTDIR)$(LIBEXEC_BIN)
diff --git a/tools/helpers/late-init-domains.c b/tools/helpers/late-init-domains.c
new file mode 100644
index 000000000000..06911d2e93d1
--- /dev/null
+++ b/tools/helpers/late-init-domains.c
@@ -0,0 +1,364 @@
+
+#include <errno.h>
+#include <getopt.h>
+#include <inttypes.h>
+#include <libxl.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <xenctrl.h>
+#include <xenguest.h>
+#include <xenstore.h>
+#include <xentoollog.h>
+#include <xen/io/xenbus.h>
+
+#include "init-dom-json.h"
+#include "late-init-domains.h"
+#include "xs-helpers.h"
+
+static struct option options[] = {
+    { "console", 0, NULL, 'c' },
+    { "xenstore", 1, NULL, 'x' },
+    { "force", 0, NULL, 'f' },
+    { "verbose", 0, NULL, 'v' },
+    { "help", 0, NULL, 'h' },
+    { NULL, 0, NULL, 0 }
+};
+
+static void usage(void)
+{
+    fprintf(stderr,
+"Usage:\n"
+"\n"
+"late-init-domains <options>\n"
+"\n"
+"where options may include:\n"
+"\n"
+"  --console <con domid>    configure the console\n"
+"  --xenstore <xs domid>    domain id of the xenstore domain\n"
+"  --force                  force domain introduction even if xenstore entries exist\n"
+"  -v[v[v]]                 verbosity constructing xenstore tree\n"
+"  --help                   help message\n");
+}
+
+#define XS_DOM_PERM(x, d, k, v)                                             \
+    ret = do_xs_write_dom_with_perm(x, d, k, v, perms, num_perms);          \
+    if ( ret != 0 ) return ret                                              \
+
+#define XS_DIR_PERM(x, p, k, v)                                             \
+    ret = do_xs_write_dir_node_with_perm(x, p, k, v, perms, num_perms);     \
+    if ( ret != 0 ) return ret                                              \
+
+static int pages_from_hvm_params(
+    struct xc_interface_core *xch, libxl_dominfo *info,
+    struct system_pages *pgs)
+{
+    int ret;
+    domid_t domid = info->domid;
+
+    ret = xc_hvm_param_get(xch, domid, HVM_PARAM_STORE_EVTCHN,
+                           &pgs->store.evtchn);
+    if (ret != 0) {
+        fprintf(stderr, "err: failed to get dom%d store evtchn\n", domid);
+        return ret;
+    }
+
+    ret = xc_hvm_param_get(xch, domid, HVM_PARAM_STORE_PFN,
+                           &pgs->store.pfn);
+    if (ret < 0) {
+        fprintf(stderr, "err: failed to get dom%d store pfn\n", domid);
+        return ret;
+    }
+
+    if ( pgs->console.enabled )
+    {
+        ret = xc_hvm_param_get(xch, domid, HVM_PARAM_CONSOLE_EVTCHN,
+                              &pgs->console.evtchn);
+        if (ret != 0) {
+            fprintf(stderr, "warn: console for dom%d not configured\n", domid);
+            pgs->console.evtchn = pgs->console.pfn = 0;
+            return 0;
+        }
+
+        ret = xc_hvm_param_get(xch, domid, HVM_PARAM_CONSOLE_PFN,
+                               &pgs->console.pfn);
+        if (ret < 0) {
+            fprintf(stderr, "warn: console for dom%d not configured\n", domid);
+            pgs->console.evtchn = pgs->console.pfn = 0;
+            return 0;
+        }
+    }
+
+    return 0;
+}
+
+static int create_xs_entries(
+    struct xs_handle *xsh, struct system_pages *pgs, libxl_dominfo *di)
+{
+    char path[128], value[16];
+    struct xs_permissions perms[2] = {
+        {.id = pgs->store.be_domid, .perms = XS_PERM_NONE},
+        {.id = di->domid, .perms = XS_PERM_READ},
+    };
+    uint32_t num_perms = (sizeof(perms) / sizeof((perms)[0]));
+    int ret = 0;
+
+    while ( do_xs_start_transaction(xsh) == 0 )
+    {
+        XS_DOM_PERM(xsh, di->domid, "", "");
+
+        snprintf(value, 16, "%d", di->domid);
+        XS_DOM_PERM(xsh, di->domid, "domid", value);
+
+        XS_DOM_PERM(xsh, di->domid, "memory", "");
+        snprintf(value, 16, "%" PRIu64, di->current_memkb);
+        XS_DOM_PERM(xsh, di->domid, "memory/target", value);
+
+        snprintf(value, 16, "%" PRIu64, di->max_memkb);
+        XS_DOM_PERM(xsh, di->domid, "memory/static-max", value);
+
+        XS_DOM_PERM(xsh, di->domid, "store", "");
+        snprintf(value, 16, "%" PRIu64, pgs->store.evtchn);
+        XS_DOM_PERM(xsh, di->domid, "store/port", value);
+
+        snprintf(value, 16, "%" PRIu64, pgs->store.pfn);
+        XS_DOM_PERM(xsh, di->domid, "store/ring-ref", value);
+
+        if ( pgs->console.enabled && pgs->console.evtchn )
+        {
+            char be_path[64], fe_path[64];
+
+            snprintf(fe_path, 64, "/local/domain/%d/console", di->domid);
+            snprintf(be_path, 64, "/local/domain/%d/backend/console/%d/0",
+                     pgs->console.be_domid, di->domid);
+
+            /* Backend entries */
+            XS_DIR_PERM(xsh, be_path, "", "");
+            snprintf(value, 16, "%d", di->domid);
+            XS_DIR_PERM(xsh, be_path, "frontend-id", value);
+            XS_DIR_PERM(xsh, be_path, "frontend", fe_path);
+            XS_DIR_PERM(xsh, be_path, "online", "1");
+            XS_DIR_PERM(xsh, be_path, "protocol", "vt100");
+
+            snprintf(value, 16, "%d", XenbusStateInitialising);
+            XS_DIR_PERM(xsh, be_path, "state", value);
+
+            /* Frontend entries */
+            XS_DOM_PERM(xsh, di->domid, "console", "");
+            snprintf(value, 16, "%d", pgs->console.be_domid);
+            XS_DIR_PERM(xsh, fe_path, "backend", be_path);
+            XS_DIR_PERM(xsh, fe_path, "backend-id", value);
+            XS_DIR_PERM(xsh, fe_path, "limit", "1048576");
+            XS_DIR_PERM(xsh, fe_path, "type", "xenconsoled");
+            XS_DIR_PERM(xsh, fe_path, "output", "pty");
+            XS_DIR_PERM(xsh, fe_path, "tty", "");
+
+            snprintf(value, 16, "%" PRIu64, pgs->console.evtchn);
+            XS_DIR_PERM(xsh, fe_path, "port", value);
+
+            snprintf(value, 16, "%" PRIu64, pgs->console.pfn);
+            XS_DIR_PERM(xsh, fe_path, "ring-ref", value);
+
+        }
+
+        snprintf(path, 128, "/libxl/%u", di->domid);
+        switch ( di->domain_type )
+        {
+        case LIBXL_DOMAIN_TYPE_PV:
+            XS_DIR_PERM(xsh, path, "type", "pv");
+            break;
+        case LIBXL_DOMAIN_TYPE_PVH:
+            XS_DIR_PERM(xsh, path, "type", "pvh");
+            break;
+        case LIBXL_DOMAIN_TYPE_HVM:
+            XS_DIR_PERM(xsh, path, "type", "hvm");
+            break;
+        default:
+            break;
+        }
+
+        ret = do_xs_end_transaction(xsh);
+        switch ( ret )
+        {
+        case 0:
+            break; /* proceed to loop break */
+        case -EAGAIN:
+            continue; /* try again */
+        default:
+            return ret; /* failed */
+        }
+
+        break;
+    }
+
+    return ret;
+}
+
+static bool init_domain(
+    struct xc_interface_core *xch, struct xs_handle *xsh,
+    struct system_pages *pgs, libxl_dominfo *di)
+{
+    xen_pfn_t con_pfn = 0L;
+    /*xc_dom_gnttab_seed will do nothing if front == back */
+    uint32_t con_domid = di->domid;
+    bool is_hvm = (di->domain_type == LIBXL_DOMAIN_TYPE_HVM ||
+                   di->domain_type == LIBXL_DOMAIN_TYPE_PVH);
+    int ret;
+
+    if ( (ret = pages_from_hvm_params(xch, di, pgs)) != 0 )
+    {
+        fprintf(stderr, "error(%d): unable to fetch dom%d system pages\n", ret,
+                di->domid);
+        return false;
+    }
+
+    if ( pgs->console.enabled && pgs->console.evtchn )
+    {
+        con_domid = pgs->console.be_domid;
+        con_pfn = pgs->console.pfn;
+    }
+
+    ret = xc_dom_gnttab_seed(xch, di->domid, is_hvm, con_pfn,
+            pgs->store.pfn, con_domid, pgs->store.be_domid);
+    if ( ret != 0 )
+    {
+        fprintf(stderr, "error (%d) setting up grant tables for dom%d\n",
+                ret, di->domid);
+        return false;
+    }
+
+    libxl_uuid_generate(&di->uuid);
+    xc_domain_sethandle(xch, di->domid,
+                        libxl_uuid_bytearray(&di->uuid));
+
+    if ( (ret = gen_stub_json_config(di->domid, &di->uuid)) != 0 )
+        fprintf(stderr, "warn(%d): unable generate dom%d json stub\n", ret,
+                di->domid);
+
+    if ( (ret = create_xs_entries(xsh, pgs, di)) != 0 )
+    {
+        fprintf(stderr, "error(%d): unable create dom%d xenstore entries\n",
+                ret, di->domid);
+        return false;
+    }
+
+    if ( !xs_introduce_domain(xsh, di->domid, pgs->store.pfn,
+                              pgs->store.evtchn) )
+    {
+        fprintf(stderr, "error introducing dom%d\n", di->domid);
+        return false;
+    }
+
+    return true;
+}
+
+int main(int argc, char** argv)
+{
+    int opt, ret, i, nb_vm = 0, count = 0;
+    bool force = false;
+    struct xs_handle *xsh = NULL;
+    struct xc_interface_core *xch = NULL;
+    xentoollog_level minmsglevel = XTL_PROGRESS;
+    xentoollog_logger *logger = NULL;
+    libxl_dominfo *info = NULL;
+    libxl_ctx *ctx;
+    struct system_pages pages = { {0} };
+
+    while ( (opt = getopt_long(argc, argv, "c:x:fv", options, NULL)) != -1 )
+    {
+        switch ( opt )
+        {
+        case 'c':
+            pages.console.be_domid = strtol(optarg, NULL, 10);
+            pages.console.enabled = true;
+            break;
+        case 'x':
+            pages.store.be_domid = strtol(optarg, NULL, 10);
+            break;
+        case 'f':
+            force = true;
+            break;
+        case 'v':
+            if ( minmsglevel > 1 )
+                minmsglevel--;
+            break;
+        case 'h':
+            usage();
+            return 0;
+        default:
+            usage();
+            return 2;
+        }
+    }
+
+    if ( optind != argc )
+    {
+        usage();
+        return 1;
+    }
+
+    logger = (xentoollog_logger *)xtl_createlogger_stdiostream(stderr,
+                                                               minmsglevel, 0);
+
+    xsh = xs_open(0);
+    xch = xc_interface_open(0, 0, 0);
+    if ( xsh == NULL || xch == NULL )
+    {
+        fprintf(stderr, "error: unable to connect to xs and/or xc interface\n");
+        ret = 1;
+        goto out;
+    }
+
+    ret = libxl_ctx_alloc(&ctx, LIBXL_VERSION, 0, NULL);
+    if (ret) {
+        fprintf(stderr, "cannot init xl context\n");
+        goto out;
+    }
+
+    info = libxl_list_domain(ctx, &nb_vm);
+    if (!info) {
+        fprintf(stderr, "libxl_list_vm failed.\n");
+        ret = 1;
+        goto out;
+    }
+
+    for (i = 0; i < nb_vm; i++) {
+        domid_t domid = info[i].domid;
+
+        /* Don't need to check for Dom0 */
+        if (!domid)
+            continue;
+
+        if ( xs_is_domain_introduced(xsh, domid) )
+        {
+            if ( !force )
+                continue;
+
+            fprintf(stderr, "warning: re-introducting domain %d\n", domid);
+        }
+
+        if ( init_domain(xch, xsh, &pages, &info[i]) )
+            count++;
+    }
+
+    printf("initialized %d out of %d domains\n", count, nb_vm);
+
+    ret = 0;
+
+out:
+    if ( info )
+        libxl_dominfo_list_free(info, nb_vm);
+
+    if ( xsh )
+        xs_close(xsh);
+
+    if ( xch )
+        xc_interface_close(xch);
+
+    if ( logger )
+        xtl_logger_destroy(logger);
+
+    return ret;
+}
diff --git a/tools/helpers/late-init-domains.h b/tools/helpers/late-init-domains.h
new file mode 100644
index 000000000000..8d071ef82ea0
--- /dev/null
+++ b/tools/helpers/late-init-domains.h
@@ -0,0 +1,18 @@
+#ifndef __LATE_INIT_PV_H
+#define __LATE_INIT_PV_H
+
+struct system_pages {
+    struct {
+        uint16_t be_domid;
+        uint64_t evtchn;
+        uint64_t pfn;
+    } store;
+    struct {
+        bool enabled;
+        uint16_t be_domid;
+        uint64_t evtchn;
+        uint64_t pfn;
+    } console;
+};
+
+#endif
diff --git a/tools/helpers/xs-helpers.c b/tools/helpers/xs-helpers.c
new file mode 100644
index 000000000000..a4d2bebbbd54
--- /dev/null
+++ b/tools/helpers/xs-helpers.c
@@ -0,0 +1,117 @@
+
+#include <err.h>
+#include <stdio.h>
+#include <string.h>
+#include <xenstore.h>
+
+#define MAX_XS_PAATH 100
+
+static xs_transaction_t t_id = XBT_NULL;
+
+int do_xs_start_transaction(struct xs_handle *xsh)
+{
+    t_id = xs_transaction_start(xsh);
+    if (t_id == XBT_NULL)
+        return -errno;
+
+    return 0;
+}
+
+int do_xs_end_transaction(struct xs_handle *xsh)
+{
+    if ( t_id == XBT_NULL )
+        return -EINVAL;
+
+    if (!xs_transaction_end(xsh, t_id, false))
+        return -errno;
+
+    return 0;
+}
+
+int do_xs_write(struct xs_handle *xsh, char *path, char *val)
+{
+    if ( !xs_write(xsh, t_id, path, val, strlen(val)) )
+    {
+        fprintf(stderr, "failed write: %s\n", path);
+        return -errno;
+    }
+
+    return 0;
+}
+
+int do_xs_perms(
+    struct xs_handle *xsh, char *path, struct xs_permissions *perms,
+    uint32_t num_perms)
+{
+    if ( !xs_set_permissions(xsh, t_id, path, perms, num_perms) )
+    {
+        fprintf(stderr, "failed set perm: %s\n", path);
+        return -errno;
+    }
+
+    return 0;
+}
+
+int do_xs_write_dir_node_with_perm(
+    struct xs_handle *xsh, char *dir, char *node, char *val,
+    struct xs_permissions *perms, uint32_t num_perms)
+{
+    char full_path[MAX_XS_PAATH];
+    int ret = 0;
+
+    /*
+     * mainly for creating a value holding node, but
+     * also support creating directory nodes.
+     */
+    if ( strlen(node) != 0 )
+        snprintf(full_path, MAX_XS_PAATH, "%s/%s", dir, node);
+    else
+        snprintf(full_path, MAX_XS_PAATH, "%s", dir);
+
+    ret = do_xs_write(xsh, full_path, val);
+    if ( ret < 0 )
+        return ret;
+
+    if ( perms != NULL && num_perms > 0 )
+        ret = do_xs_perms(xsh, full_path, perms, num_perms);
+
+    return ret;
+}
+
+int do_xs_write_dir_node(
+    struct xs_handle *xsh, char *dir, char *node, char *val)
+{
+    return do_xs_write_dir_node_with_perm(xsh, dir, node, val, NULL, 0);
+}
+
+int do_xs_write_dom_with_perm(
+    struct xs_handle *xsh, uint32_t domid, char *path, char *val,
+    struct xs_permissions *perms, uint32_t num_perms)
+{
+    char full_path[MAX_XS_PAATH];
+    int ret = 0;
+
+    /*
+     * mainly for creating a value holding node, but
+     * also support creating directory nodes.
+     */
+    if ( strlen(path) != 0 )
+        snprintf(full_path, MAX_XS_PAATH, "/local/domain/%d/%s", domid, path);
+    else
+        snprintf(full_path, MAX_XS_PAATH, "/local/domain/%d", domid);
+
+    ret = do_xs_write(xsh, full_path, val);
+    if ( ret < 0 )
+        return ret;
+
+    if ( perms != NULL && num_perms > 0 )
+        ret = do_xs_perms(xsh, full_path, perms, num_perms);
+
+    return ret;
+}
+
+int do_xs_write_dom(
+    struct xs_handle *xsh, uint32_t domid, char *path, char *val)
+{
+    return do_xs_write_dom_with_perm(xsh, domid, path, val, NULL, 0);
+}
diff --git a/tools/helpers/xs-helpers.h b/tools/helpers/xs-helpers.h
new file mode 100644
index 000000000000..89585637d4bb
--- /dev/null
+++ b/tools/helpers/xs-helpers.h
@@ -0,0 +1,26 @@
+#ifndef __XS_HELPERS_H
+#define __XS_HELPERS_H
+
+#include <xenstore.h>
+
+int do_xs_start_transaction(struct xs_handle *xsh);
+int do_xs_end_transaction(struct xs_handle *xsh);
+
+int do_xs_write(struct xs_handle *xsh, char *path, char *val);
+int do_xs_perms(
+    struct xs_handle *xsh, char *path, struct xs_permissions *perms,
+    uint32_t num_perms);
+
+int do_xs_write_dir_node_with_perm(
+    struct xs_handle *xsh, char *dir, char *node, char *val,
+    struct xs_permissions *perms, uint32_t num_perms);
+int do_xs_write_dir_node(
+    struct xs_handle *xsh, char *dir, char *node, char *val);
+
+int do_xs_write_dom_with_perm(
+    struct xs_handle *xsh, uint32_t domid, char *path, char *val,
+    struct xs_permissions *perms, uint32_t num_perms);
+int do_xs_write_dom(
+    struct xs_handle *xsh, uint32_t domid, char *path, char *val);
+
+#endif
-- 
2.30.2