[PATCH] Passing multiple modules to Domain0

[Magnus Carlsson <magnus@galois.com>]

Back in January, Samuel Thibault expressed interest in some patches
of ours for passing multiple modules from GRUB to a domain.  As it
turns out, Samuel was mostly interested in this functionality for
domU:s, whereas our patches are only for dom0.

Anyways, if anyone find them useful, here are the patches - perhaps it
can inform the development of domU support.

I also have a modified mini-os domain that demonstrates how the modules
can be picked up in Domain 0.  I can send the patches for this demo if
anyone is interested.

All the best,
Magnus

--

This is a patch for passing in all grub modules to Domain0 (with the
exception of Domain0 itself and the XSM module, if present).  It is
enabled by passing the option "multimodules" to xen.  Implemented for
x86 and x86_64.

When enabled, mod_start and mod_len in start_info_t will have a
different interpretation.  Instead of pointing to one single module
(the ramdisk), mod_start will point to an array of struct v_module's:

struct v_module {
     unsigned long mod_start;
     unsigned long mod_end;
     unsigned long string;
};

and mod_len will give the number of elements in this array.  Each
entry has the virtual addresses to the beginning and end of a module,
and the module's command line, as passed in by grub.  This all
corresponds to module_t in multiboot.h, except we use virtual
addresses instead of physical 32-bit addresses.

The argument "multimodules" is also passed on to Domain0, so that it
can know how to interpret mod_start and mod_len in start_info_t.

Signed-off-by: Magnus Carlsson <magnus@galois.com>

diff -r 08e010c3f251 xen/arch/x86/domain_build.c
--- a/xen/arch/x86/domain_build.c	Tue Apr 15 16:39:00 2008 +0100
+++ b/xen/arch/x86/domain_build.c	Tue Apr 15 13:25:37 2008 -0700
@@ -193,10 +193,104 @@ static void __init process_dom0_ioports_
      }
  }

+unsigned long __init modules_size(v_module_t *mods,
+                                  unsigned int mods_start,
+                                  unsigned int mods_end,
+                                  int multimodules,
+                                  int physical)
+{
+    unsigned long mods_size;
+    unsigned int i;
+
+    mods_size = mods_end > mods_start ?
+                  mods[mods_end-1].mod_end - mods[mods_start].mod_start : 0;
+    if ( multimodules )
+    {
+        /* Add space for module strings and a v_module_t array */
+        for ( i = mods_start; i < mods_end; i++ ) {
+            mods_size += mods[i].string
+                ? strlen(physical ? (char*)__va(mods[i].string)
+                                  : (char*)mods[i].string) + 1
+                : 0;
+            mods_size += sizeof(v_module_t);
+        }
+    }
+    return mods_size;
+}
+
+static unsigned long __init get_virtual(int physical, unsigned long addr)
+{
+    return physical ? (unsigned long)maddr_to_bootstrap_virt(addr) : addr;
+}
+
+v_module_t * __init copy_modules(unsigned long _dest,
+                                 v_module_t *mods,
+                                 unsigned int mods_start,
+                                 unsigned int mods_end,
+                                 int multimodules,
+                                 int physical)
+{
+    unsigned long dest = get_virtual(physical, _dest);
+    unsigned long len = mods[mods_end-1].mod_end - mods[mods_start].mod_start;
+    v_module_t *newmods = 0;
+
+    if ( multimodules )
+    {
+        /* Check that source and destination don't overlap in a way that
+         * causes us to step on the source as we copy, because we aren't
+         * clever enough to handle that. */
+        unsigned long mods_size = modules_size(mods, mods_start, mods_end,
+                                               multimodules, physical);
+        unsigned long source = get_virtual(physical, mods[0].mod_start);
+        BUG_ON(dest >= source && dest < source+mods_size);
+    }
+
+    if ( physical )
+        move_memory(_dest, mods[mods_start].mod_start, mods[mods_end-1].mod_end);
+    else
+        memcpy((void*)dest,
+               (void*)(get_virtual(physical, mods[mods_start].mod_start)), len);
+
+    if ( multimodules )
+    {
+        /* Create module array and copy strings */
+        v_module_t *p = (v_module_t*) ((char *)dest + len);
+        char *strings;
+        int i;
+
+        newmods = p;
+        for (i = mods_start; i < mods_end; i++,p++)
+        {
+            p->mod_start = dest + mods[i].mod_start
+                                - mods[mods_start].mod_start;
+            p->mod_end = dest + mods[i].mod_end - mods[mods_start].mod_start;
+        }
+        strings = (char*)p;
+        p = newmods;
+        for (i = mods_start; i < mods_end; i++, p++)
+        {
+            if (mods[i].string)
+            {
+                p->string = (unsigned long)strings;
+                strlcpy(strings, (physical ? __va(mods[i].string)
+                                           : (char*)mods[i].string),
+                        MAX_GUEST_CMDLINE);
+                strings += strlen(strings) + 1;
+            } else {
+                p->string = 0;
+            }
+        }
+    }
+    return newmods;
+}
+
  int __init construct_dom0(
      struct domain *d,
-    unsigned long _image_start, unsigned long image_len,
-    unsigned long _initrd_start, unsigned long initrd_len,
+    unsigned long _image_start,
+    int multimodules,
+    v_module_t *mods,
+    unsigned int mods_start,
+    unsigned int mods_end,
      char *cmdline)
  {
      int i, rc, compatible, compat32, order, machine;
@@ -207,17 +301,13 @@ int __init construct_dom0(
      unsigned long alloc_spfn;
      unsigned long alloc_epfn;
      unsigned long count;
+    unsigned long mods_size;
+    unsigned long initrd_len;
      struct page_info *page = NULL;
      start_info_t *si;
      struct vcpu *v = d->vcpu[0];
      unsigned long long value;
-#if defined(__i386__)
-    char *image_start  = (char *)_image_start;  /* use lowmem mappings */
-    char *initrd_start = (char *)_initrd_start; /* use lowmem mappings */
-#elif defined(__x86_64__)
-    char *image_start  = __va(_image_start);
-    char *initrd_start = __va(_initrd_start);
-#endif
+    char *image_start = maddr_to_bootstrap_virt((char*)_image_start);
  #if CONFIG_PAGING_LEVELS >= 4
      l4_pgentry_t *l4tab = NULL, *l4start = NULL;
  #endif
@@ -263,7 +353,10 @@ int __init construct_dom0(

      nr_pages = compute_dom0_nr_pages();

-    if ( (rc = elf_init(&elf, image_start, image_len)) != 0 )
+    mods_size = modules_size(mods, mods_start, mods_end, multimodules, 0);
+
+    if ( (rc = elf_init(&elf, image_start,
+                        mods[0].mod_end - mods[0].mod_start)) != 0 )
          return rc;
  #ifdef VERBOSE
      elf_set_verbose(&elf);
@@ -380,7 +473,7 @@ int __init construct_dom0(
      vkern_start      = parms.virt_kstart;
      vkern_end        = parms.virt_kend;
      vinitrd_start    = round_pgup(vkern_end);
-    vinitrd_end      = vinitrd_start + initrd_len;
+    vinitrd_end      = vinitrd_start + mods_size;
      vphysmap_start   = round_pgup(vinitrd_end);
      vphysmap_end     = vphysmap_start + (nr_pages * (!is_pv_32on64_domain(d) ?
                                                       sizeof(unsigned long) :
@@ -803,9 +896,17 @@ int __init construct_dom0(
          hypercall_page_initialise(d, (void *)(unsigned long)parms.virt_hypercall);
      }

-    /* Copy the initial ramdisk. */
-    if ( initrd_len != 0 )
-        memcpy((void *)vinitrd_start, initrd_start, initrd_len);
+    initrd_len = mods_size;
+
+    if ( mods_size != 0 ) {
+        v_module_t *newmods = copy_modules(vinitrd_start, mods, mods_start,
+                                           mods_end, multimodules, 0);
+        if ( multimodules )
+        {
+            vinitrd_start = (unsigned long)newmods;
+            initrd_len = mods_end - mods_start;
+        }
+    }

      /* Free temporary buffers. */
      discard_initial_images();
diff -r 08e010c3f251 xen/arch/x86/setup.c
--- a/xen/arch/x86/setup.c	Tue Apr 15 16:39:00 2008 +0100
+++ b/xen/arch/x86/setup.c	Tue Apr 15 13:25:37 2008 -0700
@@ -41,10 +41,8 @@

  #if defined(CONFIG_X86_64)
  #define BOOTSTRAP_DIRECTMAP_END (1UL << 32) /* 4GB */
-#define maddr_to_bootstrap_virt(m) maddr_to_virt(m)
  #else
  #define BOOTSTRAP_DIRECTMAP_END (1UL << 30) /* 1GB */
-#define maddr_to_bootstrap_virt(m) ((void *)(long)(m))
  #endif

  extern void generic_apic_probe(void);
@@ -74,6 +72,11 @@ integer_param("maxcpus", max_cpus);
  /* opt_watchdog: If true, run a watchdog NMI on each processor. */
  static int opt_watchdog = 0;
  boolean_param("watchdog", opt_watchdog);
+
+/* opt_multimodules: If true, pass all modules and their strings
+ * to domain0.  This option is propagated to domain0. */
+static int opt_multimodules = 0;
+boolean_param("multimodules", opt_multimodules);

  /* **** Linux config option: propagated to domain0. */
  /* "acpi=off":    Sisables both ACPI table parsing and interpreter. */
@@ -270,7 +273,7 @@ static void __init bootstrap_map(unsigne
          start >> PAGE_SHIFT, (end-start) >> PAGE_SHIFT, PAGE_HYPERVISOR);
  }

-static void __init move_memory(
+void __init move_memory(
      unsigned long dst, unsigned long src_start, unsigned long src_end)
  {
      bootstrap_map(src_start, src_end);
@@ -408,11 +411,11 @@ void __init __start_xen(unsigned long mb
  {
      char *memmap_type = NULL;
      char *cmdline, *kextra;
-    unsigned long _initrd_start = 0, _initrd_len = 0;
      unsigned int initrdidx = 1;
      multiboot_info_t *mbi = __va(mbi_p);
      module_t *mod = (module_t *)__va(mbi->mods_addr);
-    unsigned long nr_pages, modules_length;
+    v_module_t *moved_mod = 0;
+    unsigned long nr_pages;
      int i, e820_warn = 0, bytes = 0;
      struct ns16550_defaults ns16550 = {
          .data_bits = 8,
@@ -618,7 +621,6 @@ void __init __start_xen(unsigned long mb
       * we can relocate the dom0 kernel and other multiboot modules. Also, on
       * x86/64, we relocate Xen to higher memory.
       */
-    modules_length = mod[mbi->mods_count-1].mod_end - mod[0].mod_start;
      for ( i = boot_e820.nr_map-1; i >= 0; i-- )
      {
          uint64_t s, e, mask = (1UL << L2_PAGETABLE_SHIFT) - 1;
@@ -721,14 +723,25 @@ void __init __start_xen(unsigned long mb
          }
  #endif

-        /* Is the region suitable for relocating the multiboot modules? */
-        if ( !initial_images_start && (s < e) && ((e-s) >= modules_length) )
          {
-            initial_images_end = e;
-            e = (e - modules_length) & PAGE_MASK;
-            initial_images_start = e;
-            move_memory(initial_images_start,
-                        mod[0].mod_start, mod[mbi->mods_count-1].mod_end);
+            unsigned long modules_length;
+            v_module_t v_mod[mbi->mods_count];
+            for ( i = 0; i < mbi->mods_count; i++ )
+            {
+                v_mod[i].mod_start = mod[i].mod_start;
+                v_mod[i].mod_end = mod[i].mod_end;
+                v_mod[i].string = mod[i].string;
+            }
+            modules_length = modules_size(v_mod, 0, mbi->mods_count, 1, 1);
+            /* Is the region suitable for relocating the multiboot modules? */
+            if ( !initial_images_start && (s < e) && ((e-s) >= modules_length) )
+            {
+                initial_images_end = e;
+                e = (e - modules_length) & PAGE_MASK;
+                initial_images_start = e;
+                moved_mod = copy_modules(initial_images_start, v_mod, 0,
+                                         mbi->mods_count, 1, 1);
+            }
          }

          if ( !kexec_crash_area.start && (s < e) &&
@@ -1008,15 +1021,10 @@ void __init __start_xen(unsigned long mb
              safe_strcat(dom0_cmdline, " acpi=");
              safe_strcat(dom0_cmdline, acpi_param);
          }
+        if ( opt_multimodules )
+            safe_strcat(dom0_cmdline, " multimodules");

          cmdline = dom0_cmdline;
-    }
-
-    if ( (initrdidx > 0) && (initrdidx < mbi->mods_count) )
-    {
-        _initrd_start = initial_images_start +
-            (mod[initrdidx].mod_start - mod[0].mod_start);
-        _initrd_len   = mod[initrdidx].mod_end - mod[initrdidx].mod_start;
      }

      iommu_setup();
@@ -1025,13 +1033,17 @@ void __init __start_xen(unsigned long mb

      /*
       * We're going to setup domain0 using the module(s) that we stashed safely
-     * above our heap. The second module, if present, is an initrd ramdisk.
+     * above our heap. If opt_multimodules is set, pass in the remaining
+     * modules and their strings in a v_module_t array, otherwise just pass in
+     * the initrd module.
       */
      if ( construct_dom0(dom0,
                          initial_images_start,
-                        mod[0].mod_end-mod[0].mod_start,
-                        _initrd_start,
-                        _initrd_len,
+                        opt_multimodules,
+                        moved_mod,
+                        initrdidx,
+                        opt_multimodules ? mbi->mods_count
+                                         : min(initrdidx+1, mbi->mods_count),
                          cmdline) != 0)
          panic("Could not set up DOM0 guest OS\n");

diff -r 08e010c3f251 xen/include/asm-x86/page.h
--- a/xen/include/asm-x86/page.h	Tue Apr 15 16:39:00 2008 +0100
+++ b/xen/include/asm-x86/page.h	Tue Apr 15 13:25:37 2008 -0700
@@ -248,6 +248,12 @@ void clear_page_sse2(void *);
  #define pfn_to_paddr(pfn)   ((paddr_t)(pfn) << PAGE_SHIFT)
  #define paddr_to_pfn(pa)    ((unsigned long)((pa) >> PAGE_SHIFT))

+#if defined(CONFIG_X86_64)
+#define maddr_to_bootstrap_virt(m) maddr_to_virt(m)
+#else
+#define maddr_to_bootstrap_virt(m) ((void *)(long)(m))
+#endif
+
  #endif /* !defined(__ASSEMBLY__) */

  /* High table entries are reserved by the hypervisor. */
diff -r 08e010c3f251 xen/include/public/xen.h
--- a/xen/include/public/xen.h	Tue Apr 15 16:39:00 2008 +0100
+++ b/xen/include/public/xen.h	Tue Apr 15 13:25:37 2008 -0700
@@ -541,11 +541,23 @@ struct start_info {
      unsigned long pt_base;      /* VIRTUAL address of page directory.     */
      unsigned long nr_pt_frames; /* Number of bootstrap p.t. frames.       */
      unsigned long mfn_list;     /* VIRTUAL address of page-frame list.    */
-    unsigned long mod_start;    /* VIRTUAL address of pre-loaded module.  */
-    unsigned long mod_len;      /* Size (bytes) of pre-loaded module.     */
+    unsigned long mod_start;    /* VIRTUAL address of pre-loaded module,  */
+                                /* or VIRTUAL address of v_module_t array.*/
+    unsigned long mod_len;      /* Size (bytes) of pre-loaded module,     */
+                                /* or length of v_module_t array.         */
      int8_t cmd_line[MAX_GUEST_CMDLINE];
  };
  typedef struct start_info start_info_t;
+
+/* This corresponds to module_t in multiboot.h, but with virtual
+ * addresses.
+ */
+struct v_module {
+    unsigned long mod_start;
+    unsigned long mod_end;
+    unsigned long string;
+};
+typedef struct v_module v_module_t;

  /* New console union for dom0 introduced in 0x00030203. */
  #if __XEN_INTERFACE_VERSION__ < 0x00030203
diff -r 08e010c3f251 xen/include/xen/sched.h
--- a/xen/include/xen/sched.h	Tue Apr 15 16:39:00 2008 +0100
+++ b/xen/include/xen/sched.h	Tue Apr 15 13:25:37 2008 -0700
@@ -315,11 +315,38 @@ struct domain *domain_create(
  #define _DOMCRF_hap 1
  #define DOMCRF_hap  (1U<<_DOMCRF_hap)

+/* Compute size of all modules in mods, starting from mods_start,
+ * ending at mods_end - 1. Include size of module parameters and mods
+ * array itself if multimodules is set. If physical is set, assume
+ * physical addresses. */
+unsigned long modules_size(v_module_t *mods,
+                                  unsigned int mods_start,
+                                  unsigned int mods_end,
+                                  int multimodules,
+                                  int physical);
+
+/* Copy all modules in mods, starting from mods_start, ending at
+ * mods_end - 1, into dest. Copy module parameters and mods array
+ * itself if multimodules is set. If physical is set, assume physical
+ * addresses. */
+v_module_t *copy_modules(unsigned long dest,
+                         v_module_t *mods,
+                         unsigned int mods_start,
+                         unsigned int mods_end,
+                         int multimodules,
+                         int physical);
+
  int construct_dom0(
      struct domain *d,
-    unsigned long image_start, unsigned long image_len,
-    unsigned long initrd_start, unsigned long initrd_len,
+    unsigned long image_start,
+    int multimodules,
+    v_module_t *mods,
+    unsigned int mods_start,
+    unsigned int mods_end,
      char *cmdline);
+
+void move_memory(unsigned long dst,
+                 unsigned long src_start, unsigned long src_end);

  /*
   * rcu_lock_domain_by_id() is more efficient than get_domain_by_id().