[PATCH v2 0/9] xen: arm: memory mangement fixes / improvements

[PATCH v2 0/9] xen: arm: memory mangement fixes / improvements

[Ian Campbell]

The following fix a bunch of issues discovered while running on Midway
plus some random bits and bobs.

Since v1 I've fixed the "xen: support RAM at addresses 0 and 4096" to
not be stupidly broken and I've fixed the handling of memreserve regions
so that it isn't confused by overlaps with our boot modules and also
doesn't free any reserved memory in the boot modules.

Ian

[PATCH v2 1/9] xen/arm: ensure the xenheap is 32MB aligned

[Ian Campbell]

My patch 08693f5948d8 "xen: arm: reduce the size of the xen heap to max 1/8
RAM size" unintentionally violated the constraint that the xenheap must be
32MB aligned, since we only explicitly align the end of the heap and
xenheap_pages was not a multiple of 32 pages.

Round xenheap pages up to a 32MB boundary.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
---
 xen/arch/arm/setup.c |    3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
index 4b31623..1ba2eb3 100644
--- a/xen/arch/arm/setup.c
+++ b/xen/arch/arm/setup.c
@@ -323,7 +323,8 @@ static void __init setup_mm(unsigned long dtb_paddr, size_t dtb_size)
      * constraints.
      */
     heap_pages = (ram_size >> PAGE_SHIFT);
-    xenheap_pages = max(heap_pages/8, 128UL<<(20-PAGE_SHIFT));
+    xenheap_pages = (heap_pages/8 + 0x1fffUL) & ~0x1fffUL;
+    xenheap_pages = max(xenheap_pages, 128UL<<(20-PAGE_SHIFT));
 
     do
     {
-- 
1.7.10.4

[PATCH v2 2/9] xen/arm: DOMHEAP_SECOND_PAGES is arm32 specific

[Ian Campbell]

since 5263507b1b4a "xen: arm: Use a direct mapping of RAM on arm64"

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
---
 xen/include/asm-arm/config.h |    6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/xen/include/asm-arm/config.h b/xen/include/asm-arm/config.h
index 604088e..624c73e 100644
--- a/xen/include/asm-arm/config.h
+++ b/xen/include/asm-arm/config.h
@@ -136,6 +136,9 @@
 
 #define DOMHEAP_ENTRIES        1024  /* 1024 2MB mapping slots */
 
+/* Number of domheap pagetable pages required at the second level (2MB mappings) */
+#define DOMHEAP_SECOND_PAGES ((DOMHEAP_VIRT_END - DOMHEAP_VIRT_START + 1) >> FIRST_SHIFT)
+
 #else /* ARM_64 */
 
 #define SLOT0_ENTRY_BITS  39
@@ -159,9 +162,6 @@
 
 #endif
 
-/* Number of domheap pagetable pages required at the second level (2MB mappings) */
-#define DOMHEAP_SECOND_PAGES ((DOMHEAP_VIRT_END - DOMHEAP_VIRT_START + 1) >> FIRST_SHIFT)
-
 /* Fixmap slots */
 #define FIXMAP_CONSOLE  0  /* The primary UART */
 #define FIXMAP_PT       1  /* Temporary mappings of pagetable pages */
-- 
1.7.10.4

[PATCH v2 3/9] xen/arm: Reserve FDT via early module mechanism

[Ian Campbell]

This will stop us putting any heaps or relocating Xen itself over the FDT.

The devicetree will be copied to allocated memory in setup_mm and the original
copy will be freed by discard_initial_modules.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
---
 xen/arch/arm/setup.c          |    3 ++-
 xen/common/device_tree.c      |    9 ++++++++-
 xen/include/xen/device_tree.h |   11 ++++++-----
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
index 1ba2eb3..ab3d9aa 100644
--- a/xen/arch/arm/setup.c
+++ b/xen/arch/arm/setup.c
@@ -510,9 +510,10 @@ void __init start_xen(unsigned long boot_phys_offset,
 
     smp_clear_cpu_maps();
 
+    /* This is mapped by head.S */
     device_tree_flattened = (void *)BOOT_MISC_VIRT_START
         + (fdt_paddr & ((1 << SECOND_SHIFT) - 1));
-    fdt_size = device_tree_early_init(device_tree_flattened);
+    fdt_size = device_tree_early_init(device_tree_flattened, fdt_paddr);
 
     cpus = smp_get_max_cpus();
     cmdline_parse(device_tree_bootargs(device_tree_flattened));
diff --git a/xen/common/device_tree.c b/xen/common/device_tree.c
index c4f0f2c..9e0c224 100644
--- a/xen/common/device_tree.c
+++ b/xen/common/device_tree.c
@@ -510,14 +510,21 @@ static void __init early_print_info(void)
  *
  * Returns the size of the DTB.
  */
-size_t __init device_tree_early_init(const void *fdt)
+size_t __init device_tree_early_init(const void *fdt, paddr_t paddr)
 {
+    struct dt_mb_module *mod;
     int ret;
 
     ret = fdt_check_header(fdt);
     if ( ret < 0 )
         early_panic("No valid device tree\n");
 
+    mod = &early_info.modules.module[MOD_FDT];
+    mod->start = paddr & PAGE_MASK;
+    mod->size = (fdt_totalsize(fdt) + ~PAGE_MASK) & PAGE_MASK;
+
+    early_info.modules.nr_mods = max(MOD_FDT, early_info.modules.nr_mods);
+
     device_tree_for_each_node((void *)fdt, early_scan_node, NULL);
     early_print_info();
 
diff --git a/xen/include/xen/device_tree.h b/xen/include/xen/device_tree.h
index 5cc1905..3e50383 100644
--- a/xen/include/xen/device_tree.h
+++ b/xen/include/xen/device_tree.h
@@ -21,11 +21,12 @@
 #define NR_MEM_BANKS 8
 
 #define MOD_XEN 0
-#define MOD_KERNEL 1
-#define MOD_INITRD 2
-#define NR_MODULES 3
+#define MOD_FDT 1
+#define MOD_KERNEL 2
+#define MOD_INITRD 3
+#define NR_MODULES 4
 
-#define MOD_DISCARD_FIRST MOD_KERNEL
+#define MOD_DISCARD_FIRST MOD_FDT
 
 struct membank {
     paddr_t start;
@@ -166,7 +167,7 @@ typedef int (*device_tree_node_func)(const void *fdt,
 extern struct dt_early_info early_info;
 extern void *device_tree_flattened;
 
-size_t __init device_tree_early_init(const void *fdt);
+size_t __init device_tree_early_init(const void *fdt, paddr_t paddr);
 
 void __init device_tree_get_reg(const u32 **cell, u32 address_cells,
                                 u32 size_cells,
-- 
1.7.10.4

[PATCH v2 4/9] xen/arm: do not relocate Xen outside of visible RAM

[Ian Campbell]

Since we do not handle non-contiguous banks of memory lets avoid relocatting
Xen into such a bank. Avoids issues such as free_init_memory releasing pages
which are outside of the frametable.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
---
 xen/arch/arm/setup.c |    9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
index ab3d9aa..8d8028c 100644
--- a/xen/arch/arm/setup.c
+++ b/xen/arch/arm/setup.c
@@ -240,17 +240,24 @@ static paddr_t __init get_xen_paddr(void)
 {
     struct dt_mem_info *mi = &early_info.mem;
     paddr_t min_size;
-    paddr_t paddr = 0;
+    paddr_t paddr = 0, last_end;
     int i;
 
     min_size = (_end - _start + (XEN_PADDR_ALIGN-1)) & ~(XEN_PADDR_ALIGN-1);
 
+    last_end = mi->bank[0].start;
+
     /* Find the highest bank with enough space. */
     for ( i = 0; i < mi->nr_banks; i++ )
     {
         const struct membank *bank = &mi->bank[i];
         paddr_t s, e;
 
+        if ( last_end != bank->start )
+            break;
+
+        last_end = bank->start + bank->size;
+
         if ( bank->size >= min_size )
         {
             e = consider_modules(bank->start, bank->start + bank->size,
-- 
1.7.10.4

[PATCH v2 5/9] xen/arm: cope with modules outside of "visible" RAM

[Ian Campbell]

This can happen if modules are in a bank which we can't cope with e.g. due to
being non-contiguous.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
---
 xen/arch/arm/setup.c |    6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
index 8d8028c..0d9eaf7 100644
--- a/xen/arch/arm/setup.c
+++ b/xen/arch/arm/setup.c
@@ -387,6 +387,12 @@ static void __init setup_mm(unsigned long dtb_paddr, size_t dtb_size)
             e = n = ram_end;
         }
 
+        /* Module in RAM which we cannot see here, due to not handling
+         * non-contiguous memory regions yes
+         */
+        if ( e > ram_end )
+            e = ram_end;
+
         /* Avoid the xenheap */
         if ( s < ((xenheap_mfn_start+xenheap_pages) << PAGE_SHIFT)
              && (xenheap_mfn_start << PAGE_SHIFT) < e )
-- 
1.7.10.4

[PATCH v2 6/9] xen/arm: Support dtb /memreserve/ regions

[Ian Campbell]

This requires a mapping of the DTB during setup_mm. Previously this was in the
BOOT_MISC slot, which is clobbered by setup_pagetables. Split it out into its
own slot which can be preserved.

Also handle these regions as part of consider_modules() and when adding pages
to the heaps to ensure we do not locate any part of Xen or the heaps over
them.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
---
v2: drop accidentally included debug prints
    fixed handling of avoid reserved regions with heaps
    don't discard initial modules, since the FDT can (but is not required to)
    be reserved
---
 xen/arch/arm/arm32/head.S    |    2 +-
 xen/arch/arm/arm64/head.S    |    2 +-
 xen/arch/arm/mm.c            |   10 ++++++-
 xen/arch/arm/setup.c         |   67 +++++++++++++++++++++++++++++++++++++++---
 xen/arch/arm/traps.c         |    1 -
 xen/common/device_tree.c     |   13 +++++++-
 xen/include/asm-arm/config.h |    7 +++--
 xen/include/asm-arm/mm.h     |    2 ++
 8 files changed, 92 insertions(+), 12 deletions(-)

diff --git a/xen/arch/arm/arm32/head.S b/xen/arch/arm/arm32/head.S
index 79e95b6..5072e2a 100644
--- a/xen/arch/arm/arm32/head.S
+++ b/xen/arch/arm/arm32/head.S
@@ -301,7 +301,7 @@ cpu_init_done:
         orr   r2, r2, #PT_UPPER(MEM)
         orr   r2, r2, #PT_LOWER(MEM) /* r2:r3 := 2MB RAM incl. DTB */
         add   r4, r4, #8
-        strd  r2, r3, [r1, r4]       /* Map it in the early boot slot */
+        strd  r2, r3, [r1, r4]       /* Map it in the early fdt slot */
 
 pt_ready:
         PRINT("- Turning on paging -\r\n")
diff --git a/xen/arch/arm/arm64/head.S b/xen/arch/arm/arm64/head.S
index 21b7e4d..33ff489 100644
--- a/xen/arch/arm/arm64/head.S
+++ b/xen/arch/arm/arm64/head.S
@@ -255,7 +255,7 @@ skip_bss:
         mov   x3, #PT_MEM            /* x2 := 2MB RAM incl. DTB */
         orr   x2, x2, x3
         add   x4, x4, #8
-        str   x2, [x1, x4]           /* Map it in the early boot slot */
+        str   x2, [x1, x4]           /* Map it in the early fdt slot */
 
 pt_ready:
         PRINT("- Turning on paging -\r\n")
diff --git a/xen/arch/arm/mm.c b/xen/arch/arm/mm.c
index 69c157a..86e3207 100644
--- a/xen/arch/arm/mm.c
+++ b/xen/arch/arm/mm.c
@@ -361,6 +361,13 @@ static inline lpae_t pte_of_xenaddr(vaddr_t va)
     return mfn_to_xen_entry(mfn);
 }
 
+void __init remove_early_mappings(void)
+{
+    lpae_t pte = {0};
+    write_pte(xen_second + second_table_offset(BOOT_FDT_VIRT_START), pte);
+    flush_xen_data_tlb_range_va(BOOT_FDT_VIRT_START, SECOND_SIZE);
+}
+
 /* Boot-time pagetable setup.
  * Changes here may need matching changes in head.S */
 void __init setup_pagetables(unsigned long boot_phys_offset, paddr_t xen_paddr)
@@ -401,7 +408,8 @@ void __init setup_pagetables(unsigned long boot_phys_offset, paddr_t xen_paddr)
         p[second_linear_offset(va)].bits = 0;
     }
     for ( i = 0; i < 4 * LPAE_ENTRIES; i++)
-        if ( p[i].pt.valid )
+        /* The FDT is not relocated */
+        if ( p[i].pt.valid && i != second_linear_offset(BOOT_FDT_VIRT_START) )
             p[i].pt.base += (phys_offset - boot_phys_offset) >> PAGE_SHIFT;
 
     /* Change pagetables to the copy in the relocated Xen */
diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
index 0d9eaf7..5a47bda 100644
--- a/xen/arch/arm/setup.c
+++ b/xen/arch/arm/setup.c
@@ -35,6 +35,7 @@
 #include <xen/cpu.h>
 #include <xen/pfn.h>
 #include <xen/vmap.h>
+#include <xen/libfdt/libfdt.h>
 #include <asm/page.h>
 #include <asm/current.h>
 #include <asm/setup.h>
@@ -147,6 +148,33 @@ static void __init processor_id(void)
     }
 }
 
+static void dt_unreserved_regions(paddr_t s, paddr_t e,
+                                  void (*cb)(paddr_t, paddr_t), int first)
+{
+    int i, nr = fdt_num_mem_rsv(device_tree_flattened);
+
+    for ( i = first; i < nr ; i++ )
+    {
+        paddr_t r_s, r_e;
+
+        if ( fdt_get_mem_rsv(device_tree_flattened, i, &r_s, &r_e ) < 0 )
+            /* If we can't read it, pretend it doesn't exist... */
+            continue;
+
+        r_e += r_s; /* fdt_get_mem_rsc returns length */
+
+
+        if ( s < r_e && r_s < e )
+        {
+            dt_unreserved_regions(r_e, e, cb, i+1);
+            dt_unreserved_regions(s, r_s, cb, i+1);
+            return;
+        }
+    }
+
+    cb(s, e);
+}
+
 void __init discard_initial_modules(void)
 {
     struct dt_module_info *mi = &early_info.modules;
@@ -157,10 +185,12 @@ void __init discard_initial_modules(void)
         paddr_t s = mi->module[i].start;
         paddr_t e = s + PAGE_ALIGN(mi->module[i].size);
 
-        init_domheap_pages(s, e);
+        dt_unreserved_regions(s, e, init_domheap_pages, 0);
     }
 
     mi->nr_mods = 0;
+
+    remove_early_mappings();
 }
 
 /*
@@ -177,6 +207,7 @@ static paddr_t __init consider_modules(paddr_t s, paddr_t e,
 {
     const struct dt_module_info *mi = &early_info.modules;
     int i;
+    int nr_rsvd;
 
     s = (s+align-1) & ~(align-1);
     e = e & ~(align-1);
@@ -184,6 +215,7 @@ static paddr_t __init consider_modules(paddr_t s, paddr_t e,
     if ( s > e ||  e - s < size )
         return 0;
 
+    /* First check the boot modules */
     for ( i = first_mod; i <= mi->nr_mods; i++ )
     {
         paddr_t mod_s = mi->module[i].start;
@@ -199,6 +231,32 @@ static paddr_t __init consider_modules(paddr_t s, paddr_t e,
         }
     }
 
+    /* Now check any fdt reserved areas. */
+
+    nr_rsvd = fdt_num_mem_rsv(device_tree_flattened);
+
+    for ( ; i < mi->nr_mods + nr_rsvd; i++ )
+    {
+        paddr_t mod_s, mod_e;
+
+        if ( fdt_get_mem_rsv(device_tree_flattened,
+                             i - mi->nr_mods,
+                             &mod_s, &mod_e ) < 0 )
+            /* If we can't read it, pretend it doesn't exist... */
+            continue;
+
+        /* fdt_get_mem_rsv returns length */
+        mod_e += mod_s;
+
+        if ( s < mod_e && mod_s < e )
+        {
+            mod_e = consider_modules(mod_e, e, size, align, i+1);
+            if ( mod_e )
+                return mod_e;
+
+            return consider_modules(s, mod_s, size, align, i+1);
+        }
+    }
     return e;
 }
 
@@ -226,6 +284,7 @@ static paddr_t __init next_module(paddr_t s, paddr_t *n)
         lowest = mod_s;
         *n = mod_e;
     }
+
     return lowest;
 }
 
@@ -401,7 +460,7 @@ static void __init setup_mm(unsigned long dtb_paddr, size_t dtb_size)
             n = pfn_to_paddr(xenheap_mfn_start+xenheap_pages);
         }
 
-        init_boot_pages(s, e);
+        dt_unreserved_regions(s, e, init_boot_pages, 0);
 
         s = n;
     }
@@ -460,7 +519,7 @@ static void __init setup_mm(unsigned long dtb_paddr, size_t dtb_size)
 
             xenheap_mfn_end = e;
 
-            init_boot_pages(s, e);
+            dt_unreserved_regions(s, e, init_boot_pages, 0);
             s = n;
         }
     }
@@ -524,7 +583,7 @@ void __init start_xen(unsigned long boot_phys_offset,
     smp_clear_cpu_maps();
 
     /* This is mapped by head.S */
-    device_tree_flattened = (void *)BOOT_MISC_VIRT_START
+    device_tree_flattened = (void *)BOOT_FDT_VIRT_START
         + (fdt_paddr & ((1 << SECOND_SHIFT) - 1));
     fdt_size = device_tree_early_init(device_tree_flattened, fdt_paddr);
 
diff --git a/xen/arch/arm/traps.c b/xen/arch/arm/traps.c
index 0e9a141..aacbd18 100644
--- a/xen/arch/arm/traps.c
+++ b/xen/arch/arm/traps.c
@@ -810,7 +810,6 @@ void vcpu_show_execution_state(struct vcpu *v)
 
 void do_unexpected_trap(const char *msg, struct cpu_user_regs *regs)
 {
-    printk("CPU%d: Unexpected Trap: %s\n", smp_processor_id(), msg);
     show_execution_state(regs);
     while(1);
 }
diff --git a/xen/common/device_tree.c b/xen/common/device_tree.c
index 9e0c224..132a2bd 100644
--- a/xen/common/device_tree.c
+++ b/xen/common/device_tree.c
@@ -489,7 +489,7 @@ static void __init early_print_info(void)
 {
     struct dt_mem_info *mi = &early_info.mem;
     struct dt_module_info *mods = &early_info.modules;
-    int i;
+    int i, nr_rsvd;
 
     for ( i = 0; i < mi->nr_banks; i++ )
         early_printk("RAM: %"PRIpaddr" - %"PRIpaddr"\n",
@@ -502,6 +502,17 @@ static void __init early_print_info(void)
                      mods->module[i].start,
                      mods->module[i].start + mods->module[i].size,
                      mods->module[i].cmdline);
+    nr_rsvd = fdt_num_mem_rsv(device_tree_flattened);
+    for ( i = 0; i < nr_rsvd; i++ )
+    {
+        paddr_t s, e;
+        if ( fdt_get_mem_rsv(device_tree_flattened, i, &s, &e) < 0 )
+            continue;
+        /* fdt_get_mem_rsv returns length */
+        e += s;
+        early_printk(" RESVD[%d]: %"PRIpaddr" - %"PRIpaddr"\n",
+                     i, s, e);
+    }
 }
 
 /**
diff --git a/xen/include/asm-arm/config.h b/xen/include/asm-arm/config.h
index 624c73e..efeb952 100644
--- a/xen/include/asm-arm/config.h
+++ b/xen/include/asm-arm/config.h
@@ -80,10 +80,10 @@
  *   0  -   2M   Unmapped
  *   2M -   4M   Xen text, data, bss
  *   4M -   6M   Fixmap: special-purpose 4K mapping slots
- *   6M -   8M   Early boot misc (see below)
+ *   6M -   8M   Early boot mapping of FDT
+ *   8M -  10M   Early boot misc (see below)
  *
  * The early boot misc area is used:
- *   - in head.S for the DTB for device_tree_early_init().
  *   - in setup_pagetables() when relocating Xen.
  *
  * ARM32 layout:
@@ -116,7 +116,8 @@
 
 #define XEN_VIRT_START         _AT(vaddr_t,0x00200000)
 #define FIXMAP_ADDR(n)        (_AT(vaddr_t,0x00400000) + (n) * PAGE_SIZE)
-#define BOOT_MISC_VIRT_START   _AT(vaddr_t,0x00600000)
+#define BOOT_FDT_VIRT_START    _AT(vaddr_t,0x00600000)
+#define BOOT_MISC_VIRT_START   _AT(vaddr_t,0x00800000)
 
 #define HYPERVISOR_VIRT_START  XEN_VIRT_START
 
diff --git a/xen/include/asm-arm/mm.h b/xen/include/asm-arm/mm.h
index 97c2ee0..467687a 100644
--- a/xen/include/asm-arm/mm.h
+++ b/xen/include/asm-arm/mm.h
@@ -155,6 +155,8 @@ extern unsigned long total_pages;
 
 /* Boot-time pagetable setup */
 extern void setup_pagetables(unsigned long boot_phys_offset, paddr_t xen_paddr);
+/* Remove early mappings */
+extern void remove_early_mappings(void);
 /* Allocate and initialise pagetables for a secondary CPU */
 extern int __cpuinit init_secondary_pagetables(int cpu);
 /* Switch secondary CPUS to its own pagetables and finalise MMU setup */
-- 
1.7.10.4

[PATCH v2 7/9] xen/arm: rename boot misc region to boot reloc now it has a single purpose

[Ian Campbell]

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
---
 xen/arch/arm/mm.c            |    4 ++--
 xen/include/asm-arm/config.h |    7 ++-----
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/xen/arch/arm/mm.c b/xen/arch/arm/mm.c
index 86e3207..06670e0 100644
--- a/xen/arch/arm/mm.c
+++ b/xen/arch/arm/mm.c
@@ -128,7 +128,7 @@ static inline void check_memory_layout_alignment_constraints(void) {
     /* 2MB aligned regions */
     BUILD_BUG_ON(XEN_VIRT_START & ~SECOND_MASK);
     BUILD_BUG_ON(FIXMAP_ADDR(0) & ~SECOND_MASK);
-    BUILD_BUG_ON(BOOT_MISC_VIRT_START & ~SECOND_MASK);
+    BUILD_BUG_ON(BOOT_RELOC_VIRT_START & ~SECOND_MASK);
     /* 1GB aligned regions */
     BUILD_BUG_ON(XENHEAP_VIRT_START & ~FIRST_MASK);
 #ifdef CONFIG_DOMAIN_PAGE
@@ -377,7 +377,7 @@ void __init setup_pagetables(unsigned long boot_phys_offset, paddr_t xen_paddr)
     int i;
 
     /* Map the destination in the boot misc area. */
-    dest_va = BOOT_MISC_VIRT_START;
+    dest_va = BOOT_RELOC_VIRT_START;
     pte = mfn_to_xen_entry(xen_paddr >> PAGE_SHIFT);
     write_pte(xen_second + second_table_offset(dest_va), pte);
     flush_xen_data_tlb_range_va(dest_va, SECOND_SIZE);
diff --git a/xen/include/asm-arm/config.h b/xen/include/asm-arm/config.h
index efeb952..9e395c2 100644
--- a/xen/include/asm-arm/config.h
+++ b/xen/include/asm-arm/config.h
@@ -81,10 +81,7 @@
  *   2M -   4M   Xen text, data, bss
  *   4M -   6M   Fixmap: special-purpose 4K mapping slots
  *   6M -   8M   Early boot mapping of FDT
- *   8M -  10M   Early boot misc (see below)
- *
- * The early boot misc area is used:
- *   - in setup_pagetables() when relocating Xen.
+ *   8M -  10M   Early relocation address (used when relocating Xen)
  *
  * ARM32 layout:
  *   0  -   8M   <COMMON>
@@ -117,7 +114,7 @@
 #define XEN_VIRT_START         _AT(vaddr_t,0x00200000)
 #define FIXMAP_ADDR(n)        (_AT(vaddr_t,0x00400000) + (n) * PAGE_SIZE)
 #define BOOT_FDT_VIRT_START    _AT(vaddr_t,0x00600000)
-#define BOOT_MISC_VIRT_START   _AT(vaddr_t,0x00800000)
+#define BOOT_RELOC_VIRT_START  _AT(vaddr_t,0x00800000)
 
 #define HYPERVISOR_VIRT_START  XEN_VIRT_START
 
-- 
1.7.10.4

[PATCH v2 8/9] xen/arm: print the location of the Xen heap on 32 bit

[Ian Campbell]

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
---
 xen/arch/arm/setup.c |    6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
index 5a47bda..ffae16e 100644
--- a/xen/arch/arm/setup.c
+++ b/xen/arch/arm/setup.c
@@ -407,8 +407,10 @@ static void __init setup_mm(unsigned long dtb_paddr, size_t dtb_size)
 
     domheap_pages = heap_pages - xenheap_pages;
 
-    early_printk("Xen heap: %lu pages  Dom heap: %lu pages\n",
-                 xenheap_pages, domheap_pages);
+    early_printk("Xen heap: %"PRIpaddr"-%"PRIpaddr" (%lu pages)\n",
+                 e - (xenheap_pages << PAGE_SHIFT), e,
+                 xenheap_pages);
+    early_printk("Dom heap: %lu pages\n", domheap_pages);
 
     setup_xenheap_mappings((e >> PAGE_SHIFT) - xenheap_pages, xenheap_pages);
 
-- 
1.7.10.4

[PATCH v2 9/9] xen: support RAM at addresses 0 and 4096

[Ian Campbell]

Currently the mapping from pages to zones causes the page at zero to go into
zone -1 and the page at 4096 to go into zone 0, which is the Xen zone
(confusing various assertions).

Arrange instead for the mapping to be such that zone 0 is always reserved for
Xen and all other pages map to a zone >= 1.

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Cc: keir@xen.org
Cc: jbeulich@suse.com
---
v2: fixup my arithmetic
---
 xen/common/page_alloc.c |    6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/xen/common/page_alloc.c b/xen/common/page_alloc.c
index 41251b2..fb8187b 100644
--- a/xen/common/page_alloc.c
+++ b/xen/common/page_alloc.c
@@ -257,11 +257,11 @@ unsigned long __init alloc_boot_pages(
  */
 
 #define MEMZONE_XEN 0
-#define NR_ZONES    (PADDR_BITS - PAGE_SHIFT)
+#define NR_ZONES    (PADDR_BITS - PAGE_SHIFT + 1)
 
-#define bits_to_zone(b) (((b) < (PAGE_SHIFT + 1)) ? 0 : ((b) - PAGE_SHIFT - 1))
+#define bits_to_zone(b) (((b) < (PAGE_SHIFT + 1)) ? 1 : ((b) - PAGE_SHIFT))
 #define page_to_zone(pg) (is_xen_heap_page(pg) ? MEMZONE_XEN :  \
-                          (fls(page_to_mfn(pg)) - 1))
+                          (fls(page_to_mfn(pg)) ? : 1))
 
 typedef struct page_list_head heap_by_zone_and_order_t[NR_ZONES][MAX_ORDER+1];
 static heap_by_zone_and_order_t *_heap[MAX_NUMNODES];
-- 
1.7.10.4

Re: [PATCH v2 9/9] xen: support RAM at addresses 0 and 4096

[Keir Fraser]

On 13/09/2013 04:40, "Ian Campbell" <ian.campbell@citrix.com> wrote:

> Currently the mapping from pages to zones causes the page at zero to go into
> zone -1 and the page at 4096 to go into zone 0, which is the Xen zone
> (confusing various assertions).
> 
> Arrange instead for the mapping to be such that zone 0 is always reserved for
> Xen and all other pages map to a zone >= 1.
> 
> Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
> Cc: keir@xen.org
> Cc: jbeulich@suse.com

Acked-by: Keir Fraser <keir@xen.org>

> ---
> v2: fixup my arithmetic
> ---
>  xen/common/page_alloc.c |    6 +++---
>  1 file changed, 3 insertions(+), 3 deletions(-)
> 
> diff --git a/xen/common/page_alloc.c b/xen/common/page_alloc.c
> index 41251b2..fb8187b 100644
> --- a/xen/common/page_alloc.c
> +++ b/xen/common/page_alloc.c
> @@ -257,11 +257,11 @@ unsigned long __init alloc_boot_pages(
>   */
>  
>  #define MEMZONE_XEN 0
> -#define NR_ZONES    (PADDR_BITS - PAGE_SHIFT)
> +#define NR_ZONES    (PADDR_BITS - PAGE_SHIFT + 1)
>  
> -#define bits_to_zone(b) (((b) < (PAGE_SHIFT + 1)) ? 0 : ((b) - PAGE_SHIFT -
> 1))
> +#define bits_to_zone(b) (((b) < (PAGE_SHIFT + 1)) ? 1 : ((b) - PAGE_SHIFT))
>  #define page_to_zone(pg) (is_xen_heap_page(pg) ? MEMZONE_XEN :  \
> -                          (fls(page_to_mfn(pg)) - 1))
> +                          (fls(page_to_mfn(pg)) ? : 1))
>  
>  typedef struct page_list_head
> heap_by_zone_and_order_t[NR_ZONES][MAX_ORDER+1];
>  static heap_by_zone_and_order_t *_heap[MAX_NUMNODES];

Re: [PATCH v2 9/9] xen: support RAM at addresses 0 and 4096

[Jan Beulich]

>>> On 13.09.13 at 13:40, Ian Campbell <ian.campbell@citrix.com> wrote:
> --- a/xen/common/page_alloc.c
> +++ b/xen/common/page_alloc.c
> @@ -257,11 +257,11 @@ unsigned long __init alloc_boot_pages(
>   */
>  
>  #define MEMZONE_XEN 0
> -#define NR_ZONES    (PADDR_BITS - PAGE_SHIFT)
> +#define NR_ZONES    (PADDR_BITS - PAGE_SHIFT + 1)
>  
> -#define bits_to_zone(b) (((b) < (PAGE_SHIFT + 1)) ? 0 : ((b) - PAGE_SHIFT - 1))
> +#define bits_to_zone(b) (((b) < (PAGE_SHIFT + 1)) ? 1 : ((b) - PAGE_SHIFT))
>  #define page_to_zone(pg) (is_xen_heap_page(pg) ? MEMZONE_XEN :  \
> -                          (fls(page_to_mfn(pg)) - 1))
> +                          (fls(page_to_mfn(pg)) ? : 1))

So this puts both $subject pages into zone 1. If that's intended,
and you verified that it's consistent (namely in that zone 1 now
will have twice as many pages as one would expect), then I'm
certainly fine with the change as is.

Jan

Re: [PATCH v2 9/9] xen: support RAM at addresses 0 and 4096

[Ian Campbell]

On Fri, 2013-09-13 at 12:57 +0100, Jan Beulich wrote:
> >>> On 13.09.13 at 13:40, Ian Campbell <ian.campbell@citrix.com> wrote:
> > --- a/xen/common/page_alloc.c
> > +++ b/xen/common/page_alloc.c
> > @@ -257,11 +257,11 @@ unsigned long __init alloc_boot_pages(
> >   */
> >  
> >  #define MEMZONE_XEN 0
> > -#define NR_ZONES    (PADDR_BITS - PAGE_SHIFT)
> > +#define NR_ZONES    (PADDR_BITS - PAGE_SHIFT + 1)
> >  
> > -#define bits_to_zone(b) (((b) < (PAGE_SHIFT + 1)) ? 0 : ((b) - PAGE_SHIFT - 1))
> > +#define bits_to_zone(b) (((b) < (PAGE_SHIFT + 1)) ? 1 : ((b) - PAGE_SHIFT))
> >  #define page_to_zone(pg) (is_xen_heap_page(pg) ? MEMZONE_XEN :  \
> > -                          (fls(page_to_mfn(pg)) - 1))
> > +                          (fls(page_to_mfn(pg)) ? : 1))
> 
> So this puts both $subject pages into zone 1. If that's intended,
> and you verified that it's consistent (namely in that zone 1 now
> will have twice as many pages as one would expect),

Yes, they both end up in zone 1, I figured it was a minor quirk which
was unlikely to make any difference in practice.

Ian.

Re: [PATCH v2 2/9] xen/arm: DOMHEAP_SECOND_PAGES is arm32 specific

[Julien Grall]

On 09/13/2013 12:40 PM, Ian Campbell wrote:
> since 5263507b1b4a "xen: arm: Use a direct mapping of RAM on arm64"
> 
> Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Acked-by: Julien Grall <julien.grall@linaro.org>
> ---
>  xen/include/asm-arm/config.h |    6 +++---
>  1 file changed, 3 insertions(+), 3 deletions(-)
> 
> diff --git a/xen/include/asm-arm/config.h b/xen/include/asm-arm/config.h
> index 604088e..624c73e 100644
> --- a/xen/include/asm-arm/config.h
> +++ b/xen/include/asm-arm/config.h
> @@ -136,6 +136,9 @@
>  
>  #define DOMHEAP_ENTRIES        1024  /* 1024 2MB mapping slots */
>  
> +/* Number of domheap pagetable pages required at the second level (2MB mappings) */
> +#define DOMHEAP_SECOND_PAGES ((DOMHEAP_VIRT_END - DOMHEAP_VIRT_START + 1) >> FIRST_SHIFT)
> +
>  #else /* ARM_64 */
>  
>  #define SLOT0_ENTRY_BITS  39
> @@ -159,9 +162,6 @@
>  
>  #endif
>  
> -/* Number of domheap pagetable pages required at the second level (2MB mappings) */
> -#define DOMHEAP_SECOND_PAGES ((DOMHEAP_VIRT_END - DOMHEAP_VIRT_START + 1) >> FIRST_SHIFT)
> -
>  /* Fixmap slots */
>  #define FIXMAP_CONSOLE  0  /* The primary UART */
>  #define FIXMAP_PT       1  /* Temporary mappings of pagetable pages */
> 


-- 
Julien Grall

Re: [PATCH v2 3/9] xen/arm: Reserve FDT via early module mechanism

[Julien Grall]

On 09/13/2013 12:40 PM, Ian Campbell wrote:
> This will stop us putting any heaps or relocating Xen itself over the FDT.
> 
> The devicetree will be copied to allocated memory in setup_mm and the original
> copy will be freed by discard_initial_modules.
> 
> Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
> ---
>  xen/arch/arm/setup.c          |    3 ++-
>  xen/common/device_tree.c      |    9 ++++++++-
>  xen/include/xen/device_tree.h |   11 ++++++-----
>  3 files changed, 16 insertions(+), 7 deletions(-)
> 
> diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
> index 1ba2eb3..ab3d9aa 100644
> --- a/xen/arch/arm/setup.c
> +++ b/xen/arch/arm/setup.c
> @@ -510,9 +510,10 @@ void __init start_xen(unsigned long boot_phys_offset,
>  
>      smp_clear_cpu_maps();
>  
> +    /* This is mapped by head.S */
>      device_tree_flattened = (void *)BOOT_MISC_VIRT_START
>          + (fdt_paddr & ((1 << SECOND_SHIFT) - 1));
> -    fdt_size = device_tree_early_init(device_tree_flattened);
> +    fdt_size = device_tree_early_init(device_tree_flattened, fdt_paddr);
>  
>      cpus = smp_get_max_cpus();
>      cmdline_parse(device_tree_bootargs(device_tree_flattened));
> diff --git a/xen/common/device_tree.c b/xen/common/device_tree.c
> index c4f0f2c..9e0c224 100644
> --- a/xen/common/device_tree.c
> +++ b/xen/common/device_tree.c
> @@ -510,14 +510,21 @@ static void __init early_print_info(void)
>   *
>   * Returns the size of the DTB.
>   */
> -size_t __init device_tree_early_init(const void *fdt)
> +size_t __init device_tree_early_init(const void *fdt, paddr_t paddr)
>  {
> +    struct dt_mb_module *mod;
>      int ret;
>  
>      ret = fdt_check_header(fdt);
>      if ( ret < 0 )
>          early_panic("No valid device tree\n");
>  
> +    mod = &early_info.modules.module[MOD_FDT];
> +    mod->start = paddr & PAGE_MASK;

Why do we need to align the flat device tree and not the other modules
(ie: initramfs, kernel,...)?
I looked at the memory code and it seems to handle non-aligned module.

> +    mod->size = (fdt_totalsize(fdt) + ~PAGE_MASK) & PAGE_MASK;

You can use PAGE_ALIGN(fdt_totalsize(fdt)).

> +
> +    early_info.modules.nr_mods = max(MOD_FDT, early_info.modules.nr_mods);
> +
>      device_tree_for_each_node((void *)fdt, early_scan_node, NULL);
>      early_print_info();
>  
> diff --git a/xen/include/xen/device_tree.h b/xen/include/xen/device_tree.h
> index 5cc1905..3e50383 100644
> --- a/xen/include/xen/device_tree.h
> +++ b/xen/include/xen/device_tree.h
> @@ -21,11 +21,12 @@
>  #define NR_MEM_BANKS 8
>  
>  #define MOD_XEN 0
> -#define MOD_KERNEL 1
> -#define MOD_INITRD 2
> -#define NR_MODULES 3
> +#define MOD_FDT 1
> +#define MOD_KERNEL 2
> +#define MOD_INITRD 3
> +#define NR_MODULES 4
>  
> -#define MOD_DISCARD_FIRST MOD_KERNEL
> +#define MOD_DISCARD_FIRST MOD_FDT
>  
>  struct membank {
>      paddr_t start;
> @@ -166,7 +167,7 @@ typedef int (*device_tree_node_func)(const void *fdt,
>  extern struct dt_early_info early_info;
>  extern void *device_tree_flattened;
>  
> -size_t __init device_tree_early_init(const void *fdt);
> +size_t __init device_tree_early_init(const void *fdt, paddr_t paddr);
>  
>  void __init device_tree_get_reg(const u32 **cell, u32 address_cells,
>                                  u32 size_cells,
> 


-- 
Julien Grall

Re: [PATCH v2 4/9] xen/arm: do not relocate Xen outside of visible RAM

[Julien Grall]

On 09/13/2013 12:40 PM, Ian Campbell wrote:
> Since we do not handle non-contiguous banks of memory lets avoid relocatting
s/relocatting/relocating/
> Xen into such a bank. Avoids issues such as free_init_memory releasing pages
> which are outside of the frametable.
> 
> Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
> ---
>  xen/arch/arm/setup.c |    9 ++++++++-
>  1 file changed, 8 insertions(+), 1 deletion(-)
> 
> diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
> index ab3d9aa..8d8028c 100644
> --- a/xen/arch/arm/setup.c
> +++ b/xen/arch/arm/setup.c
> @@ -240,17 +240,24 @@ static paddr_t __init get_xen_paddr(void)
>  {
>      struct dt_mem_info *mi = &early_info.mem;
>      paddr_t min_size;
> -    paddr_t paddr = 0;
> +    paddr_t paddr = 0, last_end;
>      int i;
>  
>      min_size = (_end - _start + (XEN_PADDR_ALIGN-1)) & ~(XEN_PADDR_ALIGN-1);
>  
> +    last_end = mi->bank[0].start;
> +
>      /* Find the highest bank with enough space. */
>      for ( i = 0; i < mi->nr_banks; i++ )
>      {
>          const struct membank *bank = &mi->bank[i];
>          paddr_t s, e;
>  
> +        if ( last_end != bank->start )
> +            break;
> +

Can you add a comment in the code to specify that we are relocating Xen
in a "contiguous bank"?

> +        last_end = bank->start + bank->size;
> +
>          if ( bank->size >= min_size )
>          {
>              e = consider_modules(bank->start, bank->start + bank->size,
> 

-- 
Julien Grall

Re: [PATCH v2 3/9] xen/arm: Reserve FDT via early module mechanism

[Ian Campbell]

> >      ret = fdt_check_header(fdt);
> >      if ( ret < 0 )
> >          early_panic("No valid device tree\n");
> >  
> > +    mod = &early_info.modules.module[MOD_FDT];
> > +    mod->start = paddr & PAGE_MASK;
> 
> Why do we need to align the flat device tree and not the other modules
> (ie: initramfs, kernel,...)?

No reason, I just thought it might be a good idea when I wrote this.

> I looked at the memory code and it seems to handle non-aligned module.

In which case I'll drop the aligning.

> > +    mod->size = (fdt_totalsize(fdt) + ~PAGE_MASK) & PAGE_MASK;
> 
> You can use PAGE_ALIGN(fdt_totalsize(fdt)).

That's what I was searching for and failing to find!

Of course I don't need it now.

Ian.

Re: [PATCH v2 5/9] xen/arm: cope with modules outside of "visible" RAM

[Julien Grall]

On 09/13/2013 12:40 PM, Ian Campbell wrote:
> This can happen if modules are in a bank which we can't cope with e.g. due to
> being non-contiguous.
> 
> Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
> ---
>  xen/arch/arm/setup.c |    6 ++++++
>  1 file changed, 6 insertions(+)
> 
> diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
> index 8d8028c..0d9eaf7 100644
> --- a/xen/arch/arm/setup.c
> +++ b/xen/arch/arm/setup.c
> @@ -387,6 +387,12 @@ static void __init setup_mm(unsigned long dtb_paddr, size_t dtb_size)
>              e = n = ram_end;
>          }
>  
> +        /* Module in RAM which we cannot see here, due to not handling
> +         * non-contiguous memory regions yes
s/yes/yet.

Except this minor typo:

Reviewed-by: Julien Grall <julien.grall@linaro.org>

> +         */
> +        if ( e > ram_end )
> +            e = ram_end;
> +
>          /* Avoid the xenheap */
>          if ( s < ((xenheap_mfn_start+xenheap_pages) << PAGE_SHIFT)
>               && (xenheap_mfn_start << PAGE_SHIFT) < e )
> 


-- 
Julien Grall

Re: [PATCH v2 7/9] xen/arm: rename boot misc region to boot reloc now it has a single purpose

[Julien Grall]

On 09/13/2013 12:40 PM, Ian Campbell wrote:
> Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
Acked-by: Julien Grall <julien.grall@linaro.org>
> ---
>  xen/arch/arm/mm.c            |    4 ++--
>  xen/include/asm-arm/config.h |    7 ++-----
>  2 files changed, 4 insertions(+), 7 deletions(-)
> 
> diff --git a/xen/arch/arm/mm.c b/xen/arch/arm/mm.c
> index 86e3207..06670e0 100644
> --- a/xen/arch/arm/mm.c
> +++ b/xen/arch/arm/mm.c
> @@ -128,7 +128,7 @@ static inline void check_memory_layout_alignment_constraints(void) {
>      /* 2MB aligned regions */
>      BUILD_BUG_ON(XEN_VIRT_START & ~SECOND_MASK);
>      BUILD_BUG_ON(FIXMAP_ADDR(0) & ~SECOND_MASK);
> -    BUILD_BUG_ON(BOOT_MISC_VIRT_START & ~SECOND_MASK);
> +    BUILD_BUG_ON(BOOT_RELOC_VIRT_START & ~SECOND_MASK);
>      /* 1GB aligned regions */
>      BUILD_BUG_ON(XENHEAP_VIRT_START & ~FIRST_MASK);
>  #ifdef CONFIG_DOMAIN_PAGE
> @@ -377,7 +377,7 @@ void __init setup_pagetables(unsigned long boot_phys_offset, paddr_t xen_paddr)
>      int i;
>  
>      /* Map the destination in the boot misc area. */
> -    dest_va = BOOT_MISC_VIRT_START;
> +    dest_va = BOOT_RELOC_VIRT_START;
>      pte = mfn_to_xen_entry(xen_paddr >> PAGE_SHIFT);
>      write_pte(xen_second + second_table_offset(dest_va), pte);
>      flush_xen_data_tlb_range_va(dest_va, SECOND_SIZE);
> diff --git a/xen/include/asm-arm/config.h b/xen/include/asm-arm/config.h
> index efeb952..9e395c2 100644
> --- a/xen/include/asm-arm/config.h
> +++ b/xen/include/asm-arm/config.h
> @@ -81,10 +81,7 @@
>   *   2M -   4M   Xen text, data, bss
>   *   4M -   6M   Fixmap: special-purpose 4K mapping slots
>   *   6M -   8M   Early boot mapping of FDT
> - *   8M -  10M   Early boot misc (see below)
> - *
> - * The early boot misc area is used:
> - *   - in setup_pagetables() when relocating Xen.
> + *   8M -  10M   Early relocation address (used when relocating Xen)
>   *
>   * ARM32 layout:
>   *   0  -   8M   <COMMON>
> @@ -117,7 +114,7 @@
>  #define XEN_VIRT_START         _AT(vaddr_t,0x00200000)
>  #define FIXMAP_ADDR(n)        (_AT(vaddr_t,0x00400000) + (n) * PAGE_SIZE)
>  #define BOOT_FDT_VIRT_START    _AT(vaddr_t,0x00600000)
> -#define BOOT_MISC_VIRT_START   _AT(vaddr_t,0x00800000)
> +#define BOOT_RELOC_VIRT_START  _AT(vaddr_t,0x00800000)
>  
>  #define HYPERVISOR_VIRT_START  XEN_VIRT_START
>  
> 


-- 
Julien Grall

Re: [PATCH v2 8/9] xen/arm: print the location of the Xen heap on 32 bit

[Julien Grall]

On 09/13/2013 12:40 PM, Ian Campbell wrote:
> Signed-off-by: Ian Campbell <ian.campbell@citrix.com>

Reviewed-by: Julien Grall <julien.grall@linaro.org>

> ---
>  xen/arch/arm/setup.c |    6 ++++--
>  1 file changed, 4 insertions(+), 2 deletions(-)
> 
> diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
> index 5a47bda..ffae16e 100644
> --- a/xen/arch/arm/setup.c
> +++ b/xen/arch/arm/setup.c
> @@ -407,8 +407,10 @@ static void __init setup_mm(unsigned long dtb_paddr, size_t dtb_size)
>  
>      domheap_pages = heap_pages - xenheap_pages;
>  
> -    early_printk("Xen heap: %lu pages  Dom heap: %lu pages\n",
> -                 xenheap_pages, domheap_pages);
> +    early_printk("Xen heap: %"PRIpaddr"-%"PRIpaddr" (%lu pages)\n",
> +                 e - (xenheap_pages << PAGE_SHIFT), e,
> +                 xenheap_pages);
> +    early_printk("Dom heap: %lu pages\n", domheap_pages);
>  
>      setup_xenheap_mappings((e >> PAGE_SHIFT) - xenheap_pages, xenheap_pages);
>  
> 


-- 
Julien Grall