[PATCH v3 14/14] libxl: build a device tree for ARM guests

[Ian Campbell <ian.campbell@citrix.com>]

Uses xc_dom_devicetree_mem which was just added. The call to this needs to be
carefully sequenced to be after xc_dom_parse_image (so we can tell which kind
of guest we are building, although we don't use this yet) and before
xc_dom_mem_init which tries to decide where to place the FDT in guest RAM.

Removes libxl_noarch which would only have been used by IA64 after this
change. Remove IA64 as part of this patch.

There is no attempt to expose this as a configuration setting for the user.

Includes a debug hook to dump the dtb to a file for inspection.

TODO:
- v7 CPU compat is hardcoded to cortex-a15 -- may need to define something more
  generic via mach-virt dt bindngs?

Signed-off-by: Ian Campbell <ian.campbell@citrix.com>
---
v2: base addresses, irq, evtchn etc stuff is now from public API headers,
    avoiding the need to introduce domctls etc until we want to make them
    dynamic.
    fix memory node
    Improve libfdt error handling, especially for FDT_ERR_NOSPACE.
    Derive guest CPU and timer compatiblity nodes from the guest type.
---
 tools/libxl/Makefile       |    6 +-
 tools/libxl/libxl_arch.h   |    4 +
 tools/libxl/libxl_arm.c    |  553 ++++++++++++++++++++++++++++++++++++++++++++
 tools/libxl/libxl_dom.c    |    4 +
 tools/libxl/libxl_noarch.c |    8 -
 tools/libxl/libxl_x86.c    |    7 +
 6 files changed, 572 insertions(+), 10 deletions(-)
 create mode 100644 tools/libxl/libxl_arm.c
 delete mode 100644 tools/libxl/libxl_noarch.c

diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index cf214bb..d8495bb 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -28,9 +28,12 @@ CFLAGS_LIBXL += $(CFLAGS_libxenstore)
 CFLAGS_LIBXL += $(CFLAGS_libblktapctl) 
 CFLAGS_LIBXL += -Wshadow
 
+LIBXL_LIBS-$(CONFIG_ARM) += -lfdt
+
 CFLAGS += $(PTHREAD_CFLAGS)
 LDFLAGS += $(PTHREAD_LDFLAGS)
 LIBXL_LIBS += $(PTHREAD_LIBS)
+LIBXL_LIBS += $(LIBXL_LIBS-y)
 
 LIBXLU_LIBS =
 
@@ -41,8 +44,7 @@ else
 LIBXL_OBJS-y += libxl_noblktap2.o
 endif
 LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o
-LIBXL_OBJS-$(CONFIG_IA64) += libxl_nocpuid.o libxl_noarch.o
-LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_noarch.o
+LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o
 
 ifeq ($(CONFIG_NetBSD),y)
 LIBXL_OBJS-y += libxl_netbsd.o
diff --git a/tools/libxl/libxl_arch.h b/tools/libxl/libxl_arch.h
index abe6685..3de2d76 100644
--- a/tools/libxl/libxl_arch.h
+++ b/tools/libxl/libxl_arch.h
@@ -19,4 +19,8 @@
 int libxl__arch_domain_create(libxl__gc *gc, libxl_domain_config *d_config,
                uint32_t domid);
 
+/* */
+int libxl__arch_domain_configure(libxl__gc *gc,
+                                 libxl_domain_build_info *info,
+                                 struct xc_dom_image *dom);
 #endif
diff --git a/tools/libxl/libxl_arm.c b/tools/libxl/libxl_arm.c
new file mode 100644
index 0000000..6bc3536
--- /dev/null
+++ b/tools/libxl/libxl_arm.c
@@ -0,0 +1,553 @@
+#include "libxl_internal.h"
+#include "libxl_arch.h"
+
+#include <xc_dom.h>
+#include <libfdt.h>
+#include <assert.h>
+
+int libxl__arch_domain_create(libxl__gc *gc, libxl_domain_config *d_config,
+                              uint32_t domid)
+{
+    return 0;
+}
+
+static struct arch_info {
+    const char *guest_type;
+    const char *timer_compat;
+    const char *cpu_compat;
+} arch_info[] = {
+    {"xen-3.0-armv7l",  "arm,armv7-timer", "arm,cortex-a15" },
+    {"xen-3.0-aarch64", "arm,armv8-timer", "arm,armv8" },
+};
+
+enum {
+    PHANDLE_NONE = 0,
+    PHANDLE_GIC,
+};
+
+typedef uint32_t __be32;
+typedef __be32 gic_interrupt_t[3];
+
+#define ROOT_ADDRESS_CELLS 2
+#define ROOT_SIZE_CELLS 2
+
+static void set_cell(__be32 **cellp, int size, uint64_t val)
+{
+    int cells = size;
+
+    while ( size-- )
+    {
+        (*cellp)[size] = cpu_to_fdt32(val);
+        val >>= 32;
+    }
+
+    (*cellp) += cells;
+}
+
+static void set_interrupt_ppi(gic_interrupt_t interrupt, unsigned int irq,
+                              unsigned int cpumask, unsigned int level)
+{
+    __be32 *cells = interrupt;
+
+    /* See linux Documentation/devictree/bindings/arm/gic.txt */
+    set_cell(&cells, 1, 1); /* is a PPI */
+    set_cell(&cells, 1, irq - 16); /* PPIs start at 16 */
+    set_cell(&cells, 1, (cpumask << 8) | level);
+}
+
+static void set_range(__be32 **cellp,
+                      int address_cells, int size_cells,
+                      uint64_t address, uint64_t size)
+{
+    set_cell(cellp, address_cells, address);
+    set_cell(cellp, size_cells, size);
+}
+
+static int fdt_property_compat(libxl__gc *gc, void *fdt, unsigned nr_compat, ...)
+{
+    const char *compats[nr_compat];
+    int i;
+    size_t sz;
+    va_list ap;
+    char *compat, *p;
+
+    va_start(ap, nr_compat);
+    sz = 0;
+    for (i = 0; i < nr_compat; i++) {
+        const char *c = va_arg(ap, const char *);
+        compats[i] = c;
+        sz += strlen(compats[i]) + 1;
+    }
+    va_end(ap);
+
+    p = compat = libxl__zalloc(gc, sz);
+    for (i = 0; i < nr_compat; i++) {
+        strcpy(p, compats[i]);
+        p += strlen(compats[i]) + 1;
+    }
+
+    fdt_property(fdt, "compatible", compat, sz);
+
+    return 0;
+}
+
+static int fdt_property_interrupts(libxl__gc *gc, void *fdt,
+                                   gic_interrupt_t *intr,
+                                   unsigned num_irq)
+{
+    int res;
+
+    res = fdt_property(fdt, "interrupts", intr, sizeof (intr[0]) * num_irq);
+    if ( res )
+        return res;
+
+    res = fdt_property_cell(fdt, "interrupt-parent", PHANDLE_GIC);
+
+    return res;
+}
+
+static int fdt_property_regs(libxl__gc *gc, void *fdt,
+                             unsigned addr_cells,
+                             unsigned size_cells,
+                             unsigned num_regs, ...)
+{
+    uint32_t regs[num_regs*(addr_cells+size_cells)];
+    __be32 *cells = &regs[0];
+    int i;
+    va_list ap;
+    uint64_t base, size;
+
+    va_start(ap, num_regs);
+    for (i = 0 ; i < num_regs; i++) {
+        base = addr_cells ? va_arg(ap, uint64_t) : 0;
+        size = size_cells ? va_arg(ap, uint64_t) : 0;
+        set_range(&cells, addr_cells, size_cells, base, size);
+    }
+    va_end(ap);
+
+    return fdt_property(fdt, "reg", regs, sizeof(regs));
+}
+
+static int make_root_properties(libxl__gc *gc,
+                                const libxl_version_info *vers,
+                                void *fdt)
+{
+    int res;
+
+    res = fdt_property_string(fdt, "model", GCSPRINTF("XENVM-%d.%d",
+                                                      vers->xen_version_major,
+                                                      vers->xen_version_minor));
+    if ( res )
+        return res;
+
+    res = fdt_property_compat(gc, fdt, 2,
+                              GCSPRINTF("xen,xenvm-%d.%d",
+                                        vers->xen_version_major,
+                                        vers->xen_version_minor),
+                              "xen,xenvm");
+    if ( res )
+        return res;
+
+    res = fdt_property_cell(fdt, "interrupt-parent", PHANDLE_GIC);
+    if ( res )
+        return res;
+
+    res = fdt_property_cell(fdt, "#address-cells", ROOT_ADDRESS_CELLS);
+    if ( res )
+        return res;
+
+    res = fdt_property_cell(fdt, "#size-cells", ROOT_SIZE_CELLS);
+    if ( res )
+        return res;
+
+    return 0;
+}
+
+static int make_chosen_node(libxl__gc *gc, void *fdt,
+                            const libxl_domain_build_info *info)
+{
+    int res;
+
+    /* See linux Documentation/devicetree/... */
+    res = fdt_begin_node(fdt, "chosen");
+    if ( res )
+        return res;
+
+    res = fdt_property_string(fdt, "bootargs", info->u.pv.cmdline);
+    if ( res )
+        return res;
+
+    res = fdt_end_node(fdt);
+
+    return res;
+}
+
+static int make_cpus_node(libxl__gc *gc, void *fdt, int nr_cpus,
+                          const struct arch_info *ainfo)
+{
+    int res, i;
+
+    res = fdt_begin_node(fdt, "cpus");
+    if ( res )
+        return res;
+
+    res = fdt_property_cell(fdt, "#address-cells", 1);
+    if ( res )
+        return res;
+
+    res = fdt_property_cell(fdt, "#size-cells", 0);
+    if ( res )
+        return res;
+
+    for (i = 0; i < nr_cpus; i++) {
+        char name[strlen("cpu@") + 2 + 1];
+
+        if ( snprintf(name, sizeof(name)-1, "cpu@%d", i) < 0 )
+        {
+            perror("snprintf");
+            exit(1);
+        }
+
+        res = fdt_begin_node(fdt, name);
+        if ( res )
+            return res;
+
+        res = fdt_property_string(fdt, "device_type", "cpu");
+        if ( res )
+            return res;
+
+        res = fdt_property_compat(gc, fdt, 1, ainfo->cpu_compat);
+        if ( res )
+            return res;
+
+        res = fdt_property_string(fdt, "enable-method", "psci");
+        if ( res )
+            return res;
+
+        res = fdt_property_regs(gc, fdt, 1, 0, 1, (uint64_t)i);
+        if ( res )
+            return res;
+
+        res = fdt_end_node(fdt);
+        if ( res )
+            return res;
+
+    }
+
+    res = fdt_end_node(fdt);
+
+    return res;
+}
+
+static int make_psci_node(libxl__gc *gc, void *fdt)
+{
+    int res;
+
+    res = fdt_begin_node(fdt, "psci");
+    if ( res )
+        return res;
+
+    res = fdt_property_compat(gc, fdt, 1, "arm,psci");
+    if ( res )
+        return res;
+
+    res = fdt_property_string(fdt, "method", "hvc");
+    if ( res )
+        return res;
+
+    res = fdt_property_cell(fdt, "cpu_off", PSCI_cpu_off);
+    if ( res )
+        return res;
+
+    res = fdt_property_cell(fdt, "cpu_on", PSCI_cpu_on);
+    if ( res )
+        return res;
+
+    res = fdt_end_node(fdt);
+
+    return res;
+}
+
+static int make_memory_node(libxl__gc *gc, void *fdt,
+                            unsigned long long base,
+                            unsigned long long size)
+{
+    int res;
+    char name[strlen("memory@") + 8 + 1];
+
+    snprintf(name, sizeof(name), "memory@%08llx", base);
+
+    res = fdt_begin_node(fdt, name);
+    if ( res )
+        return res;
+
+    res = fdt_property_string(fdt, "device_type", "memory");
+    if ( res )
+        return res;
+
+    res = fdt_property_regs(gc, fdt, ROOT_ADDRESS_CELLS, ROOT_SIZE_CELLS,
+                            1, (uint64_t)base, (uint64_t)size);
+    if ( res )
+        return res;
+
+    res = fdt_end_node(fdt);
+
+    return res;
+}
+
+static int make_intc_node(libxl__gc *gc, void *fdt,
+                          unsigned long long gicd_base,
+                          unsigned long long gicd_size,
+                          unsigned long long gicc_base,
+                          unsigned long long gicc_size)
+{
+    int res;
+    char name[strlen("interrupt-controller@") + 8 + 1];
+
+    snprintf(name, sizeof(name), "interrupt-controller@%08llx", gicd_base);
+
+    res = fdt_begin_node(fdt, name);
+    if ( res )
+        return res;
+
+    res = fdt_property_compat(gc, fdt, 2,
+                              "arm,cortex-a15-gic",
+                              "arm,cortex-a9-gic");
+    if ( res )
+        return res;
+
+
+    res = fdt_property_cell(fdt, "#interrupt-cells", 3);
+    if ( res )
+        return res;
+
+    res = fdt_property_cell(fdt, "#address-cells", 0);
+    if ( res )
+        return res;
+
+    res = fdt_property(fdt, "interrupt-controller", NULL, 0);
+    if ( res )
+        return res;
+
+    res = fdt_property_regs(gc, fdt, ROOT_ADDRESS_CELLS, ROOT_SIZE_CELLS,
+                            2,
+                            (uint64_t)gicd_base, (uint64_t)gicd_size,
+                            (uint64_t)gicc_base, (uint64_t)gicc_size);
+    if ( res )
+        return res;
+
+    res = fdt_property_cell(fdt, "linux,phandle", PHANDLE_GIC);
+    if ( res )
+        return res;
+
+    res = fdt_property_cell(fdt, "phandle", PHANDLE_GIC);
+    if ( res )
+        return res;
+
+    res = fdt_end_node(fdt);
+
+    return res;
+}
+
+static int make_timer_node(libxl__gc *gc, void *fdt, const struct arch_info *ainfo)
+{
+    int res;
+    gic_interrupt_t ints[3];
+
+    res = fdt_begin_node(fdt, "timer");
+    if ( res )
+        return res;
+
+    res = fdt_property_compat(gc, fdt, 1, ainfo->timer_compat);
+    if ( res )
+        return res;
+
+    set_interrupt_ppi(ints[0], GUEST_TIMER_PHYS_S_PPI, 0xf, 0x8);
+    set_interrupt_ppi(ints[1], GUEST_TIMER_PHYS_NS_PPI, 0xf, 0x8);
+    set_interrupt_ppi(ints[2], GUEST_TIMER_VIRT_PPI, 0xf, 0x8);
+
+    res = fdt_property_interrupts(gc, fdt, ints, 3);
+    if ( res )
+        return res;
+
+    res = fdt_end_node(fdt);
+
+    return res;
+}
+
+static int make_hypervisor_node(libxl__gc *gc, void *fdt,
+                                const libxl_version_info *vers)
+{
+    int res;
+    gic_interrupt_t intr;
+
+    /* See linux Documentation/devicetree/bindings/arm/xen.txt */
+    res = fdt_begin_node(fdt, "hypervisor");
+    if ( res )
+        return res;
+
+    res = fdt_property_compat(gc, fdt, 2,
+                              GCSPRINTF("xen,xen-%d.%d",
+                                        vers->xen_version_major,
+                                        vers->xen_version_minor),
+                              "xen,xen");
+    if ( res )
+        return res;
+
+    /* reg 0 is grant table space */
+    res = fdt_property_regs(gc, fdt, ROOT_ADDRESS_CELLS, ROOT_SIZE_CELLS,
+                            1,GUEST_GNTTAB_BASE, GUEST_GNTTAB_SIZE);
+    if ( res )
+        return res;
+
+    /*
+     * interrupts is evtchn upcall:
+     *  - Active-low level-sensitive
+     *  - All cpus
+     */
+    set_interrupt_ppi(intr, GUEST_EVTCHN_PPI, 0xf, 0x8);
+
+    res = fdt_property_interrupts(gc, fdt, &intr, 1);
+    if ( res )
+        return res;
+
+    res = fdt_end_node(fdt);
+
+    return res;
+}
+
+static const struct arch_info *get_arch_info(libxl__gc *gc,
+                                             const struct xc_dom_image *dom)
+{
+    int i;
+
+    for (i=0; i < ARRAY_SIZE(arch_info); i++) {
+        const struct arch_info *info = &arch_info[i];
+        if (!strcmp(dom->guest_type, info->guest_type))
+            return info;
+    }
+    LOG(ERROR, "Unable to find arch FDT info for %s\n", dom->guest_type);
+    return NULL;
+}
+
+#define FDT_MAX_SIZE (1<<20)
+
+/*
+ * Call "call" handling FDR_ERR_*. Will either:
+ * - loop back to retry_resize
+ * - set rc and goto out
+ * - fall through successfully
+ *
+ * On FDT_ERR_NOSPACE we start again from scratch rather than
+ * realloc+libfdt_open_into because "call" may have failed half way
+ * through a series of steps leaving the partial tree in an
+ * inconsistent state, e.g. leaving a node open.
+ */
+#define FDT( call ) do {                                        \
+    int fdt_res = (call);                                       \
+    if (fdt_res == -FDT_ERR_NOSPACE && fdt_size < FDT_MAX_SIZE) \
+        goto retry_resize;                                      \
+    else if (fdt_res < 0) {                                     \
+        LOG(ERROR, "FDT: %s failed: %d = %s",                   \
+            #call, fdt_res, fdt_strerror(fdt_res));             \
+        rc = ERROR_FAIL;                                        \
+        goto out;                                               \
+    }                                                           \
+} while(0)
+
+int libxl__arch_domain_configure(libxl__gc *gc,
+                                 libxl_domain_build_info *info,
+                                 struct xc_dom_image *dom)
+{
+    void *fdt;
+    int rc, res, fd;
+    size_t fdt_size = 0;
+
+    const libxl_version_info *vers;
+    const struct arch_info *ainfo;
+
+    assert(info->type == LIBXL_DOMAIN_TYPE_PV);
+
+    vers = libxl_get_version_info(CTX);
+    if (vers == NULL) return ERROR_FAIL;
+
+    ainfo = get_arch_info(gc, dom);
+    if (ainfo == NULL) return ERROR_FAIL;
+
+    LOG(DEBUG, "constructing DTB for Xen version %d.%d guest",
+        vers->xen_version_major, vers->xen_version_minor);
+
+retry_resize:
+    if (fdt_size) {
+        fdt_size <<= 1;
+        LOG(DEBUG, "Increasing FDT size to %zd and retrying", fdt_size);
+    } else {
+        fdt_size = 4096;
+    }
+
+    fdt = libxl__zalloc(gc, fdt_size);
+
+    FDT( fdt_create(fdt, fdt_size) );
+
+    fdt_finish_reservemap(fdt);
+
+    FDT( fdt_begin_node(fdt, "") );
+
+    FDT( make_root_properties(gc, vers, fdt) );
+    FDT( make_chosen_node(gc, fdt, info) );
+    FDT( make_cpus_node(gc, fdt, info->max_vcpus, ainfo) );
+    FDT( make_psci_node(gc, fdt) );
+
+    FDT( make_memory_node(gc, fdt,
+                          dom->rambase_pfn << XC_PAGE_SHIFT,
+                          info->target_memkb * 1024) );
+    FDT( make_intc_node(gc, fdt,
+                        GUEST_GICD_BASE, GUEST_GICD_SIZE,
+                        GUEST_GICC_BASE, GUEST_GICD_SIZE) );
+
+    FDT( make_timer_node(gc, fdt, ainfo) );
+    FDT( make_hypervisor_node(gc, fdt, vers) );
+
+    FDT( fdt_end_node(fdt) );
+
+    FDT( fdt_finish(fdt) );
+
+    LOG(DEBUG, "fdt total size %d", fdt_totalsize(fdt));
+
+    if ( (res = xc_dom_devicetree_mem(dom, fdt, fdt_totalsize(fdt))) != 0 ) {
+        LOGE(ERROR, "xc_dom_devicetree_file failed");
+        rc = ERROR_FAIL;
+        goto out;
+    }
+
+#ifndef NDEBUG
+    if ( getenv("LIBXL_DEBUG_DUMP_DTB") ) {
+        const char *dtb = getenv("LIBXL_DEBUG_DUMP_DTB");
+
+        fd = open(dtb, O_CREAT|O_TRUNC|O_WRONLY, 0666);
+        if ( fd < 0 ) {
+            LOGE(DEBUG, "cannot open %s for LIBXL_DEBUG_DUMP_DTB", dtb);
+            goto no_debug_dump;
+        }
+
+        rc = libxl_write_exactly(CTX, fd, fdt, fdt_totalsize(fdt),
+                            dtb, "dtb");
+        if ( rc < 0 ) {
+            LOG(DEBUG, "unable to write DTB debug dump output %d", rc);
+            goto no_debug_dump;
+        }
+
+        res = close(fd);
+        if ( res < 0 ) {
+            LOGE(DEBUG, "unable to close DTB debug dump output");
+            goto no_debug_dump;
+        }
+    }
+no_debug_dump:
+#endif
+
+    rc = 0;
+
+out:
+    return rc;
+}
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 10f976c..6351ebd 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -400,6 +400,10 @@ int libxl__build_pv(libxl__gc *gc, uint32_t domid,
         LOGE(ERROR, "xc_dom_parse_image failed");
         goto out;
     }
+    if ( (ret = libxl__arch_domain_configure(gc, info, dom)) != 0 ) {
+        LOGE(ERROR, "libxl__arch_domain_configure failed");
+        goto out;
+    }
     if ( (ret = xc_dom_mem_init(dom, info->target_memkb / 1024)) != 0 ) {
         LOGE(ERROR, "xc_dom_mem_init failed");
         goto out;
diff --git a/tools/libxl/libxl_noarch.c b/tools/libxl/libxl_noarch.c
deleted file mode 100644
index 7893535..0000000
--- a/tools/libxl/libxl_noarch.c
+++ /dev/null
@@ -1,8 +0,0 @@
-#include "libxl_internal.h"
-#include "libxl_arch.h"
-
-int libxl__arch_domain_create(libxl__gc *gc, libxl_domain_config *d_config,
-                              uint32_t domid)
-{
-    return 0;
-}
diff --git a/tools/libxl/libxl_x86.c b/tools/libxl/libxl_x86.c
index a78c91d..dd13c45 100644
--- a/tools/libxl/libxl_x86.c
+++ b/tools/libxl/libxl_x86.c
@@ -308,3 +308,10 @@ int libxl__arch_domain_create(libxl__gc *gc, libxl_domain_config *d_config,
 
     return ret;
 }
+
+int libxl__arch_domain_configure(libxl__gc *gc,
+                                 libxl_domain_build_info *info,
+                                 struct xc_dom_image *dom)
+{
+    return 0;
+}
-- 
1.7.10.4