[RFC 18/19] libxl: Add support for non-PCI passthrough

[Julien Grall <julien.grall@linaro.org>]

On ARM, every non-PCI device are described in the device tree. Each of them
can be found via a path.

This path will be used to retrieved the different informations about the
device (compatible string, interrupts, MMIOs). Libxl will take care of:
    - Allocate the MMIOs regions for the device in the guest
    - Create the device node in the guest device tree
    - Map the IRQs and MMIOs range in the guest P2M

Note, that the device node won't contains specific properties for the node.
Only generic one (compatible, interrupts, regs) will be created by libxl.

In the future, per-device properties will be added. Maybe via a configuration
file listing what is needed.

Signed-off-by: Julien Grall <julien.grall@linaro.org>
Cc: Ian Jackson <ian.jackson@eu.citrix.com>
Cc: Stefano Stabellini <stefano.stabellini@eu.citrix.com>
Cc: Ian Campbell <ian.campbell@citrix.com>
---
 tools/libxl/Makefile         |    2 +-
 tools/libxl/libxl_arch.h     |    4 +-
 tools/libxl/libxl_arm.c      |  103 ++++++++++++++++++++++++++-
 tools/libxl/libxl_create.c   |   11 +++
 tools/libxl/libxl_dom.c      |   22 +++++-
 tools/libxl/libxl_dtdev.c    |  159 ++++++++++++++++++++++++++++++++++++++++++
 tools/libxl/libxl_internal.h |   31 ++++++++
 tools/libxl/libxl_types.idl  |    5 ++
 tools/libxl/libxl_x86.c      |    4 +-
 9 files changed, 335 insertions(+), 6 deletions(-)
 create mode 100644 tools/libxl/libxl_dtdev.c

diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index 4cfa275..986df08 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -76,7 +76,7 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o libxl_pci.o \
 			libxl_internal.o libxl_utils.o libxl_uuid.o \
 			libxl_json.o libxl_aoutils.o libxl_numa.o \
 			libxl_save_callout.o _libxl_save_msgs_callout.o \
-			libxl_qmp.o libxl_event.o libxl_fork.o $(LIBXL_OBJS-y)
+			libxl_qmp.o libxl_event.o libxl_fork.o libxl_dtdev.o $(LIBXL_OBJS-y)
 LIBXL_OBJS += _libxl_types.o libxl_flask.o _libxl_types_internal.o
 
 LIBXL_TESTS += timedereg
diff --git a/tools/libxl/libxl_arch.h b/tools/libxl/libxl_arch.h
index d3bc136..fea1893 100644
--- a/tools/libxl/libxl_arch.h
+++ b/tools/libxl/libxl_arch.h
@@ -17,11 +17,13 @@
 
 /* arch specific internal domain creation function */
 int libxl__arch_domain_create(libxl__gc *gc, libxl_domain_config *d_config,
-               uint32_t domid);
+                              libxl__domain_build_state *state,
+                              uint32_t domid);
 
 /* setup arch specific hardware description, i.e. DTB on ARM */
 int libxl__arch_domain_init_hw_description(libxl__gc *gc,
                                            libxl_domain_build_info *info,
+                                           libxl__domain_build_state *state,
                                            struct xc_dom_image *dom);
 /* finalize arch specific hardware description. */
 int libxl__arch_domain_finalise_hw_description(libxl__gc *gc,
diff --git a/tools/libxl/libxl_arm.c b/tools/libxl/libxl_arm.c
index e19e2f4..0eec174 100644
--- a/tools/libxl/libxl_arm.c
+++ b/tools/libxl/libxl_arm.c
@@ -24,8 +24,38 @@
 #define DT_IRQ_TYPE_LEVEL_LOW      0x00000008
 
 int libxl__arch_domain_create(libxl__gc *gc, libxl_domain_config *d_config,
+                              libxl__domain_build_state *state,
                               uint32_t domid)
 {
+    int dev_index;
+    uint32_t mmio_index;
+    uint64_t mmiocurr = GUEST_MMIO_BASE >> XC_PAGE_SHIFT;
+    /* Convenient */
+    const uint64_t mmioend = (GUEST_MMIO_BASE + GUEST_MMIO_SIZE) >> XC_PAGE_SHIFT;
+
+
+    for (dev_index = 0; dev_index < d_config->num_dtdevs; dev_index++) {
+        const libxl_device_dt *dtdev = &d_config->dtdevs[dev_index];
+        libxl__dtdev_info *info = &state->dtdevs_info[dev_index];
+
+        LOG(DEBUG, "Allocate %d MMIOs region for \"%s\"",
+            info->num_mmios, dtdev->path);
+        for (mmio_index = 0; mmio_index < info->num_mmios; mmio_index++) {
+            libxl_iomem_range *io = &info->mmios[mmio_index];
+
+            /* Check if we have enough space for the MMIO region */
+            /* TODO: Do I need to check overlap? */
+            if ((mmiocurr + io->number) > mmioend) {
+                LOG(ERROR, "Not enough space in the guest layout to allocate the MMIOs regions");
+                return -ENOMEM;
+            }
+            LOG(DEBUG, "\t0x%"PRIx64"-0x%"PRIx64,
+                mmiocurr, mmiocurr + io->number);
+            io->gfn = mmiocurr;
+            mmiocurr += io->number;
+        }
+    }
+
     return 0;
 }
 
@@ -412,6 +442,72 @@ static int make_hypervisor_node(libxl__gc *gc, void *fdt,
     return 0;
 }
 
+static int make_dtdev_node(libxl__gc *gc, void *fdt,
+                           int index, const libxl__dtdev_info *dtdev)
+{
+    const char *name;
+    uint32_t i;
+    int res;
+
+    /* The unit-address (after @) is only request when the device has MMIO */
+    if (dtdev->num_mmios > 0) {
+        uint64_t base = dtdev->mmios[0].gfn << XC_PAGE_SHIFT;
+
+        name = GCSPRINTF("dtdev-%u@%"PRIx64, index, base);
+    } else
+        name = GCSPRINTF("dtdev-%u", index);
+
+    res = fdt_begin_node(fdt, name);
+    if (res) return res;
+
+    assert(dtdev->compat_len != 0);
+    fdt_property(fdt, "compatible", dtdev->compat, dtdev->compat_len);
+
+    if (dtdev->num_mmios > 0) {
+        be32 *regs, *cells;
+        /* Convenient */
+        const unsigned addr_cells = ROOT_ADDRESS_CELLS;
+        const unsigned size_cells = ROOT_SIZE_CELLS;
+        const unsigned len = sizeof(*regs) * (addr_cells + size_cells);
+
+        regs = libxl__malloc(gc, len);
+        cells = &regs[0];
+
+        for (i = 0; i < dtdev->num_mmios; i++) {
+            uint64_t base = dtdev->mmios[i].gfn << XC_PAGE_SHIFT;
+            uint64_t size = dtdev->mmios[i].number << XC_PAGE_SHIFT;
+
+            set_range(&cells, addr_cells, size_cells, base, size);
+        }
+
+        res = fdt_property(fdt, "reg", regs, len);
+        if (res) return res;
+    }
+
+    if (dtdev->num_irqs > 0) {
+        gic_interrupt *ints;
+
+        ints = libxl__malloc(gc, sizeof(*ints) * dtdev->num_irqs);
+        for (i =0; i < dtdev->num_irqs; i++) {
+            /* TODO: Translate the IRQ type into DT type. We should
+             * not assume a 1:1 mapping */
+            /* For now, Xen is only handling SPIs passthrough and
+             * forward to VCPU0
+             */
+            assert(dtdev->irqs[i].irq >= 32);
+            set_interrupt(ints[i], dtdev->irqs[i].irq,
+                          0x1, dtdev->irqs[i].type);
+        }
+
+        res = fdt_property_interrupts(gc, fdt, ints, dtdev->num_irqs);
+    }
+
+    res = fdt_end_node(fdt);
+    if (res) return res;
+
+    return 0;
+}
+
 static const struct arch_info *get_arch_info(libxl__gc *gc,
                                              const struct xc_dom_image *dom)
 {
@@ -454,10 +550,11 @@ out:
 
 int libxl__arch_domain_init_hw_description(libxl__gc *gc,
                                            libxl_domain_build_info *info,
+                                           libxl__domain_build_state *state,
                                            struct xc_dom_image *dom)
 {
     void *fdt = NULL;
-    int rc, res;
+    int rc, res, i;
     size_t fdt_size = 0;
 
     const libxl_version_info *vers;
@@ -527,6 +624,10 @@ next_resize:
         FDT( make_timer_node(gc, fdt, ainfo) );
         FDT( make_hypervisor_node(gc, fdt, vers) );
 
+        for (i = 0; i < state->num_dtdevs; i++) {
+            FDT( make_dtdev_node(gc, fdt, i, &state->dtdevs_info[i]) );
+        }
+
         FDT( fdt_end_node(fdt) );
 
         FDT( fdt_finish(fdt) );
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index e544bbf..cad9987 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -1298,6 +1298,7 @@ static void domcreate_attach_pci(libxl__egc *egc, libxl__multidev *multidev,
 
     /* convenience aliases */
     libxl_domain_config *const d_config = dcs->guest_config;
+    libxl__domain_build_state *const state = &dcs->build_state;
 
     if (ret) {
         LOG(ERROR, "unable to add vtpm devices");
@@ -1323,6 +1324,16 @@ static void domcreate_attach_pci(libxl__egc *egc, libxl__multidev *multidev,
         }
     }
 
+    for (i = 0; i < d_config->num_dtdevs; i++) {
+
+        ret = libxl__device_dt_add(gc, domid, &state->dtdevs_info[i]);
+        if (ret < 0) {
+            LIBXL__LOG(ctx, LIBXL__LOG_ERROR,
+                       "libxl__device_dt_add failed: %d\n", ret);
+            goto error_out;
+        }
+    }
+
     domcreate_console_available(egc, dcs);
 
     domcreate_complete(egc, dcs, 0);
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 661999c..c58a4a8 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -233,6 +233,7 @@ int libxl__build_pre(libxl__gc *gc, uint32_t domid,
     libxl_domain_build_info *const info = &d_config->b_info;
     libxl_ctx *ctx = libxl__gc_owner(gc);
     char *xs_domid, *con_domid;
+    int i;
     int rc;
 
     if (xc_domain_max_vcpus(ctx->xch, domid, info->max_vcpus) != 0) {
@@ -284,7 +285,23 @@ int libxl__build_pre(libxl__gc *gc, uint32_t domid,
     if (info->type == LIBXL_DOMAIN_TYPE_HVM)
         hvm_set_conf_params(ctx->xch, domid, info);
 
-    rc = libxl__arch_domain_create(gc, d_config, domid);
+    /* We need to retrieve DT devs information before calling
+     * libxl__arch_domain_create. On ARM, the function will allocate GFN
+     * for each MMIO regions
+     */
+    state->num_dtdevs = d_config->num_dtdevs;
+    state->dtdevs_info = libxl__calloc(gc, sizeof(*state->dtdevs_info),
+                                       d_config->num_dtdevs);
+    for (i = 0; i < d_config->num_dtdevs; i++) {
+        rc = libxl__device_dt_get_info(gc, &d_config->dtdevs[i],
+                                       &state->dtdevs_info[i]);
+        if (rc) {
+            LOG(ERROR, "libxl__device_dt_get_info failed: %d", rc);
+            return ERROR_INVAL;
+        }
+    }
+
+    rc = libxl__arch_domain_create(gc, d_config, state, domid);
 
     return rc;
 }
@@ -436,7 +453,8 @@ int libxl__build_pv(libxl__gc *gc, uint32_t domid,
         LOGE(ERROR, "xc_dom_parse_image failed");
         goto out;
     }
-    if ( (ret = libxl__arch_domain_init_hw_description(gc, info, dom)) != 0 ) {
+    ret = libxl__arch_domain_init_hw_description(gc, info, state, dom);
+    if ( ret != 0 ) {
         LOGE(ERROR, "libxl__arch_domain_init_hw_description failed");
         goto out;
     }
diff --git a/tools/libxl/libxl_dtdev.c b/tools/libxl/libxl_dtdev.c
new file mode 100644
index 0000000..995de5c
--- /dev/null
+++ b/tools/libxl/libxl_dtdev.c
@@ -0,0 +1,159 @@
+/*
+ * Copyright (C) 2014      Linaro Limited.
+ * Author Julien Grall <julien.grall@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* Must come before other headers */
+
+#include "libxl_internal.h"
+
+int libxl__device_dt_add(libxl__gc *gc, uint32_t domid,
+                         const libxl__dtdev_info *dtdev)
+{
+    uint32_t i;
+    int ret;
+
+    LOG(DEBUG, "Assign device \"%s\" to dom%u", dtdev->conf->path, domid);
+
+    for (i = 0; i < dtdev->num_mmios; i++) {
+        const libxl_iomem_range *io = &dtdev->mmios[i];
+
+        ret = xc_domain_iomem_permission(CTX->xch, domid, io->start,
+                                         io->number, 1);
+        if (ret < 0) {
+            LOGE(ERROR,
+                 "%s: failed to give dom%d access to iomem range"
+                 "%"PRIx64"-%"PRIx64, dtdev->conf->path,
+                 domid, io->start, io->start + io->number + 1);
+            return ret;
+        }
+
+        ret = xc_domain_memory_mapping(CTX->xch, domid, io->gfn,
+                                       io->start, io->number, 1);
+        if (ret < 0) {
+            LOGE(ERROR,
+                "%s: failed to map to dom%u iomem range %"PRIx64"-%"PRIx64
+                " to guest address %"PRIx64,
+                dtdev->conf->path, domid, io->start,
+                io->start + io->number - 1, io->gfn);
+            return ret;
+        }
+    }
+
+    for (i = 0; i < dtdev->num_irqs; i++) {
+        int irq = dtdev->irqs[i].irq;
+
+        ret = xc_physdev_map_pirq(CTX->xch, domid, irq, &irq);
+        if (ret < 0) {
+            LOGE(ERROR, "%s: failed to map the IRQ %u into dom%u\n",
+                 dtdev->conf->path, domid, irq);
+            return ret;
+        }
+    }
+
+    return xc_assign_dt_device(CTX->xch, domid, dtdev->conf->path);
+}
+
+int libxl__device_dt_get_info(libxl__gc *gc,
+                              const libxl_device_dt *dtdev,
+                              libxl__dtdev_info *dtinfo)
+{
+    xc_dtdev_info_t info;
+    int ret = 0;
+    uint32_t i;
+    char *buff;
+    uint32_t len;
+
+    LOG(DEBUG, "Get information for DT dev \"%s\"", dtdev->path);
+
+    ret = xc_physdev_dtdev_getinfo(CTX->xch, dtdev->path, &info);
+    if (ret) {
+        LOGE(ERROR, "Unable to get the informations for \"%s\"",
+             dtdev->path);
+        return ret;
+    }
+
+    LOG(DEBUG, "num_irqs = %u num_mmios = %u compat_len = %u",
+        info.num_irqs, info.num_mmios, info.compat_len);
+
+    dtinfo->conf = dtdev;
+
+    /* Retrieve the IRQs */
+    dtinfo->num_irqs = info.num_irqs;
+    dtinfo->irqs = libxl__calloc(gc, dtinfo->num_irqs,
+                                 sizeof(*dtinfo->mmios));
+
+    LOG(DEBUG, "List of IRQs");
+    for (i = 0; i < dtinfo->num_irqs; i++) {
+        xc_dtdev_irq_t irq;
+
+        ret = xc_physdev_dtdev_getirq(CTX->xch, dtdev->path, i, &irq);
+        if (ret) {
+            LOGE(ERROR, "Unable to get IRQ%u for \"%s\"", i, dtdev->path);
+            return ret;
+        }
+
+        LOG(DEBUG, "\t- irq = %u type = %u", irq.irq, irq.type);
+        dtinfo->irqs[i].irq = irq.irq;
+        /* TODO translate the type correctly */
+        dtinfo->irqs[i].type = irq.type;
+    }
+
+    /* Retrieve the MMIOs range */
+    dtinfo->num_mmios = info.num_mmios;
+    dtinfo->mmios =  libxl__calloc(gc, dtinfo->num_mmios,
+                                   sizeof(*dtinfo->mmios));
+
+    LOG(DEBUG, "List of MMIOs");
+    for (i = 0; i < dtinfo->num_mmios; i++) {
+        xc_dtdev_mmio_t mmio;
+
+        ret = xc_physdev_dtdev_getmmio(CTX->xch, dtdev->path, i, &mmio);
+        if (ret) {
+            LOGE(ERROR, "Unable to get the MMIO range %u for \"%s\"",
+                 i, dtdev->path);
+            return ret;
+        }
+
+        LOG(DEBUG, "\t- mfn = 0x%"PRIx64" nr_mfn = 0x%"PRIx64,
+            mmio.mfn, mmio.nr_mfn);
+
+        dtinfo->mmios[i].start = mmio.mfn;
+        dtinfo->mmios[i].number = mmio.nr_mfn;
+        dtinfo->mmios[i].gfn = LIBXL_INVALID_GFN;
+    }
+
+    /* Retrieve the compatible property */
+    len = info.compat_len;
+    buff = libxl__malloc(gc, len);
+
+    ret = xc_physdev_dtdev_getcompat(CTX->xch, dtdev->path,
+                                     buff, &len);
+    if (ret) {
+        LOGE(ERROR, "Unable to get the compatible string for \"%s\"",
+             dtdev->path);
+        return ret;
+    }
+    dtinfo->compat_len = len;
+    dtinfo->compat = buff;
+
+    return 0;
+}
+
+/*
+ * Local variables:
+ * mode: C
+ * c-basic-offset: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 60d6f1d..cfe4482 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -944,6 +944,25 @@ typedef struct {
 _hidden int libxl__file_reference_map(libxl__file_reference *f);
 _hidden int libxl__file_reference_unmap(libxl__file_reference *f);
 
+/* DT dev representation */
+typedef struct {
+    uint32_t num_mmios;
+    libxl_iomem_range *mmios;
+
+    uint32_t num_irqs;
+    struct
+    {
+        uint32_t irq;
+        uint32_t type;
+    } *irqs;
+
+    const char *compat;
+    ssize_t compat_len; /* The compatible string may contain \0 */
+
+    /* Short-hand to the User configuration for this device */
+    const libxl_device_dt *conf;
+} libxl__dtdev_info;
+
 /* from xl_dom */
 _hidden libxl_domain_type libxl__domain_type(libxl__gc *gc, uint32_t domid);
 _hidden int libxl__domain_shutdown_reason(libxl__gc *gc, uint32_t domid);
@@ -969,6 +988,9 @@ typedef struct {
     libxl__file_reference pv_ramdisk;
     const char * pv_cmdline;
     bool pvh_enabled;
+
+    int num_dtdevs;
+    libxl__dtdev_info *dtdevs_info;
 } libxl__domain_build_state;
 
 _hidden int libxl__build_pre(libxl__gc *gc, uint32_t domid,
@@ -1152,6 +1174,15 @@ _hidden int libxl__create_pci_backend(libxl__gc *gc, uint32_t domid,
                                       libxl_device_pci *pcidev, int num);
 _hidden int libxl__device_pci_destroy_all(libxl__gc *gc, uint32_t domid);
 
+/* from libxl_dtdev */
+
+_hidden int libxl__device_dt_add(libxl__gc *gc, uint32_t domid,
+                                 const libxl__dtdev_info *info);
+_hidden int libxl__device_dt_get_info(libxl__gc *gc,
+                                      const libxl_device_dt *dtdev,
+                                      libxl__dtdev_info *info);
+
+
 /*----- xswait: wait for a xenstore node to be suitable -----*/
 
 typedef struct libxl__xswait_state libxl__xswait_state;
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 9201461..74ba20d 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -452,6 +452,10 @@ libxl_device_pci = Struct("device_pci", [
     ("seize", bool),
     ])
 
+libxl_device_dt = Struct("device_dt", [
+    ("path", string),
+    ])
+
 libxl_device_vtpm = Struct("device_vtpm", [
     ("backend_domid",    libxl_domid),
     ("backend_domname",  string),
@@ -466,6 +470,7 @@ libxl_domain_config = Struct("domain_config", [
     ("disks", Array(libxl_device_disk, "num_disks")),
     ("nics", Array(libxl_device_nic, "num_nics")),
     ("pcidevs", Array(libxl_device_pci, "num_pcidevs")),
+    ("dtdevs", Array(libxl_device_dt, "num_dtdevs")),
     ("vfbs", Array(libxl_device_vfb, "num_vfbs")),
     ("vkbs", Array(libxl_device_vkb, "num_vkbs")),
     ("vtpms", Array(libxl_device_vtpm, "num_vtpms")),
diff --git a/tools/libxl/libxl_x86.c b/tools/libxl/libxl_x86.c
index 7589060..3e9640b 100644
--- a/tools/libxl/libxl_x86.c
+++ b/tools/libxl/libxl_x86.c
@@ -245,7 +245,8 @@ static int libxl__e820_alloc(libxl__gc *gc, uint32_t domid,
 }
 
 int libxl__arch_domain_create(libxl__gc *gc, libxl_domain_config *d_config,
-        uint32_t domid)
+                              libxl__domain_build_state *state,
+                              uint32_t domid)
 {
     int ret = 0;
     int tsc_mode;
@@ -313,6 +314,7 @@ int libxl__arch_domain_create(libxl__gc *gc, libxl_domain_config *d_config,
 
 int libxl__arch_domain_init_hw_description(libxl__gc *gc,
                                            libxl_domain_build_info *info,
+                                           libxl__domain_build_state *state,
                                            struct xc_dom_image *dom)
 {
     return 0;
-- 
1.7.10.4