[Qemu-devel] [RFC PATCH 4/8] hw/misc: IDM Device

[Christian Pinto <c.pinto@virtualopensystems.com>]

This patch introduces the Interrupt Distribution Module (IDM) device for ARM,
x86 and X86_64 architectures, as the only currently supported ones.

The IDM device is used both as and Inter-processor interrupt routing device,
and to trigger the boot of a slave QEMU instance.

The IDM device can be instantiated either as a sysbus or PCI device.

 -device idm_ipi

or

 -device idm_ipi_pci

parameters are:
 master=[true/false] - configure the IDM device as master or slave
 num_slaves=[slaves_number] - if master is true specifies the number of slaves
 memdev=[memdev_id] - id of the shared memory backend
 socket=[socket_id] - id of the multi-client socket

Signed-off-by: Christian Pinto <c.pinto@virtualopensystems.com>
---
 default-configs/arm-softmmu.mak    |   1 +
 default-configs/i386-softmmu.mak   |   1 +
 default-configs/x86_64-softmmu.mak |   1 +
 hw/misc/Makefile.objs              |   2 +
 hw/misc/idm.c                      | 416 +++++++++++++++++++++++++++++++++++++
 include/hw/misc/idm.h              | 119 +++++++++++
 6 files changed, 540 insertions(+)
 create mode 100644 hw/misc/idm.c
 create mode 100644 include/hw/misc/idm.h

diff --git a/default-configs/arm-softmmu.mak b/default-configs/arm-softmmu.mak
index d9b90a5..44109a3 100644
--- a/default-configs/arm-softmmu.mak
+++ b/default-configs/arm-softmmu.mak
@@ -109,3 +109,4 @@ CONFIG_IOH3420=y
 CONFIG_I82801B11=y
 CONFIG_ACPI=y
 CONFIG_SMBIOS=y
+CONFIG_IDM=y
diff --git a/default-configs/i386-softmmu.mak b/default-configs/i386-softmmu.mak
index 9393cf0..f017448 100644
--- a/default-configs/i386-softmmu.mak
+++ b/default-configs/i386-softmmu.mak
@@ -51,3 +51,4 @@ CONFIG_XIO3130=y
 CONFIG_IOH3420=y
 CONFIG_I82801B11=y
 CONFIG_SMBIOS=y
+CONFIG_IDM=y
diff --git a/default-configs/x86_64-softmmu.mak b/default-configs/x86_64-softmmu.mak
index 28e2099..6977479 100644
--- a/default-configs/x86_64-softmmu.mak
+++ b/default-configs/x86_64-softmmu.mak
@@ -49,5 +49,6 @@ CONFIG_PVPANIC=y
 CONFIG_MEM_HOTPLUG=y
 CONFIG_XIO3130=y
 CONFIG_IOH3420=y
+CONFIG_IDM=y
 CONFIG_I82801B11=y
 CONFIG_SMBIOS=y
diff --git a/hw/misc/Makefile.objs b/hw/misc/Makefile.objs
index 4aa76ff..6e01c50 100644
--- a/hw/misc/Makefile.objs
+++ b/hw/misc/Makefile.objs
@@ -21,6 +21,8 @@ common-obj-$(CONFIG_MACIO) += macio/
 
 obj-$(CONFIG_IVSHMEM) += ivshmem.o
 
+obj-$(CONFIG_IDM) += idm.o
+
 obj-$(CONFIG_REALVIEW) += arm_sysctl.o
 obj-$(CONFIG_NSERIES) += cbus.o
 obj-$(CONFIG_ECCMEMCTL) += eccmemctl.o
diff --git a/hw/misc/idm.c b/hw/misc/idm.c
new file mode 100644
index 0000000..a8f408c
--- /dev/null
+++ b/hw/misc/idm.c
@@ -0,0 +1,416 @@
+/*
+ * IDM Device
+ *
+ * Copyright (C) 2015 - Virtual Open Systems
+ *
+ * Author: Christian Pinto <c.pinto@virtualopensystems.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#include "hw/sysbus.h"
+#include "qemu/error-report.h"
+#include "hw/misc/idm.h"
+
+
+static void inject_irq(IDMState *s)
+{
+    if (s->pci) {
+        PCIDevice *d = PCI_DEVICE(s);
+        pci_set_irq(d, 1);
+    } else {
+        qemu_irq_raise(s->irq);
+    }
+}
+
+static void reset_irq(IDMState *s)
+{
+    if (s->pci) {
+        PCIDevice *d = PCI_DEVICE(s);
+        pci_set_irq(d, 0);
+    } else {
+        qemu_irq_lower(s->irq);
+    }
+}
+
+static uint64_t idm_read(void *opaque, hwaddr offset, unsigned size)
+{
+    struct IDMState *s = opaque;
+    uint64_t ret = 0;
+
+    IDM_PRINTF("idm_read - offset %llu, size: %u\n",
+        (unsigned long long int)offset, size);
+
+    switch (offset) {
+    case ISR_REG:
+        /**
+         * Reading the ISR returns the whole mask, so that the driver can
+         * figure out which slaves have fired.
+         * The interrupt status register is cleared, and the interrupt lowered
+         */
+        ret = s->int_status_reg;
+        s->int_status_reg = 0;
+        reset_irq(s);
+        break;
+    default:
+        error_report("idm_read: Wrong offset");
+        break;
+    }
+
+    return ret;
+}
+
+static void send_shmem_fd(IDMState *s, MSClient *c)
+{
+    int fd, len;
+    uint32_t *message;
+    HostMemoryBackend *backend = MEMORY_BACKEND(s->hostmem);
+
+    len = strlen(SEND_MEM_FD_CMD)/4 + 3;
+    message = malloc(len * sizeof(uint32_t));
+    strcpy((char *) message, SEND_MEM_FD_CMD);
+    message[len - 2] = s->pboot_size;
+    message[len - 1] = s->pboot_offset;
+
+    fd = memory_region_get_fd(&backend->mr);
+
+    multi_socket_send_fds_to(c, &fd, 1, (char *) message, len * sizeof(uint32_t));
+
+    free(message);
+}
+
+static void idm_write(void *opaque, hwaddr offset, uint64_t value,
+    unsigned size)
+{
+    struct IDMState *s = opaque;
+
+    IDM_PRINTF("idm_write: offset %llu, size: %u, value: %llu\n",
+        (unsigned long long int)offset, size, (unsigned long long int)value);
+
+    switch (offset) {
+    case KICK_REG:
+    /**
+     * To kick another qemu instance it is sufficient to write its id into
+     * the KICK_REG. Value zero is reserved to kick the master,
+     * from 1 upwards for the slaves.
+     */
+        if (value == 0) {
+            if (!s->master) {
+                event_notifier_set(&s->master_eventfd->eventfd.out);
+                IDM_PRINTF("idm_write: triggered eventfd to master\n");
+            } else {
+                error_report("IDM module: master instance trying to kick "
+                    "the master. Wrong value written to Kick_REG");
+            }
+        } else {
+            if (s->master) {
+                event_notifier_set(&s->slaves[value - 1].eventfd.out);
+                IDM_PRINTF("idm_write: triggered eventfd to slave %llu\n",
+                    (unsigned long long int)value - 1);
+            } else {
+                error_report("IDM module: Slave trying to kick another slave."
+                    "Functionality not yet supported");
+            }
+        }
+        break;
+    case BOOT_REG:
+    /**
+     * When idm is configured as master the BOOT_REG is used to trigger
+     * the boot of a slave instance.
+     * The ID of the slave to boot is to be written in BOOT_REG
+     * Slave IDs range from 0 up-to num_slaves - 1
+     */
+        IDM_PRINTF("idm_write: triggering boot of slave %d\n",
+            (int)(value - 1));
+        send_shmem_fd(s, s->slaves[value - 1].socket_client);
+        break;
+    case PBOOT_REG:
+        if (value == PBOOT_REG_RESET)
+            s->pboot_status = 0;
+        else if (s->pboot_status == 0) {
+            s->pboot_size = value;
+            s->pboot_status = 1;
+        } else {
+            s->pboot_offset = value;
+            s->pboot_status = 0;
+        }
+        break;
+    default:
+        error_report("IDM module: wrong register to idm_write\n");
+        break;
+    }
+}
+
+static void idm_eventfd_handler(void *opaque)
+{
+    struct IDMSlave *sl = opaque;
+    struct IDMState *s = sl->idm_state;
+
+    IDM_PRINTF("idm_eventfd_handler: triggering IRQ to GuestOS\n");
+
+
+    /**
+     * Set the interrupt status register to notify which slave has fired
+     * the interrupt
+     */
+    s->int_status_reg |= 1 << (sl->slave_id + 1);
+
+    event_notifier_test_and_clear(&sl->eventfd.in);
+    inject_irq(s);
+}
+
+static void slave_register_ms_handler(MSClient *c, const char *message,
+    void *opaque)
+{
+    struct IDMState *s = opaque;
+    int fd[2], ret, id;
+
+    if (s->registered_slaves >= s->num_slaves) {
+        error_report("IDM module: All slaves already registered, "
+            "run again with bigger num_slaves");
+        exit(1);
+    }
+
+    id = s->registered_slaves;
+    s->registered_slaves++;
+
+    s->slaves[id].slave_id = id;
+    s->slaves[id].idm_state = s;
+    s->slaves[id].socket_client = c;
+    ret = event_notifier_init(&s->slaves[id].eventfd.in, 0);
+    ret |= event_notifier_init(&s->slaves[id].eventfd.out, 0);
+    if (ret) {
+        error_report("IDM module: Unable to initialize local eventfd");
+        exit(1);
+    }
+
+    /**
+     * send master eventfd to slave
+     * Set master eventfd into socket ancillary data and send
+     */
+
+    fd[0] = event_notifier_get_fd(&s->slaves[id].eventfd.in);
+    fd[1] = event_notifier_get_fd(&s->slaves[id].eventfd.out);
+
+
+    qemu_set_fd_handler(fd[0], (IOHandler *)idm_eventfd_handler, NULL,
+        &s->slaves[id]);
+
+    IDM_PRINTF("slave_register_ms_handler: master sending eventfds %d - %d "
+        "to slave %d\n", fd[0], fd[1], id);
+
+    multi_socket_send_fds_to(c, fd, 2, MASTER_EVENTFD_CMD,
+        strlen(MASTER_EVENTFD_CMD) + 1);
+}
+
+static void master_eventfd_ms_handler(MSClient *c, const char *message,
+        void *opaque)
+{
+    struct IDMState *s = opaque;
+    int fd[2];
+
+    s->master_eventfd->idm_state = s;
+    /**
+     * Master has the highest id
+     */
+    s->master_eventfd->slave_id = 0xffffffff;
+    /**
+     * Get slave eventfd from socket ancillary data
+     */
+    multi_socket_get_fds_from(c, fd);
+    IDM_PRINTF("master_eventfd_ms_handler: eventfd %d - %d "
+        "received from master\n", fd[0], fd[1]);
+
+    /**
+     * Initialize master event notifier
+     */
+    event_notifier_init_fd(&s->master_eventfd->eventfd.in, fd[1]);
+    event_notifier_init_fd(&s->master_eventfd->eventfd.out, fd[0]);
+
+    qemu_set_fd_handler(fd[1], (IOHandler *)idm_eventfd_handler, NULL,
+        s->master_eventfd);
+}
+
+
+static const MemoryRegionOps idm_ops = {
+    .read = idm_read,
+    .write = idm_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static void idm_realize_common(DeviceState *dev, Error **errp, IDMState *s)
+{
+    if (s->master && (s->num_slaves == 0)) {
+        error_report("idm_realize_common: master requires at least one slave");
+        exit(1);
+    }
+
+    if (s->master) {
+        IDM_PRINTF("idm_realize_common: Master init, num slaves %d\n",
+            s->num_slaves);
+        multi_socket_add_handler(s->master_socket, SLAVE_REGISTER_CMD,
+            slave_register_ms_handler, s);
+        s->slaves = g_new(struct IDMSlave, s->num_slaves);
+        s->registered_slaves = 0;
+    } else {
+        IDM_PRINTF("idm_realize_common: Slave init\n");
+        s->master_eventfd = g_new(struct IDMSlave, 1);
+        multi_socket_add_handler(s->master_socket, MASTER_EVENTFD_CMD,
+            master_eventfd_ms_handler, s);
+        multi_socket_write_to(&s->master_socket->listener, SLAVE_REGISTER_CMD,
+            strlen(SLAVE_REGISTER_CMD) + 1);
+    }
+
+    IDM_PRINTF("idm_realize_common: done!!\n");
+}
+
+static void idm_realize(DeviceState *dev, Error **errp)
+{
+    struct IDMState *s = IDM(dev);
+    SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
+
+    s->pci = false;
+
+    IDM_PRINTF("idm_realize\n");
+
+    /**
+     * Initialize MMIO regions with read/write functions
+     */
+    memory_region_init_io(&s->iomem, OBJECT(s), &idm_ops, s,
+                          TYPE_IDM, IDM_SIZE);
+    sysbus_init_mmio(sbd, &s->iomem);
+
+    /**
+     * Initialize IRQ
+     */
+    sysbus_init_irq(sbd, &s->irq);
+
+    idm_realize_common(dev, errp, IDM(dev));
+}
+
+static void idm_realize_pci(PCIDevice *dev, Error **errp)
+{
+    struct IDMState *s = IDM_PCI(dev);
+    uint8_t *pci_conf;
+
+    IDM_PRINTF("idm_realize_pci\n");
+
+    s->pci = true;
+
+    pci_conf = dev->config;
+    pci_conf[PCI_COMMAND] = PCI_COMMAND_MEMORY;
+
+    /**
+     * Initialize IRQ
+     */
+    pci_config_set_interrupt_pin(pci_conf, 1);
+
+    /**
+     * Initialize MMIO regions with read/write functions
+     */
+    memory_region_init_io(&s->iomem, OBJECT(s), &idm_ops, s,
+                          TYPE_IDM_PCI, IDM_SIZE);
+    pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY,
+                     &s->iomem);
+
+    idm_realize_common(&dev->qdev, errp, IDM_PCI(dev));
+}
+
+
+static Property idm_properties[] = {
+    DEFINE_PROP_UINT32("num-slaves", IDMState, num_slaves, 1),
+    DEFINE_PROP_BOOL("master", IDMState, master, true),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+
+static void idm_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    IDM_PRINTF("idm_class_init\n");
+
+    dc->props = idm_properties;
+    dc->realize = idm_realize;
+
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+}
+
+
+static void idm_pci_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    IDM_PRINTF("idm_pci_class_init\n");
+
+    dc->props = idm_properties;
+    k->realize = idm_realize_pci;
+
+    k->vendor_id = PCI_VENDOR_ID_IDM;
+    k->device_id = PCI_DEVICE_ID_IDM;
+    k->class_id = PCI_CLASS_MEMORY_RAM;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+}
+
+
+static void idm_init(Object *obj)
+{
+    struct IDMState *s = IDM(obj);
+
+    object_property_add_link(obj, IDM_MEMDEV_PROP, TYPE_MEMORY_BACKEND_SHARED,
+                             (Object **)&s->hostmem,
+                             qdev_prop_allow_set_link_before_realize,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                             &error_abort);
+
+    object_property_add_link(obj, IDM_SOCKET_PROP, TYPE_MULTI_SOCKET_BACKEND,
+                             (Object **)&s->master_socket,
+                             qdev_prop_allow_set_link_before_realize,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                             &error_abort);
+}
+
+static void idm_init_pci(Object *obj)
+{
+    IDMState *s = IDM_PCI(obj);
+
+    object_property_add_link(obj, IDM_MEMDEV_PROP, TYPE_MEMORY_BACKEND_SHARED,
+                             (Object **)&s->hostmem,
+                             qdev_prop_allow_set_link_before_realize,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                             &error_abort);
+
+    object_property_add_link(obj, IDM_SOCKET_PROP, TYPE_MULTI_SOCKET_BACKEND,
+                             (Object **)&s->master_socket,
+                             qdev_prop_allow_set_link_before_realize,
+                             OBJ_PROP_LINK_UNREF_ON_RELEASE,
+                             &error_abort);
+}
+
+static const TypeInfo idm_info = {
+    .name          = TYPE_IDM,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_init = idm_init,
+    .instance_size = sizeof(struct IDMState),
+    .class_init    = idm_class_init,
+    .class_size    = sizeof(struct IDMDeviceClass),
+};
+
+static const TypeInfo idm_info_pci = {
+    .name          = TYPE_IDM_PCI,
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_init = idm_init_pci,
+    .instance_size = sizeof(struct IDMState),
+    .class_init    = idm_pci_class_init,
+    .class_size    = sizeof(struct IDMPCIDeviceClass),
+};
+
+static void idm_register_types(void)
+{
+    type_register_static(&idm_info);
+    type_register_static(&idm_info_pci);
+}
+
+type_init(idm_register_types)
diff --git a/include/hw/misc/idm.h b/include/hw/misc/idm.h
new file mode 100644
index 0000000..ef22a4e
--- /dev/null
+++ b/include/hw/misc/idm.h
@@ -0,0 +1,119 @@
+/*
+ * IDM Device
+ *
+ * Copyright (C) 2015 - Virtual Open Systems
+ *
+ * Author: Christian Pinto <c.pinto@virtualopensystems.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ */
+
+#ifndef HW_MISC_IDM_H
+#define HW_MISC_IDM_H
+
+
+#include "hw/pci/pci.h"
+#include "qemu/multi-socket.h"
+#include "sysemu/hostmem-shared.h"
+
+typedef struct IDMState IDMState;
+
+#define DEBUG_IDM
+#ifdef DEBUG_IDM
+#define IDM_PRINTF(fmt, ...) \
+    do {printf("IDM - " fmt, ## __VA_ARGS__); fflush(stdout); } while (0)
+#else
+#define IDM_PRINTF(fmt, ...)
+#endif
+
+#define TYPE_IDM     "idm_ipi"
+#define TYPE_IDM_PCI "idm_ipi_pci"
+
+#define IDM_MEMDEV_PROP "memdev"
+#define IDM_SOCKET_PROP "socket"
+
+/*
+ * Currently using a fake PCI device ID
+ */
+#define PCI_VENDOR_ID_IDM   PCI_VENDOR_ID_REDHAT_QUMRANET
+#define PCI_DEVICE_ID_IDM   0x1111
+
+#define IDM(obj) OBJECT_CHECK(struct IDMState, (obj), TYPE_IDM)
+#define IDM_PCI(obj) OBJECT_CHECK(struct IDMState, (obj), TYPE_IDM_PCI)
+
+/*
+ * Size of the IO memory mapped region
+ * associated with IDM device registers
+ */
+#define IDM_SIZE 0x100
+
+/*
+ * Registers for IDM device
+ */
+#define ISR_REG  0x4
+#define KICK_REG 0x8
+#define BOOT_REG 0xC
+#define PBOOT_REG 0x10
+#define PBOOT_REG_RESET 0xDEAD
+
+#define SLAVE_REGISTER_CMD "SLAVE_REGISTER"
+#define MASTER_EVENTFD_CMD "MASTER_EVENTFD"
+#define SEND_MEM_FD_CMD    "send_fd"
+
+struct IDMEventfdChan {
+    EventNotifier in;
+    EventNotifier out;
+};
+
+struct IDMSlave {
+    uint32_t slave_id;
+    struct IDMEventfdChan eventfd;
+    IDMState *idm_state;
+    hwaddr boot_address;
+    MSClient *socket_client;
+};
+
+struct IDMState {
+    /*< private >*/
+    union {
+        PCIDevice pdev;
+        SysBusDevice sdev;
+    };
+
+    /*< public >*/
+    bool pci;
+    MemoryRegion iomem;
+    uint32_t num_slaves;
+    uint32_t registered_slaves;
+    bool master;
+    struct IDMSlave *master_eventfd;
+    struct IDMSlave *slaves;
+    MSBackend *master_socket;
+    qemu_irq irq;
+    HostMemoryBackendShared *hostmem;
+    /*registers*/
+    bool pboot_status;
+    uint32_t pboot_size;
+    uint32_t pboot_offset;
+
+    /*
+     * One bit is set every time an interrupt is received 0 for the master,
+     * slaves from 1 onwards
+     */
+    uint32_t int_status_reg;
+};
+
+struct IDMDeviceClass {
+    /*< private >*/
+    SysBusDeviceClass parent_class;
+    /*< public >*/
+};
+
+struct IDMPCIDeviceClass {
+    /*< private >*/
+    PCIDeviceClass parent_class;
+    /*< public >*/
+};
+
+#endif
-- 
1.9.1