[PATCH for-10.0 0/7] hw/riscv: riscv-iommu-sys device

[PATCH for-10.0 0/7] hw/riscv: riscv-iommu-sys device

[Daniel Henrique Barboza]

Hi,

Now that we have merged the base IOMMU support we can re-introduce
the riscv-iommu-sys platform device that was taken away from the initial
posting.

Aside from adding support for the device in the 'virt' machine we're
also adding MSI support for it, something that we weren't doing before.
The Linux driver is then free to choose either MSI or WSI to use the
device.

Patches based on master.

Daniel Henrique Barboza (5):
  hw/riscv/riscv-iommu.c: add riscv_iommu_instance_init()
  hw/riscv/riscv-iommu: parametrize CAP.IGS
  hw/riscv/virt.c, riscv-iommu-sys.c: add MSIx support
  hw/riscv/riscv-iommu: implement reset protocol
  docs/specs: add riscv-iommu-sys information

Sunil V L (1):
  hw/riscv/virt: Add IOMMU as platform device if the option is set

Tomasz Jeznach (1):
  hw/riscv: add riscv-iommu-sys platform device

 docs/specs/riscv-iommu.rst  |  30 ++++-
 docs/system/riscv/virt.rst  |  10 ++
 hw/riscv/meson.build        |   2 +-
 hw/riscv/riscv-iommu-bits.h |   6 +
 hw/riscv/riscv-iommu-pci.c  |  21 +++
 hw/riscv/riscv-iommu-sys.c  | 256 ++++++++++++++++++++++++++++++++++++
 hw/riscv/riscv-iommu.c      | 114 +++++++++++-----
 hw/riscv/riscv-iommu.h      |   5 +
 hw/riscv/trace-events       |   4 +
 hw/riscv/virt.c             | 108 ++++++++++++++-
 include/hw/riscv/iommu.h    |  10 +-
 include/hw/riscv/virt.h     |   6 +-
 12 files changed, 530 insertions(+), 42 deletions(-)
 create mode 100644 hw/riscv/riscv-iommu-sys.c

-- 
2.45.2

[PATCH for-10.0 1/7] hw/riscv/riscv-iommu.c: add riscv_iommu_instance_init()

[Daniel Henrique Barboza]

Move all the static initializion of the device to an init() function,
leaving only the dynamic initialization to be done during realize.

With this change s->cap is initialized with RISCV_IOMMU_CAP_DBG during
init(), and realize() will increment s->cap with the extra caps.

This will allow callers to add IOMMU capabilities before the
realization.

Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
---
 hw/riscv/riscv-iommu.c | 71 +++++++++++++++++++++++-------------------
 1 file changed, 39 insertions(+), 32 deletions(-)

diff --git a/hw/riscv/riscv-iommu.c b/hw/riscv/riscv-iommu.c
index feb650549a..1893584028 100644
--- a/hw/riscv/riscv-iommu.c
+++ b/hw/riscv/riscv-iommu.c
@@ -2096,11 +2096,48 @@ static const MemoryRegionOps riscv_iommu_trap_ops = {
     }
 };
 
+static void riscv_iommu_instance_init(Object *obj)
+{
+    RISCVIOMMUState *s = RISCV_IOMMU(obj);
+
+    /* Enable translation debug interface */
+    s->cap = RISCV_IOMMU_CAP_DBG;
+
+    /* Report QEMU target physical address space limits */
+    s->cap = set_field(s->cap, RISCV_IOMMU_CAP_PAS,
+                       TARGET_PHYS_ADDR_SPACE_BITS);
+
+    /* TODO: method to report supported PID bits */
+    s->pid_bits = 8; /* restricted to size of MemTxAttrs.pid */
+    s->cap |= RISCV_IOMMU_CAP_PD8;
+
+    /* register storage */
+    s->regs_rw = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
+    s->regs_ro = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
+    s->regs_wc = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
+
+     /* Mark all registers read-only */
+    memset(s->regs_ro, 0xff, RISCV_IOMMU_REG_SIZE);
+
+    /* Device translation context cache */
+    s->ctx_cache = g_hash_table_new_full(riscv_iommu_ctx_hash,
+                                         riscv_iommu_ctx_equal,
+                                         g_free, NULL);
+
+    s->iot_cache = g_hash_table_new_full(riscv_iommu_iot_hash,
+                                         riscv_iommu_iot_equal,
+                                         g_free, NULL);
+
+    s->iommus.le_next = NULL;
+    s->iommus.le_prev = NULL;
+    QLIST_INIT(&s->spaces);
+}
+
 static void riscv_iommu_realize(DeviceState *dev, Error **errp)
 {
     RISCVIOMMUState *s = RISCV_IOMMU(dev);
 
-    s->cap = s->version & RISCV_IOMMU_CAP_VERSION;
+    s->cap |= s->version & RISCV_IOMMU_CAP_VERSION;
     if (s->enable_msi) {
         s->cap |= RISCV_IOMMU_CAP_MSI_FLAT | RISCV_IOMMU_CAP_MSI_MRIF;
     }
@@ -2115,29 +2152,11 @@ static void riscv_iommu_realize(DeviceState *dev, Error **errp)
         s->cap |= RISCV_IOMMU_CAP_SV32X4 | RISCV_IOMMU_CAP_SV39X4 |
                   RISCV_IOMMU_CAP_SV48X4 | RISCV_IOMMU_CAP_SV57X4;
     }
-    /* Enable translation debug interface */
-    s->cap |= RISCV_IOMMU_CAP_DBG;
-
-    /* Report QEMU target physical address space limits */
-    s->cap = set_field(s->cap, RISCV_IOMMU_CAP_PAS,
-                       TARGET_PHYS_ADDR_SPACE_BITS);
-
-    /* TODO: method to report supported PID bits */
-    s->pid_bits = 8; /* restricted to size of MemTxAttrs.pid */
-    s->cap |= RISCV_IOMMU_CAP_PD8;
 
     /* Out-of-reset translation mode: OFF (DMA disabled) BARE (passthrough) */
     s->ddtp = set_field(0, RISCV_IOMMU_DDTP_MODE, s->enable_off ?
                         RISCV_IOMMU_DDTP_MODE_OFF : RISCV_IOMMU_DDTP_MODE_BARE);
 
-    /* register storage */
-    s->regs_rw = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
-    s->regs_ro = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
-    s->regs_wc = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
-
-     /* Mark all registers read-only */
-    memset(s->regs_ro, 0xff, RISCV_IOMMU_REG_SIZE);
-
     /*
      * Register complete MMIO space, including MSI/PBA registers.
      * Note, PCIDevice implementation will add overlapping MR for MSI/PBA,
@@ -2195,19 +2214,6 @@ static void riscv_iommu_realize(DeviceState *dev, Error **errp)
     memory_region_init_io(&s->trap_mr, OBJECT(dev), &riscv_iommu_trap_ops, s,
             "riscv-iommu-trap", ~0ULL);
     address_space_init(&s->trap_as, &s->trap_mr, "riscv-iommu-trap-as");
-
-    /* Device translation context cache */
-    s->ctx_cache = g_hash_table_new_full(riscv_iommu_ctx_hash,
-                                         riscv_iommu_ctx_equal,
-                                         g_free, NULL);
-
-    s->iot_cache = g_hash_table_new_full(riscv_iommu_iot_hash,
-                                         riscv_iommu_iot_equal,
-                                         g_free, NULL);
-
-    s->iommus.le_next = NULL;
-    s->iommus.le_prev = NULL;
-    QLIST_INIT(&s->spaces);
 }
 
 static void riscv_iommu_unrealize(DeviceState *dev)
@@ -2249,6 +2255,7 @@ static const TypeInfo riscv_iommu_info = {
     .name = TYPE_RISCV_IOMMU,
     .parent = TYPE_DEVICE,
     .instance_size = sizeof(RISCVIOMMUState),
+    .instance_init = riscv_iommu_instance_init,
     .class_init = riscv_iommu_class_init,
 };
 
-- 
2.45.2

[PATCH for-10.0 2/7] hw/riscv/riscv-iommu: parametrize CAP.IGS

[Daniel Henrique Barboza]

Interrupt Generation Support (IGS) is a capability that is tied to the
interrupt deliver mechanism, not with the core IOMMU emulation. We
should allow device implementations to set IGS as they wish.

A new helper is added to make it easier for device impls to set IGS. Use
it in our existing IOMMU device (riscv-iommu-pci) to set
RISCV_IOMMU_CAPS_IGS_MSI.

Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
---
 hw/riscv/riscv-iommu-bits.h | 6 ++++++
 hw/riscv/riscv-iommu-pci.c  | 1 +
 hw/riscv/riscv-iommu.c      | 5 +++++
 hw/riscv/riscv-iommu.h      | 4 ++++
 4 files changed, 16 insertions(+)

diff --git a/hw/riscv/riscv-iommu-bits.h b/hw/riscv/riscv-iommu-bits.h
index 6359ae0353..485f36b9c9 100644
--- a/hw/riscv/riscv-iommu-bits.h
+++ b/hw/riscv/riscv-iommu-bits.h
@@ -88,6 +88,12 @@ struct riscv_iommu_pq_record {
 #define RISCV_IOMMU_CAP_PD17            BIT_ULL(39)
 #define RISCV_IOMMU_CAP_PD20            BIT_ULL(40)
 
+enum riscv_iommu_igs_modes {
+    RISCV_IOMMU_CAP_IGS_MSI = 0,
+    RISCV_IOMMU_CAP_IGS_WSI,
+    RISCV_IOMMU_CAP_IGS_BOTH
+};
+
 /* 5.4 Features control register (32bits) */
 #define RISCV_IOMMU_REG_FCTL            0x0008
 #define RISCV_IOMMU_FCTL_BE             BIT(0)
diff --git a/hw/riscv/riscv-iommu-pci.c b/hw/riscv/riscv-iommu-pci.c
index a42242532d..4ce9bf6b78 100644
--- a/hw/riscv/riscv-iommu-pci.c
+++ b/hw/riscv/riscv-iommu-pci.c
@@ -155,6 +155,7 @@ static void riscv_iommu_pci_init(Object *obj)
     qdev_alias_all_properties(DEVICE(iommu), obj);
 
     iommu->icvec_avail_vectors = RISCV_IOMMU_PCI_ICVEC_VECTORS;
+    riscv_iommu_set_cap_igs(iommu, RISCV_IOMMU_CAP_IGS_MSI);
 }
 
 static Property riscv_iommu_pci_properties[] = {
diff --git a/hw/riscv/riscv-iommu.c b/hw/riscv/riscv-iommu.c
index 1893584028..d95b4b95d8 100644
--- a/hw/riscv/riscv-iommu.c
+++ b/hw/riscv/riscv-iommu.c
@@ -2096,6 +2096,11 @@ static const MemoryRegionOps riscv_iommu_trap_ops = {
     }
 };
 
+void riscv_iommu_set_cap_igs(RISCVIOMMUState *s, riscv_iommu_igs_mode mode)
+{
+    s->cap = set_field(s->cap, RISCV_IOMMU_CAP_IGS, mode);
+}
+
 static void riscv_iommu_instance_init(Object *obj)
 {
     RISCVIOMMUState *s = RISCV_IOMMU(obj);
diff --git a/hw/riscv/riscv-iommu.h b/hw/riscv/riscv-iommu.h
index da3f03440c..f9f2827808 100644
--- a/hw/riscv/riscv-iommu.h
+++ b/hw/riscv/riscv-iommu.h
@@ -21,6 +21,9 @@
 
 #include "qom/object.h"
 #include "hw/riscv/iommu.h"
+#include "hw/riscv/riscv-iommu-bits.h"
+
+typedef enum riscv_iommu_igs_modes riscv_iommu_igs_mode;
 
 struct RISCVIOMMUState {
     /*< private >*/
@@ -85,6 +88,7 @@ struct RISCVIOMMUState {
 
 void riscv_iommu_pci_setup_iommu(RISCVIOMMUState *iommu, PCIBus *bus,
          Error **errp);
+void riscv_iommu_set_cap_igs(RISCVIOMMUState *s, riscv_iommu_igs_mode mode);
 
 /* private helpers */
 
-- 
2.45.2

[PATCH for-10.0 3/7] hw/riscv: add riscv-iommu-sys platform device

[Daniel Henrique Barboza]

From: Tomasz Jeznach <tjeznach@rivosinc.com>

This device models the RISC-V IOMMU as a sysbus device. The same design
decisions taken in the riscv-iommu-pci device were kept, namely the
existence of 4 vectors are available for each interrupt cause.

The WSIs are emitted using the input of the s->notify() callback as a
index to an IRQ list. The IRQ list starts at 'base_irq' and goes until
base_irq + 3. This means that boards must have 4 contiguous IRQ lines
available, starting from 'base_irq'.

Signed-off-by: Tomasz Jeznach <tjeznach@rivosinc.com>
Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
---
 hw/riscv/meson.build       |   2 +-
 hw/riscv/riscv-iommu-sys.c | 128 +++++++++++++++++++++++++++++++++++++
 hw/riscv/riscv-iommu.c     |   3 +-
 include/hw/riscv/iommu.h   |   4 ++
 4 files changed, 134 insertions(+), 3 deletions(-)
 create mode 100644 hw/riscv/riscv-iommu-sys.c

diff --git a/hw/riscv/meson.build b/hw/riscv/meson.build
index adbef8a9b2..3be13d7774 100644
--- a/hw/riscv/meson.build
+++ b/hw/riscv/meson.build
@@ -10,6 +10,6 @@ riscv_ss.add(when: 'CONFIG_SIFIVE_U', if_true: files('sifive_u.c'))
 riscv_ss.add(when: 'CONFIG_SPIKE', if_true: files('spike.c'))
 riscv_ss.add(when: 'CONFIG_MICROCHIP_PFSOC', if_true: files('microchip_pfsoc.c'))
 riscv_ss.add(when: 'CONFIG_ACPI', if_true: files('virt-acpi-build.c'))
-riscv_ss.add(when: 'CONFIG_RISCV_IOMMU', if_true: files('riscv-iommu.c', 'riscv-iommu-pci.c'))
+riscv_ss.add(when: 'CONFIG_RISCV_IOMMU', if_true: files('riscv-iommu.c', 'riscv-iommu-pci.c', 'riscv-iommu-sys.c'))
 
 hw_arch += {'riscv': riscv_ss}
diff --git a/hw/riscv/riscv-iommu-sys.c b/hw/riscv/riscv-iommu-sys.c
new file mode 100644
index 0000000000..4b82046ce9
--- /dev/null
+++ b/hw/riscv/riscv-iommu-sys.c
@@ -0,0 +1,128 @@
+/*
+ * QEMU emulation of an RISC-V IOMMU Platform Device
+ *
+ * Copyright (C) 2022-2023 Rivos Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2 or later, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "hw/irq.h"
+#include "hw/pci/pci_bus.h"
+#include "hw/qdev-properties.h"
+#include "hw/sysbus.h"
+#include "qapi/error.h"
+#include "qemu/error-report.h"
+#include "qemu/host-utils.h"
+#include "qemu/module.h"
+#include "qom/object.h"
+
+#include "riscv-iommu.h"
+
+#define RISCV_IOMMU_SYSDEV_ICVEC_VECTORS 0x3333
+
+/* RISC-V IOMMU System Platform Device Emulation */
+
+struct RISCVIOMMUStateSys {
+    SysBusDevice     parent;
+    uint64_t         addr;
+    uint32_t         base_irq;
+    DeviceState      *irqchip;
+    RISCVIOMMUState  iommu;
+    qemu_irq         irqs[RISCV_IOMMU_INTR_COUNT];
+};
+
+static void riscv_iommu_sysdev_notify(RISCVIOMMUState *iommu,
+                                      unsigned vector)
+{
+    RISCVIOMMUStateSys *s = container_of(iommu, RISCVIOMMUStateSys, iommu);
+    uint32_t fctl =  riscv_iommu_reg_get32(iommu, RISCV_IOMMU_REG_FCTL);
+
+    /* We do not support MSIs yet */
+    if (!(fctl & RISCV_IOMMU_FCTL_WSI)) {
+        return;
+    }
+
+    qemu_irq_pulse(s->irqs[vector]);
+}
+
+static void riscv_iommu_sys_realize(DeviceState *dev, Error **errp)
+{
+    RISCVIOMMUStateSys *s = RISCV_IOMMU_SYS(dev);
+    SysBusDevice *sysdev = SYS_BUS_DEVICE(s);
+    PCIBus *pci_bus;
+    qemu_irq irq;
+
+    qdev_realize(DEVICE(&s->iommu), NULL, errp);
+    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->iommu.regs_mr);
+    if (s->addr) {
+        sysbus_mmio_map(SYS_BUS_DEVICE(s), 0, s->addr);
+    }
+
+    pci_bus = (PCIBus *) object_resolve_path_type("", TYPE_PCI_BUS, NULL);
+    if (pci_bus) {
+        riscv_iommu_pci_setup_iommu(&s->iommu, pci_bus, errp);
+    }
+
+    s->iommu.notify = riscv_iommu_sysdev_notify;
+
+    /* 4 IRQs are defined starting from s->base_irq */
+    for (int i = 0; i < RISCV_IOMMU_INTR_COUNT; i++) {
+        sysbus_init_irq(sysdev, &s->irqs[i]);
+        irq = qdev_get_gpio_in(s->irqchip, s->base_irq + i);
+        sysbus_connect_irq(sysdev, i, irq);
+    }
+}
+
+static void riscv_iommu_sys_init(Object *obj)
+{
+    RISCVIOMMUStateSys *s = RISCV_IOMMU_SYS(obj);
+    RISCVIOMMUState *iommu = &s->iommu;
+
+    object_initialize_child(obj, "iommu", iommu, TYPE_RISCV_IOMMU);
+    qdev_alias_all_properties(DEVICE(iommu), obj);
+
+    iommu->icvec_avail_vectors = RISCV_IOMMU_SYSDEV_ICVEC_VECTORS;
+    riscv_iommu_set_cap_igs(iommu, RISCV_IOMMU_CAP_IGS_WSI);
+}
+
+static Property riscv_iommu_sys_properties[] = {
+    DEFINE_PROP_UINT64("addr", RISCVIOMMUStateSys, addr, 0),
+    DEFINE_PROP_UINT32("base-irq", RISCVIOMMUStateSys, base_irq, 0),
+    DEFINE_PROP_LINK("irqchip", RISCVIOMMUStateSys, irqchip,
+                     TYPE_DEVICE, DeviceState *),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void riscv_iommu_sys_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    dc->realize = riscv_iommu_sys_realize;
+    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
+    device_class_set_props(dc, riscv_iommu_sys_properties);
+}
+
+static const TypeInfo riscv_iommu_sys = {
+    .name          = TYPE_RISCV_IOMMU_SYS,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .class_init    = riscv_iommu_sys_class_init,
+    .instance_init = riscv_iommu_sys_init,
+    .instance_size = sizeof(RISCVIOMMUStateSys),
+};
+
+static void riscv_iommu_register_sys(void)
+{
+    type_register_static(&riscv_iommu_sys);
+}
+
+type_init(riscv_iommu_register_sys)
diff --git a/hw/riscv/riscv-iommu.c b/hw/riscv/riscv-iommu.c
index d95b4b95d8..239f83f5bd 100644
--- a/hw/riscv/riscv-iommu.c
+++ b/hw/riscv/riscv-iommu.c
@@ -94,10 +94,9 @@ static uint8_t riscv_iommu_get_icvec_vector(uint32_t icvec, uint32_t vec_type)
 
 static void riscv_iommu_notify(RISCVIOMMUState *s, int vec_type)
 {
-    const uint32_t fctl = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_FCTL);
     uint32_t ipsr, icvec, vector;
 
-    if (fctl & RISCV_IOMMU_FCTL_WSI || !s->notify) {
+    if (!s->notify) {
         return;
     }
 
diff --git a/include/hw/riscv/iommu.h b/include/hw/riscv/iommu.h
index 80769a1400..fc20808553 100644
--- a/include/hw/riscv/iommu.h
+++ b/include/hw/riscv/iommu.h
@@ -33,4 +33,8 @@ typedef struct RISCVIOMMUSpace RISCVIOMMUSpace;
 OBJECT_DECLARE_SIMPLE_TYPE(RISCVIOMMUStatePci, RISCV_IOMMU_PCI)
 typedef struct RISCVIOMMUStatePci RISCVIOMMUStatePci;
 
+#define TYPE_RISCV_IOMMU_SYS "riscv-iommu-device"
+OBJECT_DECLARE_SIMPLE_TYPE(RISCVIOMMUStateSys, RISCV_IOMMU_SYS)
+typedef struct RISCVIOMMUStateSys RISCVIOMMUStateSys;
+
 #endif
-- 
2.45.2

[PATCH for-10.0 4/7] hw/riscv/virt: Add IOMMU as platform device if the option is set

[Daniel Henrique Barboza]

From: Sunil V L <sunilvl@ventanamicro.com>

Add a new machine option called 'iommu-sys' that enables a
riscv-iommu-sys platform device for the 'virt' machine. The option is
default 'off'.

The device will use IRQs 36 to 39.

We will not support both riscv-iommu-sys and riscv-iommu-pci devices in
the same board in this first implementation. If a riscv-iommu-pci device
is added in the command line we will disable the riscv-iommu-sys device.

Signed-off-by: Sunil V L <sunilvl@ventanamicro.com>
Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
---
 hw/riscv/virt.c          | 104 ++++++++++++++++++++++++++++++++++++++-
 include/hw/riscv/iommu.h |   2 +
 include/hw/riscv/virt.h  |   6 ++-
 3 files changed, 109 insertions(+), 3 deletions(-)

diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c
index 45a8c4f819..23d1380b86 100644
--- a/hw/riscv/virt.c
+++ b/hw/riscv/virt.c
@@ -33,6 +33,7 @@
 #include "target/riscv/pmu.h"
 #include "hw/riscv/riscv_hart.h"
 #include "hw/riscv/iommu.h"
+#include "hw/riscv/riscv-iommu-bits.h"
 #include "hw/riscv/virt.h"
 #include "hw/riscv/boot.h"
 #include "hw/riscv/numa.h"
@@ -76,6 +77,7 @@ static const MemMapEntry virt_memmap[] = {
     [VIRT_CLINT] =        {  0x2000000,       0x10000 },
     [VIRT_ACLINT_SSWI] =  {  0x2F00000,        0x4000 },
     [VIRT_PCIE_PIO] =     {  0x3000000,       0x10000 },
+    [VIRT_IOMMU_SYS] =    {  0x3010000,        0x1000 },
     [VIRT_PLATFORM_BUS] = {  0x4000000,     0x2000000 },
     [VIRT_PLIC] =         {  0xc000000, VIRT_PLIC_SIZE(VIRT_CPUS_MAX * 2) },
     [VIRT_APLIC_M] =      {  0xc000000, APLIC_SIZE(VIRT_CPUS_MAX) },
@@ -853,7 +855,8 @@ static void create_fdt_virtio(RISCVVirtState *s, const MemMapEntry *memmap,
 
 static void create_fdt_pcie(RISCVVirtState *s, const MemMapEntry *memmap,
                             uint32_t irq_pcie_phandle,
-                            uint32_t msi_pcie_phandle)
+                            uint32_t msi_pcie_phandle,
+                            uint32_t iommu_sys_phandle)
 {
     g_autofree char *name = NULL;
     MachineState *ms = MACHINE(s);
@@ -887,6 +890,12 @@ static void create_fdt_pcie(RISCVVirtState *s, const MemMapEntry *memmap,
         2, virt_high_pcie_memmap.base,
         2, virt_high_pcie_memmap.base, 2, virt_high_pcie_memmap.size);
 
+    if (virt_is_iommu_sys_enabled(s)) {
+        qemu_fdt_setprop_cells(ms->fdt, name, "iommu-map",
+                               0, iommu_sys_phandle, 0, 0, 0,
+                               iommu_sys_phandle, 0, 0xffff);
+    }
+
     create_pcie_irq_map(s, ms->fdt, name, irq_pcie_phandle);
 }
 
@@ -1033,6 +1042,44 @@ static void create_fdt_virtio_iommu(RISCVVirtState *s, uint16_t bdf)
                            bdf + 1, iommu_phandle, bdf + 1, 0xffff - bdf);
 }
 
+static void create_fdt_iommu_sys(RISCVVirtState *s, uint32_t irq_chip,
+                                 uint32_t *iommu_sys_phandle)
+{
+    const char comp[] = "riscv,iommu";
+    void *fdt = MACHINE(s)->fdt;
+    uint32_t iommu_phandle;
+    g_autofree char *iommu_node = NULL;
+    hwaddr addr = s->memmap[VIRT_IOMMU_SYS].base;
+    hwaddr size = s->memmap[VIRT_IOMMU_SYS].size;
+    uint32_t iommu_irq_map[RISCV_IOMMU_INTR_COUNT] = {
+        IOMMU_SYS_IRQ + RISCV_IOMMU_INTR_CQ,
+        IOMMU_SYS_IRQ + RISCV_IOMMU_INTR_FQ,
+        IOMMU_SYS_IRQ + RISCV_IOMMU_INTR_PM,
+        IOMMU_SYS_IRQ + RISCV_IOMMU_INTR_PQ,
+    };
+
+    iommu_node = g_strdup_printf("/soc/iommu@%x",
+                               (unsigned int) s->memmap[VIRT_IOMMU_SYS].base);
+    iommu_phandle = qemu_fdt_alloc_phandle(fdt);
+    qemu_fdt_add_subnode(fdt, iommu_node);
+
+    qemu_fdt_setprop(fdt, iommu_node, "compatible", comp, sizeof(comp));
+    qemu_fdt_setprop_cell(fdt, iommu_node, "#iommu-cells", 1);
+    qemu_fdt_setprop_cell(fdt, iommu_node, "phandle", iommu_phandle);
+
+    qemu_fdt_setprop_cells(fdt, iommu_node, "reg",
+                           addr >> 32, addr, size >> 32, size);
+    qemu_fdt_setprop_cell(fdt, iommu_node, "interrupt-parent", irq_chip);
+
+    qemu_fdt_setprop_cells(fdt, iommu_node, "interrupts",
+        iommu_irq_map[0], FDT_IRQ_TYPE_EDGE_LOW,
+        iommu_irq_map[1], FDT_IRQ_TYPE_EDGE_LOW,
+        iommu_irq_map[2], FDT_IRQ_TYPE_EDGE_LOW,
+        iommu_irq_map[3], FDT_IRQ_TYPE_EDGE_LOW);
+
+    *iommu_sys_phandle = iommu_phandle;
+}
+
 static void create_fdt_iommu(RISCVVirtState *s, uint16_t bdf)
 {
     const char comp[] = "riscv,pci-iommu";
@@ -1061,6 +1108,7 @@ static void finalize_fdt(RISCVVirtState *s)
 {
     uint32_t phandle = 1, irq_mmio_phandle = 1, msi_pcie_phandle = 1;
     uint32_t irq_pcie_phandle = 1, irq_virtio_phandle = 1;
+    uint32_t iommu_sys_phandle = 1;
 
     create_fdt_sockets(s, virt_memmap, &phandle, &irq_mmio_phandle,
                        &irq_pcie_phandle, &irq_virtio_phandle,
@@ -1068,7 +1116,11 @@ static void finalize_fdt(RISCVVirtState *s)
 
     create_fdt_virtio(s, virt_memmap, irq_virtio_phandle);
 
-    create_fdt_pcie(s, virt_memmap, irq_pcie_phandle, msi_pcie_phandle);
+    if (virt_is_iommu_sys_enabled(s)) {
+        create_fdt_iommu_sys(s, irq_mmio_phandle, &iommu_sys_phandle);
+    }
+    create_fdt_pcie(s, virt_memmap, irq_pcie_phandle, msi_pcie_phandle,
+                    iommu_sys_phandle);
 
     create_fdt_reset(s, virt_memmap, &phandle);
 
@@ -1650,6 +1702,22 @@ static void virt_machine_init(MachineState *machine)
         create_fdt(s, memmap);
     }
 
+    if (virt_is_iommu_sys_enabled(s)) {
+        DeviceState *iommu_sys = qdev_new(TYPE_RISCV_IOMMU_SYS);
+
+        object_property_set_uint(OBJECT(iommu_sys), "addr",
+                                 s->memmap[VIRT_IOMMU_SYS].base,
+                                 &error_fatal);
+        object_property_set_uint(OBJECT(iommu_sys), "base-irq",
+                                 IOMMU_SYS_IRQ,
+                                 &error_fatal);
+        object_property_set_link(OBJECT(iommu_sys), "irqchip",
+                                 OBJECT(mmio_irqchip),
+                                 &error_fatal);
+
+        sysbus_realize_and_unref(SYS_BUS_DEVICE(iommu_sys), &error_fatal);
+    }
+
     s->machine_done.notify = virt_machine_done;
     qemu_add_machine_init_done_notifier(&s->machine_done);
 }
@@ -1663,6 +1731,7 @@ static void virt_machine_instance_init(Object *obj)
     s->oem_id = g_strndup(ACPI_BUILD_APPNAME6, 6);
     s->oem_table_id = g_strndup(ACPI_BUILD_APPNAME8, 8);
     s->acpi = ON_OFF_AUTO_AUTO;
+    s->iommu_sys = ON_OFF_AUTO_AUTO;
 }
 
 static char *virt_get_aia_guests(Object *obj, Error **errp)
@@ -1735,6 +1804,28 @@ static void virt_set_aclint(Object *obj, bool value, Error **errp)
     s->have_aclint = value;
 }
 
+bool virt_is_iommu_sys_enabled(RISCVVirtState *s)
+{
+    return s->iommu_sys == ON_OFF_AUTO_ON;
+}
+
+static void virt_get_iommu_sys(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
+{
+    RISCVVirtState *s = RISCV_VIRT_MACHINE(obj);
+    OnOffAuto iommu_sys = s->iommu_sys;
+
+    visit_type_OnOffAuto(v, name, &iommu_sys, errp);
+}
+
+static void virt_set_iommu_sys(Object *obj, Visitor *v, const char *name,
+                               void *opaque, Error **errp)
+{
+    RISCVVirtState *s = RISCV_VIRT_MACHINE(obj);
+
+    visit_type_OnOffAuto(v, name, &s->iommu_sys, errp);
+}
+
 bool virt_is_acpi_enabled(RISCVVirtState *s)
 {
     return s->acpi != ON_OFF_AUTO_OFF;
@@ -1761,10 +1852,12 @@ static HotplugHandler *virt_machine_get_hotplug_handler(MachineState *machine,
                                                         DeviceState *dev)
 {
     MachineClass *mc = MACHINE_GET_CLASS(machine);
+    RISCVVirtState *s = RISCV_VIRT_MACHINE(machine);
 
     if (device_is_dynamic_sysbus(mc, dev) ||
         object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_IOMMU_PCI) ||
         object_dynamic_cast(OBJECT(dev), TYPE_RISCV_IOMMU_PCI)) {
+        s->iommu_sys = ON_OFF_AUTO_OFF;
         return HOTPLUG_HANDLER(machine);
     }
 
@@ -1791,6 +1884,7 @@ static void virt_machine_device_plug_cb(HotplugHandler *hotplug_dev,
 
     if (object_dynamic_cast(OBJECT(dev), TYPE_RISCV_IOMMU_PCI)) {
         create_fdt_iommu(s, pci_get_bdf(PCI_DEVICE(dev)));
+        s->iommu_sys = ON_OFF_AUTO_OFF;
     }
 }
 
@@ -1853,6 +1947,12 @@ static void virt_machine_class_init(ObjectClass *oc, void *data)
                               NULL, NULL);
     object_class_property_set_description(oc, "acpi",
                                           "Enable ACPI");
+
+    object_class_property_add(oc, "iommu-sys", "OnOffAuto",
+                              virt_get_iommu_sys, virt_set_iommu_sys,
+                              NULL, NULL);
+    object_class_property_set_description(oc, "iommu-sys",
+                                          "Enable IOMMU platform device");
 }
 
 static const TypeInfo virt_machine_typeinfo = {
diff --git a/include/hw/riscv/iommu.h b/include/hw/riscv/iommu.h
index fc20808553..8a8acfc3f0 100644
--- a/include/hw/riscv/iommu.h
+++ b/include/hw/riscv/iommu.h
@@ -37,4 +37,6 @@ typedef struct RISCVIOMMUStatePci RISCVIOMMUStatePci;
 OBJECT_DECLARE_SIMPLE_TYPE(RISCVIOMMUStateSys, RISCV_IOMMU_SYS)
 typedef struct RISCVIOMMUStateSys RISCVIOMMUStateSys;
 
+#define FDT_IRQ_TYPE_EDGE_LOW 1
+
 #endif
diff --git a/include/hw/riscv/virt.h b/include/hw/riscv/virt.h
index c0dc41ff9a..48a14bea2e 100644
--- a/include/hw/riscv/virt.h
+++ b/include/hw/riscv/virt.h
@@ -62,6 +62,7 @@ struct RISCVVirtState {
     OnOffAuto acpi;
     const MemMapEntry *memmap;
     struct GPEXHost *gpex_host;
+    OnOffAuto iommu_sys;
 };
 
 enum {
@@ -84,7 +85,8 @@ enum {
     VIRT_PCIE_MMIO,
     VIRT_PCIE_PIO,
     VIRT_PLATFORM_BUS,
-    VIRT_PCIE_ECAM
+    VIRT_PCIE_ECAM,
+    VIRT_IOMMU_SYS,
 };
 
 enum {
@@ -93,6 +95,7 @@ enum {
     VIRTIO_IRQ = 1, /* 1 to 8 */
     VIRTIO_COUNT = 8,
     PCIE_IRQ = 0x20, /* 32 to 35 */
+    IOMMU_SYS_IRQ = 0x24, /* 36-39 */
     VIRT_PLATFORM_BUS_IRQ = 64, /* 64 to 95 */
 };
 
@@ -129,6 +132,7 @@ enum {
                                  1 + FDT_APLIC_INT_CELLS)
 
 bool virt_is_acpi_enabled(RISCVVirtState *s);
+bool virt_is_iommu_sys_enabled(RISCVVirtState *s);
 void virt_acpi_setup(RISCVVirtState *vms);
 uint32_t imsic_num_bits(uint32_t count);
 
-- 
2.45.2

[PATCH for-10.0 5/7] hw/riscv/virt.c, riscv-iommu-sys.c: add MSIx support

[Daniel Henrique Barboza]

MSIx support is added in the RISC-V IOMMU platform device by including
the required MSIx facilities to alow software to properly setup the MSIx
subsystem.

We took inspiration of what is being done in the riscv-iommu-pci device,
mainly msix_init() and msix_notify(), while keeping in mind that
riscv-iommu-sys isn't a true PCI device and we don't need to copy/paste
all the contents of these MSIx functions.

Two extra MSI MemoryRegions were added: 'msix-table' and 'msix-pba'.
They are used to manage r/w of the MSI table and Pending Bit Array (PBA)
respectively. Both are subregions of the main IOMMU memory region,
iommu->regs_mr, initialized during riscv_iommu_realize(), and each one
has their own handlers for MSIx reads and writes.

This is the expected memory map when using this device in the 'virt'
machine:

    0000000003010000-0000000003010fff (prio 0, i/o): riscv-iommu-regs
      0000000003010300-000000000301034f (prio 0, i/o): msix-table
      0000000003010400-0000000003010407 (prio 0, i/o): msix-pba

We're now able to set IGS to RISCV_IOMMU_CAP_IGS_BOTH, and userspace is
free to decide which interrupt model to use.

Enabling MSIx support for this device in the 'virt' machine requires
adding 'msi-parent' in the iommu-sys DT.

Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
---
 hw/riscv/riscv-iommu-sys.c | 116 +++++++++++++++++++++++++++++++++++--
 hw/riscv/trace-events      |   2 +
 hw/riscv/virt.c            |   6 +-
 3 files changed, 119 insertions(+), 5 deletions(-)

diff --git a/hw/riscv/riscv-iommu-sys.c b/hw/riscv/riscv-iommu-sys.c
index 4b82046ce9..a0ef67a20b 100644
--- a/hw/riscv/riscv-iommu-sys.c
+++ b/hw/riscv/riscv-iommu-sys.c
@@ -26,11 +26,15 @@
 #include "qemu/host-utils.h"
 #include "qemu/module.h"
 #include "qom/object.h"
+#include "exec/exec-all.h"
+#include "trace.h"
 
 #include "riscv-iommu.h"
 
 #define RISCV_IOMMU_SYSDEV_ICVEC_VECTORS 0x3333
 
+#define RISCV_IOMMU_PCI_MSIX_VECTORS 5
+
 /* RISC-V IOMMU System Platform Device Emulation */
 
 struct RISCVIOMMUStateSys {
@@ -39,21 +43,123 @@ struct RISCVIOMMUStateSys {
     uint32_t         base_irq;
     DeviceState      *irqchip;
     RISCVIOMMUState  iommu;
+
+    /* Wired int support */
     qemu_irq         irqs[RISCV_IOMMU_INTR_COUNT];
+
+    /* Memory Regions for MSIX table and pending bit entries. */
+    MemoryRegion msix_table_mmio;
+    MemoryRegion msix_pba_mmio;
+    uint8_t *msix_table;
+    uint8_t *msix_pba;
+};
+
+static uint64_t msix_table_mmio_read(void *opaque, hwaddr addr,
+                                     unsigned size)
+{
+    RISCVIOMMUStateSys *s = opaque;
+
+    g_assert(addr + size <= RISCV_IOMMU_PCI_MSIX_VECTORS * PCI_MSIX_ENTRY_SIZE);
+    return pci_get_long(s->msix_table + addr);
+}
+
+static void msix_table_mmio_write(void *opaque, hwaddr addr,
+                                  uint64_t val, unsigned size)
+{
+    RISCVIOMMUStateSys *s = opaque;
+
+    g_assert(addr + size <= RISCV_IOMMU_PCI_MSIX_VECTORS * PCI_MSIX_ENTRY_SIZE);
+    pci_set_long(s->msix_table + addr, val);
+}
+
+static const MemoryRegionOps msix_table_mmio_ops = {
+    .read = msix_table_mmio_read,
+    .write = msix_table_mmio_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 8,
+    },
+    .impl = {
+        .max_access_size = 4,
+    },
+};
+
+static uint64_t msix_pba_mmio_read(void *opaque, hwaddr addr,
+                                   unsigned size)
+{
+    RISCVIOMMUStateSys *s = opaque;
+
+    return pci_get_long(s->msix_pba + addr);
+}
+
+static void msix_pba_mmio_write(void *opaque, hwaddr addr,
+                                uint64_t val, unsigned size)
+{
+}
+
+static const MemoryRegionOps msix_pba_mmio_ops = {
+    .read = msix_pba_mmio_read,
+    .write = msix_pba_mmio_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 8,
+    },
+    .impl = {
+        .max_access_size = 4,
+    },
 };
 
+static void riscv_iommu_sysdev_init_msi(RISCVIOMMUStateSys *s,
+                                        uint32_t n_vectors)
+{
+    RISCVIOMMUState *iommu = &s->iommu;
+    uint32_t table_size = table_size = n_vectors * PCI_MSIX_ENTRY_SIZE;
+    uint32_t table_offset = RISCV_IOMMU_REG_MSI_CONFIG;
+    uint32_t pba_size = QEMU_ALIGN_UP(n_vectors, 64) / 8;
+    uint32_t pba_offset = RISCV_IOMMU_REG_MSI_CONFIG + 256;
+
+    s->msix_table = g_malloc0(table_size);
+    s->msix_pba = g_malloc0(pba_size);
+
+    memory_region_init_io(&s->msix_table_mmio, OBJECT(s), &msix_table_mmio_ops,
+                          s, "msix-table", table_size);
+    memory_region_add_subregion(&iommu->regs_mr, table_offset,
+                                &s->msix_table_mmio);
+
+    memory_region_init_io(&s->msix_pba_mmio, OBJECT(s), &msix_pba_mmio_ops, s,
+                          "msix-pba", pba_size);
+    memory_region_add_subregion(&iommu->regs_mr, pba_offset,
+                                &s->msix_pba_mmio);
+}
+
+static void riscv_iommu_sysdev_send_MSI(RISCVIOMMUStateSys *s,
+                                        uint32_t vector)
+{
+    uint8_t *table_entry = s->msix_table + vector * PCI_MSIX_ENTRY_SIZE;
+    uint64_t msi_addr = pci_get_quad(table_entry + PCI_MSIX_ENTRY_LOWER_ADDR);
+    uint32_t msi_data = pci_get_long(table_entry + PCI_MSIX_ENTRY_DATA);
+    MemTxResult result;
+
+    address_space_stl_le(&address_space_memory, msi_addr,
+                         msi_data, MEMTXATTRS_UNSPECIFIED, &result);
+    trace_riscv_iommu_sys_msi_sent(vector, msi_addr, msi_data, result);
+}
+
 static void riscv_iommu_sysdev_notify(RISCVIOMMUState *iommu,
                                       unsigned vector)
 {
     RISCVIOMMUStateSys *s = container_of(iommu, RISCVIOMMUStateSys, iommu);
     uint32_t fctl =  riscv_iommu_reg_get32(iommu, RISCV_IOMMU_REG_FCTL);
 
-    /* We do not support MSIs yet */
-    if (!(fctl & RISCV_IOMMU_FCTL_WSI)) {
+    if (fctl & RISCV_IOMMU_FCTL_WSI) {
+        qemu_irq_pulse(s->irqs[vector]);
+        trace_riscv_iommu_sys_irq_sent(vector);
         return;
     }
 
-    qemu_irq_pulse(s->irqs[vector]);
+    riscv_iommu_sysdev_send_MSI(s, vector);
 }
 
 static void riscv_iommu_sys_realize(DeviceState *dev, Error **errp)
@@ -82,6 +188,8 @@ static void riscv_iommu_sys_realize(DeviceState *dev, Error **errp)
         irq = qdev_get_gpio_in(s->irqchip, s->base_irq + i);
         sysbus_connect_irq(sysdev, i, irq);
     }
+
+    riscv_iommu_sysdev_init_msi(s, RISCV_IOMMU_PCI_MSIX_VECTORS);
 }
 
 static void riscv_iommu_sys_init(Object *obj)
@@ -93,7 +201,7 @@ static void riscv_iommu_sys_init(Object *obj)
     qdev_alias_all_properties(DEVICE(iommu), obj);
 
     iommu->icvec_avail_vectors = RISCV_IOMMU_SYSDEV_ICVEC_VECTORS;
-    riscv_iommu_set_cap_igs(iommu, RISCV_IOMMU_CAP_IGS_WSI);
+    riscv_iommu_set_cap_igs(iommu, RISCV_IOMMU_CAP_IGS_BOTH);
 }
 
 static Property riscv_iommu_sys_properties[] = {
diff --git a/hw/riscv/trace-events b/hw/riscv/trace-events
index 0527c56c91..94facbb8b1 100644
--- a/hw/riscv/trace-events
+++ b/hw/riscv/trace-events
@@ -15,3 +15,5 @@ riscv_iommu_icvec_write(uint32_t orig, uint32_t actual) "ICVEC write: incoming 0
 riscv_iommu_ats(const char *id, unsigned b, unsigned d, unsigned f, uint64_t iova) "%s: translate request %04x:%02x.%u iova: 0x%"PRIx64
 riscv_iommu_ats_inval(const char *id) "%s: dev-iotlb invalidate"
 riscv_iommu_ats_prgr(const char *id) "%s: dev-iotlb page request group response"
+riscv_iommu_sys_irq_sent(uint32_t vector) "IRQ sent to vector %u"
+riscv_iommu_sys_msi_sent(uint32_t vector, uint64_t msi_addr, uint32_t msi_data, uint32_t result) "MSI sent to vector %u msi_addr 0x%lx msi_data 0x%x result %u"
diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c
index 23d1380b86..281fc65cc6 100644
--- a/hw/riscv/virt.c
+++ b/hw/riscv/virt.c
@@ -1043,6 +1043,7 @@ static void create_fdt_virtio_iommu(RISCVVirtState *s, uint16_t bdf)
 }
 
 static void create_fdt_iommu_sys(RISCVVirtState *s, uint32_t irq_chip,
+                                 uint32_t msi_phandle,
                                  uint32_t *iommu_sys_phandle)
 {
     const char comp[] = "riscv,iommu";
@@ -1077,6 +1078,8 @@ static void create_fdt_iommu_sys(RISCVVirtState *s, uint32_t irq_chip,
         iommu_irq_map[2], FDT_IRQ_TYPE_EDGE_LOW,
         iommu_irq_map[3], FDT_IRQ_TYPE_EDGE_LOW);
 
+    qemu_fdt_setprop_cell(fdt, iommu_node, "msi-parent", msi_phandle);
+
     *iommu_sys_phandle = iommu_phandle;
 }
 
@@ -1117,7 +1120,8 @@ static void finalize_fdt(RISCVVirtState *s)
     create_fdt_virtio(s, virt_memmap, irq_virtio_phandle);
 
     if (virt_is_iommu_sys_enabled(s)) {
-        create_fdt_iommu_sys(s, irq_mmio_phandle, &iommu_sys_phandle);
+        create_fdt_iommu_sys(s, irq_mmio_phandle, msi_pcie_phandle,
+                             &iommu_sys_phandle);
     }
     create_fdt_pcie(s, virt_memmap, irq_pcie_phandle, msi_pcie_phandle,
                     iommu_sys_phandle);
-- 
2.45.2

[PATCH for-10.0 6/7] hw/riscv/riscv-iommu: implement reset protocol

[Daniel Henrique Barboza]

Add a riscv_iommu_reset() helper in the base emulation code that
implements the expected reset behavior as defined by the riscv-iommu
spec.

Devices can then use this helper in their own reset callbacks.

Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
---
 hw/riscv/riscv-iommu-pci.c | 20 ++++++++++++++++++++
 hw/riscv/riscv-iommu-sys.c | 20 ++++++++++++++++++++
 hw/riscv/riscv-iommu.c     | 35 +++++++++++++++++++++++++++++++++++
 hw/riscv/riscv-iommu.h     |  1 +
 hw/riscv/trace-events      |  2 ++
 include/hw/riscv/iommu.h   |  6 ++++--
 6 files changed, 82 insertions(+), 2 deletions(-)

diff --git a/hw/riscv/riscv-iommu-pci.c b/hw/riscv/riscv-iommu-pci.c
index 4ce9bf6b78..51226c4877 100644
--- a/hw/riscv/riscv-iommu-pci.c
+++ b/hw/riscv/riscv-iommu-pci.c
@@ -31,6 +31,7 @@
 #include "cpu_bits.h"
 #include "riscv-iommu.h"
 #include "riscv-iommu-bits.h"
+#include "trace.h"
 
 /* RISC-V IOMMU PCI Device Emulation */
 #define RISCV_PCI_CLASS_SYSTEM_IOMMU     0x0806
@@ -66,6 +67,12 @@ typedef struct RISCVIOMMUStatePci {
     RISCVIOMMUState  iommu;   /* common IOMMU state */
 } RISCVIOMMUStatePci;
 
+struct RISCVIOMMUPciClass {
+    /*< public >*/
+    DeviceRealize parent_realize;
+    ResettablePhases parent_phases;
+};
+
 /* interrupt delivery callback */
 static void riscv_iommu_pci_notify(RISCVIOMMUState *iommu, unsigned vector)
 {
@@ -167,10 +174,23 @@ static Property riscv_iommu_pci_properties[] = {
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static void riscv_iommu_pci_reset_hold(Object *obj, ResetType type)
+{
+    RISCVIOMMUStatePci *pci = RISCV_IOMMU_PCI(obj);
+    RISCVIOMMUState *iommu = &pci->iommu;
+
+    riscv_iommu_reset(iommu);
+
+    trace_riscv_iommu_pci_reset_hold(type);
+}
+
 static void riscv_iommu_pci_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+    ResettableClass *rc = RESETTABLE_CLASS(klass);
+
+    rc->phases.hold = riscv_iommu_pci_reset_hold;
 
     k->realize = riscv_iommu_pci_realize;
     k->exit = riscv_iommu_pci_exit;
diff --git a/hw/riscv/riscv-iommu-sys.c b/hw/riscv/riscv-iommu-sys.c
index a0ef67a20b..605979a0ac 100644
--- a/hw/riscv/riscv-iommu-sys.c
+++ b/hw/riscv/riscv-iommu-sys.c
@@ -54,6 +54,12 @@ struct RISCVIOMMUStateSys {
     uint8_t *msix_pba;
 };
 
+struct RISCVIOMMUSysClass {
+    /*< public >*/
+    DeviceRealize parent_realize;
+    ResettablePhases parent_phases;
+};
+
 static uint64_t msix_table_mmio_read(void *opaque, hwaddr addr,
                                      unsigned size)
 {
@@ -212,9 +218,23 @@ static Property riscv_iommu_sys_properties[] = {
     DEFINE_PROP_END_OF_LIST(),
 };
 
+static void riscv_iommu_sys_reset_hold(Object *obj, ResetType type)
+{
+    RISCVIOMMUStateSys *sys = RISCV_IOMMU_SYS(obj);
+    RISCVIOMMUState *iommu = &sys->iommu;
+
+    riscv_iommu_reset(iommu);
+
+    trace_riscv_iommu_sys_reset_hold(type);
+}
+
 static void riscv_iommu_sys_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
+    ResettableClass *rc = RESETTABLE_CLASS(klass);
+
+    rc->phases.hold = riscv_iommu_sys_reset_hold;
+
     dc->realize = riscv_iommu_sys_realize;
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
     device_class_set_props(dc, riscv_iommu_sys_properties);
diff --git a/hw/riscv/riscv-iommu.c b/hw/riscv/riscv-iommu.c
index 239f83f5bd..cf2019d6aa 100644
--- a/hw/riscv/riscv-iommu.c
+++ b/hw/riscv/riscv-iommu.c
@@ -2228,6 +2228,41 @@ static void riscv_iommu_unrealize(DeviceState *dev)
     g_hash_table_unref(s->ctx_cache);
 }
 
+void riscv_iommu_reset(RISCVIOMMUState *s)
+{
+    uint32_t reg_clr;
+    int ddtp_mode;
+
+    /*
+     * Clear DDTP while setting DDTP_mode back to user
+     * initial setting.
+     */
+    ddtp_mode = s->enable_off ?
+                RISCV_IOMMU_DDTP_MODE_OFF : RISCV_IOMMU_DDTP_MODE_BARE;
+    s->ddtp = set_field(0, RISCV_IOMMU_DDTP_MODE, ddtp_mode);
+    riscv_iommu_reg_set64(s, RISCV_IOMMU_REG_DDTP, s->ddtp);
+
+    reg_clr = RISCV_IOMMU_CQCSR_CQEN | RISCV_IOMMU_CQCSR_CIE |
+              RISCV_IOMMU_CQCSR_CQON | RISCV_IOMMU_CQCSR_BUSY;
+    riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_CQCSR, 0, reg_clr);
+
+    reg_clr = RISCV_IOMMU_FQCSR_FQEN | RISCV_IOMMU_FQCSR_FIE |
+              RISCV_IOMMU_FQCSR_FQON | RISCV_IOMMU_FQCSR_BUSY;
+    riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_FQCSR, 0, reg_clr);
+
+    reg_clr = RISCV_IOMMU_PQCSR_PQEN | RISCV_IOMMU_PQCSR_PIE |
+              RISCV_IOMMU_PQCSR_PQON | RISCV_IOMMU_PQCSR_BUSY;
+    riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_PQCSR, 0, reg_clr);
+
+    riscv_iommu_reg_mod64(s, RISCV_IOMMU_REG_TR_REQ_CTL, 0,
+                          RISCV_IOMMU_TR_REQ_CTL_GO_BUSY);
+
+    riscv_iommu_reg_set32(s, RISCV_IOMMU_REG_IPSR, 0);
+
+    g_hash_table_remove_all(s->ctx_cache);
+    g_hash_table_remove_all(s->iot_cache);
+}
+
 static Property riscv_iommu_properties[] = {
     DEFINE_PROP_UINT32("version", RISCVIOMMUState, version,
         RISCV_IOMMU_SPEC_DOT_VER),
diff --git a/hw/riscv/riscv-iommu.h b/hw/riscv/riscv-iommu.h
index f9f2827808..9424989df4 100644
--- a/hw/riscv/riscv-iommu.h
+++ b/hw/riscv/riscv-iommu.h
@@ -89,6 +89,7 @@ struct RISCVIOMMUState {
 void riscv_iommu_pci_setup_iommu(RISCVIOMMUState *iommu, PCIBus *bus,
          Error **errp);
 void riscv_iommu_set_cap_igs(RISCVIOMMUState *s, riscv_iommu_igs_mode mode);
+void riscv_iommu_reset(RISCVIOMMUState *s);
 
 /* private helpers */
 
diff --git a/hw/riscv/trace-events b/hw/riscv/trace-events
index 94facbb8b1..aaa2c0eb94 100644
--- a/hw/riscv/trace-events
+++ b/hw/riscv/trace-events
@@ -17,3 +17,5 @@ riscv_iommu_ats_inval(const char *id) "%s: dev-iotlb invalidate"
 riscv_iommu_ats_prgr(const char *id) "%s: dev-iotlb page request group response"
 riscv_iommu_sys_irq_sent(uint32_t vector) "IRQ sent to vector %u"
 riscv_iommu_sys_msi_sent(uint32_t vector, uint64_t msi_addr, uint32_t msi_data, uint32_t result) "MSI sent to vector %u msi_addr 0x%lx msi_data 0x%x result %u"
+riscv_iommu_sys_reset_hold(int reset_type) "reset type %d"
+riscv_iommu_pci_reset_hold(int reset_type) "reset type %d"
diff --git a/include/hw/riscv/iommu.h b/include/hw/riscv/iommu.h
index 8a8acfc3f0..b03339d75c 100644
--- a/include/hw/riscv/iommu.h
+++ b/include/hw/riscv/iommu.h
@@ -30,12 +30,14 @@ typedef struct RISCVIOMMUState RISCVIOMMUState;
 typedef struct RISCVIOMMUSpace RISCVIOMMUSpace;
 
 #define TYPE_RISCV_IOMMU_PCI "riscv-iommu-pci"
-OBJECT_DECLARE_SIMPLE_TYPE(RISCVIOMMUStatePci, RISCV_IOMMU_PCI)
+OBJECT_DECLARE_TYPE(RISCVIOMMUStatePci, RISCVIOMMUPciClass, RISCV_IOMMU_PCI)
 typedef struct RISCVIOMMUStatePci RISCVIOMMUStatePci;
+typedef struct RISCVIOMMUPciClass RISCVIOMMUPciClass;
 
 #define TYPE_RISCV_IOMMU_SYS "riscv-iommu-device"
-OBJECT_DECLARE_SIMPLE_TYPE(RISCVIOMMUStateSys, RISCV_IOMMU_SYS)
+OBJECT_DECLARE_TYPE(RISCVIOMMUStateSys, RISCVIOMMUSysClass, RISCV_IOMMU_SYS)
 typedef struct RISCVIOMMUStateSys RISCVIOMMUStateSys;
+typedef struct RISCVIOMMUSysClass RISCVIOMMUSysClass;
 
 #define FDT_IRQ_TYPE_EDGE_LOW 1
 
-- 
2.45.2

[PATCH for-10.0 7/7] docs/specs: add riscv-iommu-sys information

[Daniel Henrique Barboza]

Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>
---
 docs/specs/riscv-iommu.rst | 30 +++++++++++++++++++++++++++---
 docs/system/riscv/virt.rst | 10 ++++++++++
 2 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/docs/specs/riscv-iommu.rst b/docs/specs/riscv-iommu.rst
index 463f4cffb6..b1538c9ead 100644
--- a/docs/specs/riscv-iommu.rst
+++ b/docs/specs/riscv-iommu.rst
@@ -6,9 +6,9 @@ RISC-V IOMMU support for RISC-V machines
 QEMU implements a RISC-V IOMMU emulation based on the RISC-V IOMMU spec
 version 1.0 `iommu1.0`_.
 
-The emulation includes a PCI reference device, riscv-iommu-pci, that QEMU
-RISC-V boards can use.  The 'virt' RISC-V machine is compatible with this
-device.
+The emulation includes a PCI reference device (riscv-iommu-pci) and a platform
+bus device (riscv-iommu-sys) that QEMU RISC-V boards can use.  The 'virt'
+RISC-V machine is compatible with both devices.
 
 riscv-iommu-pci reference device
 --------------------------------
@@ -83,6 +83,30 @@ Several options are available to control the capabilities of the device, namely:
 - "s-stage": enable s-stage support
 - "g-stage": enable g-stage support
 
+riscv-iommu-sys device
+----------------------
+
+This device implements the RISC-V IOMMU emulation as a platform bus device that
+RISC-V boards can use.
+
+For the 'virt' board the device is disabled by default.  To enable it use the
+'iommu-sys' machine option:
+
+.. code-block:: bash
+
+  $ qemu-system-riscv64 -M virt,iommu-sys=on (...)
+
+There is no options to configure the capabilities of this device in the 'virt'
+board using the QEMU command line.  The device is configured with the following
+riscv-iommu options:
+
+- "ioatc-limit": default value (2Mb)
+- "intremap": enabled
+- "ats": enabled
+- "off": on (DMA disabled)
+- "s-stage": enabled
+- "g-stage": enabled
+
 .. _iommu1.0: https://github.com/riscv-non-isa/riscv-iommu/releases/download/v1.0/riscv-iommu.pdf
 
 .. _linux-v8: https://lore.kernel.org/linux-riscv/cover.1718388908.git.tjeznach@rivosinc.com/
diff --git a/docs/system/riscv/virt.rst b/docs/system/riscv/virt.rst
index 8e9a2e4dda..537aac0340 100644
--- a/docs/system/riscv/virt.rst
+++ b/docs/system/riscv/virt.rst
@@ -94,6 +94,12 @@ command line:
 
   $ qemu-system-riscv64 -M virt -device riscv-iommu-pci (...)
 
+It also has support for the riscv-iommu-sys platform device:
+
+.. code-block:: bash
+
+  $ qemu-system-riscv64 -M virt,iommu-sys=on (...)
+
 Refer to :ref:`riscv-iommu` for more information on how the RISC-V IOMMU support
 works.
 
@@ -129,6 +135,10 @@ The following machine-specific options are supported:
   having AIA IMSIC (i.e. "aia=aplic-imsic" selected). When not specified,
   the default number of per-HART VS-level AIA IMSIC pages is 0.
 
+- iommu-sys=[on|off]
+
+  Enables the riscv-iommu-sys platform device. Defaults to 'off'.
+
 Running Linux kernel
 --------------------
 
-- 
2.45.2

Re: [PATCH for-10.0 0/7] hw/riscv: riscv-iommu-sys device

[Andrew Jones]

On Wed, Nov 06, 2024 at 10:34:00AM -0300, Daniel Henrique Barboza wrote:
> Hi,
> 
> Now that we have merged the base IOMMU support we can re-introduce
> the riscv-iommu-sys platform device that was taken away from the initial
> posting.
> 
> Aside from adding support for the device in the 'virt' machine we're
> also adding MSI support for it, something that we weren't doing before.
> The Linux driver is then free to choose either MSI or WSI to use the
> device.

The driver (still only in linux-next) only knows how to use WSI, but with
a series I just posted[1] MSI is also possible.

[1] https://lore.kernel.org/all/20241106175102.219923-4-ajones@ventanamicro.com/

Thanks,
drew

> 
> Patches based on master.
> 
> Daniel Henrique Barboza (5):
>   hw/riscv/riscv-iommu.c: add riscv_iommu_instance_init()
>   hw/riscv/riscv-iommu: parametrize CAP.IGS
>   hw/riscv/virt.c, riscv-iommu-sys.c: add MSIx support
>   hw/riscv/riscv-iommu: implement reset protocol
>   docs/specs: add riscv-iommu-sys information
> 
> Sunil V L (1):
>   hw/riscv/virt: Add IOMMU as platform device if the option is set
> 
> Tomasz Jeznach (1):
>   hw/riscv: add riscv-iommu-sys platform device
> 
>  docs/specs/riscv-iommu.rst  |  30 ++++-
>  docs/system/riscv/virt.rst  |  10 ++
>  hw/riscv/meson.build        |   2 +-
>  hw/riscv/riscv-iommu-bits.h |   6 +
>  hw/riscv/riscv-iommu-pci.c  |  21 +++
>  hw/riscv/riscv-iommu-sys.c  | 256 ++++++++++++++++++++++++++++++++++++
>  hw/riscv/riscv-iommu.c      | 114 +++++++++++-----
>  hw/riscv/riscv-iommu.h      |   5 +
>  hw/riscv/trace-events       |   4 +
>  hw/riscv/virt.c             | 108 ++++++++++++++-
>  include/hw/riscv/iommu.h    |  10 +-
>  include/hw/riscv/virt.h     |   6 +-
>  12 files changed, 530 insertions(+), 42 deletions(-)
>  create mode 100644 hw/riscv/riscv-iommu-sys.c
> 
> -- 
> 2.45.2
> 
> 

Re: [PATCH for-10.0 1/7] hw/riscv/riscv-iommu.c: add riscv_iommu_instance_init()

[Alistair Francis]

On Wed, Nov 6, 2024 at 11:38 PM Daniel Henrique Barboza
<dbarboza@ventanamicro.com> wrote:
>
> Move all the static initializion of the device to an init() function,
> leaving only the dynamic initialization to be done during realize.
>
> With this change s->cap is initialized with RISCV_IOMMU_CAP_DBG during
> init(), and realize() will increment s->cap with the extra caps.
>
> This will allow callers to add IOMMU capabilities before the
> realization.
>
> Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>

Reviewed-by: Alistair Francis <alistair.francis@wdc.com>

Alistair

> ---
>  hw/riscv/riscv-iommu.c | 71 +++++++++++++++++++++++-------------------
>  1 file changed, 39 insertions(+), 32 deletions(-)
>
> diff --git a/hw/riscv/riscv-iommu.c b/hw/riscv/riscv-iommu.c
> index feb650549a..1893584028 100644
> --- a/hw/riscv/riscv-iommu.c
> +++ b/hw/riscv/riscv-iommu.c
> @@ -2096,11 +2096,48 @@ static const MemoryRegionOps riscv_iommu_trap_ops = {
>      }
>  };
>
> +static void riscv_iommu_instance_init(Object *obj)
> +{
> +    RISCVIOMMUState *s = RISCV_IOMMU(obj);
> +
> +    /* Enable translation debug interface */
> +    s->cap = RISCV_IOMMU_CAP_DBG;
> +
> +    /* Report QEMU target physical address space limits */
> +    s->cap = set_field(s->cap, RISCV_IOMMU_CAP_PAS,
> +                       TARGET_PHYS_ADDR_SPACE_BITS);
> +
> +    /* TODO: method to report supported PID bits */
> +    s->pid_bits = 8; /* restricted to size of MemTxAttrs.pid */
> +    s->cap |= RISCV_IOMMU_CAP_PD8;
> +
> +    /* register storage */
> +    s->regs_rw = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
> +    s->regs_ro = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
> +    s->regs_wc = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
> +
> +     /* Mark all registers read-only */
> +    memset(s->regs_ro, 0xff, RISCV_IOMMU_REG_SIZE);
> +
> +    /* Device translation context cache */
> +    s->ctx_cache = g_hash_table_new_full(riscv_iommu_ctx_hash,
> +                                         riscv_iommu_ctx_equal,
> +                                         g_free, NULL);
> +
> +    s->iot_cache = g_hash_table_new_full(riscv_iommu_iot_hash,
> +                                         riscv_iommu_iot_equal,
> +                                         g_free, NULL);
> +
> +    s->iommus.le_next = NULL;
> +    s->iommus.le_prev = NULL;
> +    QLIST_INIT(&s->spaces);
> +}
> +
>  static void riscv_iommu_realize(DeviceState *dev, Error **errp)
>  {
>      RISCVIOMMUState *s = RISCV_IOMMU(dev);
>
> -    s->cap = s->version & RISCV_IOMMU_CAP_VERSION;
> +    s->cap |= s->version & RISCV_IOMMU_CAP_VERSION;
>      if (s->enable_msi) {
>          s->cap |= RISCV_IOMMU_CAP_MSI_FLAT | RISCV_IOMMU_CAP_MSI_MRIF;
>      }
> @@ -2115,29 +2152,11 @@ static void riscv_iommu_realize(DeviceState *dev, Error **errp)
>          s->cap |= RISCV_IOMMU_CAP_SV32X4 | RISCV_IOMMU_CAP_SV39X4 |
>                    RISCV_IOMMU_CAP_SV48X4 | RISCV_IOMMU_CAP_SV57X4;
>      }
> -    /* Enable translation debug interface */
> -    s->cap |= RISCV_IOMMU_CAP_DBG;
> -
> -    /* Report QEMU target physical address space limits */
> -    s->cap = set_field(s->cap, RISCV_IOMMU_CAP_PAS,
> -                       TARGET_PHYS_ADDR_SPACE_BITS);
> -
> -    /* TODO: method to report supported PID bits */
> -    s->pid_bits = 8; /* restricted to size of MemTxAttrs.pid */
> -    s->cap |= RISCV_IOMMU_CAP_PD8;
>
>      /* Out-of-reset translation mode: OFF (DMA disabled) BARE (passthrough) */
>      s->ddtp = set_field(0, RISCV_IOMMU_DDTP_MODE, s->enable_off ?
>                          RISCV_IOMMU_DDTP_MODE_OFF : RISCV_IOMMU_DDTP_MODE_BARE);
>
> -    /* register storage */
> -    s->regs_rw = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
> -    s->regs_ro = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
> -    s->regs_wc = g_new0(uint8_t, RISCV_IOMMU_REG_SIZE);
> -
> -     /* Mark all registers read-only */
> -    memset(s->regs_ro, 0xff, RISCV_IOMMU_REG_SIZE);
> -
>      /*
>       * Register complete MMIO space, including MSI/PBA registers.
>       * Note, PCIDevice implementation will add overlapping MR for MSI/PBA,
> @@ -2195,19 +2214,6 @@ static void riscv_iommu_realize(DeviceState *dev, Error **errp)
>      memory_region_init_io(&s->trap_mr, OBJECT(dev), &riscv_iommu_trap_ops, s,
>              "riscv-iommu-trap", ~0ULL);
>      address_space_init(&s->trap_as, &s->trap_mr, "riscv-iommu-trap-as");
> -
> -    /* Device translation context cache */
> -    s->ctx_cache = g_hash_table_new_full(riscv_iommu_ctx_hash,
> -                                         riscv_iommu_ctx_equal,
> -                                         g_free, NULL);
> -
> -    s->iot_cache = g_hash_table_new_full(riscv_iommu_iot_hash,
> -                                         riscv_iommu_iot_equal,
> -                                         g_free, NULL);
> -
> -    s->iommus.le_next = NULL;
> -    s->iommus.le_prev = NULL;
> -    QLIST_INIT(&s->spaces);
>  }
>
>  static void riscv_iommu_unrealize(DeviceState *dev)
> @@ -2249,6 +2255,7 @@ static const TypeInfo riscv_iommu_info = {
>      .name = TYPE_RISCV_IOMMU,
>      .parent = TYPE_DEVICE,
>      .instance_size = sizeof(RISCVIOMMUState),
> +    .instance_init = riscv_iommu_instance_init,
>      .class_init = riscv_iommu_class_init,
>  };
>
> --
> 2.45.2
>
>

Re: [PATCH for-10.0 2/7] hw/riscv/riscv-iommu: parametrize CAP.IGS

[Alistair Francis]

On Wed, Nov 6, 2024 at 11:36 PM Daniel Henrique Barboza
<dbarboza@ventanamicro.com> wrote:
>
> Interrupt Generation Support (IGS) is a capability that is tied to the
> interrupt deliver mechanism, not with the core IOMMU emulation. We
> should allow device implementations to set IGS as they wish.
>
> A new helper is added to make it easier for device impls to set IGS. Use
> it in our existing IOMMU device (riscv-iommu-pci) to set
> RISCV_IOMMU_CAPS_IGS_MSI.
>
> Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>

Reviewed-by: Alistair Francis <alistair.francis@wdc.com>

Alistair

> ---
>  hw/riscv/riscv-iommu-bits.h | 6 ++++++
>  hw/riscv/riscv-iommu-pci.c  | 1 +
>  hw/riscv/riscv-iommu.c      | 5 +++++
>  hw/riscv/riscv-iommu.h      | 4 ++++
>  4 files changed, 16 insertions(+)
>
> diff --git a/hw/riscv/riscv-iommu-bits.h b/hw/riscv/riscv-iommu-bits.h
> index 6359ae0353..485f36b9c9 100644
> --- a/hw/riscv/riscv-iommu-bits.h
> +++ b/hw/riscv/riscv-iommu-bits.h
> @@ -88,6 +88,12 @@ struct riscv_iommu_pq_record {
>  #define RISCV_IOMMU_CAP_PD17            BIT_ULL(39)
>  #define RISCV_IOMMU_CAP_PD20            BIT_ULL(40)
>
> +enum riscv_iommu_igs_modes {
> +    RISCV_IOMMU_CAP_IGS_MSI = 0,
> +    RISCV_IOMMU_CAP_IGS_WSI,
> +    RISCV_IOMMU_CAP_IGS_BOTH
> +};
> +
>  /* 5.4 Features control register (32bits) */
>  #define RISCV_IOMMU_REG_FCTL            0x0008
>  #define RISCV_IOMMU_FCTL_BE             BIT(0)
> diff --git a/hw/riscv/riscv-iommu-pci.c b/hw/riscv/riscv-iommu-pci.c
> index a42242532d..4ce9bf6b78 100644
> --- a/hw/riscv/riscv-iommu-pci.c
> +++ b/hw/riscv/riscv-iommu-pci.c
> @@ -155,6 +155,7 @@ static void riscv_iommu_pci_init(Object *obj)
>      qdev_alias_all_properties(DEVICE(iommu), obj);
>
>      iommu->icvec_avail_vectors = RISCV_IOMMU_PCI_ICVEC_VECTORS;
> +    riscv_iommu_set_cap_igs(iommu, RISCV_IOMMU_CAP_IGS_MSI);
>  }
>
>  static Property riscv_iommu_pci_properties[] = {
> diff --git a/hw/riscv/riscv-iommu.c b/hw/riscv/riscv-iommu.c
> index 1893584028..d95b4b95d8 100644
> --- a/hw/riscv/riscv-iommu.c
> +++ b/hw/riscv/riscv-iommu.c
> @@ -2096,6 +2096,11 @@ static const MemoryRegionOps riscv_iommu_trap_ops = {
>      }
>  };
>
> +void riscv_iommu_set_cap_igs(RISCVIOMMUState *s, riscv_iommu_igs_mode mode)
> +{
> +    s->cap = set_field(s->cap, RISCV_IOMMU_CAP_IGS, mode);
> +}
> +
>  static void riscv_iommu_instance_init(Object *obj)
>  {
>      RISCVIOMMUState *s = RISCV_IOMMU(obj);
> diff --git a/hw/riscv/riscv-iommu.h b/hw/riscv/riscv-iommu.h
> index da3f03440c..f9f2827808 100644
> --- a/hw/riscv/riscv-iommu.h
> +++ b/hw/riscv/riscv-iommu.h
> @@ -21,6 +21,9 @@
>
>  #include "qom/object.h"
>  #include "hw/riscv/iommu.h"
> +#include "hw/riscv/riscv-iommu-bits.h"
> +
> +typedef enum riscv_iommu_igs_modes riscv_iommu_igs_mode;
>
>  struct RISCVIOMMUState {
>      /*< private >*/
> @@ -85,6 +88,7 @@ struct RISCVIOMMUState {
>
>  void riscv_iommu_pci_setup_iommu(RISCVIOMMUState *iommu, PCIBus *bus,
>           Error **errp);
> +void riscv_iommu_set_cap_igs(RISCVIOMMUState *s, riscv_iommu_igs_mode mode);
>
>  /* private helpers */
>
> --
> 2.45.2
>
>

Re: [PATCH for-10.0 3/7] hw/riscv: add riscv-iommu-sys platform device

[Alistair Francis]

On Wed, Nov 6, 2024 at 11:39 PM Daniel Henrique Barboza
<dbarboza@ventanamicro.com> wrote:
>
> From: Tomasz Jeznach <tjeznach@rivosinc.com>
>
> This device models the RISC-V IOMMU as a sysbus device. The same design
> decisions taken in the riscv-iommu-pci device were kept, namely the
> existence of 4 vectors are available for each interrupt cause.
>
> The WSIs are emitted using the input of the s->notify() callback as a
> index to an IRQ list. The IRQ list starts at 'base_irq' and goes until
> base_irq + 3. This means that boards must have 4 contiguous IRQ lines
> available, starting from 'base_irq'.
>
> Signed-off-by: Tomasz Jeznach <tjeznach@rivosinc.com>
> Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>

Reviewed-by: Alistair Francis <alistair.francis@wdc.com>

Alistair

> ---
>  hw/riscv/meson.build       |   2 +-
>  hw/riscv/riscv-iommu-sys.c | 128 +++++++++++++++++++++++++++++++++++++
>  hw/riscv/riscv-iommu.c     |   3 +-
>  include/hw/riscv/iommu.h   |   4 ++
>  4 files changed, 134 insertions(+), 3 deletions(-)
>  create mode 100644 hw/riscv/riscv-iommu-sys.c
>
> diff --git a/hw/riscv/meson.build b/hw/riscv/meson.build
> index adbef8a9b2..3be13d7774 100644
> --- a/hw/riscv/meson.build
> +++ b/hw/riscv/meson.build
> @@ -10,6 +10,6 @@ riscv_ss.add(when: 'CONFIG_SIFIVE_U', if_true: files('sifive_u.c'))
>  riscv_ss.add(when: 'CONFIG_SPIKE', if_true: files('spike.c'))
>  riscv_ss.add(when: 'CONFIG_MICROCHIP_PFSOC', if_true: files('microchip_pfsoc.c'))
>  riscv_ss.add(when: 'CONFIG_ACPI', if_true: files('virt-acpi-build.c'))
> -riscv_ss.add(when: 'CONFIG_RISCV_IOMMU', if_true: files('riscv-iommu.c', 'riscv-iommu-pci.c'))
> +riscv_ss.add(when: 'CONFIG_RISCV_IOMMU', if_true: files('riscv-iommu.c', 'riscv-iommu-pci.c', 'riscv-iommu-sys.c'))
>
>  hw_arch += {'riscv': riscv_ss}
> diff --git a/hw/riscv/riscv-iommu-sys.c b/hw/riscv/riscv-iommu-sys.c
> new file mode 100644
> index 0000000000..4b82046ce9
> --- /dev/null
> +++ b/hw/riscv/riscv-iommu-sys.c
> @@ -0,0 +1,128 @@
> +/*
> + * QEMU emulation of an RISC-V IOMMU Platform Device
> + *
> + * Copyright (C) 2022-2023 Rivos Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify it
> + * under the terms and conditions of the GNU General Public License,
> + * version 2 or later, as published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License along
> + * with this program; if not, see <http://www.gnu.org/licenses/>.
> + */
> +
> +#include "qemu/osdep.h"
> +#include "hw/irq.h"
> +#include "hw/pci/pci_bus.h"
> +#include "hw/qdev-properties.h"
> +#include "hw/sysbus.h"
> +#include "qapi/error.h"
> +#include "qemu/error-report.h"
> +#include "qemu/host-utils.h"
> +#include "qemu/module.h"
> +#include "qom/object.h"
> +
> +#include "riscv-iommu.h"
> +
> +#define RISCV_IOMMU_SYSDEV_ICVEC_VECTORS 0x3333
> +
> +/* RISC-V IOMMU System Platform Device Emulation */
> +
> +struct RISCVIOMMUStateSys {
> +    SysBusDevice     parent;
> +    uint64_t         addr;
> +    uint32_t         base_irq;
> +    DeviceState      *irqchip;
> +    RISCVIOMMUState  iommu;
> +    qemu_irq         irqs[RISCV_IOMMU_INTR_COUNT];
> +};
> +
> +static void riscv_iommu_sysdev_notify(RISCVIOMMUState *iommu,
> +                                      unsigned vector)
> +{
> +    RISCVIOMMUStateSys *s = container_of(iommu, RISCVIOMMUStateSys, iommu);
> +    uint32_t fctl =  riscv_iommu_reg_get32(iommu, RISCV_IOMMU_REG_FCTL);
> +
> +    /* We do not support MSIs yet */
> +    if (!(fctl & RISCV_IOMMU_FCTL_WSI)) {
> +        return;
> +    }
> +
> +    qemu_irq_pulse(s->irqs[vector]);
> +}
> +
> +static void riscv_iommu_sys_realize(DeviceState *dev, Error **errp)
> +{
> +    RISCVIOMMUStateSys *s = RISCV_IOMMU_SYS(dev);
> +    SysBusDevice *sysdev = SYS_BUS_DEVICE(s);
> +    PCIBus *pci_bus;
> +    qemu_irq irq;
> +
> +    qdev_realize(DEVICE(&s->iommu), NULL, errp);
> +    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &s->iommu.regs_mr);
> +    if (s->addr) {
> +        sysbus_mmio_map(SYS_BUS_DEVICE(s), 0, s->addr);
> +    }
> +
> +    pci_bus = (PCIBus *) object_resolve_path_type("", TYPE_PCI_BUS, NULL);
> +    if (pci_bus) {
> +        riscv_iommu_pci_setup_iommu(&s->iommu, pci_bus, errp);
> +    }
> +
> +    s->iommu.notify = riscv_iommu_sysdev_notify;
> +
> +    /* 4 IRQs are defined starting from s->base_irq */
> +    for (int i = 0; i < RISCV_IOMMU_INTR_COUNT; i++) {
> +        sysbus_init_irq(sysdev, &s->irqs[i]);
> +        irq = qdev_get_gpio_in(s->irqchip, s->base_irq + i);
> +        sysbus_connect_irq(sysdev, i, irq);
> +    }
> +}
> +
> +static void riscv_iommu_sys_init(Object *obj)
> +{
> +    RISCVIOMMUStateSys *s = RISCV_IOMMU_SYS(obj);
> +    RISCVIOMMUState *iommu = &s->iommu;
> +
> +    object_initialize_child(obj, "iommu", iommu, TYPE_RISCV_IOMMU);
> +    qdev_alias_all_properties(DEVICE(iommu), obj);
> +
> +    iommu->icvec_avail_vectors = RISCV_IOMMU_SYSDEV_ICVEC_VECTORS;
> +    riscv_iommu_set_cap_igs(iommu, RISCV_IOMMU_CAP_IGS_WSI);
> +}
> +
> +static Property riscv_iommu_sys_properties[] = {
> +    DEFINE_PROP_UINT64("addr", RISCVIOMMUStateSys, addr, 0),
> +    DEFINE_PROP_UINT32("base-irq", RISCVIOMMUStateSys, base_irq, 0),
> +    DEFINE_PROP_LINK("irqchip", RISCVIOMMUStateSys, irqchip,
> +                     TYPE_DEVICE, DeviceState *),
> +    DEFINE_PROP_END_OF_LIST(),
> +};
> +
> +static void riscv_iommu_sys_class_init(ObjectClass *klass, void *data)
> +{
> +    DeviceClass *dc = DEVICE_CLASS(klass);
> +    dc->realize = riscv_iommu_sys_realize;
> +    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
> +    device_class_set_props(dc, riscv_iommu_sys_properties);
> +}
> +
> +static const TypeInfo riscv_iommu_sys = {
> +    .name          = TYPE_RISCV_IOMMU_SYS,
> +    .parent        = TYPE_SYS_BUS_DEVICE,
> +    .class_init    = riscv_iommu_sys_class_init,
> +    .instance_init = riscv_iommu_sys_init,
> +    .instance_size = sizeof(RISCVIOMMUStateSys),
> +};
> +
> +static void riscv_iommu_register_sys(void)
> +{
> +    type_register_static(&riscv_iommu_sys);
> +}
> +
> +type_init(riscv_iommu_register_sys)
> diff --git a/hw/riscv/riscv-iommu.c b/hw/riscv/riscv-iommu.c
> index d95b4b95d8..239f83f5bd 100644
> --- a/hw/riscv/riscv-iommu.c
> +++ b/hw/riscv/riscv-iommu.c
> @@ -94,10 +94,9 @@ static uint8_t riscv_iommu_get_icvec_vector(uint32_t icvec, uint32_t vec_type)
>
>  static void riscv_iommu_notify(RISCVIOMMUState *s, int vec_type)
>  {
> -    const uint32_t fctl = riscv_iommu_reg_get32(s, RISCV_IOMMU_REG_FCTL);
>      uint32_t ipsr, icvec, vector;
>
> -    if (fctl & RISCV_IOMMU_FCTL_WSI || !s->notify) {
> +    if (!s->notify) {
>          return;
>      }
>
> diff --git a/include/hw/riscv/iommu.h b/include/hw/riscv/iommu.h
> index 80769a1400..fc20808553 100644
> --- a/include/hw/riscv/iommu.h
> +++ b/include/hw/riscv/iommu.h
> @@ -33,4 +33,8 @@ typedef struct RISCVIOMMUSpace RISCVIOMMUSpace;
>  OBJECT_DECLARE_SIMPLE_TYPE(RISCVIOMMUStatePci, RISCV_IOMMU_PCI)
>  typedef struct RISCVIOMMUStatePci RISCVIOMMUStatePci;
>
> +#define TYPE_RISCV_IOMMU_SYS "riscv-iommu-device"
> +OBJECT_DECLARE_SIMPLE_TYPE(RISCVIOMMUStateSys, RISCV_IOMMU_SYS)
> +typedef struct RISCVIOMMUStateSys RISCVIOMMUStateSys;
> +
>  #endif
> --
> 2.45.2
>
>

Re: [PATCH for-10.0 4/7] hw/riscv/virt: Add IOMMU as platform device if the option is set

[Alistair Francis]

On Wed, Nov 6, 2024 at 11:36 PM Daniel Henrique Barboza
<dbarboza@ventanamicro.com> wrote:
>
> From: Sunil V L <sunilvl@ventanamicro.com>
>
> Add a new machine option called 'iommu-sys' that enables a
> riscv-iommu-sys platform device for the 'virt' machine. The option is
> default 'off'.
>
> The device will use IRQs 36 to 39.
>
> We will not support both riscv-iommu-sys and riscv-iommu-pci devices in
> the same board in this first implementation. If a riscv-iommu-pci device
> is added in the command line we will disable the riscv-iommu-sys device.
>
> Signed-off-by: Sunil V L <sunilvl@ventanamicro.com>
> Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>

Reviewed-by: Alistair Francis <alistair.francis@wdc.com>

Alistair

> ---
>  hw/riscv/virt.c          | 104 ++++++++++++++++++++++++++++++++++++++-
>  include/hw/riscv/iommu.h |   2 +
>  include/hw/riscv/virt.h  |   6 ++-
>  3 files changed, 109 insertions(+), 3 deletions(-)
>
> diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c
> index 45a8c4f819..23d1380b86 100644
> --- a/hw/riscv/virt.c
> +++ b/hw/riscv/virt.c
> @@ -33,6 +33,7 @@
>  #include "target/riscv/pmu.h"
>  #include "hw/riscv/riscv_hart.h"
>  #include "hw/riscv/iommu.h"
> +#include "hw/riscv/riscv-iommu-bits.h"
>  #include "hw/riscv/virt.h"
>  #include "hw/riscv/boot.h"
>  #include "hw/riscv/numa.h"
> @@ -76,6 +77,7 @@ static const MemMapEntry virt_memmap[] = {
>      [VIRT_CLINT] =        {  0x2000000,       0x10000 },
>      [VIRT_ACLINT_SSWI] =  {  0x2F00000,        0x4000 },
>      [VIRT_PCIE_PIO] =     {  0x3000000,       0x10000 },
> +    [VIRT_IOMMU_SYS] =    {  0x3010000,        0x1000 },
>      [VIRT_PLATFORM_BUS] = {  0x4000000,     0x2000000 },
>      [VIRT_PLIC] =         {  0xc000000, VIRT_PLIC_SIZE(VIRT_CPUS_MAX * 2) },
>      [VIRT_APLIC_M] =      {  0xc000000, APLIC_SIZE(VIRT_CPUS_MAX) },
> @@ -853,7 +855,8 @@ static void create_fdt_virtio(RISCVVirtState *s, const MemMapEntry *memmap,
>
>  static void create_fdt_pcie(RISCVVirtState *s, const MemMapEntry *memmap,
>                              uint32_t irq_pcie_phandle,
> -                            uint32_t msi_pcie_phandle)
> +                            uint32_t msi_pcie_phandle,
> +                            uint32_t iommu_sys_phandle)
>  {
>      g_autofree char *name = NULL;
>      MachineState *ms = MACHINE(s);
> @@ -887,6 +890,12 @@ static void create_fdt_pcie(RISCVVirtState *s, const MemMapEntry *memmap,
>          2, virt_high_pcie_memmap.base,
>          2, virt_high_pcie_memmap.base, 2, virt_high_pcie_memmap.size);
>
> +    if (virt_is_iommu_sys_enabled(s)) {
> +        qemu_fdt_setprop_cells(ms->fdt, name, "iommu-map",
> +                               0, iommu_sys_phandle, 0, 0, 0,
> +                               iommu_sys_phandle, 0, 0xffff);
> +    }
> +
>      create_pcie_irq_map(s, ms->fdt, name, irq_pcie_phandle);
>  }
>
> @@ -1033,6 +1042,44 @@ static void create_fdt_virtio_iommu(RISCVVirtState *s, uint16_t bdf)
>                             bdf + 1, iommu_phandle, bdf + 1, 0xffff - bdf);
>  }
>
> +static void create_fdt_iommu_sys(RISCVVirtState *s, uint32_t irq_chip,
> +                                 uint32_t *iommu_sys_phandle)
> +{
> +    const char comp[] = "riscv,iommu";
> +    void *fdt = MACHINE(s)->fdt;
> +    uint32_t iommu_phandle;
> +    g_autofree char *iommu_node = NULL;
> +    hwaddr addr = s->memmap[VIRT_IOMMU_SYS].base;
> +    hwaddr size = s->memmap[VIRT_IOMMU_SYS].size;
> +    uint32_t iommu_irq_map[RISCV_IOMMU_INTR_COUNT] = {
> +        IOMMU_SYS_IRQ + RISCV_IOMMU_INTR_CQ,
> +        IOMMU_SYS_IRQ + RISCV_IOMMU_INTR_FQ,
> +        IOMMU_SYS_IRQ + RISCV_IOMMU_INTR_PM,
> +        IOMMU_SYS_IRQ + RISCV_IOMMU_INTR_PQ,
> +    };
> +
> +    iommu_node = g_strdup_printf("/soc/iommu@%x",
> +                               (unsigned int) s->memmap[VIRT_IOMMU_SYS].base);
> +    iommu_phandle = qemu_fdt_alloc_phandle(fdt);
> +    qemu_fdt_add_subnode(fdt, iommu_node);
> +
> +    qemu_fdt_setprop(fdt, iommu_node, "compatible", comp, sizeof(comp));
> +    qemu_fdt_setprop_cell(fdt, iommu_node, "#iommu-cells", 1);
> +    qemu_fdt_setprop_cell(fdt, iommu_node, "phandle", iommu_phandle);
> +
> +    qemu_fdt_setprop_cells(fdt, iommu_node, "reg",
> +                           addr >> 32, addr, size >> 32, size);
> +    qemu_fdt_setprop_cell(fdt, iommu_node, "interrupt-parent", irq_chip);
> +
> +    qemu_fdt_setprop_cells(fdt, iommu_node, "interrupts",
> +        iommu_irq_map[0], FDT_IRQ_TYPE_EDGE_LOW,
> +        iommu_irq_map[1], FDT_IRQ_TYPE_EDGE_LOW,
> +        iommu_irq_map[2], FDT_IRQ_TYPE_EDGE_LOW,
> +        iommu_irq_map[3], FDT_IRQ_TYPE_EDGE_LOW);
> +
> +    *iommu_sys_phandle = iommu_phandle;
> +}
> +
>  static void create_fdt_iommu(RISCVVirtState *s, uint16_t bdf)
>  {
>      const char comp[] = "riscv,pci-iommu";
> @@ -1061,6 +1108,7 @@ static void finalize_fdt(RISCVVirtState *s)
>  {
>      uint32_t phandle = 1, irq_mmio_phandle = 1, msi_pcie_phandle = 1;
>      uint32_t irq_pcie_phandle = 1, irq_virtio_phandle = 1;
> +    uint32_t iommu_sys_phandle = 1;
>
>      create_fdt_sockets(s, virt_memmap, &phandle, &irq_mmio_phandle,
>                         &irq_pcie_phandle, &irq_virtio_phandle,
> @@ -1068,7 +1116,11 @@ static void finalize_fdt(RISCVVirtState *s)
>
>      create_fdt_virtio(s, virt_memmap, irq_virtio_phandle);
>
> -    create_fdt_pcie(s, virt_memmap, irq_pcie_phandle, msi_pcie_phandle);
> +    if (virt_is_iommu_sys_enabled(s)) {
> +        create_fdt_iommu_sys(s, irq_mmio_phandle, &iommu_sys_phandle);
> +    }
> +    create_fdt_pcie(s, virt_memmap, irq_pcie_phandle, msi_pcie_phandle,
> +                    iommu_sys_phandle);
>
>      create_fdt_reset(s, virt_memmap, &phandle);
>
> @@ -1650,6 +1702,22 @@ static void virt_machine_init(MachineState *machine)
>          create_fdt(s, memmap);
>      }
>
> +    if (virt_is_iommu_sys_enabled(s)) {
> +        DeviceState *iommu_sys = qdev_new(TYPE_RISCV_IOMMU_SYS);
> +
> +        object_property_set_uint(OBJECT(iommu_sys), "addr",
> +                                 s->memmap[VIRT_IOMMU_SYS].base,
> +                                 &error_fatal);
> +        object_property_set_uint(OBJECT(iommu_sys), "base-irq",
> +                                 IOMMU_SYS_IRQ,
> +                                 &error_fatal);
> +        object_property_set_link(OBJECT(iommu_sys), "irqchip",
> +                                 OBJECT(mmio_irqchip),
> +                                 &error_fatal);
> +
> +        sysbus_realize_and_unref(SYS_BUS_DEVICE(iommu_sys), &error_fatal);
> +    }
> +
>      s->machine_done.notify = virt_machine_done;
>      qemu_add_machine_init_done_notifier(&s->machine_done);
>  }
> @@ -1663,6 +1731,7 @@ static void virt_machine_instance_init(Object *obj)
>      s->oem_id = g_strndup(ACPI_BUILD_APPNAME6, 6);
>      s->oem_table_id = g_strndup(ACPI_BUILD_APPNAME8, 8);
>      s->acpi = ON_OFF_AUTO_AUTO;
> +    s->iommu_sys = ON_OFF_AUTO_AUTO;
>  }
>
>  static char *virt_get_aia_guests(Object *obj, Error **errp)
> @@ -1735,6 +1804,28 @@ static void virt_set_aclint(Object *obj, bool value, Error **errp)
>      s->have_aclint = value;
>  }
>
> +bool virt_is_iommu_sys_enabled(RISCVVirtState *s)
> +{
> +    return s->iommu_sys == ON_OFF_AUTO_ON;
> +}
> +
> +static void virt_get_iommu_sys(Object *obj, Visitor *v, const char *name,
> +                               void *opaque, Error **errp)
> +{
> +    RISCVVirtState *s = RISCV_VIRT_MACHINE(obj);
> +    OnOffAuto iommu_sys = s->iommu_sys;
> +
> +    visit_type_OnOffAuto(v, name, &iommu_sys, errp);
> +}
> +
> +static void virt_set_iommu_sys(Object *obj, Visitor *v, const char *name,
> +                               void *opaque, Error **errp)
> +{
> +    RISCVVirtState *s = RISCV_VIRT_MACHINE(obj);
> +
> +    visit_type_OnOffAuto(v, name, &s->iommu_sys, errp);
> +}
> +
>  bool virt_is_acpi_enabled(RISCVVirtState *s)
>  {
>      return s->acpi != ON_OFF_AUTO_OFF;
> @@ -1761,10 +1852,12 @@ static HotplugHandler *virt_machine_get_hotplug_handler(MachineState *machine,
>                                                          DeviceState *dev)
>  {
>      MachineClass *mc = MACHINE_GET_CLASS(machine);
> +    RISCVVirtState *s = RISCV_VIRT_MACHINE(machine);
>
>      if (device_is_dynamic_sysbus(mc, dev) ||
>          object_dynamic_cast(OBJECT(dev), TYPE_VIRTIO_IOMMU_PCI) ||
>          object_dynamic_cast(OBJECT(dev), TYPE_RISCV_IOMMU_PCI)) {
> +        s->iommu_sys = ON_OFF_AUTO_OFF;
>          return HOTPLUG_HANDLER(machine);
>      }
>
> @@ -1791,6 +1884,7 @@ static void virt_machine_device_plug_cb(HotplugHandler *hotplug_dev,
>
>      if (object_dynamic_cast(OBJECT(dev), TYPE_RISCV_IOMMU_PCI)) {
>          create_fdt_iommu(s, pci_get_bdf(PCI_DEVICE(dev)));
> +        s->iommu_sys = ON_OFF_AUTO_OFF;
>      }
>  }
>
> @@ -1853,6 +1947,12 @@ static void virt_machine_class_init(ObjectClass *oc, void *data)
>                                NULL, NULL);
>      object_class_property_set_description(oc, "acpi",
>                                            "Enable ACPI");
> +
> +    object_class_property_add(oc, "iommu-sys", "OnOffAuto",
> +                              virt_get_iommu_sys, virt_set_iommu_sys,
> +                              NULL, NULL);
> +    object_class_property_set_description(oc, "iommu-sys",
> +                                          "Enable IOMMU platform device");
>  }
>
>  static const TypeInfo virt_machine_typeinfo = {
> diff --git a/include/hw/riscv/iommu.h b/include/hw/riscv/iommu.h
> index fc20808553..8a8acfc3f0 100644
> --- a/include/hw/riscv/iommu.h
> +++ b/include/hw/riscv/iommu.h
> @@ -37,4 +37,6 @@ typedef struct RISCVIOMMUStatePci RISCVIOMMUStatePci;
>  OBJECT_DECLARE_SIMPLE_TYPE(RISCVIOMMUStateSys, RISCV_IOMMU_SYS)
>  typedef struct RISCVIOMMUStateSys RISCVIOMMUStateSys;
>
> +#define FDT_IRQ_TYPE_EDGE_LOW 1
> +
>  #endif
> diff --git a/include/hw/riscv/virt.h b/include/hw/riscv/virt.h
> index c0dc41ff9a..48a14bea2e 100644
> --- a/include/hw/riscv/virt.h
> +++ b/include/hw/riscv/virt.h
> @@ -62,6 +62,7 @@ struct RISCVVirtState {
>      OnOffAuto acpi;
>      const MemMapEntry *memmap;
>      struct GPEXHost *gpex_host;
> +    OnOffAuto iommu_sys;
>  };
>
>  enum {
> @@ -84,7 +85,8 @@ enum {
>      VIRT_PCIE_MMIO,
>      VIRT_PCIE_PIO,
>      VIRT_PLATFORM_BUS,
> -    VIRT_PCIE_ECAM
> +    VIRT_PCIE_ECAM,
> +    VIRT_IOMMU_SYS,
>  };
>
>  enum {
> @@ -93,6 +95,7 @@ enum {
>      VIRTIO_IRQ = 1, /* 1 to 8 */
>      VIRTIO_COUNT = 8,
>      PCIE_IRQ = 0x20, /* 32 to 35 */
> +    IOMMU_SYS_IRQ = 0x24, /* 36-39 */
>      VIRT_PLATFORM_BUS_IRQ = 64, /* 64 to 95 */
>  };
>
> @@ -129,6 +132,7 @@ enum {
>                                   1 + FDT_APLIC_INT_CELLS)
>
>  bool virt_is_acpi_enabled(RISCVVirtState *s);
> +bool virt_is_iommu_sys_enabled(RISCVVirtState *s);
>  void virt_acpi_setup(RISCVVirtState *vms);
>  uint32_t imsic_num_bits(uint32_t count);
>
> --
> 2.45.2
>
>

Re: [PATCH for-10.0 5/7] hw/riscv/virt.c, riscv-iommu-sys.c: add MSIx support

[Alistair Francis]

On Wed, Nov 6, 2024 at 11:36 PM Daniel Henrique Barboza
<dbarboza@ventanamicro.com> wrote:
>
> MSIx support is added in the RISC-V IOMMU platform device by including
> the required MSIx facilities to alow software to properly setup the MSIx
> subsystem.
>
> We took inspiration of what is being done in the riscv-iommu-pci device,
> mainly msix_init() and msix_notify(), while keeping in mind that
> riscv-iommu-sys isn't a true PCI device and we don't need to copy/paste
> all the contents of these MSIx functions.
>
> Two extra MSI MemoryRegions were added: 'msix-table' and 'msix-pba'.
> They are used to manage r/w of the MSI table and Pending Bit Array (PBA)
> respectively. Both are subregions of the main IOMMU memory region,
> iommu->regs_mr, initialized during riscv_iommu_realize(), and each one
> has their own handlers for MSIx reads and writes.
>
> This is the expected memory map when using this device in the 'virt'
> machine:
>
>     0000000003010000-0000000003010fff (prio 0, i/o): riscv-iommu-regs
>       0000000003010300-000000000301034f (prio 0, i/o): msix-table
>       0000000003010400-0000000003010407 (prio 0, i/o): msix-pba
>
> We're now able to set IGS to RISCV_IOMMU_CAP_IGS_BOTH, and userspace is
> free to decide which interrupt model to use.
>
> Enabling MSIx support for this device in the 'virt' machine requires
> adding 'msi-parent' in the iommu-sys DT.
>
> Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>

Acked-by: Alistair Francis <alistair.francis@wdc.com>

Alistair

> ---
>  hw/riscv/riscv-iommu-sys.c | 116 +++++++++++++++++++++++++++++++++++--
>  hw/riscv/trace-events      |   2 +
>  hw/riscv/virt.c            |   6 +-
>  3 files changed, 119 insertions(+), 5 deletions(-)
>
> diff --git a/hw/riscv/riscv-iommu-sys.c b/hw/riscv/riscv-iommu-sys.c
> index 4b82046ce9..a0ef67a20b 100644
> --- a/hw/riscv/riscv-iommu-sys.c
> +++ b/hw/riscv/riscv-iommu-sys.c
> @@ -26,11 +26,15 @@
>  #include "qemu/host-utils.h"
>  #include "qemu/module.h"
>  #include "qom/object.h"
> +#include "exec/exec-all.h"
> +#include "trace.h"
>
>  #include "riscv-iommu.h"
>
>  #define RISCV_IOMMU_SYSDEV_ICVEC_VECTORS 0x3333
>
> +#define RISCV_IOMMU_PCI_MSIX_VECTORS 5
> +
>  /* RISC-V IOMMU System Platform Device Emulation */
>
>  struct RISCVIOMMUStateSys {
> @@ -39,21 +43,123 @@ struct RISCVIOMMUStateSys {
>      uint32_t         base_irq;
>      DeviceState      *irqchip;
>      RISCVIOMMUState  iommu;
> +
> +    /* Wired int support */
>      qemu_irq         irqs[RISCV_IOMMU_INTR_COUNT];
> +
> +    /* Memory Regions for MSIX table and pending bit entries. */
> +    MemoryRegion msix_table_mmio;
> +    MemoryRegion msix_pba_mmio;
> +    uint8_t *msix_table;
> +    uint8_t *msix_pba;
> +};
> +
> +static uint64_t msix_table_mmio_read(void *opaque, hwaddr addr,
> +                                     unsigned size)
> +{
> +    RISCVIOMMUStateSys *s = opaque;
> +
> +    g_assert(addr + size <= RISCV_IOMMU_PCI_MSIX_VECTORS * PCI_MSIX_ENTRY_SIZE);
> +    return pci_get_long(s->msix_table + addr);
> +}
> +
> +static void msix_table_mmio_write(void *opaque, hwaddr addr,
> +                                  uint64_t val, unsigned size)
> +{
> +    RISCVIOMMUStateSys *s = opaque;
> +
> +    g_assert(addr + size <= RISCV_IOMMU_PCI_MSIX_VECTORS * PCI_MSIX_ENTRY_SIZE);
> +    pci_set_long(s->msix_table + addr, val);
> +}
> +
> +static const MemoryRegionOps msix_table_mmio_ops = {
> +    .read = msix_table_mmio_read,
> +    .write = msix_table_mmio_write,
> +    .endianness = DEVICE_LITTLE_ENDIAN,
> +    .valid = {
> +        .min_access_size = 4,
> +        .max_access_size = 8,
> +    },
> +    .impl = {
> +        .max_access_size = 4,
> +    },
> +};
> +
> +static uint64_t msix_pba_mmio_read(void *opaque, hwaddr addr,
> +                                   unsigned size)
> +{
> +    RISCVIOMMUStateSys *s = opaque;
> +
> +    return pci_get_long(s->msix_pba + addr);
> +}
> +
> +static void msix_pba_mmio_write(void *opaque, hwaddr addr,
> +                                uint64_t val, unsigned size)
> +{
> +}
> +
> +static const MemoryRegionOps msix_pba_mmio_ops = {
> +    .read = msix_pba_mmio_read,
> +    .write = msix_pba_mmio_write,
> +    .endianness = DEVICE_LITTLE_ENDIAN,
> +    .valid = {
> +        .min_access_size = 4,
> +        .max_access_size = 8,
> +    },
> +    .impl = {
> +        .max_access_size = 4,
> +    },
>  };
>
> +static void riscv_iommu_sysdev_init_msi(RISCVIOMMUStateSys *s,
> +                                        uint32_t n_vectors)
> +{
> +    RISCVIOMMUState *iommu = &s->iommu;
> +    uint32_t table_size = table_size = n_vectors * PCI_MSIX_ENTRY_SIZE;
> +    uint32_t table_offset = RISCV_IOMMU_REG_MSI_CONFIG;
> +    uint32_t pba_size = QEMU_ALIGN_UP(n_vectors, 64) / 8;
> +    uint32_t pba_offset = RISCV_IOMMU_REG_MSI_CONFIG + 256;
> +
> +    s->msix_table = g_malloc0(table_size);
> +    s->msix_pba = g_malloc0(pba_size);
> +
> +    memory_region_init_io(&s->msix_table_mmio, OBJECT(s), &msix_table_mmio_ops,
> +                          s, "msix-table", table_size);
> +    memory_region_add_subregion(&iommu->regs_mr, table_offset,
> +                                &s->msix_table_mmio);
> +
> +    memory_region_init_io(&s->msix_pba_mmio, OBJECT(s), &msix_pba_mmio_ops, s,
> +                          "msix-pba", pba_size);
> +    memory_region_add_subregion(&iommu->regs_mr, pba_offset,
> +                                &s->msix_pba_mmio);
> +}
> +
> +static void riscv_iommu_sysdev_send_MSI(RISCVIOMMUStateSys *s,
> +                                        uint32_t vector)
> +{
> +    uint8_t *table_entry = s->msix_table + vector * PCI_MSIX_ENTRY_SIZE;
> +    uint64_t msi_addr = pci_get_quad(table_entry + PCI_MSIX_ENTRY_LOWER_ADDR);
> +    uint32_t msi_data = pci_get_long(table_entry + PCI_MSIX_ENTRY_DATA);
> +    MemTxResult result;
> +
> +    address_space_stl_le(&address_space_memory, msi_addr,
> +                         msi_data, MEMTXATTRS_UNSPECIFIED, &result);
> +    trace_riscv_iommu_sys_msi_sent(vector, msi_addr, msi_data, result);
> +}
> +
>  static void riscv_iommu_sysdev_notify(RISCVIOMMUState *iommu,
>                                        unsigned vector)
>  {
>      RISCVIOMMUStateSys *s = container_of(iommu, RISCVIOMMUStateSys, iommu);
>      uint32_t fctl =  riscv_iommu_reg_get32(iommu, RISCV_IOMMU_REG_FCTL);
>
> -    /* We do not support MSIs yet */
> -    if (!(fctl & RISCV_IOMMU_FCTL_WSI)) {
> +    if (fctl & RISCV_IOMMU_FCTL_WSI) {
> +        qemu_irq_pulse(s->irqs[vector]);
> +        trace_riscv_iommu_sys_irq_sent(vector);
>          return;
>      }
>
> -    qemu_irq_pulse(s->irqs[vector]);
> +    riscv_iommu_sysdev_send_MSI(s, vector);
>  }
>
>  static void riscv_iommu_sys_realize(DeviceState *dev, Error **errp)
> @@ -82,6 +188,8 @@ static void riscv_iommu_sys_realize(DeviceState *dev, Error **errp)
>          irq = qdev_get_gpio_in(s->irqchip, s->base_irq + i);
>          sysbus_connect_irq(sysdev, i, irq);
>      }
> +
> +    riscv_iommu_sysdev_init_msi(s, RISCV_IOMMU_PCI_MSIX_VECTORS);
>  }
>
>  static void riscv_iommu_sys_init(Object *obj)
> @@ -93,7 +201,7 @@ static void riscv_iommu_sys_init(Object *obj)
>      qdev_alias_all_properties(DEVICE(iommu), obj);
>
>      iommu->icvec_avail_vectors = RISCV_IOMMU_SYSDEV_ICVEC_VECTORS;
> -    riscv_iommu_set_cap_igs(iommu, RISCV_IOMMU_CAP_IGS_WSI);
> +    riscv_iommu_set_cap_igs(iommu, RISCV_IOMMU_CAP_IGS_BOTH);
>  }
>
>  static Property riscv_iommu_sys_properties[] = {
> diff --git a/hw/riscv/trace-events b/hw/riscv/trace-events
> index 0527c56c91..94facbb8b1 100644
> --- a/hw/riscv/trace-events
> +++ b/hw/riscv/trace-events
> @@ -15,3 +15,5 @@ riscv_iommu_icvec_write(uint32_t orig, uint32_t actual) "ICVEC write: incoming 0
>  riscv_iommu_ats(const char *id, unsigned b, unsigned d, unsigned f, uint64_t iova) "%s: translate request %04x:%02x.%u iova: 0x%"PRIx64
>  riscv_iommu_ats_inval(const char *id) "%s: dev-iotlb invalidate"
>  riscv_iommu_ats_prgr(const char *id) "%s: dev-iotlb page request group response"
> +riscv_iommu_sys_irq_sent(uint32_t vector) "IRQ sent to vector %u"
> +riscv_iommu_sys_msi_sent(uint32_t vector, uint64_t msi_addr, uint32_t msi_data, uint32_t result) "MSI sent to vector %u msi_addr 0x%lx msi_data 0x%x result %u"
> diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c
> index 23d1380b86..281fc65cc6 100644
> --- a/hw/riscv/virt.c
> +++ b/hw/riscv/virt.c
> @@ -1043,6 +1043,7 @@ static void create_fdt_virtio_iommu(RISCVVirtState *s, uint16_t bdf)
>  }
>
>  static void create_fdt_iommu_sys(RISCVVirtState *s, uint32_t irq_chip,
> +                                 uint32_t msi_phandle,
>                                   uint32_t *iommu_sys_phandle)
>  {
>      const char comp[] = "riscv,iommu";
> @@ -1077,6 +1078,8 @@ static void create_fdt_iommu_sys(RISCVVirtState *s, uint32_t irq_chip,
>          iommu_irq_map[2], FDT_IRQ_TYPE_EDGE_LOW,
>          iommu_irq_map[3], FDT_IRQ_TYPE_EDGE_LOW);
>
> +    qemu_fdt_setprop_cell(fdt, iommu_node, "msi-parent", msi_phandle);
> +
>      *iommu_sys_phandle = iommu_phandle;
>  }
>
> @@ -1117,7 +1120,8 @@ static void finalize_fdt(RISCVVirtState *s)
>      create_fdt_virtio(s, virt_memmap, irq_virtio_phandle);
>
>      if (virt_is_iommu_sys_enabled(s)) {
> -        create_fdt_iommu_sys(s, irq_mmio_phandle, &iommu_sys_phandle);
> +        create_fdt_iommu_sys(s, irq_mmio_phandle, msi_pcie_phandle,
> +                             &iommu_sys_phandle);
>      }
>      create_fdt_pcie(s, virt_memmap, irq_pcie_phandle, msi_pcie_phandle,
>                      iommu_sys_phandle);
> --
> 2.45.2
>
>

Re: [PATCH for-10.0 6/7] hw/riscv/riscv-iommu: implement reset protocol

[Alistair Francis]

On Wed, Nov 6, 2024 at 11:35 PM Daniel Henrique Barboza
<dbarboza@ventanamicro.com> wrote:
>
> Add a riscv_iommu_reset() helper in the base emulation code that
> implements the expected reset behavior as defined by the riscv-iommu
> spec.
>
> Devices can then use this helper in their own reset callbacks.
>
> Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>

Acked-by: Alistair Francis <alistair.francis@wdc.com>

Alistair

> ---
>  hw/riscv/riscv-iommu-pci.c | 20 ++++++++++++++++++++
>  hw/riscv/riscv-iommu-sys.c | 20 ++++++++++++++++++++
>  hw/riscv/riscv-iommu.c     | 35 +++++++++++++++++++++++++++++++++++
>  hw/riscv/riscv-iommu.h     |  1 +
>  hw/riscv/trace-events      |  2 ++
>  include/hw/riscv/iommu.h   |  6 ++++--
>  6 files changed, 82 insertions(+), 2 deletions(-)
>
> diff --git a/hw/riscv/riscv-iommu-pci.c b/hw/riscv/riscv-iommu-pci.c
> index 4ce9bf6b78..51226c4877 100644
> --- a/hw/riscv/riscv-iommu-pci.c
> +++ b/hw/riscv/riscv-iommu-pci.c
> @@ -31,6 +31,7 @@
>  #include "cpu_bits.h"
>  #include "riscv-iommu.h"
>  #include "riscv-iommu-bits.h"
> +#include "trace.h"
>
>  /* RISC-V IOMMU PCI Device Emulation */
>  #define RISCV_PCI_CLASS_SYSTEM_IOMMU     0x0806
> @@ -66,6 +67,12 @@ typedef struct RISCVIOMMUStatePci {
>      RISCVIOMMUState  iommu;   /* common IOMMU state */
>  } RISCVIOMMUStatePci;
>
> +struct RISCVIOMMUPciClass {
> +    /*< public >*/
> +    DeviceRealize parent_realize;
> +    ResettablePhases parent_phases;
> +};
> +
>  /* interrupt delivery callback */
>  static void riscv_iommu_pci_notify(RISCVIOMMUState *iommu, unsigned vector)
>  {
> @@ -167,10 +174,23 @@ static Property riscv_iommu_pci_properties[] = {
>      DEFINE_PROP_END_OF_LIST(),
>  };
>
> +static void riscv_iommu_pci_reset_hold(Object *obj, ResetType type)
> +{
> +    RISCVIOMMUStatePci *pci = RISCV_IOMMU_PCI(obj);
> +    RISCVIOMMUState *iommu = &pci->iommu;
> +
> +    riscv_iommu_reset(iommu);
> +
> +    trace_riscv_iommu_pci_reset_hold(type);
> +}
> +
>  static void riscv_iommu_pci_class_init(ObjectClass *klass, void *data)
>  {
>      DeviceClass *dc = DEVICE_CLASS(klass);
>      PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
> +    ResettableClass *rc = RESETTABLE_CLASS(klass);
> +
> +    rc->phases.hold = riscv_iommu_pci_reset_hold;
>
>      k->realize = riscv_iommu_pci_realize;
>      k->exit = riscv_iommu_pci_exit;
> diff --git a/hw/riscv/riscv-iommu-sys.c b/hw/riscv/riscv-iommu-sys.c
> index a0ef67a20b..605979a0ac 100644
> --- a/hw/riscv/riscv-iommu-sys.c
> +++ b/hw/riscv/riscv-iommu-sys.c
> @@ -54,6 +54,12 @@ struct RISCVIOMMUStateSys {
>      uint8_t *msix_pba;
>  };
>
> +struct RISCVIOMMUSysClass {
> +    /*< public >*/
> +    DeviceRealize parent_realize;
> +    ResettablePhases parent_phases;
> +};
> +
>  static uint64_t msix_table_mmio_read(void *opaque, hwaddr addr,
>                                       unsigned size)
>  {
> @@ -212,9 +218,23 @@ static Property riscv_iommu_sys_properties[] = {
>      DEFINE_PROP_END_OF_LIST(),
>  };
>
> +static void riscv_iommu_sys_reset_hold(Object *obj, ResetType type)
> +{
> +    RISCVIOMMUStateSys *sys = RISCV_IOMMU_SYS(obj);
> +    RISCVIOMMUState *iommu = &sys->iommu;
> +
> +    riscv_iommu_reset(iommu);
> +
> +    trace_riscv_iommu_sys_reset_hold(type);
> +}
> +
>  static void riscv_iommu_sys_class_init(ObjectClass *klass, void *data)
>  {
>      DeviceClass *dc = DEVICE_CLASS(klass);
> +    ResettableClass *rc = RESETTABLE_CLASS(klass);
> +
> +    rc->phases.hold = riscv_iommu_sys_reset_hold;
> +
>      dc->realize = riscv_iommu_sys_realize;
>      set_bit(DEVICE_CATEGORY_MISC, dc->categories);
>      device_class_set_props(dc, riscv_iommu_sys_properties);
> diff --git a/hw/riscv/riscv-iommu.c b/hw/riscv/riscv-iommu.c
> index 239f83f5bd..cf2019d6aa 100644
> --- a/hw/riscv/riscv-iommu.c
> +++ b/hw/riscv/riscv-iommu.c
> @@ -2228,6 +2228,41 @@ static void riscv_iommu_unrealize(DeviceState *dev)
>      g_hash_table_unref(s->ctx_cache);
>  }
>
> +void riscv_iommu_reset(RISCVIOMMUState *s)
> +{
> +    uint32_t reg_clr;
> +    int ddtp_mode;
> +
> +    /*
> +     * Clear DDTP while setting DDTP_mode back to user
> +     * initial setting.
> +     */
> +    ddtp_mode = s->enable_off ?
> +                RISCV_IOMMU_DDTP_MODE_OFF : RISCV_IOMMU_DDTP_MODE_BARE;
> +    s->ddtp = set_field(0, RISCV_IOMMU_DDTP_MODE, ddtp_mode);
> +    riscv_iommu_reg_set64(s, RISCV_IOMMU_REG_DDTP, s->ddtp);
> +
> +    reg_clr = RISCV_IOMMU_CQCSR_CQEN | RISCV_IOMMU_CQCSR_CIE |
> +              RISCV_IOMMU_CQCSR_CQON | RISCV_IOMMU_CQCSR_BUSY;
> +    riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_CQCSR, 0, reg_clr);
> +
> +    reg_clr = RISCV_IOMMU_FQCSR_FQEN | RISCV_IOMMU_FQCSR_FIE |
> +              RISCV_IOMMU_FQCSR_FQON | RISCV_IOMMU_FQCSR_BUSY;
> +    riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_FQCSR, 0, reg_clr);
> +
> +    reg_clr = RISCV_IOMMU_PQCSR_PQEN | RISCV_IOMMU_PQCSR_PIE |
> +              RISCV_IOMMU_PQCSR_PQON | RISCV_IOMMU_PQCSR_BUSY;
> +    riscv_iommu_reg_mod32(s, RISCV_IOMMU_REG_PQCSR, 0, reg_clr);
> +
> +    riscv_iommu_reg_mod64(s, RISCV_IOMMU_REG_TR_REQ_CTL, 0,
> +                          RISCV_IOMMU_TR_REQ_CTL_GO_BUSY);
> +
> +    riscv_iommu_reg_set32(s, RISCV_IOMMU_REG_IPSR, 0);
> +
> +    g_hash_table_remove_all(s->ctx_cache);
> +    g_hash_table_remove_all(s->iot_cache);
> +}
> +
>  static Property riscv_iommu_properties[] = {
>      DEFINE_PROP_UINT32("version", RISCVIOMMUState, version,
>          RISCV_IOMMU_SPEC_DOT_VER),
> diff --git a/hw/riscv/riscv-iommu.h b/hw/riscv/riscv-iommu.h
> index f9f2827808..9424989df4 100644
> --- a/hw/riscv/riscv-iommu.h
> +++ b/hw/riscv/riscv-iommu.h
> @@ -89,6 +89,7 @@ struct RISCVIOMMUState {
>  void riscv_iommu_pci_setup_iommu(RISCVIOMMUState *iommu, PCIBus *bus,
>           Error **errp);
>  void riscv_iommu_set_cap_igs(RISCVIOMMUState *s, riscv_iommu_igs_mode mode);
> +void riscv_iommu_reset(RISCVIOMMUState *s);
>
>  /* private helpers */
>
> diff --git a/hw/riscv/trace-events b/hw/riscv/trace-events
> index 94facbb8b1..aaa2c0eb94 100644
> --- a/hw/riscv/trace-events
> +++ b/hw/riscv/trace-events
> @@ -17,3 +17,5 @@ riscv_iommu_ats_inval(const char *id) "%s: dev-iotlb invalidate"
>  riscv_iommu_ats_prgr(const char *id) "%s: dev-iotlb page request group response"
>  riscv_iommu_sys_irq_sent(uint32_t vector) "IRQ sent to vector %u"
>  riscv_iommu_sys_msi_sent(uint32_t vector, uint64_t msi_addr, uint32_t msi_data, uint32_t result) "MSI sent to vector %u msi_addr 0x%lx msi_data 0x%x result %u"
> +riscv_iommu_sys_reset_hold(int reset_type) "reset type %d"
> +riscv_iommu_pci_reset_hold(int reset_type) "reset type %d"
> diff --git a/include/hw/riscv/iommu.h b/include/hw/riscv/iommu.h
> index 8a8acfc3f0..b03339d75c 100644
> --- a/include/hw/riscv/iommu.h
> +++ b/include/hw/riscv/iommu.h
> @@ -30,12 +30,14 @@ typedef struct RISCVIOMMUState RISCVIOMMUState;
>  typedef struct RISCVIOMMUSpace RISCVIOMMUSpace;
>
>  #define TYPE_RISCV_IOMMU_PCI "riscv-iommu-pci"
> -OBJECT_DECLARE_SIMPLE_TYPE(RISCVIOMMUStatePci, RISCV_IOMMU_PCI)
> +OBJECT_DECLARE_TYPE(RISCVIOMMUStatePci, RISCVIOMMUPciClass, RISCV_IOMMU_PCI)
>  typedef struct RISCVIOMMUStatePci RISCVIOMMUStatePci;
> +typedef struct RISCVIOMMUPciClass RISCVIOMMUPciClass;
>
>  #define TYPE_RISCV_IOMMU_SYS "riscv-iommu-device"
> -OBJECT_DECLARE_SIMPLE_TYPE(RISCVIOMMUStateSys, RISCV_IOMMU_SYS)
> +OBJECT_DECLARE_TYPE(RISCVIOMMUStateSys, RISCVIOMMUSysClass, RISCV_IOMMU_SYS)
>  typedef struct RISCVIOMMUStateSys RISCVIOMMUStateSys;
> +typedef struct RISCVIOMMUSysClass RISCVIOMMUSysClass;
>
>  #define FDT_IRQ_TYPE_EDGE_LOW 1
>
> --
> 2.45.2
>
>

Re: [PATCH for-10.0 7/7] docs/specs: add riscv-iommu-sys information

[Alistair Francis]

On Wed, Nov 6, 2024 at 11:35 PM Daniel Henrique Barboza
<dbarboza@ventanamicro.com> wrote:
>
> Signed-off-by: Daniel Henrique Barboza <dbarboza@ventanamicro.com>

Reviewed-by: Alistair Francis <alistair.francis@wdc.com>

Alistair

> ---
>  docs/specs/riscv-iommu.rst | 30 +++++++++++++++++++++++++++---
>  docs/system/riscv/virt.rst | 10 ++++++++++
>  2 files changed, 37 insertions(+), 3 deletions(-)
>
> diff --git a/docs/specs/riscv-iommu.rst b/docs/specs/riscv-iommu.rst
> index 463f4cffb6..b1538c9ead 100644
> --- a/docs/specs/riscv-iommu.rst
> +++ b/docs/specs/riscv-iommu.rst
> @@ -6,9 +6,9 @@ RISC-V IOMMU support for RISC-V machines
>  QEMU implements a RISC-V IOMMU emulation based on the RISC-V IOMMU spec
>  version 1.0 `iommu1.0`_.
>
> -The emulation includes a PCI reference device, riscv-iommu-pci, that QEMU
> -RISC-V boards can use.  The 'virt' RISC-V machine is compatible with this
> -device.
> +The emulation includes a PCI reference device (riscv-iommu-pci) and a platform
> +bus device (riscv-iommu-sys) that QEMU RISC-V boards can use.  The 'virt'
> +RISC-V machine is compatible with both devices.
>
>  riscv-iommu-pci reference device
>  --------------------------------
> @@ -83,6 +83,30 @@ Several options are available to control the capabilities of the device, namely:
>  - "s-stage": enable s-stage support
>  - "g-stage": enable g-stage support
>
> +riscv-iommu-sys device
> +----------------------
> +
> +This device implements the RISC-V IOMMU emulation as a platform bus device that
> +RISC-V boards can use.
> +
> +For the 'virt' board the device is disabled by default.  To enable it use the
> +'iommu-sys' machine option:
> +
> +.. code-block:: bash
> +
> +  $ qemu-system-riscv64 -M virt,iommu-sys=on (...)
> +
> +There is no options to configure the capabilities of this device in the 'virt'
> +board using the QEMU command line.  The device is configured with the following
> +riscv-iommu options:
> +
> +- "ioatc-limit": default value (2Mb)
> +- "intremap": enabled
> +- "ats": enabled
> +- "off": on (DMA disabled)
> +- "s-stage": enabled
> +- "g-stage": enabled
> +
>  .. _iommu1.0: https://github.com/riscv-non-isa/riscv-iommu/releases/download/v1.0/riscv-iommu.pdf
>
>  .. _linux-v8: https://lore.kernel.org/linux-riscv/cover.1718388908.git.tjeznach@rivosinc.com/
> diff --git a/docs/system/riscv/virt.rst b/docs/system/riscv/virt.rst
> index 8e9a2e4dda..537aac0340 100644
> --- a/docs/system/riscv/virt.rst
> +++ b/docs/system/riscv/virt.rst
> @@ -94,6 +94,12 @@ command line:
>
>    $ qemu-system-riscv64 -M virt -device riscv-iommu-pci (...)
>
> +It also has support for the riscv-iommu-sys platform device:
> +
> +.. code-block:: bash
> +
> +  $ qemu-system-riscv64 -M virt,iommu-sys=on (...)
> +
>  Refer to :ref:`riscv-iommu` for more information on how the RISC-V IOMMU support
>  works.
>
> @@ -129,6 +135,10 @@ The following machine-specific options are supported:
>    having AIA IMSIC (i.e. "aia=aplic-imsic" selected). When not specified,
>    the default number of per-HART VS-level AIA IMSIC pages is 0.
>
> +- iommu-sys=[on|off]
> +
> +  Enables the riscv-iommu-sys platform device. Defaults to 'off'.
> +
>  Running Linux kernel
>  --------------------
>
> --
> 2.45.2
>
>

Re: [PATCH for-10.0 0/7] hw/riscv: riscv-iommu-sys device

[Alistair Francis]

On Wed, Nov 6, 2024 at 11:38 PM Daniel Henrique Barboza
<dbarboza@ventanamicro.com> wrote:
>
> Hi,
>
> Now that we have merged the base IOMMU support we can re-introduce
> the riscv-iommu-sys platform device that was taken away from the initial
> posting.
>
> Aside from adding support for the device in the 'virt' machine we're
> also adding MSI support for it, something that we weren't doing before.
> The Linux driver is then free to choose either MSI or WSI to use the
> device.
>
> Patches based on master.
>
> Daniel Henrique Barboza (5):
>   hw/riscv/riscv-iommu.c: add riscv_iommu_instance_init()
>   hw/riscv/riscv-iommu: parametrize CAP.IGS
>   hw/riscv/virt.c, riscv-iommu-sys.c: add MSIx support
>   hw/riscv/riscv-iommu: implement reset protocol
>   docs/specs: add riscv-iommu-sys information
>
> Sunil V L (1):
>   hw/riscv/virt: Add IOMMU as platform device if the option is set
>
> Tomasz Jeznach (1):
>   hw/riscv: add riscv-iommu-sys platform device

Thanks!

Applied to riscv-to-apply.next

Alistair

>
>  docs/specs/riscv-iommu.rst  |  30 ++++-
>  docs/system/riscv/virt.rst  |  10 ++
>  hw/riscv/meson.build        |   2 +-
>  hw/riscv/riscv-iommu-bits.h |   6 +
>  hw/riscv/riscv-iommu-pci.c  |  21 +++
>  hw/riscv/riscv-iommu-sys.c  | 256 ++++++++++++++++++++++++++++++++++++
>  hw/riscv/riscv-iommu.c      | 114 +++++++++++-----
>  hw/riscv/riscv-iommu.h      |   5 +
>  hw/riscv/trace-events       |   4 +
>  hw/riscv/virt.c             | 108 ++++++++++++++-
>  include/hw/riscv/iommu.h    |  10 +-
>  include/hw/riscv/virt.h     |   6 +-
>  12 files changed, 530 insertions(+), 42 deletions(-)
>  create mode 100644 hw/riscv/riscv-iommu-sys.c
>
> --
> 2.45.2
>
>