[PATCH v2 0/8] sparc/leon3: Add support for -smp

[PATCH v2 0/8] sparc/leon3: Add support for -smp

[Clément Chigot]

V2 modifications
 - Patch1: Add SPDX copyright tags.
 - Patch3: Add defines for MP_STATUS fields. Improve comments.
 - Patch4: Improve a comment.
 - Patch6: Dropped as already merged.

---

This series allows leon3 emulations to record up 4 CPUs.

It requires some enhancements in the grlib_irqmp device and adding the
cpu_index field in the asr17 instruction.

It has been tested locally with various bareboard runtimes.


Clément Chigot (8):
  sparc/grlib: split out the headers for each peripherals
  intc/grlib_irqmp: add ncpus property
  intc/grlib_irqmp: implements the multiprocessor status register
  intc/grlib_irqmp: implements multicore irq
  target/sparc: implement asr17 feature for smp
  leon3: implement multiprocessor
  leon3: check cpu_id in the tiny bootloader
  MAINTAINERS: replace Fabien by myself as Leon3 maintainer

 MAINTAINERS                                   |   2 +-
 hw/char/grlib_apbuart.c                       |   6 +-
 hw/intc/grlib_irqmp.c                         | 112 ++++++++++-----
 hw/sparc/leon3.c                              | 135 +++++++++++++-----
 hw/timer/grlib_gptimer.c                      |   6 +-
 include/hw/char/grlib_uart.h                  |  32 +++++
 .../hw/{sparc/grlib.h => intc/grlib_irqmp.h}  |  18 +--
 include/hw/timer/grlib_gptimer.h              |  32 +++++
 target/sparc/helper.c                         |  16 +++
 target/sparc/helper.h                         |   1 +
 target/sparc/translate.c                      |  13 +-
 11 files changed, 278 insertions(+), 95 deletions(-)
 create mode 100644 include/hw/char/grlib_uart.h
 rename include/hw/{sparc/grlib.h => intc/grlib_irqmp.h} (83%)
 create mode 100644 include/hw/timer/grlib_gptimer.h

-- 
2.25.1

[PATCH v2 1/8] sparc/grlib: split out the headers for each peripherals

[Clément Chigot]

... and move them in their right hardware directory.

Update Copyright and add SPDX-License-Identifier at the same time.

Co-developed-by: Frederic Konrad <konrad.frederic@yahoo.fr>
Signed-off-by: Clément Chigot <chigot@adacore.com>
---
 hw/char/grlib_apbuart.c                       |  6 ++--
 hw/intc/grlib_irqmp.c                         |  6 ++--
 hw/sparc/leon3.c                              |  8 +++--
 hw/timer/grlib_gptimer.c                      |  6 ++--
 include/hw/char/grlib_uart.h                  | 32 +++++++++++++++++++
 .../hw/{sparc/grlib.h => intc/grlib_irqmp.h}  | 16 ++++------
 include/hw/timer/grlib_gptimer.h              | 32 +++++++++++++++++++
 7 files changed, 88 insertions(+), 18 deletions(-)
 create mode 100644 include/hw/char/grlib_uart.h
 rename include/hw/{sparc/grlib.h => intc/grlib_irqmp.h} (86%)
 create mode 100644 include/hw/timer/grlib_gptimer.h

diff --git a/hw/char/grlib_apbuart.c b/hw/char/grlib_apbuart.c
index 82ff40a530..515b65bc07 100644
--- a/hw/char/grlib_apbuart.c
+++ b/hw/char/grlib_apbuart.c
@@ -1,7 +1,9 @@
 /*
  * QEMU GRLIB APB UART Emulator
  *
- * Copyright (c) 2010-2019 AdaCore
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright (c) 2010-2024 AdaCore
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -26,7 +28,7 @@
 #include "hw/irq.h"
 #include "hw/qdev-properties.h"
 #include "hw/qdev-properties-system.h"
-#include "hw/sparc/grlib.h"
+#include "hw/char/grlib_uart.h"
 #include "hw/sysbus.h"
 #include "qemu/module.h"
 #include "chardev/char-fe.h"
diff --git a/hw/intc/grlib_irqmp.c b/hw/intc/grlib_irqmp.c
index 3bfe2544b7..11eef62457 100644
--- a/hw/intc/grlib_irqmp.c
+++ b/hw/intc/grlib_irqmp.c
@@ -3,7 +3,9 @@
  *
  * (Multiprocessor and extended interrupt not supported)
  *
- * Copyright (c) 2010-2019 AdaCore
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright (c) 2010-2024 AdaCore
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -29,7 +31,7 @@
 #include "hw/sysbus.h"
 
 #include "hw/qdev-properties.h"
-#include "hw/sparc/grlib.h"
+#include "hw/intc/grlib_irqmp.h"
 
 #include "trace.h"
 #include "qapi/error.h"
diff --git a/hw/sparc/leon3.c b/hw/sparc/leon3.c
index 2dfb742566..b7d81c76f3 100644
--- a/hw/sparc/leon3.c
+++ b/hw/sparc/leon3.c
@@ -1,7 +1,9 @@
 /*
  * QEMU Leon3 System Emulator
  *
- * Copyright (c) 2010-2019 AdaCore
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright (c) 2010-2024 AdaCore
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -40,7 +42,9 @@
 #include "elf.h"
 #include "trace.h"
 
-#include "hw/sparc/grlib.h"
+#include "hw/timer/grlib_gptimer.h"
+#include "hw/char/grlib_uart.h"
+#include "hw/intc/grlib_irqmp.h"
 #include "hw/misc/grlib_ahb_apb_pnp.h"
 
 /* Default system clock.  */
diff --git a/hw/timer/grlib_gptimer.c b/hw/timer/grlib_gptimer.c
index 5c4923c1e0..4990885451 100644
--- a/hw/timer/grlib_gptimer.c
+++ b/hw/timer/grlib_gptimer.c
@@ -1,7 +1,9 @@
 /*
  * QEMU GRLIB GPTimer Emulator
  *
- * Copyright (c) 2010-2019 AdaCore
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright (c) 2010-2024 AdaCore
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -23,7 +25,7 @@
  */
 
 #include "qemu/osdep.h"
-#include "hw/sparc/grlib.h"
+#include "hw/timer/grlib_gptimer.h"
 #include "hw/sysbus.h"
 #include "qemu/timer.h"
 #include "hw/irq.h"
diff --git a/include/hw/char/grlib_uart.h b/include/hw/char/grlib_uart.h
new file mode 100644
index 0000000000..7496f8fd5e
--- /dev/null
+++ b/include/hw/char/grlib_uart.h
@@ -0,0 +1,32 @@
+/*
+ * QEMU GRLIB UART
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright (c) 2024 AdaCore
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef GRLIB_UART_H
+#define GRLIB_UART_H
+
+#define TYPE_GRLIB_APB_UART "grlib-apbuart"
+
+#endif
diff --git a/include/hw/sparc/grlib.h b/include/hw/intc/grlib_irqmp.h
similarity index 86%
rename from include/hw/sparc/grlib.h
rename to include/hw/intc/grlib_irqmp.h
index ef1946c7f8..c5a90cbb3e 100644
--- a/include/hw/sparc/grlib.h
+++ b/include/hw/intc/grlib_irqmp.h
@@ -1,7 +1,9 @@
 /*
  * QEMU GRLIB Components
  *
- * Copyright (c) 2010-2019 AdaCore
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright (c) 2010-2024 AdaCore
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -22,8 +24,8 @@
  * THE SOFTWARE.
  */
 
-#ifndef GRLIB_H
-#define GRLIB_H
+#ifndef GRLIB_IRQMP_H
+#define GRLIB_IRQMP_H
 
 #include "hw/sysbus.h"
 
@@ -36,10 +38,4 @@
 
 void grlib_irqmp_ack(DeviceState *dev, int intno);
 
-/* GPTimer */
-#define TYPE_GRLIB_GPTIMER "grlib-gptimer"
-
-/* APB UART */
-#define TYPE_GRLIB_APB_UART "grlib-apbuart"
-
-#endif /* GRLIB_H */
+#endif /* GRLIB_IRQMP_H */
diff --git a/include/hw/timer/grlib_gptimer.h b/include/hw/timer/grlib_gptimer.h
new file mode 100644
index 0000000000..e56f1b8bf3
--- /dev/null
+++ b/include/hw/timer/grlib_gptimer.h
@@ -0,0 +1,32 @@
+/*
+ * QEMU GRLIB GPTimer
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Copyright (c) 2024 AdaCore
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifndef GRLIB_GPTIMER_H
+#define GRLIB_GPTIMER_H
+
+#define TYPE_GRLIB_GPTIMER "grlib-gptimer"
+
+#endif
-- 
2.25.1

[PATCH v2 2/8] intc/grlib_irqmp: add ncpus property

[Clément Chigot]

This adds a "ncpus" property to the "grlib-irqmp" device to be used later,
this required a little refactoring of how we initialize the device (ie: use
realize instead of init).

Co-developed-by: Frederic Konrad <konrad.frederic@yahoo.fr>
Signed-off-by: Clément Chigot <chigot@adacore.com>
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>
---
 hw/intc/grlib_irqmp.c | 30 +++++++++++++++++++++---------
 hw/sparc/leon3.c      |  2 +-
 2 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/hw/intc/grlib_irqmp.c b/hw/intc/grlib_irqmp.c
index 11eef62457..744cd64c58 100644
--- a/hw/intc/grlib_irqmp.c
+++ b/hw/intc/grlib_irqmp.c
@@ -1,7 +1,7 @@
 /*
  * QEMU GRLIB IRQMP Emulator
  *
- * (Multiprocessor and extended interrupt not supported)
+ * (Extended interrupt not supported)
  *
  * SPDX-License-Identifier: MIT
  *
@@ -63,6 +63,7 @@ struct IRQMP {
 
     MemoryRegion iomem;
 
+    unsigned int ncpus;
     IRQMPState *state;
     qemu_irq irq;
 };
@@ -326,33 +327,44 @@ static void grlib_irqmp_reset(DeviceState *d)
     irqmp->state->parent = irqmp;
 }
 
-static void grlib_irqmp_init(Object *obj)
+static void grlib_irqmp_realize(DeviceState *dev, Error **errp)
 {
-    IRQMP *irqmp = GRLIB_IRQMP(obj);
-    SysBusDevice *dev = SYS_BUS_DEVICE(obj);
+    IRQMP *irqmp = GRLIB_IRQMP(dev);
 
-    qdev_init_gpio_in(DEVICE(obj), grlib_irqmp_set_irq, MAX_PILS);
-    qdev_init_gpio_out_named(DEVICE(obj), &irqmp->irq, "grlib-irq", 1);
-    memory_region_init_io(&irqmp->iomem, obj, &grlib_irqmp_ops, irqmp,
+    if ((!irqmp->ncpus) || (irqmp->ncpus > IRQMP_MAX_CPU)) {
+        error_setg(errp, "Invalid ncpus properties: "
+                   "%u, must be 0 < ncpus =< %u.", irqmp->ncpus,
+                   IRQMP_MAX_CPU);
+    }
+
+    qdev_init_gpio_in(dev, grlib_irqmp_set_irq, MAX_PILS);
+    qdev_init_gpio_out_named(dev, &irqmp->irq, "grlib-irq", 1);
+    memory_region_init_io(&irqmp->iomem, OBJECT(dev), &grlib_irqmp_ops, irqmp,
                           "irqmp", IRQMP_REG_SIZE);
 
     irqmp->state = g_malloc0(sizeof *irqmp->state);
 
-    sysbus_init_mmio(dev, &irqmp->iomem);
+    sysbus_init_mmio(SYS_BUS_DEVICE(dev), &irqmp->iomem);
 }
 
+static Property grlib_irqmp_properties[] = {
+    DEFINE_PROP_UINT32("ncpus", IRQMP, ncpus, 1),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void grlib_irqmp_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
 
+    dc->realize = grlib_irqmp_realize;
     dc->reset = grlib_irqmp_reset;
+    device_class_set_props(dc, grlib_irqmp_properties);
 }
 
 static const TypeInfo grlib_irqmp_info = {
     .name          = TYPE_GRLIB_IRQMP,
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(IRQMP),
-    .instance_init = grlib_irqmp_init,
     .class_init    = grlib_irqmp_class_init,
 };
 
diff --git a/hw/sparc/leon3.c b/hw/sparc/leon3.c
index b7d81c76f3..b72761b959 100644
--- a/hw/sparc/leon3.c
+++ b/hw/sparc/leon3.c
@@ -265,11 +265,11 @@ static void leon3_generic_hw_init(MachineState *machine)
 
     /* Allocate IRQ manager */
     irqmpdev = qdev_new(TYPE_GRLIB_IRQMP);
+    sysbus_realize_and_unref(SYS_BUS_DEVICE(irqmpdev), &error_fatal);
     qdev_init_gpio_in_named_with_opaque(DEVICE(cpu), leon3_set_pil_in,
                                         env, "pil", 1);
     qdev_connect_gpio_out_named(irqmpdev, "grlib-irq", 0,
                                 qdev_get_gpio_in_named(DEVICE(cpu), "pil", 0));
-    sysbus_realize_and_unref(SYS_BUS_DEVICE(irqmpdev), &error_fatal);
     sysbus_mmio_map(SYS_BUS_DEVICE(irqmpdev), 0, LEON3_IRQMP_OFFSET);
     env->irq_manager = irqmpdev;
     env->qemu_irq_ack = leon3_irq_manager;
-- 
2.25.1

[PATCH v2 3/8] intc/grlib_irqmp: implements the multiprocessor status register

[Clément Chigot]

This implements the multiprocessor status register in grlib-irqmp and bind
it to a start signal, which will be later wired in leon3-generic to
start a cpu.

The EIRQ and BA bits are not implemented.

Based on https://gaisler.com/doc/gr712rc-usermanual.pdf, §8.3.5.

Co-developed-by: Frederic Konrad <konrad.frederic@yahoo.fr>
Signed-off-by: Clément Chigot <chigot@adacore.com>
---
 hw/intc/grlib_irqmp.c | 35 ++++++++++++++++++++++++++++++++---
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/hw/intc/grlib_irqmp.c b/hw/intc/grlib_irqmp.c
index 744cd64c58..fa689e7a1f 100644
--- a/hw/intc/grlib_irqmp.c
+++ b/hw/intc/grlib_irqmp.c
@@ -52,6 +52,10 @@
 #define FORCE_OFFSET     0x80
 #define EXTENDED_OFFSET  0xC0
 
+/* Multiprocessor Status Register  */
+#define MP_STATUS_CPU_STATUS_MASK ((1 << IRQMP_MAX_CPU)-2)
+#define MP_STATUS_NCPU_SHIFT      28
+
 #define MAX_PILS 16
 
 OBJECT_DECLARE_SIMPLE_TYPE(IRQMP, GRLIB_IRQMP)
@@ -65,6 +69,7 @@ struct IRQMP {
 
     unsigned int ncpus;
     IRQMPState *state;
+    qemu_irq start_signal[IRQMP_MAX_CPU];
     qemu_irq irq;
 };
 
@@ -72,6 +77,7 @@ struct IRQMPState {
     uint32_t level;
     uint32_t pending;
     uint32_t clear;
+    uint32_t mpstatus;
     uint32_t broadcast;
 
     uint32_t mask[IRQMP_MAX_CPU];
@@ -182,10 +188,12 @@ static uint64_t grlib_irqmp_read(void *opaque, hwaddr addr,
         return state->force[0];
 
     case CLEAR_OFFSET:
-    case MP_STATUS_OFFSET:
         /* Always read as 0 */
         return 0;
 
+    case MP_STATUS_OFFSET:
+        return state->mpstatus;
+
     case BROADCAST_OFFSET:
         return state->broadcast;
 
@@ -224,8 +232,9 @@ static uint64_t grlib_irqmp_read(void *opaque, hwaddr addr,
 static void grlib_irqmp_write(void *opaque, hwaddr addr,
                               uint64_t value, unsigned size)
 {
-    IRQMP      *irqmp = opaque;
+    IRQMP *irqmp = opaque;
     IRQMPState *state;
+    int i;
 
     assert(irqmp != NULL);
     state = irqmp->state;
@@ -258,7 +267,18 @@ static void grlib_irqmp_write(void *opaque, hwaddr addr,
         return;
 
     case MP_STATUS_OFFSET:
-        /* Read Only (no SMP support) */
+        /*
+         * Writing and reading operations are reversed for the CPU status.
+         * Writing "1" will start the CPU, but reading "1" means that the CPU
+         * is power-down.
+         */
+        value &= MP_STATUS_CPU_STATUS_MASK;
+        for (i = 0; i < irqmp->ncpus; i++) {
+            if ((value >> i) & 1) {
+                qemu_set_irq(irqmp->start_signal[i], 1);
+                state->mpstatus &= ~(1 << i);
+            }
+        }
         return;
 
     case BROADCAST_OFFSET:
@@ -325,6 +345,8 @@ static void grlib_irqmp_reset(DeviceState *d)
 
     memset(irqmp->state, 0, sizeof *irqmp->state);
     irqmp->state->parent = irqmp;
+    irqmp->state->mpstatus = ((irqmp->ncpus - 1) << MP_STATUS_NCPU_SHIFT)
+        | ((1 << irqmp->ncpus) - 2);
 }
 
 static void grlib_irqmp_realize(DeviceState *dev, Error **errp)
@@ -338,6 +360,13 @@ static void grlib_irqmp_realize(DeviceState *dev, Error **errp)
     }
 
     qdev_init_gpio_in(dev, grlib_irqmp_set_irq, MAX_PILS);
+
+    /*
+     * Transitionning from 0 to 1 starts the CPUs. The opposite can't
+     * happen.
+     */
+    qdev_init_gpio_out_named(dev, irqmp->start_signal, "grlib-start-cpu",
+                             IRQMP_MAX_CPU);
     qdev_init_gpio_out_named(dev, &irqmp->irq, "grlib-irq", 1);
     memory_region_init_io(&irqmp->iomem, OBJECT(dev), &grlib_irqmp_ops, irqmp,
                           "irqmp", IRQMP_REG_SIZE);
-- 
2.25.1

[PATCH v2 4/8] intc/grlib_irqmp: implements multicore irq

[Clément Chigot]

Now there is an ncpus property, use it in order to deliver the IRQ to
multiple CPU.

Co-developed-by: Frederic Konrad <konrad.frederic@yahoo.fr>
Signed-off-by: Clément Chigot <chigot@adacore.com>
---
 hw/intc/grlib_irqmp.c         | 43 ++++++++++++++++++-----------------
 hw/sparc/leon3.c              |  3 ++-
 include/hw/intc/grlib_irqmp.h |  2 +-
 3 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/hw/intc/grlib_irqmp.c b/hw/intc/grlib_irqmp.c
index 8299ac183e..91237e6c44 100644
--- a/hw/intc/grlib_irqmp.c
+++ b/hw/intc/grlib_irqmp.c
@@ -70,7 +70,7 @@ struct IRQMP {
     unsigned int ncpus;
     IRQMPState *state;
     qemu_irq start_signal[IRQMP_MAX_CPU];
-    qemu_irq irq;
+    qemu_irq irq[IRQMP_MAX_CPU];
 };
 
 struct IRQMPState {
@@ -89,37 +89,37 @@ struct IRQMPState {
 
 static void grlib_irqmp_check_irqs(IRQMPState *state)
 {
-    uint32_t      pend   = 0;
-    uint32_t      level0 = 0;
-    uint32_t      level1 = 0;
+    uint32_t pend = 0;
+    uint32_t level0 = 0;
+    uint32_t level1 = 0;
+    int i;
 
     assert(state != NULL);
     assert(state->parent != NULL);
 
-    /* IRQ for CPU 0 (no SMP support) */
-    pend = (state->pending | state->force[0])
-        & state->mask[0];
-
-    level0 = pend & ~state->level;
-    level1 = pend &  state->level;
+    for (i = 0; i < state->parent->ncpus; i++) {
+        pend = (state->pending | state->force[i]) & state->mask[i];
+        level0 = pend & ~state->level;
+        level1 = pend &  state->level;
 
-    trace_grlib_irqmp_check_irqs(state->pending, state->force[0],
-                                 state->mask[0], level1, level0);
+        trace_grlib_irqmp_check_irqs(state->pending, state->force[i],
+                                     state->mask[i], level1, level0);
 
-    /* Trigger level1 interrupt first and level0 if there is no level1 */
-    qemu_set_irq(state->parent->irq, level1 ?: level0);
+        /* Trigger level1 interrupt first and level0 if there is no level1 */
+        qemu_set_irq(state->parent->irq[i], level1 ?: level0);
+    }
 }
 
-static void grlib_irqmp_ack_mask(IRQMPState *state, uint32_t mask)
+static void grlib_irqmp_ack_mask(IRQMPState *state, int cpu, uint32_t mask)
 {
     /* Clear registers */
     state->pending  &= ~mask;
-    state->force[0] &= ~mask; /* Only CPU 0 (No SMP support) */
+    state->force[cpu] &= ~mask;
 
     grlib_irqmp_check_irqs(state);
 }
 
-void grlib_irqmp_ack(DeviceState *dev, int intno)
+void grlib_irqmp_ack(DeviceState *dev, int cpu, int intno)
 {
     IRQMP        *irqmp = GRLIB_IRQMP(dev);
     IRQMPState   *state;
@@ -133,7 +133,7 @@ void grlib_irqmp_ack(DeviceState *dev, int intno)
 
     trace_grlib_irqmp_ack(intno);
 
-    grlib_irqmp_ack_mask(state, mask);
+    grlib_irqmp_ack_mask(state, cpu, mask);
 }
 
 static void grlib_irqmp_set_irq(void *opaque, int irq, int level)
@@ -159,7 +159,6 @@ static void grlib_irqmp_set_irq(void *opaque, int irq, int level)
             s->pending |= 1 << irq;
         }
         grlib_irqmp_check_irqs(s);
-
     }
 }
 
@@ -263,7 +262,9 @@ static void grlib_irqmp_write(void *opaque, hwaddr addr,
 
     case CLEAR_OFFSET:
         value &= ~1; /* clean up the value */
-        grlib_irqmp_ack_mask(state, value);
+        for (i = 0; i < irqmp->ncpus; i++) {
+            grlib_irqmp_ack_mask(state, i, value);
+        }
         return;
 
     case MP_STATUS_OFFSET:
@@ -367,7 +368,7 @@ static void grlib_irqmp_realize(DeviceState *dev, Error **errp)
      */
     qdev_init_gpio_out_named(dev, irqmp->start_signal, "grlib-start-cpu",
                              IRQMP_MAX_CPU);
-    qdev_init_gpio_out_named(dev, &irqmp->irq, "grlib-irq", 1);
+    qdev_init_gpio_out_named(dev, irqmp->irq, "grlib-irq", irqmp->ncpus);
     memory_region_init_io(&irqmp->iomem, OBJECT(dev), &grlib_irqmp_ops, irqmp,
                           "irqmp", IRQMP_REG_SIZE);
 
diff --git a/hw/sparc/leon3.c b/hw/sparc/leon3.c
index b72761b959..7866f0a049 100644
--- a/hw/sparc/leon3.c
+++ b/hw/sparc/leon3.c
@@ -170,7 +170,8 @@ static void leon3_cache_control_int(CPUSPARCState *env)
 
 static void leon3_irq_ack(CPUSPARCState *env, int intno)
 {
-    grlib_irqmp_ack(env->irq_manager, intno);
+    /* No SMP support yet, only CPU #0 available so far.  */
+    grlib_irqmp_ack(env->irq_manager, 0, intno);
 }
 
 /*
diff --git a/include/hw/intc/grlib_irqmp.h b/include/hw/intc/grlib_irqmp.h
index c5a90cbb3e..b564a0009f 100644
--- a/include/hw/intc/grlib_irqmp.h
+++ b/include/hw/intc/grlib_irqmp.h
@@ -36,6 +36,6 @@
 /* IRQMP */
 #define TYPE_GRLIB_IRQMP "grlib-irqmp"
 
-void grlib_irqmp_ack(DeviceState *dev, int intno);
+void grlib_irqmp_ack(DeviceState *dev, int cpu, int intno);
 
 #endif /* GRLIB_IRQMP_H */
-- 
2.25.1

[PATCH v2 5/8] target/sparc: implement asr17 feature for smp

[Clément Chigot]

This allows the guest program to know its cpu id.

Co-developed-by: Frederic Konrad <konrad.frederic@yahoo.fr>
Signed-off-by: Clément Chigot <chigot@adacore.com>
Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
---
 target/sparc/helper.c    | 16 ++++++++++++++++
 target/sparc/helper.h    |  1 +
 target/sparc/translate.c | 13 +++----------
 3 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/target/sparc/helper.c b/target/sparc/helper.c
index bd10b60e4b..2247e243b5 100644
--- a/target/sparc/helper.c
+++ b/target/sparc/helper.c
@@ -212,4 +212,20 @@ void helper_power_down(CPUSPARCState *env)
     env->npc = env->pc + 4;
     cpu_loop_exit(cs);
 }
+
+target_ulong helper_rdasr17(CPUSPARCState *env)
+{
+    CPUState *cs = env_cpu(env);
+    target_ulong val;
+
+    /*
+     * TODO: There are many more fields to be filled,
+     * some of which are writable.
+     */
+    val = env->def.nwindows - 1;    /* [4:0]   NWIN   */
+    val |= 1 << 8;                  /* [8]      V8    */
+    val |= (cs->cpu_index) << 28;   /* [31:28] INDEX  */
+
+    return val;
+}
 #endif
diff --git a/target/sparc/helper.h b/target/sparc/helper.h
index 55eff66283..fc818b8678 100644
--- a/target/sparc/helper.h
+++ b/target/sparc/helper.h
@@ -2,6 +2,7 @@
 DEF_HELPER_1(rett, void, env)
 DEF_HELPER_2(wrpsr, void, env, tl)
 DEF_HELPER_1(rdpsr, tl, env)
+DEF_HELPER_1(rdasr17, tl, env)
 DEF_HELPER_1(power_down, void, env)
 #else
 DEF_HELPER_FLAGS_2(wrpil, TCG_CALL_NO_RWG, void, env, tl)
diff --git a/target/sparc/translate.c b/target/sparc/translate.c
index 9387299559..1cabda9565 100644
--- a/target/sparc/translate.c
+++ b/target/sparc/translate.c
@@ -37,6 +37,7 @@
 
 #ifdef TARGET_SPARC64
 # define gen_helper_rdpsr(D, E)                 qemu_build_not_reached()
+# define gen_helper_rdasr17(D, E)               qemu_build_not_reached()
 # define gen_helper_rett(E)                     qemu_build_not_reached()
 # define gen_helper_power_down(E)               qemu_build_not_reached()
 # define gen_helper_wrpsr(E, S)                 qemu_build_not_reached()
@@ -2681,16 +2682,8 @@ static bool trans_RDY(DisasContext *dc, arg_RDY *a)
 
 static TCGv do_rd_leon3_config(DisasContext *dc, TCGv dst)
 {
-    uint32_t val;
-
-    /*
-     * TODO: There are many more fields to be filled,
-     * some of which are writable.
-     */
-    val = dc->def->nwindows - 1;   /* [4:0] NWIN */
-    val |= 1 << 8;                 /* [8]   V8   */
-
-    return tcg_constant_tl(val);
+    gen_helper_rdasr17(dst, tcg_env);
+    return dst;
 }
 
 TRANS(RDASR17, ASR17, do_rd_special, true, a->rd, do_rd_leon3_config)
-- 
2.25.1

[PATCH v2 6/8] leon3: implement multiprocessor

[Clément Chigot]

This allows to register more than one CPU on the leon3_generic machine.

Co-developed-by: Frederic Konrad <konrad.frederic@yahoo.fr>
Signed-off-by: Clément Chigot <chigot@adacore.com>
---
 hw/sparc/leon3.c | 106 +++++++++++++++++++++++++++++++++--------------
 1 file changed, 74 insertions(+), 32 deletions(-)

diff --git a/hw/sparc/leon3.c b/hw/sparc/leon3.c
index 7866f0a049..eacd85ee4f 100644
--- a/hw/sparc/leon3.c
+++ b/hw/sparc/leon3.c
@@ -54,6 +54,8 @@
 #define LEON3_PROM_OFFSET    (0x00000000)
 #define LEON3_RAM_OFFSET     (0x40000000)
 
+#define MAX_CPUS  4
+
 #define LEON3_UART_OFFSET  (0x80000100)
 #define LEON3_UART_IRQ     (3)
 
@@ -67,9 +69,12 @@
 #define LEON3_AHB_PNP_OFFSET (0xFFFFF000)
 
 typedef struct ResetData {
-    SPARCCPU *cpu;
-    uint32_t  entry;            /* save kernel entry in case of reset */
-    target_ulong sp;            /* initial stack pointer */
+    struct CPUResetData {
+        int id;
+        SPARCCPU *cpu;
+        target_ulong sp;  /* initial stack pointer */
+    } info[MAX_CPUS];
+    uint32_t entry;             /* save kernel entry in case of reset */
 } ResetData;
 
 static uint32_t *gen_store_u32(uint32_t *code, hwaddr addr, uint32_t val)
@@ -125,18 +130,19 @@ static void write_bootloader(CPUSPARCState *env, uint8_t *base,
     stl_p(p++, 0x01000000); /* nop */
 }
 
-static void main_cpu_reset(void *opaque)
+static void leon3_cpu_reset(void *opaque)
 {
-    ResetData *s   = (ResetData *)opaque;
-    CPUState *cpu = CPU(s->cpu);
-    CPUSPARCState  *env = &s->cpu->env;
+    struct CPUResetData *info = (struct CPUResetData *) opaque;
+    int id = info->id;
+    ResetData *s = (ResetData *)DO_UPCAST(ResetData, info[id], info);
+    CPUState *cpu = CPU(s->info[id].cpu);
+    CPUSPARCState *env = cpu_env(cpu);
 
     cpu_reset(cpu);
-
-    cpu->halted = 0;
-    env->pc     = s->entry;
-    env->npc    = s->entry + 4;
-    env->regbase[6] = s->sp;
+    cpu->halted = cpu->cpu_index != 0;
+    env->pc = s->entry;
+    env->npc = s->entry + 4;
+    env->regbase[6] = s->info[id].sp;
 }
 
 static void leon3_cache_control_int(CPUSPARCState *env)
@@ -170,8 +176,8 @@ static void leon3_cache_control_int(CPUSPARCState *env)
 
 static void leon3_irq_ack(CPUSPARCState *env, int intno)
 {
-    /* No SMP support yet, only CPU #0 available so far.  */
-    grlib_irqmp_ack(env->irq_manager, 0, intno);
+    CPUState *cpu = CPU(env_cpu(env));
+    grlib_irqmp_ack(env->irq_manager, cpu->cpu_index, intno);
 }
 
 /*
@@ -213,6 +219,20 @@ static void leon3_set_pil_in(void *opaque, int n, int level)
     }
 }
 
+static void leon3_start_cpu_async_work(CPUState *cpu, run_on_cpu_data data)
+{
+    cpu->halted = 0;
+}
+
+static void leon3_start_cpu(void *opaque, int n, int level)
+{
+    CPUState *cs = CPU(opaque);
+
+    if (level) {
+        async_run_on_cpu(cs, leon3_start_cpu_async_work, RUN_ON_CPU_NULL);
+    }
+}
+
 static void leon3_irq_manager(CPUSPARCState *env, int intno)
 {
     leon3_irq_ack(env, intno);
@@ -238,17 +258,21 @@ static void leon3_generic_hw_init(MachineState *machine)
     AHBPnp *ahb_pnp;
     APBPnp *apb_pnp;
 
-    /* Init CPU */
-    cpu = SPARC_CPU(cpu_create(machine->cpu_type));
-    env = &cpu->env;
+    reset_info = g_malloc0(sizeof(ResetData));
 
-    cpu_sparc_set_id(env, 0);
+    for (i = 0; i < machine->smp.cpus; i++) {
+        /* Init CPU */
+        cpu = SPARC_CPU(cpu_create(machine->cpu_type));
+        env = &cpu->env;
 
-    /* Reset data */
-    reset_info        = g_new0(ResetData, 1);
-    reset_info->cpu   = cpu;
-    reset_info->sp    = LEON3_RAM_OFFSET + ram_size;
-    qemu_register_reset(main_cpu_reset, reset_info);
+        cpu_sparc_set_id(env, i);
+
+        /* Reset data */
+        reset_info->info[i].id = i;
+        reset_info->info[i].cpu = cpu;
+        reset_info->info[i].sp = LEON3_RAM_OFFSET + ram_size;
+        qemu_register_reset(leon3_cpu_reset, &reset_info->info[i]);
+    }
 
     ahb_pnp = GRLIB_AHB_PNP(qdev_new(TYPE_GRLIB_AHB_PNP));
     sysbus_realize_and_unref(SYS_BUS_DEVICE(ahb_pnp), &error_fatal);
@@ -266,14 +290,28 @@ static void leon3_generic_hw_init(MachineState *machine)
 
     /* Allocate IRQ manager */
     irqmpdev = qdev_new(TYPE_GRLIB_IRQMP);
+    object_property_set_int(OBJECT(irqmpdev), "ncpus", machine->smp.cpus,
+                            &error_fatal);
     sysbus_realize_and_unref(SYS_BUS_DEVICE(irqmpdev), &error_fatal);
-    qdev_init_gpio_in_named_with_opaque(DEVICE(cpu), leon3_set_pil_in,
-                                        env, "pil", 1);
-    qdev_connect_gpio_out_named(irqmpdev, "grlib-irq", 0,
-                                qdev_get_gpio_in_named(DEVICE(cpu), "pil", 0));
+
+    for (i = 0; i < machine->smp.cpus; i++) {
+        cpu = reset_info->info[i].cpu;
+        env = &cpu->env;
+        qdev_init_gpio_in_named_with_opaque(DEVICE(cpu), leon3_start_cpu,
+                                            cpu, "start_cpu", 1);
+        qdev_connect_gpio_out_named(irqmpdev, "grlib-start-cpu", i,
+                                    qdev_get_gpio_in_named(DEVICE(cpu),
+                                                           "start_cpu", 0));
+        qdev_init_gpio_in_named_with_opaque(DEVICE(cpu), leon3_set_pil_in,
+                                            env, "pil", 1);
+        qdev_connect_gpio_out_named(irqmpdev, "grlib-irq", i,
+                                    qdev_get_gpio_in_named(DEVICE(cpu),
+                                                           "pil", 0));
+        env->irq_manager = irqmpdev;
+        env->qemu_irq_ack = leon3_irq_manager;
+    }
+
     sysbus_mmio_map(SYS_BUS_DEVICE(irqmpdev), 0, LEON3_IRQMP_OFFSET);
-    env->irq_manager = irqmpdev;
-    env->qemu_irq_ack = leon3_irq_manager;
     grlib_apb_pnp_add_entry(apb_pnp, LEON3_IRQMP_OFFSET, 0xFFF,
                             GRLIB_VENDOR_GAISLER, GRLIB_IRQMP_DEV,
                             2, 0, GRLIB_APBIO_AREA);
@@ -347,10 +385,13 @@ static void leon3_generic_hw_init(MachineState *machine)
             uint8_t *bootloader_entry;
 
             bootloader_entry = memory_region_get_ram_ptr(prom);
-            write_bootloader(env, bootloader_entry, entry);
-            env->pc = LEON3_PROM_OFFSET;
-            env->npc = LEON3_PROM_OFFSET + 4;
+            write_bootloader(&reset_info->info[0].cpu->env, bootloader_entry,
+                             entry);
             reset_info->entry = LEON3_PROM_OFFSET;
+            for (i = 0; i < machine->smp.cpus; i++) {
+                reset_info->info[i].cpu->env.pc = LEON3_PROM_OFFSET;
+                reset_info->info[i].cpu->env.npc = LEON3_PROM_OFFSET + 4;
+            }
         }
     }
 
@@ -389,6 +430,7 @@ static void leon3_generic_machine_init(MachineClass *mc)
     mc->init = leon3_generic_hw_init;
     mc->default_cpu_type = SPARC_CPU_TYPE_NAME("LEON3");
     mc->default_ram_id = "leon3.ram";
+    mc->max_cpus = MAX_CPUS;
 }
 
 DEFINE_MACHINE("leon3_generic", leon3_generic_machine_init)
-- 
2.25.1

[PATCH v2 7/8] leon3: check cpu_id in the tiny bootloader

[Clément Chigot]

Now that SMP is possible, the asr17 must be checked in the little boot code
or the secondary CPU will reinitialize the Timer and the Uart.

Co-developed-by: Frederic Konrad <konrad.frederic@yahoo.fr>
Signed-off-by: Clément Chigot <chigot@adacore.com>
---
 hw/sparc/leon3.c | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/hw/sparc/leon3.c b/hw/sparc/leon3.c
index eacd85ee4f..87a8044a3e 100644
--- a/hw/sparc/leon3.c
+++ b/hw/sparc/leon3.c
@@ -100,13 +100,27 @@ static uint32_t *gen_store_u32(uint32_t *code, hwaddr addr, uint32_t val)
 
 /*
  * When loading a kernel in RAM the machine is expected to be in a different
- * state (eg: initialized by the bootloader). This little code reproduces
- * this behavior.
+ * state (eg: initialized by the bootloader).  This little code reproduces
+ * this behavior.  Also this code can be executed by the secondary cpus as
+ * well since it looks at the %asr17 register before doing any
+ * initialization, it allows to use the same reset address for all the
+ * cpus.
  */
 static void write_bootloader(CPUSPARCState *env, uint8_t *base,
                              hwaddr kernel_addr)
 {
     uint32_t *p = (uint32_t *) base;
+    uint32_t *sec_cpu_branch_p = NULL;
+
+    /* If we are running on a secondary CPU, jump directly to the kernel.  */
+
+    stl_p(p++, 0x85444000); /* rd %asr17, %g2      */
+    stl_p(p++, 0x8530a01c); /* srl  %g2, 0x1c, %g2 */
+    stl_p(p++, 0x80908000); /* tst  %g2            */
+    /* Fill that later.  */
+    sec_cpu_branch_p = p;
+    stl_p(p++, 0x0BADC0DE); /* bne xxx             */
+    stl_p(p++, 0x01000000); /* nop */
 
     /* Initialize the UARTs                                        */
     /* *UART_CONTROL = UART_RECEIVE_ENABLE | UART_TRANSMIT_ENABLE; */
@@ -120,6 +134,10 @@ static void write_bootloader(CPUSPARCState *env, uint8_t *base,
     /* *GPTIMER0_CONFIG = GPTIMER_ENABLE | GPTIMER_RESTART;        */
     p = gen_store_u32(p, 0x80000318, 3);
 
+    /* Now, the relative branch above can be computed.  */
+    stl_p(sec_cpu_branch_p, 0x12800000
+          + (p - sec_cpu_branch_p));
+
     /* JUMP to the entry point                                     */
     stl_p(p++, 0x82100000); /* mov %g0, %g1 */
     stl_p(p++, 0x03000000 + extract32(kernel_addr, 10, 22));
-- 
2.25.1

[PATCH v2 8/8] MAINTAINERS: replace Fabien by myself as Leon3 maintainer

[Clément Chigot]

CC: Fabien Chouteau <chouteau@adacore.com>
Signed-off-by: Clément Chigot <chigot@adacore.com>
Reviewed-by: Fabien Chouteau <chouteau@adacore.com>
---
 MAINTAINERS | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/MAINTAINERS b/MAINTAINERS
index b406fb20c0..b4e78e7748 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -1708,7 +1708,7 @@ F: hw/rtc/sun4v-rtc.c
 F: include/hw/rtc/sun4v-rtc.h
 
 Leon3
-M: Fabien Chouteau <chouteau@adacore.com>
+M: Clément Chigot <chigot@adacore.com>
 M: Frederic Konrad <konrad.frederic@yahoo.fr>
 S: Maintained
 F: hw/sparc/leon3.c
-- 
2.25.1

Re: [PATCH v2 0/8] sparc/leon3: Add support for -smp

[Clément Chigot]

Hey Philippe,

Gentle ping on this serie

Thanks,
Clément



On Tue, Jan 16, 2024 at 2:02 PM Clément Chigot <chigot@adacore.com> wrote:
>
> V2 modifications
>  - Patch1: Add SPDX copyright tags.
>  - Patch3: Add defines for MP_STATUS fields. Improve comments.
>  - Patch4: Improve a comment.
>  - Patch6: Dropped as already merged.
>
> ---
>
> This series allows leon3 emulations to record up 4 CPUs.
>
> It requires some enhancements in the grlib_irqmp device and adding the
> cpu_index field in the asr17 instruction.
>
> It has been tested locally with various bareboard runtimes.
>
>
> Clément Chigot (8):
>   sparc/grlib: split out the headers for each peripherals
>   intc/grlib_irqmp: add ncpus property
>   intc/grlib_irqmp: implements the multiprocessor status register
>   intc/grlib_irqmp: implements multicore irq
>   target/sparc: implement asr17 feature for smp
>   leon3: implement multiprocessor
>   leon3: check cpu_id in the tiny bootloader
>   MAINTAINERS: replace Fabien by myself as Leon3 maintainer
>
>  MAINTAINERS                                   |   2 +-
>  hw/char/grlib_apbuart.c                       |   6 +-
>  hw/intc/grlib_irqmp.c                         | 112 ++++++++++-----
>  hw/sparc/leon3.c                              | 135 +++++++++++++-----
>  hw/timer/grlib_gptimer.c                      |   6 +-
>  include/hw/char/grlib_uart.h                  |  32 +++++
>  .../hw/{sparc/grlib.h => intc/grlib_irqmp.h}  |  18 +--
>  include/hw/timer/grlib_gptimer.h              |  32 +++++
>  target/sparc/helper.c                         |  16 +++
>  target/sparc/helper.h                         |   1 +
>  target/sparc/translate.c                      |  13 +-
>  11 files changed, 278 insertions(+), 95 deletions(-)
>  create mode 100644 include/hw/char/grlib_uart.h
>  rename include/hw/{sparc/grlib.h => intc/grlib_irqmp.h} (83%)
>  create mode 100644 include/hw/timer/grlib_gptimer.h
>
> --
> 2.25.1
>

Re: [PATCH v2 1/8] sparc/grlib: split out the headers for each peripherals

[Philippe Mathieu-Daudé]

On 16/1/24 14:02, Clément Chigot wrote:
> ... and move them in their right hardware directory.
> 
> Update Copyright and add SPDX-License-Identifier at the same time.
> 
> Co-developed-by: Frederic Konrad <konrad.frederic@yahoo.fr>
> Signed-off-by: Clément Chigot <chigot@adacore.com>
> ---
>   hw/char/grlib_apbuart.c                       |  6 ++--
>   hw/intc/grlib_irqmp.c                         |  6 ++--
>   hw/sparc/leon3.c                              |  8 +++--
>   hw/timer/grlib_gptimer.c                      |  6 ++--
>   include/hw/char/grlib_uart.h                  | 32 +++++++++++++++++++
>   .../hw/{sparc/grlib.h => intc/grlib_irqmp.h}  | 16 ++++------
>   include/hw/timer/grlib_gptimer.h              | 32 +++++++++++++++++++
>   7 files changed, 88 insertions(+), 18 deletions(-)
>   create mode 100644 include/hw/char/grlib_uart.h
>   rename include/hw/{sparc/grlib.h => intc/grlib_irqmp.h} (86%)
>   create mode 100644 include/hw/timer/grlib_gptimer.h

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>

Re: [PATCH v2 4/8] intc/grlib_irqmp: implements multicore irq

[Philippe Mathieu-Daudé]

On 16/1/24 14:02, Clément Chigot wrote:
> Now there is an ncpus property, use it in order to deliver the IRQ to
> multiple CPU.
> 
> Co-developed-by: Frederic Konrad <konrad.frederic@yahoo.fr>
> Signed-off-by: Clément Chigot <chigot@adacore.com>
> ---
>   hw/intc/grlib_irqmp.c         | 43 ++++++++++++++++++-----------------
>   hw/sparc/leon3.c              |  3 ++-
>   include/hw/intc/grlib_irqmp.h |  2 +-
>   3 files changed, 25 insertions(+), 23 deletions(-)


>   static void grlib_irqmp_check_irqs(IRQMPState *state)
>   {
> -    uint32_t      pend   = 0;
> -    uint32_t      level0 = 0;
> -    uint32_t      level1 = 0;
> +    uint32_t pend = 0;
> +    uint32_t level0 = 0;
> +    uint32_t level1 = 0;
> +    int i;
>   
>       assert(state != NULL);
>       assert(state->parent != NULL);
>   
> -    /* IRQ for CPU 0 (no SMP support) */
> -    pend = (state->pending | state->force[0])
> -        & state->mask[0];
> -
> -    level0 = pend & ~state->level;
> -    level1 = pend &  state->level;
> +    for (i = 0; i < state->parent->ncpus; i++) {
> +        pend = (state->pending | state->force[i]) & state->mask[i];
> +        level0 = pend & ~state->level;
> +        level1 = pend &  state->level;

     for (unsigned i = 0; i < state->parent->ncpus; i++) {
         uint32_t pend = (state->pending | state->force[i])
                         & state->mask[i];
         uint32_t level0 = pend & ~state->level;
         uint32_t level1 = pend &  state->level;

>   
> -    trace_grlib_irqmp_check_irqs(state->pending, state->force[0],
> -                                 state->mask[0], level1, level0);
> +        trace_grlib_irqmp_check_irqs(state->pending, state->force[i],
> +                                     state->mask[i], level1, level0);
>   
> -    /* Trigger level1 interrupt first and level0 if there is no level1 */
> -    qemu_set_irq(state->parent->irq, level1 ?: level0);
> +        /* Trigger level1 interrupt first and level0 if there is no level1 */
> +        qemu_set_irq(state->parent->irq[i], level1 ?: level0);
> +    }
>   }
>   
> -static void grlib_irqmp_ack_mask(IRQMPState *state, uint32_t mask)
> +static void grlib_irqmp_ack_mask(IRQMPState *state, int cpu, uint32_t mask)

unsigned cpu, ...

>   {
>       /* Clear registers */
>       state->pending  &= ~mask;
> -    state->force[0] &= ~mask; /* Only CPU 0 (No SMP support) */
> +    state->force[cpu] &= ~mask;
>   
>       grlib_irqmp_check_irqs(state);
>   }
>   
> -void grlib_irqmp_ack(DeviceState *dev, int intno)
> +void grlib_irqmp_ack(DeviceState *dev, int cpu, int intno)

unsigned cpu, ...

>   {
>       IRQMP        *irqmp = GRLIB_IRQMP(dev);
>       IRQMPState   *state;
> @@ -133,7 +133,7 @@ void grlib_irqmp_ack(DeviceState *dev, int intno)
>   
>       trace_grlib_irqmp_ack(intno);
>   
> -    grlib_irqmp_ack_mask(state, mask);
> +    grlib_irqmp_ack_mask(state, cpu, mask);
>   }
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>

Re: [PATCH v2 7/8] leon3: check cpu_id in the tiny bootloader

[Philippe Mathieu-Daudé]

On 16/1/24 14:02, Clément Chigot wrote:
> Now that SMP is possible, the asr17 must be checked in the little boot code
> or the secondary CPU will reinitialize the Timer and the Uart.
> 
> Co-developed-by: Frederic Konrad <konrad.frederic@yahoo.fr>
> Signed-off-by: Clément Chigot <chigot@adacore.com>
> ---
>   hw/sparc/leon3.c | 22 ++++++++++++++++++++--
>   1 file changed, 20 insertions(+), 2 deletions(-)
> 
> diff --git a/hw/sparc/leon3.c b/hw/sparc/leon3.c
> index eacd85ee4f..87a8044a3e 100644
> --- a/hw/sparc/leon3.c
> +++ b/hw/sparc/leon3.c
> @@ -100,13 +100,27 @@ static uint32_t *gen_store_u32(uint32_t *code, hwaddr addr, uint32_t val)
>   
>   /*
>    * When loading a kernel in RAM the machine is expected to be in a different
> - * state (eg: initialized by the bootloader). This little code reproduces
> - * this behavior.
> + * state (eg: initialized by the bootloader).  This little code reproduces
> + * this behavior.  Also this code can be executed by the secondary cpus as
> + * well since it looks at the %asr17 register before doing any
> + * initialization, it allows to use the same reset address for all the
> + * cpus.
>    */
>   static void write_bootloader(CPUSPARCState *env, uint8_t *base,
>                                hwaddr kernel_addr)
>   {
>       uint32_t *p = (uint32_t *) base;
> +    uint32_t *sec_cpu_branch_p = NULL;
> +
> +    /* If we are running on a secondary CPU, jump directly to the kernel.  */
> +
> +    stl_p(p++, 0x85444000); /* rd %asr17, %g2      */
> +    stl_p(p++, 0x8530a01c); /* srl  %g2, 0x1c, %g2 */
> +    stl_p(p++, 0x80908000); /* tst  %g2            */
> +    /* Fill that later.  */

       /* Filled below. */

> +    sec_cpu_branch_p = p;
> +    stl_p(p++, 0x0BADC0DE); /* bne xxx             */
> +    stl_p(p++, 0x01000000); /* nop */
>   
>       /* Initialize the UARTs                                        */
>       /* *UART_CONTROL = UART_RECEIVE_ENABLE | UART_TRANSMIT_ENABLE; */
> @@ -120,6 +134,10 @@ static void write_bootloader(CPUSPARCState *env, uint8_t *base,
>       /* *GPTIMER0_CONFIG = GPTIMER_ENABLE | GPTIMER_RESTART;        */
>       p = gen_store_u32(p, 0x80000318, 3);
>   
> +    /* Now, the relative branch above can be computed.  */
> +    stl_p(sec_cpu_branch_p, 0x12800000
> +          + (p - sec_cpu_branch_p));
> +
>       /* JUMP to the entry point                                     */
>       stl_p(p++, 0x82100000); /* mov %g0, %g1 */
>       stl_p(p++, 0x03000000 + extract32(kernel_addr, 10, 22));

Alternatively have main_cpu_reset / secondary_cpu_reset handlers.
You could split BL in HWINIT / JUMP, have HWINIT() return # instr
used and adjust secondary_cpu_reset entry.

Anyway,
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>

Re: [PATCH v2 6/8] leon3: implement multiprocessor

[Philippe Mathieu-Daudé]

Hi Clément,

On 16/1/24 14:02, Clément Chigot wrote:
> This allows to register more than one CPU on the leon3_generic machine.
> 
> Co-developed-by: Frederic Konrad <konrad.frederic@yahoo.fr>
> Signed-off-by: Clément Chigot <chigot@adacore.com>
> ---
>   hw/sparc/leon3.c | 106 +++++++++++++++++++++++++++++++++--------------
>   1 file changed, 74 insertions(+), 32 deletions(-)
> 
> diff --git a/hw/sparc/leon3.c b/hw/sparc/leon3.c
> index 7866f0a049..eacd85ee4f 100644
> --- a/hw/sparc/leon3.c
> +++ b/hw/sparc/leon3.c
> @@ -54,6 +54,8 @@
>   #define LEON3_PROM_OFFSET    (0x00000000)
>   #define LEON3_RAM_OFFSET     (0x40000000)
>   
> +#define MAX_CPUS  4
> +
>   #define LEON3_UART_OFFSET  (0x80000100)
>   #define LEON3_UART_IRQ     (3)
>   
> @@ -67,9 +69,12 @@
>   #define LEON3_AHB_PNP_OFFSET (0xFFFFF000)
>   
>   typedef struct ResetData {
> -    SPARCCPU *cpu;
> -    uint32_t  entry;            /* save kernel entry in case of reset */
> -    target_ulong sp;            /* initial stack pointer */
> +    struct CPUResetData {
> +        int id;
> +        SPARCCPU *cpu;
> +        target_ulong sp;  /* initial stack pointer */
> +    } info[MAX_CPUS];
> +    uint32_t entry;             /* save kernel entry in case of reset */
>   } ResetData;
>   
>   static uint32_t *gen_store_u32(uint32_t *code, hwaddr addr, uint32_t val)
> @@ -125,18 +130,19 @@ static void write_bootloader(CPUSPARCState *env, uint8_t *base,
>       stl_p(p++, 0x01000000); /* nop */
>   }
>   
> -static void main_cpu_reset(void *opaque)
> +static void leon3_cpu_reset(void *opaque)
>   {
> -    ResetData *s   = (ResetData *)opaque;
> -    CPUState *cpu = CPU(s->cpu);
> -    CPUSPARCState  *env = &s->cpu->env;
> +    struct CPUResetData *info = (struct CPUResetData *) opaque;
> +    int id = info->id;
> +    ResetData *s = (ResetData *)DO_UPCAST(ResetData, info[id], info);
> +    CPUState *cpu = CPU(s->info[id].cpu);
> +    CPUSPARCState *env = cpu_env(cpu);
>   
>       cpu_reset(cpu);
> -
> -    cpu->halted = 0;
> -    env->pc     = s->entry;
> -    env->npc    = s->entry + 4;
> -    env->regbase[6] = s->sp;
> +    cpu->halted = cpu->cpu_index != 0;
> +    env->pc = s->entry;
> +    env->npc = s->entry + 4;
> +    env->regbase[6] = s->info[id].sp;

You take care to initialize with different stack, ...

>   }
>   
>   static void leon3_cache_control_int(CPUSPARCState *env)
> @@ -170,8 +176,8 @@ static void leon3_cache_control_int(CPUSPARCState *env)
>   
>   static void leon3_irq_ack(CPUSPARCState *env, int intno)
>   {
> -    /* No SMP support yet, only CPU #0 available so far.  */
> -    grlib_irqmp_ack(env->irq_manager, 0, intno);
> +    CPUState *cpu = CPU(env_cpu(env));
> +    grlib_irqmp_ack(env->irq_manager, cpu->cpu_index, intno);
>   }
>   
>   /*
> @@ -213,6 +219,20 @@ static void leon3_set_pil_in(void *opaque, int n, int level)
>       }
>   }
>   
> +static void leon3_start_cpu_async_work(CPUState *cpu, run_on_cpu_data data)
> +{
> +    cpu->halted = 0;
> +}
> +
> +static void leon3_start_cpu(void *opaque, int n, int level)
> +{
> +    CPUState *cs = CPU(opaque);
> +
> +    if (level) {
> +        async_run_on_cpu(cs, leon3_start_cpu_async_work, RUN_ON_CPU_NULL);
> +    }
> +}
> +
>   static void leon3_irq_manager(CPUSPARCState *env, int intno)
>   {
>       leon3_irq_ack(env, intno);
> @@ -238,17 +258,21 @@ static void leon3_generic_hw_init(MachineState *machine)
>       AHBPnp *ahb_pnp;
>       APBPnp *apb_pnp;
>   
> -    /* Init CPU */
> -    cpu = SPARC_CPU(cpu_create(machine->cpu_type));
> -    env = &cpu->env;
> +    reset_info = g_malloc0(sizeof(ResetData));
>   
> -    cpu_sparc_set_id(env, 0);
> +    for (i = 0; i < machine->smp.cpus; i++) {
> +        /* Init CPU */
> +        cpu = SPARC_CPU(cpu_create(machine->cpu_type));
> +        env = &cpu->env;
>   
> -    /* Reset data */
> -    reset_info        = g_new0(ResetData, 1);
> -    reset_info->cpu   = cpu;
> -    reset_info->sp    = LEON3_RAM_OFFSET + ram_size;
> -    qemu_register_reset(main_cpu_reset, reset_info);
> +        cpu_sparc_set_id(env, i);
> +
> +        /* Reset data */
> +        reset_info->info[i].id = i;
> +        reset_info->info[i].cpu = cpu;
> +        reset_info->info[i].sp = LEON3_RAM_OFFSET + ram_size;

... but all CPUs are initialized with the same stack. Is this
expected?

> +        qemu_register_reset(leon3_cpu_reset, &reset_info->info[i]);
> +    }
>   
>       ahb_pnp = GRLIB_AHB_PNP(qdev_new(TYPE_GRLIB_AHB_PNP));
>       sysbus_realize_and_unref(SYS_BUS_DEVICE(ahb_pnp), &error_fatal);
> @@ -266,14 +290,28 @@ static void leon3_generic_hw_init(MachineState *machine)
>   
>       /* Allocate IRQ manager */
>       irqmpdev = qdev_new(TYPE_GRLIB_IRQMP);
> +    object_property_set_int(OBJECT(irqmpdev), "ncpus", machine->smp.cpus,
> +                            &error_fatal);
>       sysbus_realize_and_unref(SYS_BUS_DEVICE(irqmpdev), &error_fatal);
> -    qdev_init_gpio_in_named_with_opaque(DEVICE(cpu), leon3_set_pil_in,
> -                                        env, "pil", 1);
> -    qdev_connect_gpio_out_named(irqmpdev, "grlib-irq", 0,
> -                                qdev_get_gpio_in_named(DEVICE(cpu), "pil", 0));
> +
> +    for (i = 0; i < machine->smp.cpus; i++) {
> +        cpu = reset_info->info[i].cpu;
> +        env = &cpu->env;
> +        qdev_init_gpio_in_named_with_opaque(DEVICE(cpu), leon3_start_cpu,
> +                                            cpu, "start_cpu", 1);
> +        qdev_connect_gpio_out_named(irqmpdev, "grlib-start-cpu", i,
> +                                    qdev_get_gpio_in_named(DEVICE(cpu),
> +                                                           "start_cpu", 0));
> +        qdev_init_gpio_in_named_with_opaque(DEVICE(cpu), leon3_set_pil_in,
> +                                            env, "pil", 1);
> +        qdev_connect_gpio_out_named(irqmpdev, "grlib-irq", i,
> +                                    qdev_get_gpio_in_named(DEVICE(cpu),
> +                                                           "pil", 0));
> +        env->irq_manager = irqmpdev;
> +        env->qemu_irq_ack = leon3_irq_manager;
> +    }
> +
>       sysbus_mmio_map(SYS_BUS_DEVICE(irqmpdev), 0, LEON3_IRQMP_OFFSET);
> -    env->irq_manager = irqmpdev;
> -    env->qemu_irq_ack = leon3_irq_manager;
>       grlib_apb_pnp_add_entry(apb_pnp, LEON3_IRQMP_OFFSET, 0xFFF,
>                               GRLIB_VENDOR_GAISLER, GRLIB_IRQMP_DEV,
>                               2, 0, GRLIB_APBIO_AREA);
> @@ -347,10 +385,13 @@ static void leon3_generic_hw_init(MachineState *machine)
>               uint8_t *bootloader_entry;
>   
>               bootloader_entry = memory_region_get_ram_ptr(prom);
> -            write_bootloader(env, bootloader_entry, entry);
> -            env->pc = LEON3_PROM_OFFSET;
> -            env->npc = LEON3_PROM_OFFSET + 4;
> +            write_bootloader(&reset_info->info[0].cpu->env, bootloader_entry,
> +                             entry);
>               reset_info->entry = LEON3_PROM_OFFSET;
> +            for (i = 0; i < machine->smp.cpus; i++) {
> +                reset_info->info[i].cpu->env.pc = LEON3_PROM_OFFSET;
> +                reset_info->info[i].cpu->env.npc = LEON3_PROM_OFFSET + 4;
> +            }
>           }
>       }
>   
> @@ -389,6 +430,7 @@ static void leon3_generic_machine_init(MachineClass *mc)
>       mc->init = leon3_generic_hw_init;
>       mc->default_cpu_type = SPARC_CPU_TYPE_NAME("LEON3");
>       mc->default_ram_id = "leon3.ram";
> +    mc->max_cpus = MAX_CPUS;
>   }
>   
>   DEFINE_MACHINE("leon3_generic", leon3_generic_machine_init)

Re: [PATCH v2 5/8] target/sparc: implement asr17 feature for smp

[Philippe Mathieu-Daudé]

On 16/1/24 14:02, Clément Chigot wrote:
> This allows the guest program to know its cpu id.
> 
> Co-developed-by: Frederic Konrad <konrad.frederic@yahoo.fr>
> Signed-off-by: Clément Chigot <chigot@adacore.com>
> Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
> ---
>   target/sparc/helper.c    | 16 ++++++++++++++++
>   target/sparc/helper.h    |  1 +
>   target/sparc/translate.c | 13 +++----------
>   3 files changed, 20 insertions(+), 10 deletions(-)

Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>

Re: [PATCH v2 6/8] leon3: implement multiprocessor

[Philippe Mathieu-Daudé]

On 16/1/24 14:02, Clément Chigot wrote:
> This allows to register more than one CPU on the leon3_generic machine.
> 
> Co-developed-by: Frederic Konrad <konrad.frederic@yahoo.fr>
> Signed-off-by: Clément Chigot <chigot@adacore.com>
> ---
>   hw/sparc/leon3.c | 106 +++++++++++++++++++++++++++++++++--------------
>   1 file changed, 74 insertions(+), 32 deletions(-)


> +static void leon3_start_cpu(void *opaque, int n, int level)
> +{
> +    CPUState *cs = CPU(opaque);
> +
> +    if (level) {
> +        async_run_on_cpu(cs, leon3_start_cpu_async_work, RUN_ON_CPU_NULL);
> +    }


What about instead:

     assert(level == 1);
     async_run_on_cpu(cs, leon3_start_cpu_async_work, RUN_ON_CPU_NULL);

since per patch #3:

  +    /*
  +     * Transitionning from 0 to 1 starts the CPUs. The opposite can't
  +     * happen.
  +     */

> +}

Re: [PATCH v2 3/8] intc/grlib_irqmp: implements the multiprocessor status register

[Philippe Mathieu-Daudé]

On 16/1/24 14:02, Clément Chigot wrote:
> This implements the multiprocessor status register in grlib-irqmp and bind
> it to a start signal, which will be later wired in leon3-generic to
> start a cpu.
> 
> The EIRQ and BA bits are not implemented.
> 
> Based on https://gaisler.com/doc/gr712rc-usermanual.pdf, §8.3.5.
> 
> Co-developed-by: Frederic Konrad <konrad.frederic@yahoo.fr>
> Signed-off-by: Clément Chigot <chigot@adacore.com>
> ---
>   hw/intc/grlib_irqmp.c | 35 ++++++++++++++++++++++++++++++++---
>   1 file changed, 32 insertions(+), 3 deletions(-)


> @@ -325,6 +345,8 @@ static void grlib_irqmp_reset(DeviceState *d)
>   
>       memset(irqmp->state, 0, sizeof *irqmp->state);
>       irqmp->state->parent = irqmp;
> +    irqmp->state->mpstatus = ((irqmp->ncpus - 1) << MP_STATUS_NCPU_SHIFT)
> +        | ((1 << irqmp->ncpus) - 2);

Indentation off, otherwise:
Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>

>   }

Re: [PATCH v2 0/8] sparc/leon3: Add support for -smp

[Philippe Mathieu-Daudé]

On 16/1/24 14:02, Clément Chigot wrote:
> V2 modifications
>   - Patch1: Add SPDX copyright tags.
>   - Patch3: Add defines for MP_STATUS fields. Improve comments.
>   - Patch4: Improve a comment.
>   - Patch6: Dropped as already merged.
> 
> ---
> 
> This series allows leon3 emulations to record up 4 CPUs.
> 
> It requires some enhancements in the grlib_irqmp device and adding the
> cpu_index field in the asr17 instruction.
> 
> It has been tested locally with various bareboard runtimes.
> 
> 
> Clément Chigot (8):
>    sparc/grlib: split out the headers for each peripherals
>    intc/grlib_irqmp: add ncpus property
>    intc/grlib_irqmp: implements the multiprocessor status register
>    intc/grlib_irqmp: implements multicore irq
>    target/sparc: implement asr17 feature for smp
>    leon3: implement multiprocessor
>    leon3: check cpu_id in the tiny bootloader
>    MAINTAINERS: replace Fabien by myself as Leon3 maintainer

Cc'ing maintainers per:

$ ./scripts/get_maintainer.pl -f hw/sparc/leon3.c
"Clément Chigot" <chigot@adacore.com> (maintainer:Leon3)
Frederic Konrad <konrad.frederic@yahoo.fr> (maintainer:Leon3)
Mark Cave-Ayland <mark.cave-ayland@ilande.co.uk> (maintainer:SPARC TCG CPUs)
Artyom Tarasenko <atar4qemu@gmail.com> (maintainer:SPARC TCG CPUs)

Mark / Artyom if you don't have PR planned I can take the v3 if you Ack.

Regards,

Phil.

Re: [PATCH v2 7/8] leon3: check cpu_id in the tiny bootloader

[Clément Chigot]

On Tue, Jan 30, 2024 at 10:15 AM Philippe Mathieu-Daudé
<philmd@linaro.org> wrote:
>
> On 16/1/24 14:02, Clément Chigot wrote:
> > Now that SMP is possible, the asr17 must be checked in the little boot code
> > or the secondary CPU will reinitialize the Timer and the Uart.
> >
> > Co-developed-by: Frederic Konrad <konrad.frederic@yahoo.fr>
> > Signed-off-by: Clément Chigot <chigot@adacore.com>
> > ---
> >   hw/sparc/leon3.c | 22 ++++++++++++++++++++--
> >   1 file changed, 20 insertions(+), 2 deletions(-)
> >
> > diff --git a/hw/sparc/leon3.c b/hw/sparc/leon3.c
> > index eacd85ee4f..87a8044a3e 100644
> > --- a/hw/sparc/leon3.c
> > +++ b/hw/sparc/leon3.c
> > @@ -100,13 +100,27 @@ static uint32_t *gen_store_u32(uint32_t *code, hwaddr addr, uint32_t val)
> >
> >   /*
> >    * When loading a kernel in RAM the machine is expected to be in a different
> > - * state (eg: initialized by the bootloader). This little code reproduces
> > - * this behavior.
> > + * state (eg: initialized by the bootloader).  This little code reproduces
> > + * this behavior.  Also this code can be executed by the secondary cpus as
> > + * well since it looks at the %asr17 register before doing any
> > + * initialization, it allows to use the same reset address for all the
> > + * cpus.
> >    */
> >   static void write_bootloader(CPUSPARCState *env, uint8_t *base,
> >                                hwaddr kernel_addr)
> >   {
> >       uint32_t *p = (uint32_t *) base;
> > +    uint32_t *sec_cpu_branch_p = NULL;
> > +
> > +    /* If we are running on a secondary CPU, jump directly to the kernel.  */
> > +
> > +    stl_p(p++, 0x85444000); /* rd %asr17, %g2      */
> > +    stl_p(p++, 0x8530a01c); /* srl  %g2, 0x1c, %g2 */
> > +    stl_p(p++, 0x80908000); /* tst  %g2            */
> > +    /* Fill that later.  */
>
>        /* Filled below. */
>
> > +    sec_cpu_branch_p = p;
> > +    stl_p(p++, 0x0BADC0DE); /* bne xxx             */
> > +    stl_p(p++, 0x01000000); /* nop */
> >
> >       /* Initialize the UARTs                                        */
> >       /* *UART_CONTROL = UART_RECEIVE_ENABLE | UART_TRANSMIT_ENABLE; */
> > @@ -120,6 +134,10 @@ static void write_bootloader(CPUSPARCState *env, uint8_t *base,
> >       /* *GPTIMER0_CONFIG = GPTIMER_ENABLE | GPTIMER_RESTART;        */
> >       p = gen_store_u32(p, 0x80000318, 3);
> >
> > +    /* Now, the relative branch above can be computed.  */
> > +    stl_p(sec_cpu_branch_p, 0x12800000
> > +          + (p - sec_cpu_branch_p));
> > +
> >       /* JUMP to the entry point                                     */
> >       stl_p(p++, 0x82100000); /* mov %g0, %g1 */
> >       stl_p(p++, 0x03000000 + extract32(kernel_addr, 10, 22));
>
> Alternatively have main_cpu_reset / secondary_cpu_reset handlers.
> You could split BL in HWINIT / JUMP, have HWINIT() return # instr
> used and adjust secondary_cpu_reset entry.

Indeed that would make this code a bit easier to maintain. I'll keep
it for a future MR if you don't mind.
Thanks for the idea anyway.

> Anyway,
> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org>

Re: [PATCH v2 6/8] leon3: implement multiprocessor

[Clément Chigot]

On Tue, Jan 30, 2024 at 12:43 PM Philippe Mathieu-Daudé
<philmd@linaro.org> wrote:
>
> Hi Clément,
>
> On 16/1/24 14:02, Clément Chigot wrote:
> > This allows to register more than one CPU on the leon3_generic machine.
> >
> > Co-developed-by: Frederic Konrad <konrad.frederic@yahoo.fr>
> > Signed-off-by: Clément Chigot <chigot@adacore.com>
> > ---
> >   hw/sparc/leon3.c | 106 +++++++++++++++++++++++++++++++++--------------
> >   1 file changed, 74 insertions(+), 32 deletions(-)
> >
> > diff --git a/hw/sparc/leon3.c b/hw/sparc/leon3.c
> > index 7866f0a049..eacd85ee4f 100644
> > --- a/hw/sparc/leon3.c
> > +++ b/hw/sparc/leon3.c
> > @@ -54,6 +54,8 @@
> >   #define LEON3_PROM_OFFSET    (0x00000000)
> >   #define LEON3_RAM_OFFSET     (0x40000000)
> >
> > +#define MAX_CPUS  4
> > +
> >   #define LEON3_UART_OFFSET  (0x80000100)
> >   #define LEON3_UART_IRQ     (3)
> >
> > @@ -67,9 +69,12 @@
> >   #define LEON3_AHB_PNP_OFFSET (0xFFFFF000)
> >
> >   typedef struct ResetData {
> > -    SPARCCPU *cpu;
> > -    uint32_t  entry;            /* save kernel entry in case of reset */
> > -    target_ulong sp;            /* initial stack pointer */
> > +    struct CPUResetData {
> > +        int id;
> > +        SPARCCPU *cpu;
> > +        target_ulong sp;  /* initial stack pointer */
> > +    } info[MAX_CPUS];
> > +    uint32_t entry;             /* save kernel entry in case of reset */
> >   } ResetData;
> >
> >   static uint32_t *gen_store_u32(uint32_t *code, hwaddr addr, uint32_t val)
> > @@ -125,18 +130,19 @@ static void write_bootloader(CPUSPARCState *env, uint8_t *base,
> >       stl_p(p++, 0x01000000); /* nop */
> >   }
> >
> > -static void main_cpu_reset(void *opaque)
> > +static void leon3_cpu_reset(void *opaque)
> >   {
> > -    ResetData *s   = (ResetData *)opaque;
> > -    CPUState *cpu = CPU(s->cpu);
> > -    CPUSPARCState  *env = &s->cpu->env;
> > +    struct CPUResetData *info = (struct CPUResetData *) opaque;
> > +    int id = info->id;
> > +    ResetData *s = (ResetData *)DO_UPCAST(ResetData, info[id], info);
> > +    CPUState *cpu = CPU(s->info[id].cpu);
> > +    CPUSPARCState *env = cpu_env(cpu);
> >
> >       cpu_reset(cpu);
> > -
> > -    cpu->halted = 0;
> > -    env->pc     = s->entry;
> > -    env->npc    = s->entry + 4;
> > -    env->regbase[6] = s->sp;
> > +    cpu->halted = cpu->cpu_index != 0;
> > +    env->pc = s->entry;
> > +    env->npc = s->entry + 4;
> > +    env->regbase[6] = s->info[id].sp;
>
> You take care to initialize with different stack, ...
>
> >   }
> >
> >   static void leon3_cache_control_int(CPUSPARCState *env)
> > @@ -170,8 +176,8 @@ static void leon3_cache_control_int(CPUSPARCState *env)
> >
> >   static void leon3_irq_ack(CPUSPARCState *env, int intno)
> >   {
> > -    /* No SMP support yet, only CPU #0 available so far.  */
> > -    grlib_irqmp_ack(env->irq_manager, 0, intno);
> > +    CPUState *cpu = CPU(env_cpu(env));
> > +    grlib_irqmp_ack(env->irq_manager, cpu->cpu_index, intno);
> >   }
> >
> >   /*
> > @@ -213,6 +219,20 @@ static void leon3_set_pil_in(void *opaque, int n, int level)
> >       }
> >   }
> >
> > +static void leon3_start_cpu_async_work(CPUState *cpu, run_on_cpu_data data)
> > +{
> > +    cpu->halted = 0;
> > +}
> > +
> > +static void leon3_start_cpu(void *opaque, int n, int level)
> > +{
> > +    CPUState *cs = CPU(opaque);
> > +
> > +    if (level) {
> > +        async_run_on_cpu(cs, leon3_start_cpu_async_work, RUN_ON_CPU_NULL);
> > +    }
> > +}
> > +
> >   static void leon3_irq_manager(CPUSPARCState *env, int intno)
> >   {
> >       leon3_irq_ack(env, intno);
> > @@ -238,17 +258,21 @@ static void leon3_generic_hw_init(MachineState *machine)
> >       AHBPnp *ahb_pnp;
> >       APBPnp *apb_pnp;
> >
> > -    /* Init CPU */
> > -    cpu = SPARC_CPU(cpu_create(machine->cpu_type));
> > -    env = &cpu->env;
> > +    reset_info = g_malloc0(sizeof(ResetData));
> >
> > -    cpu_sparc_set_id(env, 0);
> > +    for (i = 0; i < machine->smp.cpus; i++) {
> > +        /* Init CPU */
> > +        cpu = SPARC_CPU(cpu_create(machine->cpu_type));
> > +        env = &cpu->env;
> >
> > -    /* Reset data */
> > -    reset_info        = g_new0(ResetData, 1);
> > -    reset_info->cpu   = cpu;
> > -    reset_info->sp    = LEON3_RAM_OFFSET + ram_size;
> > -    qemu_register_reset(main_cpu_reset, reset_info);
> > +        cpu_sparc_set_id(env, i);
> > +
> > +        /* Reset data */
> > +        reset_info->info[i].id = i;
> > +        reset_info->info[i].cpu = cpu;
> > +        reset_info->info[i].sp = LEON3_RAM_OFFSET + ram_size;
>
> ... but all CPUs are initialized with the same stack. Is this
> expected?

Indeed, I've just blindly updated the existing code.
The official doc (see [1] §4.2.15) does not mention anything about SP
when a reset occurs. The program loaded should take care of their
initialization.
I'll remove that. Thanks for the notice.

[1] https://gaisler.com/doc/gr712rc-usermanual.pdf

> > +        qemu_register_reset(leon3_cpu_reset, &reset_info->info[i]);
> > +    }
> >
> >       ahb_pnp = GRLIB_AHB_PNP(qdev_new(TYPE_GRLIB_AHB_PNP));
> >       sysbus_realize_and_unref(SYS_BUS_DEVICE(ahb_pnp), &error_fatal);
> > @@ -266,14 +290,28 @@ static void leon3_generic_hw_init(MachineState *machine)
> >
> >       /* Allocate IRQ manager */
> >       irqmpdev = qdev_new(TYPE_GRLIB_IRQMP);
> > +    object_property_set_int(OBJECT(irqmpdev), "ncpus", machine->smp.cpus,
> > +                            &error_fatal);
> >       sysbus_realize_and_unref(SYS_BUS_DEVICE(irqmpdev), &error_fatal);
> > -    qdev_init_gpio_in_named_with_opaque(DEVICE(cpu), leon3_set_pil_in,
> > -                                        env, "pil", 1);
> > -    qdev_connect_gpio_out_named(irqmpdev, "grlib-irq", 0,
> > -                                qdev_get_gpio_in_named(DEVICE(cpu), "pil", 0));
> > +
> > +    for (i = 0; i < machine->smp.cpus; i++) {
> > +        cpu = reset_info->info[i].cpu;
> > +        env = &cpu->env;
> > +        qdev_init_gpio_in_named_with_opaque(DEVICE(cpu), leon3_start_cpu,
> > +                                            cpu, "start_cpu", 1);
> > +        qdev_connect_gpio_out_named(irqmpdev, "grlib-start-cpu", i,
> > +                                    qdev_get_gpio_in_named(DEVICE(cpu),
> > +                                                           "start_cpu", 0));
> > +        qdev_init_gpio_in_named_with_opaque(DEVICE(cpu), leon3_set_pil_in,
> > +                                            env, "pil", 1);
> > +        qdev_connect_gpio_out_named(irqmpdev, "grlib-irq", i,
> > +                                    qdev_get_gpio_in_named(DEVICE(cpu),
> > +                                                           "pil", 0));
> > +        env->irq_manager = irqmpdev;
> > +        env->qemu_irq_ack = leon3_irq_manager;
> > +    }
> > +
> >       sysbus_mmio_map(SYS_BUS_DEVICE(irqmpdev), 0, LEON3_IRQMP_OFFSET);
> > -    env->irq_manager = irqmpdev;
> > -    env->qemu_irq_ack = leon3_irq_manager;
> >       grlib_apb_pnp_add_entry(apb_pnp, LEON3_IRQMP_OFFSET, 0xFFF,
> >                               GRLIB_VENDOR_GAISLER, GRLIB_IRQMP_DEV,
> >                               2, 0, GRLIB_APBIO_AREA);
> > @@ -347,10 +385,13 @@ static void leon3_generic_hw_init(MachineState *machine)
> >               uint8_t *bootloader_entry;
> >
> >               bootloader_entry = memory_region_get_ram_ptr(prom);
> > -            write_bootloader(env, bootloader_entry, entry);
> > -            env->pc = LEON3_PROM_OFFSET;
> > -            env->npc = LEON3_PROM_OFFSET + 4;
> > +            write_bootloader(&reset_info->info[0].cpu->env, bootloader_entry,
> > +                             entry);
> >               reset_info->entry = LEON3_PROM_OFFSET;
> > +            for (i = 0; i < machine->smp.cpus; i++) {
> > +                reset_info->info[i].cpu->env.pc = LEON3_PROM_OFFSET;
> > +                reset_info->info[i].cpu->env.npc = LEON3_PROM_OFFSET + 4;
> > +            }
> >           }
> >       }
> >
> > @@ -389,6 +430,7 @@ static void leon3_generic_machine_init(MachineClass *mc)
> >       mc->init = leon3_generic_hw_init;
> >       mc->default_cpu_type = SPARC_CPU_TYPE_NAME("LEON3");
> >       mc->default_ram_id = "leon3.ram";
> > +    mc->max_cpus = MAX_CPUS;
> >   }
> >
> >   DEFINE_MACHINE("leon3_generic", leon3_generic_machine_init)
>