[Qemu-devel] [PATCH 0/3] hw/arm: add Fixed Virtual Platform VE support

[Qemu-devel] [PATCH 0/3] hw/arm: add Fixed Virtual Platform VE support

[Sergey Fedorov]

These patches add support for ARM Fixed Virtual Platform Versatile Express
board. ARM VFP VE is similar to hardware Versatile Express boards. So these
changes rely largely on existing QEMU VE support code. First two patches are
prerequisites for the final patch which adds FVP VE board support itself.

The changes were tested by running a Linux system in SMP mode using
'arch/arm64/boot/dts/rtsm_ve-aemv8a.dts' from Linux kernel source code for
generating device tree blob.

Sergey Fedorov (3):
  hw/arm/boot: add secondary loader for AArch64
  hw/arm/vexpress: introduce VEDBoardInfo::smp_bootreg_addr
  hw/arm/vexpress: add FVP VE board support

 hw/arm/boot.c     |  26 +++++++++-
 hw/arm/vexpress.c | 142 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 165 insertions(+), 3 deletions(-)

-- 
2.3.4

[Qemu-devel] [PATCH 1/3] hw/arm/boot: add secondary loader for AArch64

[Sergey Fedorov]

This default secondary loader is used to bring up secondary CPUs using
spin table boot method.

Signed-off-by: Sergey Fedorov <serge.fdrv@gmail.com>
---
 hw/arm/boot.c | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/hw/arm/boot.c b/hw/arm/boot.c
index a48d1b2..2bff2f2 100644
--- a/hw/arm/boot.c
+++ b/hw/arm/boot.c
@@ -57,6 +57,18 @@ static const ARMInsnFixup bootloader_aarch64[] = {
     { 0, FIXUP_TERMINATOR }
 };
 
+static const ARMInsnFixup smpboot_aarch64[] = {
+    { 0xd503205f }, /* wfe */
+    { 0x580000a0 }, /* ldr x0, bootreg_addr */
+    { 0xf9400000 }, /* ldr x0, [x0] */
+    { 0xb4ffffa0 }, /* cbz x0, <wfe> */
+    { 0xd61f0000 }, /* br  x0 */
+    { 0xd503201f }, /* nop */
+    { 0, FIXUP_BOOTREG }, /* bootreg_addr: .word 0x... */
+    { 0 }, /* .word 0 */
+    { 0, FIXUP_TERMINATOR }
+};
+
 /* The worlds second smallest bootloader.  Set r0-r2, then jump to kernel.  */
 static const ARMInsnFixup bootloader[] = {
     { 0xe3a00000 }, /* mov     r0, #0 */
@@ -152,6 +164,7 @@ static void default_write_secondary(ARMCPU *cpu,
                                     const struct arm_boot_info *info)
 {
     uint32_t fixupcontext[FIXUP_MAX];
+    const ARMInsnFixup *secondary_loader;
 
     fixupcontext[FIXUP_GIC_CPU_IF] = info->gic_cpu_if_addr;
     fixupcontext[FIXUP_BOOTREG] = info->smp_bootreg_addr;
@@ -161,8 +174,13 @@ static void default_write_secondary(ARMCPU *cpu,
         fixupcontext[FIXUP_DSB] = CP15_DSB_INSN;
     }
 
+    if (arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
+        secondary_loader = smpboot_aarch64;
+    } else {
+        secondary_loader = smpboot;
+    }
     write_bootloader("smpboot", info->smp_loader_start,
-                     smpboot, fixupcontext);
+                     secondary_loader, fixupcontext);
 }
 
 static void default_reset_secondary(ARMCPU *cpu,
@@ -171,7 +189,11 @@ static void default_reset_secondary(ARMCPU *cpu,
     CPUARMState *env = &cpu->env;
 
     stl_phys_notdirty(&address_space_memory, info->smp_bootreg_addr, 0);
-    env->regs[15] = info->smp_loader_start;
+    if (is_a64(env)) {
+        env->pc = info->smp_loader_start;
+    } else {
+        env->regs[15] = info->smp_loader_start;
+    }
 }
 
 static inline bool have_dtb(const struct arm_boot_info *info)
-- 
2.3.4

[Qemu-devel] [PATCH 2/3] hw/arm/vexpress: introduce VEDBoardInfo::smp_bootreg_addr

[Sergey Fedorov]

Require secondary CPU release address to be specified explicitly in each
daughterboard info structure.

Signed-off-by: Sergey Fedorov <serge.fdrv@gmail.com>
---
 hw/arm/vexpress.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/hw/arm/vexpress.c b/hw/arm/vexpress.c
index 8496c16..97ccf15 100644
--- a/hw/arm/vexpress.c
+++ b/hw/arm/vexpress.c
@@ -186,6 +186,7 @@ struct VEDBoardInfo {
     struct arm_boot_info bootinfo;
     const hwaddr *motherboard_map;
     hwaddr loader_start;
+    hwaddr smp_bootreg_addr;
     const hwaddr gic_cpu_if_addr;
     uint32_t proc_id;
     uint32_t num_voltage_sensors;
@@ -337,6 +338,7 @@ static const uint32_t a9_clocks[] = {
 static VEDBoardInfo a9_daughterboard = {
     .motherboard_map = motherboard_legacy_map,
     .loader_start = 0x60000000,
+    .smp_bootreg_addr = 0x10000030,
     .gic_cpu_if_addr = 0x1e000100,
     .proc_id = 0x0c000191,
     .num_voltage_sensors = ARRAY_SIZE(a9_voltages),
@@ -418,6 +420,7 @@ static const uint32_t a15_clocks[] = {
 static VEDBoardInfo a15_daughterboard = {
     .motherboard_map = motherboard_aseries_map,
     .loader_start = 0x80000000,
+    .smp_bootreg_addr = 0x1c010030,
     .gic_cpu_if_addr = 0x2c002000,
     .proc_id = 0x14000237,
     .num_voltage_sensors = ARRAY_SIZE(a15_voltages),
@@ -700,7 +703,7 @@ static void vexpress_common_init(MachineState *machine)
     daughterboard->bootinfo.board_id = VEXPRESS_BOARD_ID;
     daughterboard->bootinfo.loader_start = daughterboard->loader_start;
     daughterboard->bootinfo.smp_loader_start = map[VE_SRAM];
-    daughterboard->bootinfo.smp_bootreg_addr = map[VE_SYSREGS] + 0x30;
+    daughterboard->bootinfo.smp_bootreg_addr = daughterboard->smp_bootreg_addr;
     daughterboard->bootinfo.gic_cpu_if_addr = daughterboard->gic_cpu_if_addr;
     daughterboard->bootinfo.modify_dtb = vexpress_modify_dtb;
     /* Indicate that when booting Linux we should be in secure state */
-- 
2.3.4

[Qemu-devel] [PATCH 3/3] hw/arm/vexpress: add FVP VE board support

[Sergey Fedorov]

This patch allows to boot AA64 linux kernel in SMP mode with DTB
generated from 'arch/arm64/boot/dts/rtsm_ve-aemv8a.dts' of Linux kernel
source code.

CPU and GIC creation parts are adaptation of code from "hw/arm/virt.c".

Signed-off-by: Sergey Fedorov <serge.fdrv@gmail.com>
---
 hw/arm/vexpress.c | 137 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 137 insertions(+)

diff --git a/hw/arm/vexpress.c b/hw/arm/vexpress.c
index 97ccf15..8ab4e8a 100644
--- a/hw/arm/vexpress.c
+++ b/hw/arm/vexpress.c
@@ -170,6 +170,7 @@ typedef struct {
 #define TYPE_VEXPRESS_MACHINE   "vexpress"
 #define TYPE_VEXPRESS_A9_MACHINE   "vexpress-a9"
 #define TYPE_VEXPRESS_A15_MACHINE   "vexpress-a15"
+#define TYPE_FVP_VE_MACHINE   "fvp_ve"
 #define VEXPRESS_MACHINE(obj) \
     OBJECT_CHECK(VexpressMachineState, (obj), TYPE_VEXPRESS_MACHINE)
 #define VEXPRESS_MACHINE_GET_CLASS(obj) \
@@ -430,6 +431,124 @@ static VEDBoardInfo a15_daughterboard = {
     .init = a15_daughterboard_init,
 };
 
+static void fvp_ve_create_gic(const VexpressMachineState *vms,
+                              hwaddr periphbase, qemu_irq *pic)
+{
+    /* We create a standalone GIC v2 */
+    DeviceState *gicdev;
+    SysBusDevice *gicbusdev;
+    int i;
+
+    gicdev = qdev_create(NULL, "arm_gic");
+    qdev_prop_set_uint32(gicdev, "revision", 2);
+    qdev_prop_set_uint32(gicdev, "num-cpu", smp_cpus);
+    qdev_prop_set_uint32(gicdev, "num-irq", 96);
+    qdev_init_nofail(gicdev);
+    gicbusdev = SYS_BUS_DEVICE(gicdev);
+    sysbus_mmio_map(gicbusdev, 0, periphbase + 0x1000);
+    sysbus_mmio_map(gicbusdev, 1, periphbase + 0x2000);
+
+    /* Wire the outputs from each CPU's generic timer to the
+     * appropriate GIC PPI inputs, and the GIC's IRQ output to
+     * the CPU's IRQ input.
+     */
+    for (i = 0; i < smp_cpus; i++) {
+        DeviceState *cpudev = DEVICE(qemu_get_cpu(i));
+        int ppibase = 64 + i * 32;
+        /* physical timer; we wire it up to the non-secure timer's ID,
+         * since QEMU doesn't have TrustZone support so far.
+         */
+        qdev_connect_gpio_out(cpudev, 0,
+                              qdev_get_gpio_in(gicdev, ppibase + 30));
+        /* virtual timer */
+        qdev_connect_gpio_out(cpudev, 1,
+                              qdev_get_gpio_in(gicdev, ppibase + 27));
+
+        sysbus_connect_irq(gicbusdev, i, qdev_get_gpio_in(cpudev, ARM_CPU_IRQ));
+    }
+
+    for (i = 0; i < 64; i++) {
+        pic[i] = qdev_get_gpio_in(gicdev, i);
+    }
+}
+
+static void fvp_ve_daughterboard_init(const VexpressMachineState *vms,
+                                      ram_addr_t ram_size,
+                                      const char *cpu_model,
+                                      qemu_irq *pic)
+{
+    MemoryRegion *sysmem = get_system_memory();
+    MemoryRegion *ram = g_new(MemoryRegion, 1);
+    MemoryRegion *sram = g_new(MemoryRegion, 1);
+    const hwaddr periphbase = 0x2c000000;
+    int n;
+
+    if (!cpu_model) {
+        cpu_model = "cortex-a57";
+    }
+
+    {
+        /* We have to use a separate 64 bit variable here to avoid the gcc
+         * "comparison is always false due to limited range of data type"
+         * warning if we are on a host where ram_addr_t is 32 bits.
+         */
+        uint64_t rsz = ram_size;
+        if (rsz > (30ULL * 1024 * 1024 * 1024)) {
+            fprintf(stderr, "fvp_ve: cannot model more than 30GB RAM\n");
+            exit(1);
+        }
+    }
+
+    memory_region_init_ram(ram, NULL, "fvp.highmem", ram_size, &error_abort);
+    vmstate_register_ram_global(ram);
+    memory_region_add_subregion(sysmem, 0x80000000, ram);
+
+    for (n = 0; n < smp_cpus; n++) {
+        ObjectClass *oc = cpu_class_by_name(TYPE_ARM_CPU, cpu_model);
+        Object *cpuobj;
+
+        if (!oc) {
+            fprintf(stderr, "Unable to find CPU definition\n");
+            exit(1);
+        }
+        cpuobj = object_new(object_class_get_name(oc));
+
+        if (!vms->secure) {
+            object_property_set_bool(cpuobj, false, "has_el3", NULL);
+        }
+
+        if (object_property_find(cpuobj, "reset-cbar", NULL)) {
+            object_property_set_int(cpuobj, periphbase,
+                                    "reset-cbar", &error_abort);
+        }
+
+        object_property_set_bool(cpuobj, true, "realized", NULL);
+    }
+
+    fvp_ve_create_gic(vms, periphbase, pic);
+
+    memory_region_init_ram(sram, NULL, "fvp.sram", 0x10000, &error_abort);
+    vmstate_register_ram_global(sram);
+    memory_region_add_subregion(sysmem, 0x2e000000, sram);
+}
+
+static const uint32_t fvp_ve_clocks[] = {
+    32000, /* refclk */
+    1000000, /* timclk */
+    24000000, /* apb_pclk */
+};
+
+static VEDBoardInfo fvp_ve_daughterboard = {
+    .motherboard_map = motherboard_aseries_map,
+    .loader_start = 0x80000000,
+    .smp_bootreg_addr = 0x8000fff8,
+    .gic_cpu_if_addr = 0x2c002000,
+    .num_voltage_sensors = 0,
+    .num_clocks = ARRAY_SIZE(fvp_ve_clocks),
+    .clocks = fvp_ve_clocks,
+    .init = fvp_ve_daughterboard_init,
+};
+
 static int add_virtio_mmio_node(void *fdt, uint32_t acells, uint32_t scells,
                                 hwaddr addr, hwaddr size, uint32_t intc,
                                 int irq)
@@ -772,6 +891,17 @@ static void vexpress_a15_class_init(ObjectClass *oc, void *data)
     vmc->daughterboard = &a15_daughterboard;
 }
 
+static void fvp_ve_class_init(ObjectClass *oc, void *data)
+{
+    MachineClass *mc = MACHINE_CLASS(oc);
+    VexpressMachineClass *vmc = VEXPRESS_MACHINE_CLASS(oc);
+
+    mc->name = TYPE_FVP_VE_MACHINE;
+    mc->desc = "ARM Versatile Express for Cortex-A15";
+
+    vmc->daughterboard = &fvp_ve_daughterboard;
+}
+
 static const TypeInfo vexpress_info = {
     .name = TYPE_VEXPRESS_MACHINE,
     .parent = TYPE_MACHINE,
@@ -794,11 +924,18 @@ static const TypeInfo vexpress_a15_info = {
     .class_init = vexpress_a15_class_init,
 };
 
+static const TypeInfo fvp_ve_info = {
+    .name = TYPE_FVP_VE_MACHINE,
+    .parent = TYPE_VEXPRESS_MACHINE,
+    .class_init = fvp_ve_class_init,
+};
+
 static void vexpress_machine_init(void)
 {
     type_register_static(&vexpress_info);
     type_register_static(&vexpress_a9_info);
     type_register_static(&vexpress_a15_info);
+    type_register_static(&fvp_ve_info);
 }
 
 machine_init(vexpress_machine_init);
-- 
2.3.4

Re: [Qemu-devel] [PATCH 0/3] hw/arm: add Fixed Virtual Platform VE support

[Peter Maydell]

On 24 March 2015 at 23:55, Sergey Fedorov <serge.fdrv@gmail.com> wrote:
> These patches add support for ARM Fixed Virtual Platform Versatile Express
> board. ARM VFP VE is similar to hardware Versatile Express boards. So these
> changes rely largely on existing QEMU VE support code. First two patches are
> prerequisites for the final patch which adds FVP VE board support itself.
>
> The changes were tested by running a Linux system in SMP mode using
> 'arch/arm64/boot/dts/rtsm_ve-aemv8a.dts' from Linux kernel source code for
> generating device tree blob.

...so what's the motivation for our attempting to model
somebody else's board model? What do we gain over our
existing VE hardware model?

-- PMM

Re: [Qemu-devel] [PATCH 0/3] hw/arm: add Fixed Virtual Platform VE support

[Sergey Fedorov]

On 24.03.2015 17:13, Peter Maydell wrote:
> On 24 March 2015 at 23:55, Sergey Fedorov <serge.fdrv@gmail.com> wrote:
>> These patches add support for ARM Fixed Virtual Platform Versatile Express
>> board. ARM VFP VE is similar to hardware Versatile Express boards. So these
>> changes rely largely on existing QEMU VE support code. First two patches are
>> prerequisites for the final patch which adds FVP VE board support itself.
>>
>> The changes were tested by running a Linux system in SMP mode using
>> 'arch/arm64/boot/dts/rtsm_ve-aemv8a.dts' from Linux kernel source code for
>> generating device tree blob.
> ...so what's the motivation for our attempting to model
> somebody else's board model? What do we gain over our
> existing VE hardware model?
>
> -- PMM

This model uses spin table boot method. So it enables SMP on AArch64 in
TCG mode.

Best regards,
Sergey.

Re: [Qemu-devel] [PATCH 0/3] hw/arm: add Fixed Virtual Platform VE support

[Peter Maydell]

On 25 March 2015 at 00:18, Sergey Fedorov <serge.fdrv@gmail.com> wrote:
> This model uses spin table boot method.

Yes, I noticed, that's a strong reason why I don't want to
add it if we can avoid it :-)

> So it enables SMP on AArch64 in TCG mode.

We already support SMP on AArch64 in TCG mode using PSCI.

-- PMM

Re: [Qemu-devel] [PATCH 0/3] hw/arm: add Fixed Virtual Platform VE support

[Sergey Fedorov]

On 24.03.2015 17:23, Peter Maydell wrote:
> On 25 March 2015 at 00:18, Sergey Fedorov <serge.fdrv@gmail.com> wrote:
>> This model uses spin table boot method.
> Yes, I noticed, that's a strong reason why I don't want to
> add it if we can avoid it :-)

Why not? :-)

>
>> So it enables SMP on AArch64 in TCG mode.
> We already support SMP on AArch64 in TCG mode using PSCI.

Ah, seems I missed this. But isn't it supported only in 'virt' board?

Regards,
Sergey

Re: [Qemu-devel] [PATCH 0/3] hw/arm: add Fixed Virtual Platform VE support

[Peter Maydell]

On 25 March 2015 at 00:31, Sergey Fedorov <serge.fdrv@gmail.com> wrote:
> On 24.03.2015 17:23, Peter Maydell wrote:
>> On 25 March 2015 at 00:18, Sergey Fedorov <serge.fdrv@gmail.com> wrote:
>>> This model uses spin table boot method.
>> Yes, I noticed, that's a strong reason why I don't want to
>> add it if we can avoid it :-)
>
> Why not? :-)

Because PSCI is the expected method for SMP support for
new 64-bit boards (as I understand it) and so I'd rather
not add a lot of legacy spintable code unless we really
need the board support for some other reason.

>> We already support SMP on AArch64 in TCG mode using PSCI.
>
> Ah, seems I missed this. But isn't it supported only in 'virt'
> board?

Yes, at the moment. But then 'virt' is our only 64 bit board...

-- PMM

Re: [Qemu-devel] [PATCH 0/3] hw/arm: add Fixed Virtual Platform VE support

[Sergey Fedorov]

On 24.03.2015 17:48, Peter Maydell wrote:
> On 25 March 2015 at 00:31, Sergey Fedorov <serge.fdrv@gmail.com> wrote:
>> On 24.03.2015 17:23, Peter Maydell wrote:
>>> On 25 March 2015 at 00:18, Sergey Fedorov <serge.fdrv@gmail.com> wrote:
>>>> This model uses spin table boot method.
>>> Yes, I noticed, that's a strong reason why I don't want to
>>> add it if we can avoid it :-)
>> Why not? :-)
> Because PSCI is the expected method for SMP support for
> new 64-bit boards (as I understand it) and so I'd rather
> not add a lot of legacy spintable code unless we really
> need the board support for some other reason.
>
>>> We already support SMP on AArch64 in TCG mode using PSCI.
>> Ah, seems I missed this. But isn't it supported only in 'virt'
>> board?
> Yes, at the moment. But then 'virt' is our only 64 bit board...

So if I understand you correctly, it would be suitable to implement a
model like Juno ARM Development Platform in order to get AArch64 VE
model with SMP support in system mode?

Re: [Qemu-devel] [PATCH 0/3] hw/arm: add Fixed Virtual Platform VE support

[Sergey Fedorov]

On 24.03.2015 18:30, Sergey Fedorov wrote:
> On 24.03.2015 17:48, Peter Maydell wrote:
>> On 25 March 2015 at 00:31, Sergey Fedorov <serge.fdrv@gmail.com> wrote:
>>> On 24.03.2015 17:23, Peter Maydell wrote:
>>>> On 25 March 2015 at 00:18, Sergey Fedorov <serge.fdrv@gmail.com> wrote:
>>>>> This model uses spin table boot method.
>>>> Yes, I noticed, that's a strong reason why I don't want to
>>>> add it if we can avoid it :-)
>>> Why not? :-)
>> Because PSCI is the expected method for SMP support for
>> new 64-bit boards (as I understand it) and so I'd rather
>> not add a lot of legacy spintable code unless we really
>> need the board support for some other reason.
>>
>>>> We already support SMP on AArch64 in TCG mode using PSCI.
>>> Ah, seems I missed this. But isn't it supported only in 'virt'
>>> board?
>> Yes, at the moment. But then 'virt' is our only 64 bit board...
> So if I understand you correctly, it would be suitable to implement a
> model like Juno ARM Development Platform in order to get AArch64 VE
> model with SMP support in system mode?

As you can guess, my objective is to get a model based on Versatile
Express with Cortex-A57 CPU which could be used to run an SMP Linux
kernel on system mode QEMU. So I decided to use the same model as ARM
FastModels provides. I could get ARMv8-A Foundation Platform model as
well, but it also uses spin table boot method. Seems there's no much choice.

Best regards,
Sergey

Re: [Qemu-devel] [PATCH 0/3] hw/arm: add Fixed Virtual Platform VE support

[Peter Maydell]

On 25 March 2015 at 01:43, Sergey Fedorov <serge.fdrv@gmail.com> wrote:
> On 24.03.2015 18:30, Sergey Fedorov wrote:
>> So if I understand you correctly, it would be suitable to implement a
>> model like Juno ARM Development Platform in order to get AArch64 VE
>> model with SMP support in system mode?
>
> As you can guess, my objective is to get a model based on Versatile
> Express with Cortex-A57 CPU which could be used to run an SMP Linux
> kernel on system mode QEMU. So I decided to use the same model as ARM
> FastModels provides. I could get ARMv8-A Foundation Platform model as
> well, but it also uses spin table boot method. Seems there's no much choice.

Basically I'd rather we modelled some bit of real hardware.
I don't see the point in providing a model of somebody
else's model -- if you don't care about matching real
hardware we have 'virt' already. So I don't really want
to model either the Foundation Platform or the AEM,
unless there's a really compelling reason to do that.

thanks
-- PMM

Re: [Qemu-devel] [PATCH 0/3] hw/arm: add Fixed Virtual Platform VE support

[Sergey Fedorov]

On 25.03.2015 04:06, Peter Maydell wrote:
> On 25 March 2015 at 01:43, Sergey Fedorov <serge.fdrv@gmail.com> wrote:
>> On 24.03.2015 18:30, Sergey Fedorov wrote:
>>> So if I understand you correctly, it would be suitable to implement a
>>> model like Juno ARM Development Platform in order to get AArch64 VE
>>> model with SMP support in system mode?
>> As you can guess, my objective is to get a model based on Versatile
>> Express with Cortex-A57 CPU which could be used to run an SMP Linux
>> kernel on system mode QEMU. So I decided to use the same model as ARM
>> FastModels provides. I could get ARMv8-A Foundation Platform model as
>> well, but it also uses spin table boot method. Seems there's no much choice.
> Basically I'd rather we modelled some bit of real hardware.
> I don't see the point in providing a model of somebody
> else's model -- if you don't care about matching real
> hardware we have 'virt' already. So I don't really want
> to model either the Foundation Platform or the AEM,
> unless there's a really compelling reason to do that.
>
> thanks
> -- PMM

I understand your view. Unfortunately, I'm not going to implement any
other model so far. I'd rather focus on EL2 implementation for AArch64.
But maybe someone will find useful these patches :)

Thanks,
Sergey