[Qemu-devel] [RFC v1 0/2] EL3 support for AArch64 Linux bootloader

[Qemu-devel] [RFC v1 0/2] EL3 support for AArch64 Linux bootloader

[Peter Crosthwaite]

Hi Peter, Edgar,

These patches add bootloader support for Edgars upcomming EL2/3 work.
This allows for Linux boot from EL3 without a bootloader or wrapper.

Regards,
Peter


Peter Crosthwaite (2):
  arm: boot: Add partial machine code fixup
  arm: boot: Add EL jump-down code for Linux

 hw/arm/boot.c | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

-- 
2.0.1.1.gfbfc394

[Qemu-devel] [RFC v1 1/2] arm: boot: Add partial machine code fixup

[Peter Crosthwaite]

Allow a fixup to be used to deposit a field in a machine-code
instruction. The option shift and length fixup fields indicate the
field being deposited by the fixup.

Signed-off-by: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
---

 hw/arm/boot.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/hw/arm/boot.c b/hw/arm/boot.c
index 1241761..840f5da 100644
--- a/hw/arm/boot.c
+++ b/hw/arm/boot.c
@@ -41,6 +41,8 @@ typedef enum {
 typedef struct ARMInsnFixup {
     uint32_t insn;
     FixupType fixup;
+    int shift;
+    int length;
 } ARMInsnFixup;
 
 static const ARMInsnFixup bootloader_aarch64[] = {
@@ -125,6 +127,10 @@ static void write_bootloader(const char *name, hwaddr addr,
     for (i = 0; i < len; i++) {
         uint32_t insn = insns[i].insn;
         FixupType fixup = insns[i].fixup;
+        int shift = insns[i].shift;
+        int length = insns[i].length ? insns[i].length : 32;
+
+        assert(shift + length <= 32);
 
         switch (fixup) {
         case FIXUP_NONE:
@@ -135,7 +141,7 @@ static void write_bootloader(const char *name, hwaddr addr,
         case FIXUP_GIC_CPU_IF:
         case FIXUP_BOOTREG:
         case FIXUP_DSB:
-            insn = fixupcontext[fixup];
+            insn = deposit32(insn, shift, length, fixupcontext[fixup]);
             break;
         default:
             abort();
-- 
2.0.1.1.gfbfc394

[Qemu-devel] [RFC v1 2/2] arm: boot: Add EL jump-down code for Linux

[Peter Crosthwaite]

Linux should boot in EL2 or EL1. If in EL3, jump down before handing
off to Linux.

Signed-off-by: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
---

 hw/arm/boot.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/hw/arm/boot.c b/hw/arm/boot.c
index 840f5da..f1f6365 100644
--- a/hw/arm/boot.c
+++ b/hw/arm/boot.c
@@ -35,6 +35,7 @@ typedef enum {
     FIXUP_GIC_CPU_IF, /* overwrite with GIC CPU interface address */
     FIXUP_BOOTREG,    /* overwrite with boot register address */
     FIXUP_DSB,        /* overwrite with correct DSB insn for cpu */
+    FIXUP_EL,         /* overwrite with kernel entry EL */
     FIXUP_MAX,
 } FixupType;
 
@@ -46,6 +47,20 @@ typedef struct ARMInsnFixup {
 } ARMInsnFixup;
 
 static const ARMInsnFixup bootloader_aarch64[] = {
+    { 0xd5384240 }, /* mrs x0, currentel */
+    { 0x7100301f }, /* cmp w0, #0xc */
+    { 0x54000001 + (9 << 5) }, /* b.ne ELx_start */
+/* Jump down from EL3 to ELx */
+    { 0x10000001 + (9 << 5) }, /* adr x1, ELx_start */
+    { 0xd53e1100 }, /* mrs x0, scr_el3 */
+    { 0xb2400000 }, /* orr x0, x0, #0x1 - SCR.NS */
+    { 0xb2780000 }, /* orr x0, x0, #0x80 - SCR.HCE */
+    { 0xd51e1100 }, /* msr scr_el3, x0 */
+    { 0xd2807820, FIXUP_EL, 7, 2 }, /* movz x0, 0x3c1 (+ EL<<2) */
+    { 0xd51e4000 }, /* msr spsr_el3, x0 */
+    { 0xd51e4021 }, /* msr elr_el3, x1 */
+    { 0xd69f03e0 }, /* eret */
+/* ELx_start: */
     { 0x580000c0 }, /* ldr x0, arg ; Load the lower 32-bits of DTB */
     { 0xaa1f03e1 }, /* mov x1, xzr */
     { 0xaa1f03e2 }, /* mov x2, xzr */
@@ -141,6 +156,7 @@ static void write_bootloader(const char *name, hwaddr addr,
         case FIXUP_GIC_CPU_IF:
         case FIXUP_BOOTREG:
         case FIXUP_DSB:
+        case FIXUP_EL:
             insn = deposit32(insn, shift, length, fixupcontext[fixup]);
             break;
         default:
@@ -583,6 +599,11 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
         }
         fixupcontext[FIXUP_ENTRYPOINT] = entry;
 
+        fixupcontext[FIXUP_EL] = 1;
+        if (arm_feature(&cpu->env, ARM_FEATURE_EL2)) {
+            fixupcontext[FIXUP_EL] = 2;
+        }
+
         write_bootloader("bootloader", info->loader_start,
                          primary_loader, fixupcontext);
 
-- 
2.0.1.1.gfbfc394

Re: [Qemu-devel] [RFC v1 2/2] arm: boot: Add EL jump-down code for Linux

[Christopher Covington]

Hi Peter,

On 08/18/2014 08:54 PM, Peter Crosthwaite wrote:
> Linux should boot in EL2 or EL1. If in EL3, jump down before handing
> off to Linux.
> 
> Signed-off-by: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
> ---
> 
>  hw/arm/boot.c | 21 +++++++++++++++++++++
>  1 file changed, 21 insertions(+)
> 
> diff --git a/hw/arm/boot.c b/hw/arm/boot.c
> index 840f5da..f1f6365 100644
> --- a/hw/arm/boot.c
> +++ b/hw/arm/boot.c
> @@ -35,6 +35,7 @@ typedef enum {
>      FIXUP_GIC_CPU_IF, /* overwrite with GIC CPU interface address */
>      FIXUP_BOOTREG,    /* overwrite with boot register address */
>      FIXUP_DSB,        /* overwrite with correct DSB insn for cpu */
> +    FIXUP_EL,         /* overwrite with kernel entry EL */
>      FIXUP_MAX,
>  } FixupType;
>  
> @@ -46,6 +47,20 @@ typedef struct ARMInsnFixup {
>  } ARMInsnFixup;
>  
>  static const ARMInsnFixup bootloader_aarch64[] = {
> +    { 0xd5384240 }, /* mrs x0, currentel */
> +    { 0x7100301f }, /* cmp w0, #0xc */
> +    { 0x54000001 + (9 << 5) }, /* b.ne ELx_start */
> +/* Jump down from EL3 to ELx */
> +    { 0x10000001 + (9 << 5) }, /* adr x1, ELx_start */

Nit: Consider putting this right before the usage to make the code slightly
easier to follow. Always using a distinct GPR per SR (such as x2 for SPSR) or
reusing the same GPR for all the SRs (such as x0 for ELR) might also make the
code easier to follow.

> +    { 0xd53e1100 }, /* mrs x0, scr_el3 */
> +    { 0xb2400000 }, /* orr x0, x0, #0x1 - SCR.NS */
> +    { 0xb2780000 }, /* orr x0, x0, #0x80 - SCR.HCE */
> +    { 0xd51e1100 }, /* msr scr_el3, x0 */
> +    { 0xd2807820, FIXUP_EL, 7, 2 }, /* movz x0, 0x3c1 (+ EL<<2) */
> +    { 0xd51e4000 }, /* msr spsr_el3, x0 */
> +    { 0xd51e4021 }, /* msr elr_el3, x1 */
> +    { 0xd69f03e0 }, /* eret */
> +/* ELx_start: */
>      { 0x580000c0 }, /* ldr x0, arg ; Load the lower 32-bits of DTB */
>      { 0xaa1f03e1 }, /* mov x1, xzr */
>      { 0xaa1f03e2 }, /* mov x2, xzr */
> @@ -141,6 +156,7 @@ static void write_bootloader(const char *name, hwaddr addr,
>          case FIXUP_GIC_CPU_IF:
>          case FIXUP_BOOTREG:
>          case FIXUP_DSB:
> +        case FIXUP_EL:
>              insn = deposit32(insn, shift, length, fixupcontext[fixup]);
>              break;
>          default:
> @@ -583,6 +599,11 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
>          }
>          fixupcontext[FIXUP_ENTRYPOINT] = entry;
>  
> +        fixupcontext[FIXUP_EL] = 1;
> +        if (arm_feature(&cpu->env, ARM_FEATURE_EL2)) {
> +            fixupcontext[FIXUP_EL] = 2;
> +        }
> +
>          write_bootloader("bootloader", info->loader_start,
>                           primary_loader, fixupcontext);
>  
> 

Thanks,
Christopher

-- 
Employee of Qualcomm Innovation Center, Inc.
Qualcomm Innovation Center, Inc. is a member of Code Aurora Forum,
hosted by the Linux Foundation.

Re: [Qemu-devel] [RFC v1 2/2] arm: boot: Add EL jump-down code for Linux

[Peter Maydell]

On 19 August 2014 01:54, Peter Crosthwaite <peter.crosthwaite@xilinx.com> wrote:
> Linux should boot in EL2 or EL1. If in EL3, jump down before handing
> off to Linux.
>
> Signed-off-by: Peter Crosthwaite <peter.crosthwaite@xilinx.com>

The other way we could do this would be to have the
do_cpu_reset() hook put the CPU into EL2 or EL1 before
starting the boot code. (This would be in line with how it
would work for KVM).

Have you thought about how this would work for
secondary CPUs (which boot via PSCI poweron)?

thanks
-- PMM

Re: [Qemu-devel] [RFC v1 2/2] arm: boot: Add EL jump-down code for Linux

[Peter Crosthwaite]

On Tue, Sep 2, 2014 at 3:15 AM, Peter Maydell <peter.maydell@linaro.org> wrote:
> On 19 August 2014 01:54, Peter Crosthwaite <peter.crosthwaite@xilinx.com> wrote:
>> Linux should boot in EL2 or EL1. If in EL3, jump down before handing
>> off to Linux.
>>
>> Signed-off-by: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
>
> The other way we could do this would be to have the
> do_cpu_reset() hook put the CPU into EL2 or EL1 before
> starting the boot code. (This would be in line with how it
> would work for KVM).
>

Tried that at first, and the problem we found was it introduced a
reset order dep. The arm-boot and CPU race on who sets the EL.

> Have you thought about how this would work for
> secondary CPUs (which boot via PSCI poweron)?
>

Not yet although I expect something similar. Currently I use a spin
table elf which I load and entry-points for the secondaries (I have a
bootloader patch out-of-tree that can do this) which does the
reduction. It's pretty much the same routine as here so there might be
some code share.

Regards,
Peter

> thanks
> -- PMM
>

Re: [Qemu-devel] [RFC v1 2/2] arm: boot: Add EL jump-down code for Linux

[Peter Maydell]

On 1 September 2014 23:41, Peter Crosthwaite
<peter.crosthwaite@xilinx.com> wrote:
> On Tue, Sep 2, 2014 at 3:15 AM, Peter Maydell <peter.maydell@linaro.org> wrote:
>> On 19 August 2014 01:54, Peter Crosthwaite <peter.crosthwaite@xilinx.com> wrote:
>>> Linux should boot in EL2 or EL1. If in EL3, jump down before handing
>>> off to Linux.
>>>
>>> Signed-off-by: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
>>
>> The other way we could do this would be to have the
>> do_cpu_reset() hook put the CPU into EL2 or EL1 before
>> starting the boot code. (This would be in line with how it
>> would work for KVM).
>>
>
> Tried that at first, and the problem we found was it introduced a
> reset order dep. The arm-boot and CPU race on who sets the EL.

Surely there's already a reset order dependency, because
both the arm-boot reset hook and CPU reset proper want
to initialise the PC ?

>> Have you thought about how this would work for
>> secondary CPUs (which boot via PSCI poweron)?

> Not yet although I expect something similar.

Hmm. I guess PSCI poweron won't even go through
the boot.c code, thinking a bit more about it. I need to
look at the TCG PSCI patches too.

-- PMM

Re: [Qemu-devel] [RFC v1 2/2] arm: boot: Add EL jump-down code for Linux

[Peter Crosthwaite]

On Tue, Sep 2, 2014 at 8:56 AM, Peter Maydell <peter.maydell@linaro.org> wrote:
> On 1 September 2014 23:41, Peter Crosthwaite
> <peter.crosthwaite@xilinx.com> wrote:
>> On Tue, Sep 2, 2014 at 3:15 AM, Peter Maydell <peter.maydell@linaro.org> wrote:
>>> On 19 August 2014 01:54, Peter Crosthwaite <peter.crosthwaite@xilinx.com> wrote:
>>>> Linux should boot in EL2 or EL1. If in EL3, jump down before handing
>>>> off to Linux.
>>>>
>>>> Signed-off-by: Peter Crosthwaite <peter.crosthwaite@xilinx.com>
>>>
>>> The other way we could do this would be to have the
>>> do_cpu_reset() hook put the CPU into EL2 or EL1 before
>>> starting the boot code. (This would be in line with how it
>>> would work for KVM).
>>>
>>
>> Tried that at first, and the problem we found was it introduced a
>> reset order dep. The arm-boot and CPU race on who sets the EL.
>
> Surely there's already a reset order dependency, because
> both the arm-boot reset hook and CPU reset proper want
> to initialise the PC ?
>

Yes. I had a PC specific solution to this though when it became a
problem. I have another series that made the reset ordering a little
less predictable (specifically implementation of CPU reset GPIOs). So
i'm trying to reduce the footprint of this issue as much as possible.

Regards,
Peter

>>> Have you thought about how this would work for
>>> secondary CPUs (which boot via PSCI poweron)?
>
>> Not yet although I expect something similar.
>
> Hmm. I guess PSCI poweron won't even go through
> the boot.c code, thinking a bit more about it. I need to
> look at the TCG PSCI patches too.
>
> -- PMM
>

Re: [Qemu-devel] [RFC v1 2/2] arm: boot: Add EL jump-down code for Linux

[Peter Maydell]

On 2 September 2014 00:06, Peter Crosthwaite
<peter.crosthwaite@xilinx.com> wrote:
> On Tue, Sep 2, 2014 at 8:56 AM, Peter Maydell <peter.maydell@linaro.org> wrote:
>> Surely there's already a reset order dependency, because
>> both the arm-boot reset hook and CPU reset proper want
>> to initialise the PC ?

> Yes. I had a PC specific solution to this though when it became a
> problem. I have another series that made the reset ordering a little
> less predictable (specifically implementation of CPU reset GPIOs)

Actually, I'm confused now where the reset ordering issue comes
in, because (ignoring realize, which will call reset) as far as I
could make out the only place that the ARM CPU reset gets
called is when the do_cpu_reset() hook in boot.c calls
cpu_reset(CPU(cpu)) explicitly. This is unfortunate because
it means if you didn't use -kernel then we won't reset the CPU
if you do a system reset after first bootup...but on the other
hand it means there's no reset order issue?

Thinking about reset gives me a headache, though.

thanks
-- PMM

Re: [Qemu-devel] [RFC v1 2/2] arm: boot: Add EL jump-down code for Linux

[Peter Crosthwaite]

On Tue, Sep 2, 2014 at 9:12 AM, Peter Maydell <peter.maydell@linaro.org> wrote:
> On 2 September 2014 00:06, Peter Crosthwaite
> <peter.crosthwaite@xilinx.com> wrote:
>> On Tue, Sep 2, 2014 at 8:56 AM, Peter Maydell <peter.maydell@linaro.org> wrote:
>>> Surely there's already a reset order dependency, because
>>> both the arm-boot reset hook and CPU reset proper want
>>> to initialise the PC ?
>
>> Yes. I had a PC specific solution to this though when it became a
>> problem. I have another series that made the reset ordering a little
>> less predictable (specifically implementation of CPU reset GPIOs)
>
> Actually, I'm confused now where the reset ordering issue comes
> in, because (ignoring realize, which will call reset) as far as I
> could make out the only place that the ARM CPU reset gets
> called is when the do_cpu_reset() hook in boot.c calls
> cpu_reset(CPU(cpu)) explicitly. This is unfortunate because
> it means if you didn't use -kernel then we won't reset the CPU

Yeh, my setup fixes this via assertion of an actual CPU reset GPIO
that connects to cpu_reset(). so I guess for me this problem is solved
...

> if you do a system reset after first bootup...but on the other
> hand it means there's no reset order issue?
>

... but then the solution introduces the reset order issue for me.

The other solution worth discussing now is whether we simply want to
acknowledge that devs can have reset order deps and that's ok.
Implementing on the qdev level:

void qdev_register_reset_dep(DeviceState *dependant, Device *dependee);

You then QOMify the bootloader and get rid of these explicit
bootloader cpu_reset calls.

Regards,
Peter

> Thinking about reset gives me a headache, though.
>
> thanks
> -- PMM
>

Re: [Qemu-devel] [RFC v1 2/2] arm: boot: Add EL jump-down code for Linux

[Peter Maydell]

On 2 September 2014 00:22, Peter Crosthwaite
<peter.crosthwaite@xilinx.com> wrote:
> On Tue, Sep 2, 2014 at 9:12 AM, Peter Maydell <peter.maydell@linaro.org> wrote:
>> Actually, I'm confused now where the reset ordering issue comes
>> in, because (ignoring realize, which will call reset) as far as I
>> could make out the only place that the ARM CPU reset gets
>> called is when the do_cpu_reset() hook in boot.c calls
>> cpu_reset(CPU(cpu)) explicitly. This is unfortunate because
>> it means if you didn't use -kernel then we won't reset the CPU
>
> Yeh, my setup fixes this via assertion of an actual CPU reset GPIO
> that connects to cpu_reset(). so I guess for me this problem is solved
> ...
>
>> if you do a system reset after first bootup...but on the other
>> hand it means there's no reset order issue?
>>
>
> ... but then the solution introduces the reset order issue for me.

Aha. (I haven't figured out how we ought to have the cpu
reset get called on system reset yet -- I was vaguely under
the impression that all QOM/qdev devices got their reset
hook called, but obviously not.)

> The other solution worth discussing now is whether we simply want to
> acknowledge that devs can have reset order deps and that's ok.
> Implementing on the qdev level:
>
> void qdev_register_reset_dep(DeviceState *dependant, Device *dependee);

I guess the question is what devices have a reset dependency
other than this odd bootloader special case? (I should try to
dig our previous discussions out of the list archives tomorrow.)

> You then QOMify the bootloader and get rid of these explicit
> bootloader cpu_reset calls.

It would certainly be nice not to have the somewhat funky
bootloader handling of reset...

-- PMM