[U-Boot] [RFD] OMAP5: Working HYP mode

[U-Boot] [RFD] OMAP5: Working HYP mode

[Alexander Tarasikov]

Hello, u-boot developers!
I am attaching a patch to enable the HYP (hypervisor)
mode on the OMAP5 CPU. It is based on a previous patch by
Ian Molton. For some reason, Ian's patch was incorrect
(loading PC with the wrong address, missing memory barriers,
and, most importantly, using the wrong SMC call). I think it 
is because Ian was using some early omap5 prototype, and I was
using the SVT OMAP5432UEVM board which is commercially available.

Since this is a gross hack, I neither expect nor want it to be merged,
but I think people experimenting with ARM virtualization will find
it useful.

I have KVM working on OMAP5 as of now (using KVM patches from 
openvirtualization and some linux kernel bugfixes of my own) and
I plan to write a blog post soon about how to repeat my steps.
My plan is to publish the patches to fix the bugs I've found.

I would like to receive comments on what is the planned way to implement
the HYP switching in u-boot and whether it is possible without using the
TrustZone SMC calls.

Have fun

[U-Boot] [PATCH] HACK: enable HYP mode on OMAP5

[Alexander Tarasikov]

---
 arch/arm/lib/bootm.c | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 87 insertions(+), 2 deletions(-)

diff --git a/arch/arm/lib/bootm.c b/arch/arm/lib/bootm.c
index eefb456..63395c3 100644
--- a/arch/arm/lib/bootm.c
+++ b/arch/arm/lib/bootm.c
@@ -8,6 +8,9 @@
  * Marius Groeger <mgroeger@sysgo.de>
  *
  * Copyright (C) 2001  Erik Mouw (J.A.K.Mouw at its.tudelft.nl)
+ * HYP entry (C) 2012  Ian Molton <ian.molton@codethink.co.uk>
+ *                and  Clemens Fischer <clemens.fischer@h-da.de>
+ *           (C) 2013 Alexander Tarasikov <alexander.tarasikov@gmail.com>
  *
  * SPDX-License-Identifier:	GPL-2.0+
  */
@@ -26,6 +29,89 @@ DECLARE_GLOBAL_DATA_PTR;
 
 static struct tag *params;
 
+/*
+ * function called by cpu 1 after wakeup
+ */
+extern void __hyp_init_sec(void);
+
+asm (
+	".pushsection .text\n"
+	".global __hyp_init_sec\n"
+	"__hyp_init_sec:\n"
+		"ldr r12, =0x102\n"
+		"adr r0, hyp_init_cont\n"
+		"dsb\n"
+		"isb\n"
+		"dmb\n"
+		"smc #0\n"
+		"hyp_init_cont: ldr r1, =0x48281800\n" // AUX_CORE_BOOT_0
+		"mov r2, #0\n"
+		"str r2, [r1, #4]\n"
+		"dsb\n"
+		"isb\n"
+		"dmb\n"
+		"wait: wfe\n"
+		"ldr r2, [r1, #4]\n"
+		"cmp r2, #0\n"
+		"movne pc, r2\n"
+		"b wait\n"
+	".popsection\n"
+);
+
+/*
+ * Enable HYP mode on the OMAP5 CPU
+ *
+ * FIXME: this needs to test to make sure its running on an OMAP5
+ *
+ * We wake up CPU1 at __hyp_init_sec which allows us to put it into HYP
+ * mode.
+ *
+ * CPU1 then clears AUX_CORE_BOOT_0 and enters WFE, until the kernel wakes it.
+ *
+ * In order to avoid CPU1 continuing execution on just about any event, we
+ * wait for AUX_CORE_BOOT_0 to contain a non-zero address, at which point
+ * we continue execution at that address.
+ *
+ */
+
+static void hyp_enable(void) {
+	 /*Wake up CPU1 and enable HYP on CPU0. */
+	asm(
+		"ldr r1, =0x48281800\n"     // AUX_CORE_BOOT_1
+		"ldr r2, =__hyp_init_sec\n"
+		"str r2, [r1, #4]\n"
+		"mov r2, #0x20\n"
+		"str r2, [r1]\n"            // AUX_CORE_BOOT_0
+		"isb\n"
+		"dmb\n"
+		"dsb\n"
+		"sev\n"                     // Wake CPU1
+		"adr r1, hyp_stack\n"
+		"stm r1, {r4-r14}\n" //save registers on the temporary stack
+		"ldr r12, =0x102\n" //start hypervisor SMC code
+		"adr r0, hyp_ena_cont\n" //address after SMC instruction
+		"dsb\n"
+		"isb\n"
+		"dmb\n"
+		"smc #0\n"                 // CPU0 -> HYP mode
+		"hyp_stack:\n"
+		".word 0x0\n"
+		".word 0x0\n"
+		".word 0x0\n"
+		".word 0x0\n"
+		".word 0x0\n"
+		".word 0x0\n"
+		".word 0x0\n"
+		".word 0x0\n"
+		".word 0x0\n"
+		".word 0x0\n"
+		".word 0x0\n"
+		"hyp_ena_cont: adr r1,hyp_stack\n"
+		"ldm   r1, {r4-r14}\n"
+		:::"r0", "r1", "r2", "r3", "cc", "memory"
+	);
+};
+
 static ulong get_sp(void)
 {
 	ulong ret;
@@ -144,7 +230,6 @@ static void setup_initrd_tag(bd_t *bd, ulong initrd_start, ulong initrd_end)
 
 	params->u.initrd.start = initrd_start;
 	params->u.initrd.size = initrd_end - initrd_start;
-
 	params = tag_next (params);
 }
 
@@ -185,7 +270,6 @@ __weak void setup_board_tags(struct tag **in_params) {}
 static void boot_prep_linux(bootm_headers_t *images)
 {
 	char *commandline = getenv("bootargs");
-
 	if (IMAGE_ENABLE_OF_LIBFDT && images->ft_len) {
 #ifdef CONFIG_OF_LIBFDT
 		debug("using: FDT\n");
@@ -246,6 +330,7 @@ static void boot_jump_linux(bootm_headers_t *images, int flag)
 	else
 		r2 = gd->bd->bi_boot_params;
 
+	hyp_enable();
 	if (!fake)
 		kernel_entry(0, machid, r2);
 }
-- 
1.8.4

[U-Boot] [PATCH] HACK: enable HYP mode on OMAP5

[MJ embd]

Few quick comments
1) wouldnt it be better to create a .S file and move big chunk of
inline assembly there
2) can the multiple occurences of  .word be substituted with something else?


On 9/17/13, Alexander Tarasikov <alexander.tarasikov@gmail.com> wrote:
> ---
>  arch/arm/lib/bootm.c | 89
> ++++++++++++++++++++++++++++++++++++++++++++++++++--
>  1 file changed, 87 insertions(+), 2 deletions(-)
>
> diff --git a/arch/arm/lib/bootm.c b/arch/arm/lib/bootm.c
> index eefb456..63395c3 100644
> --- a/arch/arm/lib/bootm.c
> +++ b/arch/arm/lib/bootm.c
> @@ -8,6 +8,9 @@
>   * Marius Groeger <mgroeger@sysgo.de>
>   *
>   * Copyright (C) 2001  Erik Mouw (J.A.K.Mouw at its.tudelft.nl)
> + * HYP entry (C) 2012  Ian Molton <ian.molton@codethink.co.uk>
> + *                and  Clemens Fischer <clemens.fischer@h-da.de>
> + *           (C) 2013 Alexander Tarasikov <alexander.tarasikov@gmail.com>
>   *
>   * SPDX-License-Identifier:	GPL-2.0+
>   */
> @@ -26,6 +29,89 @@ DECLARE_GLOBAL_DATA_PTR;
>
>  static struct tag *params;
>
> +/*
> + * function called by cpu 1 after wakeup
> + */
> +extern void __hyp_init_sec(void);
> +
> +asm (
> +	".pushsection .text\n"
> +	".global __hyp_init_sec\n"
> +	"__hyp_init_sec:\n"
> +		"ldr r12, =0x102\n"
> +		"adr r0, hyp_init_cont\n"
> +		"dsb\n"
> +		"isb\n"
> +		"dmb\n"
> +		"smc #0\n"
> +		"hyp_init_cont: ldr r1, =0x48281800\n" // AUX_CORE_BOOT_0
> +		"mov r2, #0\n"
> +		"str r2, [r1, #4]\n"
> +		"dsb\n"
> +		"isb\n"
> +		"dmb\n"
> +		"wait: wfe\n"
> +		"ldr r2, [r1, #4]\n"
> +		"cmp r2, #0\n"
> +		"movne pc, r2\n"
> +		"b wait\n"
> +	".popsection\n"
> +);
> +
> +/*
> + * Enable HYP mode on the OMAP5 CPU
> + *
> + * FIXME: this needs to test to make sure its running on an OMAP5
> + *
> + * We wake up CPU1 at __hyp_init_sec which allows us to put it into HYP
> + * mode.
> + *
> + * CPU1 then clears AUX_CORE_BOOT_0 and enters WFE, until the kernel wakes
> it.
> + *
> + * In order to avoid CPU1 continuing execution on just about any event, we
> + * wait for AUX_CORE_BOOT_0 to contain a non-zero address, at which point
> + * we continue execution at that address.
> + *
> + */
> +
> +static void hyp_enable(void) {
> +	 /*Wake up CPU1 and enable HYP on CPU0. */
> +	asm(
> +		"ldr r1, =0x48281800\n"     // AUX_CORE_BOOT_1
> +		"ldr r2, =__hyp_init_sec\n"
> +		"str r2, [r1, #4]\n"
> +		"mov r2, #0x20\n"
> +		"str r2, [r1]\n"            // AUX_CORE_BOOT_0
> +		"isb\n"
> +		"dmb\n"
> +		"dsb\n"
> +		"sev\n"                     // Wake CPU1
> +		"adr r1, hyp_stack\n"
> +		"stm r1, {r4-r14}\n" //save registers on the temporary stack
> +		"ldr r12, =0x102\n" //start hypervisor SMC code
> +		"adr r0, hyp_ena_cont\n" //address after SMC instruction
> +		"dsb\n"
> +		"isb\n"
> +		"dmb\n"
> +		"smc #0\n"                 // CPU0 -> HYP mode
> +		"hyp_stack:\n"
> +		".word 0x0\n"
> +		".word 0x0\n"
> +		".word 0x0\n"
> +		".word 0x0\n"
> +		".word 0x0\n"
> +		".word 0x0\n"
> +		".word 0x0\n"
> +		".word 0x0\n"
> +		".word 0x0\n"
> +		".word 0x0\n"
> +		".word 0x0\n"
> +		"hyp_ena_cont: adr r1,hyp_stack\n"
> +		"ldm   r1, {r4-r14}\n"
> +		:::"r0", "r1", "r2", "r3", "cc", "memory"
> +	);
> +};
> +
>  static ulong get_sp(void)
>  {
>  	ulong ret;
> @@ -144,7 +230,6 @@ static void setup_initrd_tag(bd_t *bd, ulong
> initrd_start, ulong initrd_end)
>
>  	params->u.initrd.start = initrd_start;
>  	params->u.initrd.size = initrd_end - initrd_start;
> -
>  	params = tag_next (params);
>  }
>
> @@ -185,7 +270,6 @@ __weak void setup_board_tags(struct tag **in_params) {}
>  static void boot_prep_linux(bootm_headers_t *images)
>  {
>  	char *commandline = getenv("bootargs");
> -
>  	if (IMAGE_ENABLE_OF_LIBFDT && images->ft_len) {
>  #ifdef CONFIG_OF_LIBFDT
>  		debug("using: FDT\n");
> @@ -246,6 +330,7 @@ static void boot_jump_linux(bootm_headers_t *images, int
> flag)
>  	else
>  		r2 = gd->bd->bi_boot_params;
>
> +	hyp_enable();
>  	if (!fake)
>  		kernel_entry(0, machid, r2);
>  }
> --
> 1.8.4
>
> _______________________________________________
> U-Boot mailing list
> U-Boot at lists.denx.de
> http://lists.denx.de/mailman/listinfo/u-boot
>


-- 
-mj

[U-Boot] [RFD] OMAP5: Working HYP mode

[Andre Przywara]

On 09/17/2013 12:28 AM, Alexander Tarasikov wrote:
> Hello, u-boot developers!
> I am attaching a patch to enable the HYP (hypervisor)
> mode on the OMAP5 CPU. It is based on a previous patch by
> Ian Molton. For some reason, Ian's patch was incorrect
> (loading PC with the wrong address, missing memory barriers,
> and, most importantly, using the wrong SMC call). I think it
> is because Ian was using some early omap5 prototype, and I was
> using the SVT OMAP5432UEVM board which is commercially available.
>
> Since this is a gross hack, I neither expect nor want it to be merged,
> but I think people experimenting with ARM virtualization will find
> it useful.
>
> I have KVM working on OMAP5 as of now (using KVM patches from
> openvirtualization and some linux kernel bugfixes of my own) and
> I plan to write a blog post soon about how to repeat my steps.
> My plan is to publish the patches to fix the bugs I've found.
>
> I would like to receive comments on what is the planned way to implement
> the HYP switching in u-boot and whether it is possible without using the
> TrustZone SMC calls.

Alexander,

have you seen my series?

http://lists.denx.de/pipermail/u-boot/2013-September/163019.html
[U-Boot] [PATCH v5 0/8] ARMv7: Add HYP mode switching support

I am about to support more boards, but OMAP5 seems very different, 
probably because it starts u-boot in non-secure state already, right?
So all the magic is in the firmware, which just needs to be called by 
smc calls, right?
So it looks like one could at least use some infrastructure 
(CONFIG_ARMV7_VIRT and the higher level .c files) to properly merge 
OMAP5 support in. Also needed would be to make most code of my patch 
optional by using the proper defines.
What do you think?

Regards,
Andre.

[U-Boot] [RFD] OMAP5: Working HYP mode

[Alexander Tarasikov]

On Wed, Sep 25, 2013 at 11:57 AM, Andre Przywara
<andre.przywara@linaro.org> wrote:
>
> Alexander,
>
> have you seen my series?

Hi and sorry for the late reply. Yes, seen that.

>
> http://lists.denx.de/pipermail/u-boot/2013-September/163019.html
> [U-Boot] [PATCH v5 0/8] ARMv7: Add HYP mode switching support
>
> I am about to support more boards, but OMAP5 seems very different, probably
> because it starts u-boot in non-secure state already, right?

Correct!

> So all the magic is in the firmware, which just needs to be called by smc
> calls, right?
> So it looks like one could at least use some infrastructure
> (CONFIG_ARMV7_VIRT and the higher level .c files) to properly merge OMAP5
> support in. Also needed would be to make most code of my patch optional by
> using the proper defines.
> What do you think?

I don't know, maybe just add a callback (or define the HYP
initialization routine
as a weak symbol). OMAP5 just works without modifying VBAR or touching anything
except what's mentioned in the patch, so I probably need to read the manual
about virtualization to comment on it.

>
> Regards,
> Andre.
>



-- 
Regards, Alexander