Re: [PATCH RFC 15/18] arm: Add ability to relocate Xen in over 4GB space

[Julien Grall <julien.grall@arm.com>]

Hello,

On 18/05/16 17:32, Andrii Anisov wrote:
> From: Iurii Konovalenko <iurii.konovalenko@globallogic.com>
>
> Move Xen to the end of physical memory

Can you explain why? As Ian campbell said on the previous version sent 
last year [1], Xen itself (i.e .text, .bss and .data, not the xenheap 
itself) is at most 2MB. Are your constraints such that at a 4GB-2MB 
(i.e. 4096M) of lowmem would not be acceptable?

Also, this patch looks very similar to the one you have sent last year 
[2]. Most of the comments in this thread are still valid, so I will not 
repeat them here.

Regards,

>
> Signed-off-by: Iurii Konovalenko <iurii.konovalenko@globallogic.com>
> ---
>   xen/Rules.mk                   |  1 +
>   xen/arch/arm/arm32/head.S      | 21 ++++++++++++++++++++-
>   xen/arch/arm/domain_build.c    |  2 +-
>   xen/arch/arm/platforms/omap5.c | 17 ++++++++++++++---
>   xen/arch/arm/platforms/rcar2.c |  9 ++++++++-
>   xen/arch/arm/setup.c           | 21 ++++++++++
>++++++++++-
>   xen/arch/arm/smpboot.c         | 33 +++++++++++++++++++++++++++++----
>   7 files changed, 93 insertions(+), 11 deletions(-)
>
> diff --git a/xen/Rules.mk b/xen/Rules.mk
> index feb08d6..fbd34a5 100644
> --- a/xen/Rules.mk
> +++ b/xen/Rules.mk
> @@ -64,6 +64,7 @@ CFLAGS-$(HAS_PCI)       += -DHAS_PCI
>   CFLAGS-$(HAS_IOPORTS)   += -DHAS_IOPORTS
>   CFLAGS-$(HAS_PDX)       += -DHAS_PDX
>   CFLAGS-$(frame_pointer) += -fno-omit-frame-pointer -DCONFIG_FRAME_POINTER
> +CFLAGS-$(ARM32_RELOCATE_OVER_4GB) += -DARM32_RELOCATE_OVER_4GB
>
>   ifneq ($(max_phys_cpus),)
>   CFLAGS-y                += -DMAX_PHYS_CPUS=$(max_phys_cpus)
> diff --git a/xen/arch/arm/arm32/head.S b/xen/arch/arm/arm32/head.S
> index e1f29bd..a644d6d 100644
> --- a/xen/arch/arm/arm32/head.S
> +++ b/xen/arch/arm/arm32/head.S
> @@ -262,8 +262,21 @@ cpu_init_done:
>           add   r4, r4, r10            /* r4 := paddr (boot_pagetable) */
>           mov   r5, #0                 /* r4:r5 is paddr (boot_pagetable) */
>           mcrr  CP64(r4, r5, HTTBR)
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +        teq   r7, #0
> +        beq   1f                     /* construct pagetable if CPU0 */
>
> -        /* Setup boot_pgtable: */
> +        /*Skip constructing TLBs for secondary CPUs, use constracted by CPU0*/
> +        PRINT("- Skip construction pagetable, using CPU0 table @")
> +        mov   r0, r5
> +        bl    putn
> +        mov   r0, r4
> +        bl    putn
> +        PRINT("  -\r\n")
> +        bne   skip_constructing
> +#endif
> +
> +1:      /* Setup boot_pgtable: */
>           ldr   r1, =boot_second
>           add   r1, r1, r10            /* r1 := paddr (boot_second) */
>
> @@ -346,6 +359,7 @@ virtphys_clash:
>           PRINT("- Unable to build boot page tables - virt and phys addresses clash. -\r\n")
>           b     fail
>
> +skip_constructing:
>   1:
>           PRINT("- Turning on paging -\r\n")
>
> @@ -427,6 +441,11 @@ paging:
>            * setup in init_secondary_pagetables. */
>
>           ldr   r4, =init_ttbr         /* VA of HTTBR value stashed by CPU 0 */
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +        ldr   r1, =_start
> +        sub r4, r1
> +        add r4, #BOOT_RELOC_VIRT_START
> +#endif
>           ldrd  r4, r5, [r4]           /* Actual value */
>           dsb
>           mcrr  CP64(r4, r5, HTTBR)
> diff --git a/xen/arch/arm/domain_build.c b/xen/arch/arm/domain_build.c
> index b48718d..f06792e 100644
> --- a/xen/arch/arm/domain_build.c
> +++ b/xen/arch/arm/domain_build.c
> @@ -1487,7 +1487,7 @@ static void __init find_gnttab_region(struct domain *d,
>       if ( (kinfo->gnttab_size >> PAGE_SHIFT) < max_grant_frames )
>           panic("Cannot find a space for the grant table region\n");
>
> -#ifdef CONFIG_ARM_32
> +#if defined(CONFIG_ARM_32) && !defined(ARM32_RELOCATE_OVER_4GB)
>       /*
>        * The gnttab region must be under 4GB in order to work with DOM0
>        * using short page table.
> diff --git a/xen/arch/arm/platforms/omap5.c b/xen/arch/arm/platforms/omap5.c
> index a49ba62..fe77397 100644
> --- a/xen/arch/arm/platforms/omap5.c
> +++ b/xen/arch/arm/platforms/omap5.c
> @@ -25,6 +25,10 @@
>   #include <xen/vmap.h>
>   #include <asm/io.h>
>
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +extern paddr_t xen_relocation_offset;
> +#endif
> +
>   static uint16_t num_den[8][2] = {
>       {         0,          0 },  /* not used */
>       {  26 *  64,  26 *  125 },  /* 12.0 Mhz */
> @@ -132,9 +136,16 @@ static int __init omap5_smp_init(void)
>       }
>
>       printk("Set AuxCoreBoot1 to %"PRIpaddr" (%p)\n",
> -           __pa(init_secondary), init_secondary);
> -    writel(__pa(init_secondary), wugen_base + OMAP_AUX_CORE_BOOT_1_OFFSET);
> -
> +           __pa(init_secondary)
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +            - xen_relocation_offset
> +#endif
> +			, init_secondary);
> +    writel(__pa(init_secondary)
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +            - xen_relocation_offset
> +#endif
> +			, wugen_base + OMAP_AUX_CORE_BOOT_1_OFFSET);
>       printk("Set AuxCoreBoot0 to 0x20\n");
>       writel(0x20, wugen_base + OMAP_AUX_CORE_BOOT_0_OFFSET);
>
> diff --git a/xen/arch/arm/platforms/rcar2.c b/xen/arch/arm/platforms/rcar2.c
> index bb25751..26973f6 100644
> --- a/xen/arch/arm/platforms/rcar2.c
> +++ b/xen/arch/arm/platforms/rcar2.c
> @@ -25,6 +25,9 @@
>   #define RCAR2_RAM_SIZE                         0x1000
>   #define RCAR2_SMP_START_OFFSET                 0xFFC
>
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +extern paddr_t xen_relocation_offset;
> +#endif
>   static int __init rcar2_smp_init(void)
>   {
>       void __iomem *pram;
> @@ -38,7 +41,11 @@ static int __init rcar2_smp_init(void)
>       }
>
>       /* setup reset vectors */
> -    writel(__pa(init_secondary), pram + RCAR2_SMP_START_OFFSET);
> +    writel(__pa(init_secondary)
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +            - xen_relocation_offset
> +#endif
> +            , pram + RCAR2_SMP_START_OFFSET);
>       iounmap(pram);
>
>       sev();
> diff --git a/xen/arch/arm/setup.c b/xen/arch/arm/setup.c
> index 48f734f..7e507bc 100644
> --- a/xen/arch/arm/setup.c
> +++ b/xen/arch/arm/setup.c
> @@ -394,9 +394,13 @@ static paddr_t __init get_xen_paddr(void)
>           const struct membank *bank = &mi->bank[i];
>           paddr_t s, e;
>
> +        //TODO: investigate reason why we need contiguous memory
> +        //and how it can affect relocation of Xen in over 4GB space
> +#ifndef ARM32_RELOCATE_OVER_4GB
>           /* We can only deal with contiguous memory at the moment */
>           if ( last_end != bank->start )
>               break;
> +#endif
>
>           last_end = bank->start + bank->size;
>
> @@ -407,7 +411,7 @@ static paddr_t __init get_xen_paddr(void)
>               if ( !e )
>                   continue;
>
> -#ifdef CONFIG_ARM_32
> +#if defined (CONFIG_ARM_32) && !defined(ARM32_RELOCATE_OVER_4GB)
>               /* Xen must be under 4GB */
>               if ( e > 0x100000000ULL )
>                   e = 0x100000000ULL;
> @@ -698,6 +702,9 @@ void __init setup_cache(void)
>       cacheline_bytes = 1U << (4 + (ccsid & 0x7));
>   }
>
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +paddr_t xen_relocation_offset;
> +#endif
>   /* C entry point for boot CPU */
>   void __init start_xen(unsigned long boot_phys_offset,
>                         unsigned long fdt_paddr,
> @@ -739,8 +746,20 @@ void __init start_xen(unsigned long boot_phys_offset,
>                                (paddr_t)(uintptr_t)(_end - _start + 1), NULL);
>       BUG_ON(!xen_bootmodule);
>
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +    //save physical address of init_secondary
> +    xen_relocation_offset = __pa(init_secondary);
> +#endif
>       xen_paddr = get_xen_paddr();
>       setup_pagetables(boot_phys_offset, xen_paddr);
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +    //Now Xen is relocated
> +    //Calculate offset of Xen relocation
> +    //It is difference between new physical address of init_secondary an old one
> +    //This offset is needed in several places when we have to write to old Xen location
> +    //(secondary CPUs run on old-located Xen and rely on some variables from CPU0)
> +    xen_relocation_offset = __pa(init_secondary) - xen_relocation_offset;
> +#endif
>
>       /* Update Xen's address now that we have relocated. */
>       printk("Update BOOTMOD_XEN from %"PRIpaddr"-%"PRIpaddr" => %"PRIpaddr"-%"PRIpaddr"\n",
> diff --git a/xen/arch/arm/smpboot.c b/xen/arch/arm/smpboot.c
> index a96cda2..731144c 100644
> --- a/xen/arch/arm/smpboot.c
> +++ b/xen/arch/arm/smpboot.c
> @@ -31,6 +31,9 @@
>   #include <xen/console.h>
>   #include <asm/gic.h>
>   #include <asm/psci.h>
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +#include <xen/vmap.h>
> +#endif
>
>   cpumask_t cpu_online_map;
>   cpumask_t cpu_present_map;
> @@ -353,17 +356,33 @@ int __init cpu_up_send_sgi(int cpu)
>       return 0;
>   }
>
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +extern paddr_t xen_relocation_offset;
> +#endif
>   /* Bring up a remote CPU */
>   int __cpu_up(unsigned int cpu)
>   {
>       int rc;
>       s_time_t deadline;
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +    paddr_t p_info = __pa(&smp_up_cpu) - xen_relocation_offset;
> +    unsigned long* info = ioremap_nocache(p_info, sizeof(unsigned long));
> +#else
> +    unsigned long* info = &smp_up_cpu;
> +#endif
>
>       printk("Bringing up CPU%d\n", cpu);
>
>       rc = init_secondary_pagetables(cpu);
>       if ( rc < 0 )
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +    {
> +        iounmap(info);
> +        return rc;
> +    }
> +#else
>           return rc;
> +#endif
>
>       console_start_sync(); /* Secondary may use early_printk */
>
> @@ -374,8 +393,8 @@ int __cpu_up(unsigned int cpu)
>       init_data.cpuid = cpu;
>
>       /* Open the gate for this CPU */
> -    smp_up_cpu = cpu_logical_map(cpu);
> -    clean_dcache(smp_up_cpu);
> +    *info = cpu_logical_map(cpu);
> +    clean_dcache(*info);
>
>       rc = arch_cpu_up(cpu);
>
> @@ -383,6 +402,9 @@ int __cpu_up(unsigned int cpu)
>
>       if ( rc < 0 )
>       {
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +        iounmap(info);
> +#endif
>           printk("Failed to bring up CPU%d\n", cpu);
>           return rc;
>       }
> @@ -406,8 +428,11 @@ int __cpu_up(unsigned int cpu)
>        */
>       init_data.stack = NULL;
>       init_data.cpuid = ~0;
> -    smp_up_cpu = MPIDR_INVALID;
> -    clean_dcache(smp_up_cpu);
> +    *info = MPIDR_INVALID;
> +    clean_dcache(*info);
> +#ifdef ARM32_RELOCATE_OVER_4GB
> +    iounmap(info);
> +#endif
>
>       if ( !cpu_online(cpu) )
>       {
>

[1] 
http://lists.xenproject.org/archives/html/xen-devel/2015-05/msg01031.html
[2] 
http://lists.xenproject.org/archives/html/xen-devel/2015-04/msg00662.html

-- 
Julien Grall

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel