Re: [PATCH v2 5/5] x86: provide build time option to support up to 123Tb of memory

[Andrew Cooper <andrew.cooper3@citrix.com>]

On 02/02/15 11:21, Jan Beulich wrote:
> As this requires growing struct page_info from 32 to 48 bytes as well
> as shrinking the always accessible direct mapped memory range from 5Tb
> to 3.5Tb, this isn't being introduced as a general or default enabled
> feature.
>
> For now setting "bigmem=y" implies "shadow-paging=n", as the shadow
> paging code otherwise fails to build (see
> http://lists.xenproject.org/archives/html/xen-devel/2015-01/msg03165.html).

Stale commit message?

>
> A side effect of the change to x86's mm.h is that asm/mm.h may no
> longer be included directly. Hence in the few places where this was done,
> xen/mm.h is being substituted (indirectly in the hvm/mtrr.h case).
>
> Signed-off-by: Jan Beulich <jbeulich@suse.com>

Content Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>

> ---
> v2: "bigmem=y" no longer implies "shadow-paging=n".
>
> --- a/xen/arch/x86/Rules.mk
> +++ b/xen/arch/x86/Rules.mk
> @@ -33,6 +33,7 @@ x86 := y
>  x86_64 := y
>  
>  shadow-paging ?= y
> +bigmem        ?= n
>  
>  CFLAGS += -mno-red-zone -mno-sse -fpic
>  CFLAGS += -fno-asynchronous-unwind-tables
> @@ -42,3 +43,4 @@ CFLAGS += -DGCC_HAS_VISIBILITY_ATTRIBUTE
>  endif
>  
>  CFLAGS-$(shadow-paging) += -DCONFIG_SHADOW_PAGING
> +CFLAGS-$(bigmem)        += -DCONFIG_BIGMEM
> --- a/xen/arch/x86/hvm/mtrr.c
> +++ b/xen/arch/x86/hvm/mtrr.c
> @@ -18,13 +18,11 @@
>   */
>  
>  #include <public/hvm/e820.h>
> -#include <xen/types.h>
> +#include <xen/domain_page.h>
>  #include <asm/e820.h>
>  #include <asm/iocap.h>
> -#include <asm/mm.h>
>  #include <asm/paging.h>
>  #include <asm/p2m.h>
> -#include <xen/domain_page.h>
>  #include <asm/mtrr.h>
>  #include <asm/hvm/support.h>
>  #include <asm/hvm/cacheattr.h>
> --- a/xen/arch/x86/setup.c
> +++ b/xen/arch/x86/setup.c
> @@ -386,8 +386,13 @@ static void __init setup_max_pdx(unsigne
>      if ( max_pdx > FRAMETABLE_NR )
>          max_pdx = FRAMETABLE_NR;
>  
> +    if ( max_pdx > MPT_VIRT_SIZE / sizeof(unsigned long) )
> +        max_pdx = MPT_VIRT_SIZE / sizeof(unsigned long);
> +
> +#ifdef PAGE_LIST_NULL
>      if ( max_pdx >= PAGE_LIST_NULL )
>          max_pdx = PAGE_LIST_NULL - 1;
> +#endif
>  
>      max_page = pdx_to_pfn(max_pdx - 1) + 1;
>  }
> --- a/xen/include/asm-x86/config.h
> +++ b/xen/include/asm-x86/config.h
> @@ -161,6 +161,7 @@ extern unsigned char boot_edid_info[128]
>   *    High read-only compatibility machine-to-phys translation table.
>   *  0xffff82d080000000 - 0xffff82d0bfffffff [1GB,   2^30 bytes, PML4:261]
>   *    Xen text, static data, bss.
> +#ifndef CONFIG_BIGMEM
>   *  0xffff82d0c0000000 - 0xffff82dffbffffff [61GB - 64MB,       PML4:261]
>   *    Reserved for future use.
>   *  0xffff82dffc000000 - 0xffff82dfffffffff [64MB,  2^26 bytes, PML4:261]
> @@ -169,6 +170,16 @@ extern unsigned char boot_edid_info[128]
>   *    Page-frame information array.
>   *  0xffff830000000000 - 0xffff87ffffffffff [5TB, 5*2^40 bytes, PML4:262-271]
>   *    1:1 direct mapping of all physical memory.
> +#else
> + *  0xffff82d0c0000000 - 0xffff82ffdfffffff [188.5GB,           PML4:261]
> + *    Reserved for future use.
> + *  0xffff82ffe0000000 - 0xffff82ffffffffff [512MB, 2^29 bytes, PML4:261]
> + *    Super-page information array.
> + *  0xffff830000000000 - 0xffff847fffffffff [1.5TB, 3*2^39 bytes, PML4:262-264]
> + *    Page-frame information array.
> + *  0xffff848000000000 - 0xffff87ffffffffff [3.5TB, 7*2^39 bytes, PML4:265-271]
> + *    1:1 direct mapping of all physical memory.
> +#endif
>   *  0xffff880000000000 - 0xffffffffffffffff [120TB,             PML4:272-511]
>   *    PV: Guest-defined use.
>   *  0xffff880000000000 - 0xffffff7fffffffff [119.5TB,           PML4:272-510]
> @@ -237,21 +248,35 @@ extern unsigned char boot_edid_info[128]
>  /* Slot 261: xen text, static data and bss (1GB). */
>  #define XEN_VIRT_START          (HIRO_COMPAT_MPT_VIRT_END)
>  #define XEN_VIRT_END            (XEN_VIRT_START + GB(1))
> -/* Slot 261: superpage information array (64MB). */
> +
> +/* Slot 261: superpage information array (64MB or 512MB). */
>  #define SPAGETABLE_VIRT_END     FRAMETABLE_VIRT_START
>  #define SPAGETABLE_NR           (((FRAMETABLE_NR - 1) >> (SUPERPAGE_SHIFT - \
>                                                            PAGE_SHIFT)) + 1)
>  #define SPAGETABLE_SIZE         (SPAGETABLE_NR * sizeof(struct spage_info))
>  #define SPAGETABLE_VIRT_START   ((SPAGETABLE_VIRT_END - SPAGETABLE_SIZE) & \
>                                   (_AC(-1,UL) << SUPERPAGE_SHIFT))
> +
> +#ifndef CONFIG_BIGMEM
>  /* Slot 261: page-frame information array (128GB). */
> -#define FRAMETABLE_VIRT_END     DIRECTMAP_VIRT_START
>  #define FRAMETABLE_SIZE         GB(128)
> +#else
> +/* Slot 262-264: page-frame information array (1.5TB). */
> +#define FRAMETABLE_SIZE         GB(1536)
> +#endif
> +#define FRAMETABLE_VIRT_END     DIRECTMAP_VIRT_START
>  #define FRAMETABLE_NR           (FRAMETABLE_SIZE / sizeof(*frame_table))
>  #define FRAMETABLE_VIRT_START   (FRAMETABLE_VIRT_END - FRAMETABLE_SIZE)
> +
> +#ifndef CONFIG_BIGMEM
>  /* Slot 262-271/510: A direct 1:1 mapping of all of physical memory. */
>  #define DIRECTMAP_VIRT_START    (PML4_ADDR(262))
>  #define DIRECTMAP_SIZE          (PML4_ENTRY_BYTES * (511 - 262))
> +#else
> +/* Slot 265-271/510: A direct 1:1 mapping of all of physical memory. */
> +#define DIRECTMAP_VIRT_START    (PML4_ADDR(265))
> +#define DIRECTMAP_SIZE          (PML4_ENTRY_BYTES * (511 - 265))
> +#endif
>  #define DIRECTMAP_VIRT_END      (DIRECTMAP_VIRT_START + DIRECTMAP_SIZE)
>  
>  #ifndef __ASSEMBLY__
> --- a/xen/include/asm-x86/mm.h
> +++ b/xen/include/asm-x86/mm.h
> @@ -17,6 +17,7 @@
>   */
>  #define PFN_ORDER(_pfn) ((_pfn)->v.free.order)
>  
> +#ifndef CONFIG_BIGMEM
>  /*
>   * This definition is solely for the use in struct page_info (and
>   * struct page_list_head), intended to allow easy adjustment once x86-64
> @@ -30,6 +31,9 @@ struct page_list_entry
>  {
>      __pdx_t next, prev;
>  };
> +#else
> +#define __pdx_t unsigned long
> +#endif
>  
>  struct page_sharing_info;
>  
> --- a/xen/include/asm-x86/mtrr.h
> +++ b/xen/include/asm-x86/mtrr.h
> @@ -1,8 +1,7 @@
>  #ifndef __ASM_X86_MTRR_H__
>  #define __ASM_X86_MTRR_H__
>  
> -#include <xen/config.h>
> -#include <asm/mm.h>
> +#include <xen/mm.h>
>  
>  /* These are the region types. They match the architectural specification. */
>  #define MTRR_TYPE_UNCACHABLE 0
>
>