[PATCH v2 0/6] grub-xen: support booting huge pv-domains

[PATCH v2 0/6] grub-xen: support booting huge pv-domains

[Juergen Gross]

The Xen hypervisor supports starting a dom0 with large memory (up to
the TB range) by not including the initrd and p2m list in the initial
kernel mapping. Especially the p2m list can grow larger than the
available virtual space in the initial mapping.

The started kernel is indicating the support of each feature via
elf notes.

This series enables grub-xen to do the same as the hypervisor.

Tested with:
- 32 bit domU (kernel not supporting unmapped initrd)
- 32 bit domU (kernel supporting unmapped initrd)
- 1 GB 64 bit domU (kernel supporting unmapped initrd, not p2m)
- 1 GB 64 bit domU (kernel supporting unmapped initrd and p2m)
- 900GB 64 bit domU (kernel supporting unmapped initrd and p2m)

Changes in V2:
- rebased patch 5 to current master

Juergen Gross (6):
  xen: factor out p2m list allocation into separate function
  xen: factor out allocation of special pages into separate function
  xen: factor out allocation of page tables into separate function
  xen: add capability to load initrd outside of initial mapping
  xen: modify page table construction
  xen: add capability to load p2m list outside of kernel mapping

 grub-core/lib/i386/xen/relocator.S   |  47 ++--
 grub-core/lib/x86_64/xen/relocator.S |  44 ++-
 grub-core/lib/xen/relocator.c        |  22 +-
 grub-core/loader/i386/xen.c          | 521 +++++++++++++++++++++++++----------
 grub-core/loader/i386/xen_fileXX.c   |   7 +
 include/grub/xen/relocator.h         |   6 +-
 include/grub/xen_file.h              |   3 +
 7 files changed, 447 insertions(+), 203 deletions(-)

-- 
2.6.2

[PATCH v2 1/6] xen: factor out p2m list allocation into separate function

[Juergen Gross]

Do the p2m list allocation of the to be loaded kernel in a separate
function. This will allow doing the p2m list allocation at different
times of the boot preparations depending on the features the kernel
is supporting.

While at this remove superfluous setting of first_p2m_pfn and
nr_p2m_frames as those are needed only in case of the p2m list not
being mapped by the initial kernel mapping.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
 grub-core/loader/i386/xen.c | 70 ++++++++++++++++++++++++++-------------------
 1 file changed, 40 insertions(+), 30 deletions(-)

diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
index c4d9689..42ed7c7 100644
--- a/grub-core/loader/i386/xen.c
+++ b/grub-core/loader/i386/xen.c
@@ -52,6 +52,8 @@ static struct grub_xen_file_info xen_inf;
 static struct xen_multiboot_mod_list *xen_module_info_page;
 static grub_uint64_t modules_target_start;
 static grub_size_t n_modules;
+static struct grub_relocator_xen_state state;
+static grub_xen_mfn_t *virt_mfn_list;
 
 #define PAGE_SIZE 4096
 #define MAX_MODULES (PAGE_SIZE / sizeof (struct xen_multiboot_mod_list))
@@ -166,7 +168,7 @@ generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
 }
 
 static grub_err_t
-set_mfns (grub_xen_mfn_t * new_mfn_list, grub_xen_mfn_t pfn)
+set_mfns (grub_xen_mfn_t pfn)
 {
   grub_xen_mfn_t i, t;
   grub_xen_mfn_t cn_pfn = -1, st_pfn = -1;
@@ -175,32 +177,32 @@ set_mfns (grub_xen_mfn_t * new_mfn_list, grub_xen_mfn_t pfn)
 
   for (i = 0; i < grub_xen_start_page_addr->nr_pages; i++)
     {
-      if (new_mfn_list[i] == grub_xen_start_page_addr->console.domU.mfn)
+      if (virt_mfn_list[i] == grub_xen_start_page_addr->console.domU.mfn)
 	cn_pfn = i;
-      if (new_mfn_list[i] == grub_xen_start_page_addr->store_mfn)
+      if (virt_mfn_list[i] == grub_xen_start_page_addr->store_mfn)
 	st_pfn = i;
     }
   if (cn_pfn == (grub_xen_mfn_t)-1)
     return grub_error (GRUB_ERR_BUG, "no console");
   if (st_pfn == (grub_xen_mfn_t)-1)
     return grub_error (GRUB_ERR_BUG, "no store");
-  t = new_mfn_list[pfn];
-  new_mfn_list[pfn] = new_mfn_list[cn_pfn];
-  new_mfn_list[cn_pfn] = t;
-  t = new_mfn_list[pfn + 1];
-  new_mfn_list[pfn + 1] = new_mfn_list[st_pfn];
-  new_mfn_list[st_pfn] = t;
-
-  m2p_updates[0].ptr = page2offset (new_mfn_list[pfn]) | MMU_MACHPHYS_UPDATE;
+  t = virt_mfn_list[pfn];
+  virt_mfn_list[pfn] = virt_mfn_list[cn_pfn];
+  virt_mfn_list[cn_pfn] = t;
+  t = virt_mfn_list[pfn + 1];
+  virt_mfn_list[pfn + 1] = virt_mfn_list[st_pfn];
+  virt_mfn_list[st_pfn] = t;
+
+  m2p_updates[0].ptr = page2offset (virt_mfn_list[pfn]) | MMU_MACHPHYS_UPDATE;
   m2p_updates[0].val = pfn;
   m2p_updates[1].ptr =
-    page2offset (new_mfn_list[pfn + 1]) | MMU_MACHPHYS_UPDATE;
+    page2offset (virt_mfn_list[pfn + 1]) | MMU_MACHPHYS_UPDATE;
   m2p_updates[1].val = pfn + 1;
   m2p_updates[2].ptr =
-    page2offset (new_mfn_list[cn_pfn]) | MMU_MACHPHYS_UPDATE;
+    page2offset (virt_mfn_list[cn_pfn]) | MMU_MACHPHYS_UPDATE;
   m2p_updates[2].val = cn_pfn;
   m2p_updates[3].ptr =
-    page2offset (new_mfn_list[st_pfn]) | MMU_MACHPHYS_UPDATE;
+    page2offset (virt_mfn_list[st_pfn]) | MMU_MACHPHYS_UPDATE;
   m2p_updates[3].val = st_pfn;
 
   grub_xen_mmu_update (m2p_updates, 4, NULL, DOMID_SELF);
@@ -209,34 +211,43 @@ set_mfns (grub_xen_mfn_t * new_mfn_list, grub_xen_mfn_t pfn)
 }
 
 static grub_err_t
+grub_xen_p2m_alloc (void)
+{
+  grub_relocator_chunk_t ch;
+  grub_size_t p2msize;
+  grub_err_t err;
+
+  state.mfn_list = max_addr;
+  next_start.mfn_list = max_addr + xen_inf.virt_base;
+  p2msize = sizeof (grub_xen_mfn_t) * grub_xen_start_page_addr->nr_pages;
+  err = grub_relocator_alloc_chunk_addr (relocator, &ch, max_addr, p2msize);
+  if (err)
+    return err;
+  virt_mfn_list = get_virtual_current_address (ch);
+  grub_memcpy (virt_mfn_list,
+	       (void *) grub_xen_start_page_addr->mfn_list, p2msize);
+  max_addr = ALIGN_UP (max_addr + p2msize, PAGE_SIZE);
+
+  return GRUB_ERR_NONE;
+}
+
+static grub_err_t
 grub_xen_boot (void)
 {
-  struct grub_relocator_xen_state state;
   grub_relocator_chunk_t ch;
   grub_err_t err;
-  grub_size_t pgtsize;
   struct start_info *nst;
   grub_uint64_t nr_info_pages;
   grub_uint64_t nr_pages, nr_pt_pages, nr_need_pages;
   struct gnttab_set_version gnttab_setver;
-  grub_xen_mfn_t *new_mfn_list;
   grub_size_t i;
 
   if (grub_xen_n_allocated_shared_pages)
     return grub_error (GRUB_ERR_BUG, "active grants");
 
-  state.mfn_list = max_addr;
-  next_start.mfn_list = max_addr + xen_inf.virt_base;
-  next_start.first_p2m_pfn = max_addr >> PAGE_SHIFT;	/* Is this right? */
-  pgtsize = sizeof (grub_xen_mfn_t) * grub_xen_start_page_addr->nr_pages;
-  err = grub_relocator_alloc_chunk_addr (relocator, &ch, max_addr, pgtsize);
-  next_start.nr_p2m_frames = (pgtsize + PAGE_SIZE - 1) >> PAGE_SHIFT;
+  err = grub_xen_p2m_alloc ();
   if (err)
     return err;
-  new_mfn_list = get_virtual_current_address (ch);
-  grub_memcpy (new_mfn_list,
-	       (void *) grub_xen_start_page_addr->mfn_list, pgtsize);
-  max_addr = ALIGN_UP (max_addr + pgtsize, PAGE_SIZE);
 
   err = grub_relocator_alloc_chunk_addr (relocator, &ch,
 					 max_addr, sizeof (next_start));
@@ -254,7 +265,7 @@ grub_xen_boot (void)
   next_start.console.domU = grub_xen_start_page_addr->console.domU;
   next_start.shared_info = grub_xen_start_page_addr->shared_info;
 
-  err = set_mfns (new_mfn_list, max_addr >> PAGE_SHIFT);
+  err = set_mfns (max_addr >> PAGE_SHIFT);
   if (err)
     return err;
   max_addr += 2 * PAGE_SIZE;
@@ -288,13 +299,12 @@ grub_xen_boot (void)
 
   generate_page_table (get_virtual_current_address (ch),
 		       max_addr >> PAGE_SHIFT, nr_pages,
-		       xen_inf.virt_base, new_mfn_list);
+		       xen_inf.virt_base, virt_mfn_list);
 
   max_addr += page2offset (nr_pt_pages);
   state.stack = max_addr + STACK_SIZE + xen_inf.virt_base;
   state.entry_point = xen_inf.entry_point;
 
-  next_start.nr_p2m_frames += nr_pt_pages;
   next_start.nr_pt_frames = nr_pt_pages;
   state.paging_size = nr_pt_pages;
 
-- 
2.6.2

Re: [PATCH v2 1/6] xen: factor out p2m list allocation into separate function

[Daniel Kiper]

On Thu, Feb 11, 2016 at 08:53:21AM +0100, Juergen Gross wrote:
> Do the p2m list allocation of the to be loaded kernel in a separate
> function. This will allow doing the p2m list allocation at different
> times of the boot preparations depending on the features the kernel
> is supporting.
>
> While at this remove superfluous setting of first_p2m_pfn and
> nr_p2m_frames as those are needed only in case of the p2m list not
> being mapped by the initial kernel mapping.
>
> Signed-off-by: Juergen Gross <jgross@suse.com>
> ---
>  grub-core/loader/i386/xen.c | 70 ++++++++++++++++++++++++++-------------------
>  1 file changed, 40 insertions(+), 30 deletions(-)
>
> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
> index c4d9689..42ed7c7 100644
> --- a/grub-core/loader/i386/xen.c
> +++ b/grub-core/loader/i386/xen.c
> @@ -52,6 +52,8 @@ static struct grub_xen_file_info xen_inf;
>  static struct xen_multiboot_mod_list *xen_module_info_page;
>  static grub_uint64_t modules_target_start;
>  static grub_size_t n_modules;
> +static struct grub_relocator_xen_state state;
> +static grub_xen_mfn_t *virt_mfn_list;

Do we strongly need this as globals? I suppose that
both of them could be local to grub_xen_boot.

>  #define PAGE_SIZE 4096
>  #define MAX_MODULES (PAGE_SIZE / sizeof (struct xen_multiboot_mod_list))
> @@ -166,7 +168,7 @@ generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
>  }
>
>  static grub_err_t
> -set_mfns (grub_xen_mfn_t * new_mfn_list, grub_xen_mfn_t pfn)
> +set_mfns (grub_xen_mfn_t pfn)
>  {
>    grub_xen_mfn_t i, t;
>    grub_xen_mfn_t cn_pfn = -1, st_pfn = -1;
> @@ -175,32 +177,32 @@ set_mfns (grub_xen_mfn_t * new_mfn_list, grub_xen_mfn_t pfn)
>
>    for (i = 0; i < grub_xen_start_page_addr->nr_pages; i++)
>      {
> -      if (new_mfn_list[i] == grub_xen_start_page_addr->console.domU.mfn)
> +      if (virt_mfn_list[i] == grub_xen_start_page_addr->console.domU.mfn)
>  	cn_pfn = i;
> -      if (new_mfn_list[i] == grub_xen_start_page_addr->store_mfn)
> +      if (virt_mfn_list[i] == grub_xen_start_page_addr->store_mfn)
>  	st_pfn = i;
>      }
>    if (cn_pfn == (grub_xen_mfn_t)-1)
>      return grub_error (GRUB_ERR_BUG, "no console");
>    if (st_pfn == (grub_xen_mfn_t)-1)
>      return grub_error (GRUB_ERR_BUG, "no store");
> -  t = new_mfn_list[pfn];
> -  new_mfn_list[pfn] = new_mfn_list[cn_pfn];
> -  new_mfn_list[cn_pfn] = t;
> -  t = new_mfn_list[pfn + 1];
> -  new_mfn_list[pfn + 1] = new_mfn_list[st_pfn];
> -  new_mfn_list[st_pfn] = t;
> -
> -  m2p_updates[0].ptr = page2offset (new_mfn_list[pfn]) | MMU_MACHPHYS_UPDATE;
> +  t = virt_mfn_list[pfn];
> +  virt_mfn_list[pfn] = virt_mfn_list[cn_pfn];
> +  virt_mfn_list[cn_pfn] = t;
> +  t = virt_mfn_list[pfn + 1];
> +  virt_mfn_list[pfn + 1] = virt_mfn_list[st_pfn];
> +  virt_mfn_list[st_pfn] = t;
> +
> +  m2p_updates[0].ptr = page2offset (virt_mfn_list[pfn]) | MMU_MACHPHYS_UPDATE;
>    m2p_updates[0].val = pfn;
>    m2p_updates[1].ptr =
> -    page2offset (new_mfn_list[pfn + 1]) | MMU_MACHPHYS_UPDATE;
> +    page2offset (virt_mfn_list[pfn + 1]) | MMU_MACHPHYS_UPDATE;
>    m2p_updates[1].val = pfn + 1;
>    m2p_updates[2].ptr =
> -    page2offset (new_mfn_list[cn_pfn]) | MMU_MACHPHYS_UPDATE;
> +    page2offset (virt_mfn_list[cn_pfn]) | MMU_MACHPHYS_UPDATE;
>    m2p_updates[2].val = cn_pfn;
>    m2p_updates[3].ptr =
> -    page2offset (new_mfn_list[st_pfn]) | MMU_MACHPHYS_UPDATE;
> +    page2offset (virt_mfn_list[st_pfn]) | MMU_MACHPHYS_UPDATE;
>    m2p_updates[3].val = st_pfn;
>
>    grub_xen_mmu_update (m2p_updates, 4, NULL, DOMID_SELF);
> @@ -209,34 +211,43 @@ set_mfns (grub_xen_mfn_t * new_mfn_list, grub_xen_mfn_t pfn)
>  }
>
>  static grub_err_t
> +grub_xen_p2m_alloc (void)
> +{
> +  grub_relocator_chunk_t ch;
> +  grub_size_t p2msize;
> +  grub_err_t err;
> +
> +  state.mfn_list = max_addr;
> +  next_start.mfn_list = max_addr + xen_inf.virt_base;
> +  p2msize = sizeof (grub_xen_mfn_t) * grub_xen_start_page_addr->nr_pages;
> +  err = grub_relocator_alloc_chunk_addr (relocator, &ch, max_addr, p2msize);

Hmmm.... Where relocator is defined?

> +  if (err)
> +    return err;
> +  virt_mfn_list = get_virtual_current_address (ch);
> +  grub_memcpy (virt_mfn_list,
> +	       (void *) grub_xen_start_page_addr->mfn_list, p2msize);
> +  max_addr = ALIGN_UP (max_addr + p2msize, PAGE_SIZE);
> +
> +  return GRUB_ERR_NONE;
> +}
> +
> +static grub_err_t
>  grub_xen_boot (void)
>  {
> -  struct grub_relocator_xen_state state;
>    grub_relocator_chunk_t ch;
>    grub_err_t err;
> -  grub_size_t pgtsize;
>    struct start_info *nst;
>    grub_uint64_t nr_info_pages;
>    grub_uint64_t nr_pages, nr_pt_pages, nr_need_pages;
>    struct gnttab_set_version gnttab_setver;
> -  grub_xen_mfn_t *new_mfn_list;
>    grub_size_t i;
>
>    if (grub_xen_n_allocated_shared_pages)
>      return grub_error (GRUB_ERR_BUG, "active grants");
>
> -  state.mfn_list = max_addr;
> -  next_start.mfn_list = max_addr + xen_inf.virt_base;
> -  next_start.first_p2m_pfn = max_addr >> PAGE_SHIFT;	/* Is this right? */
> -  pgtsize = sizeof (grub_xen_mfn_t) * grub_xen_start_page_addr->nr_pages;
> -  err = grub_relocator_alloc_chunk_addr (relocator, &ch, max_addr, pgtsize);
> -  next_start.nr_p2m_frames = (pgtsize + PAGE_SIZE - 1) >> PAGE_SHIFT;
> +  err = grub_xen_p2m_alloc ();
>    if (err)
>      return err;
> -  new_mfn_list = get_virtual_current_address (ch);
> -  grub_memcpy (new_mfn_list,
> -	       (void *) grub_xen_start_page_addr->mfn_list, pgtsize);
> -  max_addr = ALIGN_UP (max_addr + pgtsize, PAGE_SIZE);
>
>    err = grub_relocator_alloc_chunk_addr (relocator, &ch,

Ditto. I think that this could be passed as a grub_xen_boot() argument.
Otherwise this appear from nowhere and reading/understanding is more
difficult.

Daniel

Re: [PATCH v2 1/6] xen: factor out p2m list allocation into separate function

[Juergen Gross]

On 11/02/16 13:19, Daniel Kiper wrote:
> On Thu, Feb 11, 2016 at 08:53:21AM +0100, Juergen Gross wrote:
>> Do the p2m list allocation of the to be loaded kernel in a separate
>> function. This will allow doing the p2m list allocation at different
>> times of the boot preparations depending on the features the kernel
>> is supporting.
>>
>> While at this remove superfluous setting of first_p2m_pfn and
>> nr_p2m_frames as those are needed only in case of the p2m list not
>> being mapped by the initial kernel mapping.
>>
>> Signed-off-by: Juergen Gross <jgross@suse.com>
>> ---
>>  grub-core/loader/i386/xen.c | 70 ++++++++++++++++++++++++++-------------------
>>  1 file changed, 40 insertions(+), 30 deletions(-)
>>
>> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
>> index c4d9689..42ed7c7 100644
>> --- a/grub-core/loader/i386/xen.c
>> +++ b/grub-core/loader/i386/xen.c
>> @@ -52,6 +52,8 @@ static struct grub_xen_file_info xen_inf;
>>  static struct xen_multiboot_mod_list *xen_module_info_page;
>>  static grub_uint64_t modules_target_start;
>>  static grub_size_t n_modules;
>> +static struct grub_relocator_xen_state state;
>> +static grub_xen_mfn_t *virt_mfn_list;
> 
> Do we strongly need this as globals? I suppose that
> both of them could be local to grub_xen_boot.

This would require passing the state pointer to many other functions.
Same applies to virt_mfn_list.

I just followed the style used in the source already: variables used in
multiple functions are mostly defined globally (there are even some
which are used in one function only).

> 
>>  #define PAGE_SIZE 4096
>>  #define MAX_MODULES (PAGE_SIZE / sizeof (struct xen_multiboot_mod_list))
>> @@ -166,7 +168,7 @@ generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
>>  }
>>
>>  static grub_err_t
>> -set_mfns (grub_xen_mfn_t * new_mfn_list, grub_xen_mfn_t pfn)
>> +set_mfns (grub_xen_mfn_t pfn)
>>  {
>>    grub_xen_mfn_t i, t;
>>    grub_xen_mfn_t cn_pfn = -1, st_pfn = -1;
>> @@ -175,32 +177,32 @@ set_mfns (grub_xen_mfn_t * new_mfn_list, grub_xen_mfn_t pfn)
>>
>>    for (i = 0; i < grub_xen_start_page_addr->nr_pages; i++)
>>      {
>> -      if (new_mfn_list[i] == grub_xen_start_page_addr->console.domU.mfn)
>> +      if (virt_mfn_list[i] == grub_xen_start_page_addr->console.domU.mfn)
>>  	cn_pfn = i;
>> -      if (new_mfn_list[i] == grub_xen_start_page_addr->store_mfn)
>> +      if (virt_mfn_list[i] == grub_xen_start_page_addr->store_mfn)
>>  	st_pfn = i;
>>      }
>>    if (cn_pfn == (grub_xen_mfn_t)-1)
>>      return grub_error (GRUB_ERR_BUG, "no console");
>>    if (st_pfn == (grub_xen_mfn_t)-1)
>>      return grub_error (GRUB_ERR_BUG, "no store");
>> -  t = new_mfn_list[pfn];
>> -  new_mfn_list[pfn] = new_mfn_list[cn_pfn];
>> -  new_mfn_list[cn_pfn] = t;
>> -  t = new_mfn_list[pfn + 1];
>> -  new_mfn_list[pfn + 1] = new_mfn_list[st_pfn];
>> -  new_mfn_list[st_pfn] = t;
>> -
>> -  m2p_updates[0].ptr = page2offset (new_mfn_list[pfn]) | MMU_MACHPHYS_UPDATE;
>> +  t = virt_mfn_list[pfn];
>> +  virt_mfn_list[pfn] = virt_mfn_list[cn_pfn];
>> +  virt_mfn_list[cn_pfn] = t;
>> +  t = virt_mfn_list[pfn + 1];
>> +  virt_mfn_list[pfn + 1] = virt_mfn_list[st_pfn];
>> +  virt_mfn_list[st_pfn] = t;
>> +
>> +  m2p_updates[0].ptr = page2offset (virt_mfn_list[pfn]) | MMU_MACHPHYS_UPDATE;
>>    m2p_updates[0].val = pfn;
>>    m2p_updates[1].ptr =
>> -    page2offset (new_mfn_list[pfn + 1]) | MMU_MACHPHYS_UPDATE;
>> +    page2offset (virt_mfn_list[pfn + 1]) | MMU_MACHPHYS_UPDATE;
>>    m2p_updates[1].val = pfn + 1;
>>    m2p_updates[2].ptr =
>> -    page2offset (new_mfn_list[cn_pfn]) | MMU_MACHPHYS_UPDATE;
>> +    page2offset (virt_mfn_list[cn_pfn]) | MMU_MACHPHYS_UPDATE;
>>    m2p_updates[2].val = cn_pfn;
>>    m2p_updates[3].ptr =
>> -    page2offset (new_mfn_list[st_pfn]) | MMU_MACHPHYS_UPDATE;
>> +    page2offset (virt_mfn_list[st_pfn]) | MMU_MACHPHYS_UPDATE;
>>    m2p_updates[3].val = st_pfn;
>>
>>    grub_xen_mmu_update (m2p_updates, 4, NULL, DOMID_SELF);
>> @@ -209,34 +211,43 @@ set_mfns (grub_xen_mfn_t * new_mfn_list, grub_xen_mfn_t pfn)
>>  }
>>
>>  static grub_err_t
>> +grub_xen_p2m_alloc (void)
>> +{
>> +  grub_relocator_chunk_t ch;
>> +  grub_size_t p2msize;
>> +  grub_err_t err;
>> +
>> +  state.mfn_list = max_addr;
>> +  next_start.mfn_list = max_addr + xen_inf.virt_base;
>> +  p2msize = sizeof (grub_xen_mfn_t) * grub_xen_start_page_addr->nr_pages;
>> +  err = grub_relocator_alloc_chunk_addr (relocator, &ch, max_addr, p2msize);
> 
> Hmmm.... Where relocator is defined?

First global variable in this source.

> 
>> +  if (err)
>> +    return err;
>> +  virt_mfn_list = get_virtual_current_address (ch);
>> +  grub_memcpy (virt_mfn_list,
>> +	       (void *) grub_xen_start_page_addr->mfn_list, p2msize);
>> +  max_addr = ALIGN_UP (max_addr + p2msize, PAGE_SIZE);
>> +
>> +  return GRUB_ERR_NONE;
>> +}
>> +
>> +static grub_err_t
>>  grub_xen_boot (void)
>>  {
>> -  struct grub_relocator_xen_state state;
>>    grub_relocator_chunk_t ch;
>>    grub_err_t err;
>> -  grub_size_t pgtsize;
>>    struct start_info *nst;
>>    grub_uint64_t nr_info_pages;
>>    grub_uint64_t nr_pages, nr_pt_pages, nr_need_pages;
>>    struct gnttab_set_version gnttab_setver;
>> -  grub_xen_mfn_t *new_mfn_list;
>>    grub_size_t i;
>>
>>    if (grub_xen_n_allocated_shared_pages)
>>      return grub_error (GRUB_ERR_BUG, "active grants");
>>
>> -  state.mfn_list = max_addr;
>> -  next_start.mfn_list = max_addr + xen_inf.virt_base;
>> -  next_start.first_p2m_pfn = max_addr >> PAGE_SHIFT;	/* Is this right? */
>> -  pgtsize = sizeof (grub_xen_mfn_t) * grub_xen_start_page_addr->nr_pages;
>> -  err = grub_relocator_alloc_chunk_addr (relocator, &ch, max_addr, pgtsize);
>> -  next_start.nr_p2m_frames = (pgtsize + PAGE_SIZE - 1) >> PAGE_SHIFT;
>> +  err = grub_xen_p2m_alloc ();
>>    if (err)
>>      return err;
>> -  new_mfn_list = get_virtual_current_address (ch);
>> -  grub_memcpy (new_mfn_list,
>> -	       (void *) grub_xen_start_page_addr->mfn_list, pgtsize);
>> -  max_addr = ALIGN_UP (max_addr + pgtsize, PAGE_SIZE);
>>
>>    err = grub_relocator_alloc_chunk_addr (relocator, &ch,
> 
> Ditto. I think that this could be passed as a grub_xen_boot() argument.
> Otherwise this appear from nowhere and reading/understanding is more
> difficult.

Huh? How that? grub_xen_boot() can't have any parameters (see prototype
of grub_loader_set() which is used with grub_xen_boot() as parameter).

BTW: I didn't introduce this kind of usage of relocator.


Juergen

Re: [PATCH v2 1/6] xen: factor out p2m list allocation into separate function

[Daniel Kiper]

On Thu, Feb 11, 2016 at 01:38:10PM +0100, Juergen Gross wrote:
> On 11/02/16 13:19, Daniel Kiper wrote:
> > On Thu, Feb 11, 2016 at 08:53:21AM +0100, Juergen Gross wrote:
> >> Do the p2m list allocation of the to be loaded kernel in a separate
> >> function. This will allow doing the p2m list allocation at different
> >> times of the boot preparations depending on the features the kernel
> >> is supporting.
> >>
> >> While at this remove superfluous setting of first_p2m_pfn and
> >> nr_p2m_frames as those are needed only in case of the p2m list not
> >> being mapped by the initial kernel mapping.
> >>
> >> Signed-off-by: Juergen Gross <jgross@suse.com>
> >> ---
> >>  grub-core/loader/i386/xen.c | 70 ++++++++++++++++++++++++++-------------------
> >>  1 file changed, 40 insertions(+), 30 deletions(-)
> >>
> >> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
> >> index c4d9689..42ed7c7 100644
> >> --- a/grub-core/loader/i386/xen.c
> >> +++ b/grub-core/loader/i386/xen.c
> >> @@ -52,6 +52,8 @@ static struct grub_xen_file_info xen_inf;
> >>  static struct xen_multiboot_mod_list *xen_module_info_page;
> >>  static grub_uint64_t modules_target_start;
> >>  static grub_size_t n_modules;
> >> +static struct grub_relocator_xen_state state;
> >> +static grub_xen_mfn_t *virt_mfn_list;
> >
> > Do we strongly need this as globals? I suppose that
> > both of them could be local to grub_xen_boot.
>
> This would require passing the state pointer to many other functions.
> Same applies to virt_mfn_list.
>
> I just followed the style used in the source already: variables used in
> multiple functions are mostly defined globally (there are even some
> which are used in one function only).

Well, I do not like that style but if maintainer do not object I will
do not complain more here about that.

> >>  #define PAGE_SIZE 4096
> >>  #define MAX_MODULES (PAGE_SIZE / sizeof (struct xen_multiboot_mod_list))
> >> @@ -166,7 +168,7 @@ generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
> >>  }
> >>
> >>  static grub_err_t
> >> -set_mfns (grub_xen_mfn_t * new_mfn_list, grub_xen_mfn_t pfn)
> >> +set_mfns (grub_xen_mfn_t pfn)
> >>  {
> >>    grub_xen_mfn_t i, t;
> >>    grub_xen_mfn_t cn_pfn = -1, st_pfn = -1;
> >> @@ -175,32 +177,32 @@ set_mfns (grub_xen_mfn_t * new_mfn_list, grub_xen_mfn_t pfn)
> >>
> >>    for (i = 0; i < grub_xen_start_page_addr->nr_pages; i++)
> >>      {
> >> -      if (new_mfn_list[i] == grub_xen_start_page_addr->console.domU.mfn)
> >> +      if (virt_mfn_list[i] == grub_xen_start_page_addr->console.domU.mfn)
> >>  	cn_pfn = i;
> >> -      if (new_mfn_list[i] == grub_xen_start_page_addr->store_mfn)
> >> +      if (virt_mfn_list[i] == grub_xen_start_page_addr->store_mfn)
> >>  	st_pfn = i;
> >>      }
> >>    if (cn_pfn == (grub_xen_mfn_t)-1)
> >>      return grub_error (GRUB_ERR_BUG, "no console");
> >>    if (st_pfn == (grub_xen_mfn_t)-1)
> >>      return grub_error (GRUB_ERR_BUG, "no store");
> >> -  t = new_mfn_list[pfn];
> >> -  new_mfn_list[pfn] = new_mfn_list[cn_pfn];
> >> -  new_mfn_list[cn_pfn] = t;
> >> -  t = new_mfn_list[pfn + 1];
> >> -  new_mfn_list[pfn + 1] = new_mfn_list[st_pfn];
> >> -  new_mfn_list[st_pfn] = t;
> >> -
> >> -  m2p_updates[0].ptr = page2offset (new_mfn_list[pfn]) | MMU_MACHPHYS_UPDATE;
> >> +  t = virt_mfn_list[pfn];
> >> +  virt_mfn_list[pfn] = virt_mfn_list[cn_pfn];
> >> +  virt_mfn_list[cn_pfn] = t;
> >> +  t = virt_mfn_list[pfn + 1];
> >> +  virt_mfn_list[pfn + 1] = virt_mfn_list[st_pfn];
> >> +  virt_mfn_list[st_pfn] = t;
> >> +
> >> +  m2p_updates[0].ptr = page2offset (virt_mfn_list[pfn]) | MMU_MACHPHYS_UPDATE;
> >>    m2p_updates[0].val = pfn;
> >>    m2p_updates[1].ptr =
> >> -    page2offset (new_mfn_list[pfn + 1]) | MMU_MACHPHYS_UPDATE;
> >> +    page2offset (virt_mfn_list[pfn + 1]) | MMU_MACHPHYS_UPDATE;
> >>    m2p_updates[1].val = pfn + 1;
> >>    m2p_updates[2].ptr =
> >> -    page2offset (new_mfn_list[cn_pfn]) | MMU_MACHPHYS_UPDATE;
> >> +    page2offset (virt_mfn_list[cn_pfn]) | MMU_MACHPHYS_UPDATE;
> >>    m2p_updates[2].val = cn_pfn;
> >>    m2p_updates[3].ptr =
> >> -    page2offset (new_mfn_list[st_pfn]) | MMU_MACHPHYS_UPDATE;
> >> +    page2offset (virt_mfn_list[st_pfn]) | MMU_MACHPHYS_UPDATE;
> >>    m2p_updates[3].val = st_pfn;
> >>
> >>    grub_xen_mmu_update (m2p_updates, 4, NULL, DOMID_SELF);
> >> @@ -209,34 +211,43 @@ set_mfns (grub_xen_mfn_t * new_mfn_list, grub_xen_mfn_t pfn)
> >>  }
> >>
> >>  static grub_err_t
> >> +grub_xen_p2m_alloc (void)
> >> +{
> >> +  grub_relocator_chunk_t ch;
> >> +  grub_size_t p2msize;
> >> +  grub_err_t err;
> >> +
> >> +  state.mfn_list = max_addr;
> >> +  next_start.mfn_list = max_addr + xen_inf.virt_base;
> >> +  p2msize = sizeof (grub_xen_mfn_t) * grub_xen_start_page_addr->nr_pages;
> >> +  err = grub_relocator_alloc_chunk_addr (relocator, &ch, max_addr, p2msize);
> >
> > Hmmm.... Where relocator is defined?
>
> First global variable in this source.
>
> >
> >> +  if (err)
> >> +    return err;
> >> +  virt_mfn_list = get_virtual_current_address (ch);
> >> +  grub_memcpy (virt_mfn_list,
> >> +	       (void *) grub_xen_start_page_addr->mfn_list, p2msize);
> >> +  max_addr = ALIGN_UP (max_addr + p2msize, PAGE_SIZE);
> >> +
> >> +  return GRUB_ERR_NONE;
> >> +}
> >> +
> >> +static grub_err_t
> >>  grub_xen_boot (void)
> >>  {
> >> -  struct grub_relocator_xen_state state;
> >>    grub_relocator_chunk_t ch;
> >>    grub_err_t err;
> >> -  grub_size_t pgtsize;
> >>    struct start_info *nst;
> >>    grub_uint64_t nr_info_pages;
> >>    grub_uint64_t nr_pages, nr_pt_pages, nr_need_pages;
> >>    struct gnttab_set_version gnttab_setver;
> >> -  grub_xen_mfn_t *new_mfn_list;
> >>    grub_size_t i;
> >>
> >>    if (grub_xen_n_allocated_shared_pages)
> >>      return grub_error (GRUB_ERR_BUG, "active grants");
> >>
> >> -  state.mfn_list = max_addr;
> >> -  next_start.mfn_list = max_addr + xen_inf.virt_base;
> >> -  next_start.first_p2m_pfn = max_addr >> PAGE_SHIFT;	/* Is this right? */
> >> -  pgtsize = sizeof (grub_xen_mfn_t) * grub_xen_start_page_addr->nr_pages;
> >> -  err = grub_relocator_alloc_chunk_addr (relocator, &ch, max_addr, pgtsize);
> >> -  next_start.nr_p2m_frames = (pgtsize + PAGE_SIZE - 1) >> PAGE_SHIFT;
> >> +  err = grub_xen_p2m_alloc ();
> >>    if (err)
> >>      return err;
> >> -  new_mfn_list = get_virtual_current_address (ch);
> >> -  grub_memcpy (new_mfn_list,
> >> -	       (void *) grub_xen_start_page_addr->mfn_list, pgtsize);
> >> -  max_addr = ALIGN_UP (max_addr + pgtsize, PAGE_SIZE);
> >>
> >>    err = grub_relocator_alloc_chunk_addr (relocator, &ch,
> >
> > Ditto. I think that this could be passed as a grub_xen_boot() argument.
> > Otherwise this appear from nowhere and reading/understanding is more
> > difficult.
>
> Huh? How that? grub_xen_boot() can't have any parameters (see prototype
> of grub_loader_set() which is used with grub_xen_boot() as parameter).
>
> BTW: I didn't introduce this kind of usage of relocator.

OK, I have checked other loaders and it looks that they use relocator in
the same way. Let's leave it as is. However, this does not waive my
concerns expressed above in regards to other global variables.

Daniel

Re: [PATCH v2 1/6] xen: factor out p2m list allocation into separate function

[Juergen Gross]

On 11/02/16 18:09, Daniel Kiper wrote:
> On Thu, Feb 11, 2016 at 01:38:10PM +0100, Juergen Gross wrote:
>> On 11/02/16 13:19, Daniel Kiper wrote:
>>> On Thu, Feb 11, 2016 at 08:53:21AM +0100, Juergen Gross wrote:
>>>> Do the p2m list allocation of the to be loaded kernel in a separate
>>>> function. This will allow doing the p2m list allocation at different
>>>> times of the boot preparations depending on the features the kernel
>>>> is supporting.
>>>>
>>>> While at this remove superfluous setting of first_p2m_pfn and
>>>> nr_p2m_frames as those are needed only in case of the p2m list not
>>>> being mapped by the initial kernel mapping.
>>>>
>>>> Signed-off-by: Juergen Gross <jgross@suse.com>
>>>> ---
>>>>  grub-core/loader/i386/xen.c | 70 ++++++++++++++++++++++++++-------------------
>>>>  1 file changed, 40 insertions(+), 30 deletions(-)
>>>>
>>>> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
>>>> index c4d9689..42ed7c7 100644
>>>> --- a/grub-core/loader/i386/xen.c
>>>> +++ b/grub-core/loader/i386/xen.c
>>>> @@ -52,6 +52,8 @@ static struct grub_xen_file_info xen_inf;
>>>>  static struct xen_multiboot_mod_list *xen_module_info_page;
>>>>  static grub_uint64_t modules_target_start;
>>>>  static grub_size_t n_modules;
>>>> +static struct grub_relocator_xen_state state;
>>>> +static grub_xen_mfn_t *virt_mfn_list;
>>>
>>> Do we strongly need this as globals? I suppose that
>>> both of them could be local to grub_xen_boot.
>>
>> This would require passing the state pointer to many other functions.
>> Same applies to virt_mfn_list.
>>
>> I just followed the style used in the source already: variables used in
>> multiple functions are mostly defined globally (there are even some
>> which are used in one function only).
> 
> Well, I do not like that style but if maintainer do not object I will
> do not complain more here about that.
> 
>>>>  #define PAGE_SIZE 4096
>>>>  #define MAX_MODULES (PAGE_SIZE / sizeof (struct xen_multiboot_mod_list))
>>>> @@ -166,7 +168,7 @@ generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
>>>>  }
>>>>
>>>>  static grub_err_t
>>>> -set_mfns (grub_xen_mfn_t * new_mfn_list, grub_xen_mfn_t pfn)
>>>> +set_mfns (grub_xen_mfn_t pfn)
>>>>  {
>>>>    grub_xen_mfn_t i, t;
>>>>    grub_xen_mfn_t cn_pfn = -1, st_pfn = -1;
>>>> @@ -175,32 +177,32 @@ set_mfns (grub_xen_mfn_t * new_mfn_list, grub_xen_mfn_t pfn)
>>>>
>>>>    for (i = 0; i < grub_xen_start_page_addr->nr_pages; i++)
>>>>      {
>>>> -      if (new_mfn_list[i] == grub_xen_start_page_addr->console.domU.mfn)
>>>> +      if (virt_mfn_list[i] == grub_xen_start_page_addr->console.domU.mfn)
>>>>  	cn_pfn = i;
>>>> -      if (new_mfn_list[i] == grub_xen_start_page_addr->store_mfn)
>>>> +      if (virt_mfn_list[i] == grub_xen_start_page_addr->store_mfn)
>>>>  	st_pfn = i;
>>>>      }
>>>>    if (cn_pfn == (grub_xen_mfn_t)-1)
>>>>      return grub_error (GRUB_ERR_BUG, "no console");
>>>>    if (st_pfn == (grub_xen_mfn_t)-1)
>>>>      return grub_error (GRUB_ERR_BUG, "no store");
>>>> -  t = new_mfn_list[pfn];
>>>> -  new_mfn_list[pfn] = new_mfn_list[cn_pfn];
>>>> -  new_mfn_list[cn_pfn] = t;
>>>> -  t = new_mfn_list[pfn + 1];
>>>> -  new_mfn_list[pfn + 1] = new_mfn_list[st_pfn];
>>>> -  new_mfn_list[st_pfn] = t;
>>>> -
>>>> -  m2p_updates[0].ptr = page2offset (new_mfn_list[pfn]) | MMU_MACHPHYS_UPDATE;
>>>> +  t = virt_mfn_list[pfn];
>>>> +  virt_mfn_list[pfn] = virt_mfn_list[cn_pfn];
>>>> +  virt_mfn_list[cn_pfn] = t;
>>>> +  t = virt_mfn_list[pfn + 1];
>>>> +  virt_mfn_list[pfn + 1] = virt_mfn_list[st_pfn];
>>>> +  virt_mfn_list[st_pfn] = t;
>>>> +
>>>> +  m2p_updates[0].ptr = page2offset (virt_mfn_list[pfn]) | MMU_MACHPHYS_UPDATE;
>>>>    m2p_updates[0].val = pfn;
>>>>    m2p_updates[1].ptr =
>>>> -    page2offset (new_mfn_list[pfn + 1]) | MMU_MACHPHYS_UPDATE;
>>>> +    page2offset (virt_mfn_list[pfn + 1]) | MMU_MACHPHYS_UPDATE;
>>>>    m2p_updates[1].val = pfn + 1;
>>>>    m2p_updates[2].ptr =
>>>> -    page2offset (new_mfn_list[cn_pfn]) | MMU_MACHPHYS_UPDATE;
>>>> +    page2offset (virt_mfn_list[cn_pfn]) | MMU_MACHPHYS_UPDATE;
>>>>    m2p_updates[2].val = cn_pfn;
>>>>    m2p_updates[3].ptr =
>>>> -    page2offset (new_mfn_list[st_pfn]) | MMU_MACHPHYS_UPDATE;
>>>> +    page2offset (virt_mfn_list[st_pfn]) | MMU_MACHPHYS_UPDATE;
>>>>    m2p_updates[3].val = st_pfn;
>>>>
>>>>    grub_xen_mmu_update (m2p_updates, 4, NULL, DOMID_SELF);
>>>> @@ -209,34 +211,43 @@ set_mfns (grub_xen_mfn_t * new_mfn_list, grub_xen_mfn_t pfn)
>>>>  }
>>>>
>>>>  static grub_err_t
>>>> +grub_xen_p2m_alloc (void)
>>>> +{
>>>> +  grub_relocator_chunk_t ch;
>>>> +  grub_size_t p2msize;
>>>> +  grub_err_t err;
>>>> +
>>>> +  state.mfn_list = max_addr;
>>>> +  next_start.mfn_list = max_addr + xen_inf.virt_base;
>>>> +  p2msize = sizeof (grub_xen_mfn_t) * grub_xen_start_page_addr->nr_pages;
>>>> +  err = grub_relocator_alloc_chunk_addr (relocator, &ch, max_addr, p2msize);
>>>
>>> Hmmm.... Where relocator is defined?
>>
>> First global variable in this source.
>>
>>>
>>>> +  if (err)
>>>> +    return err;
>>>> +  virt_mfn_list = get_virtual_current_address (ch);
>>>> +  grub_memcpy (virt_mfn_list,
>>>> +	       (void *) grub_xen_start_page_addr->mfn_list, p2msize);
>>>> +  max_addr = ALIGN_UP (max_addr + p2msize, PAGE_SIZE);
>>>> +
>>>> +  return GRUB_ERR_NONE;
>>>> +}
>>>> +
>>>> +static grub_err_t
>>>>  grub_xen_boot (void)
>>>>  {
>>>> -  struct grub_relocator_xen_state state;
>>>>    grub_relocator_chunk_t ch;
>>>>    grub_err_t err;
>>>> -  grub_size_t pgtsize;
>>>>    struct start_info *nst;
>>>>    grub_uint64_t nr_info_pages;
>>>>    grub_uint64_t nr_pages, nr_pt_pages, nr_need_pages;
>>>>    struct gnttab_set_version gnttab_setver;
>>>> -  grub_xen_mfn_t *new_mfn_list;
>>>>    grub_size_t i;
>>>>
>>>>    if (grub_xen_n_allocated_shared_pages)
>>>>      return grub_error (GRUB_ERR_BUG, "active grants");
>>>>
>>>> -  state.mfn_list = max_addr;
>>>> -  next_start.mfn_list = max_addr + xen_inf.virt_base;
>>>> -  next_start.first_p2m_pfn = max_addr >> PAGE_SHIFT;	/* Is this right? */
>>>> -  pgtsize = sizeof (grub_xen_mfn_t) * grub_xen_start_page_addr->nr_pages;
>>>> -  err = grub_relocator_alloc_chunk_addr (relocator, &ch, max_addr, pgtsize);
>>>> -  next_start.nr_p2m_frames = (pgtsize + PAGE_SIZE - 1) >> PAGE_SHIFT;
>>>> +  err = grub_xen_p2m_alloc ();
>>>>    if (err)
>>>>      return err;
>>>> -  new_mfn_list = get_virtual_current_address (ch);
>>>> -  grub_memcpy (new_mfn_list,
>>>> -	       (void *) grub_xen_start_page_addr->mfn_list, pgtsize);
>>>> -  max_addr = ALIGN_UP (max_addr + pgtsize, PAGE_SIZE);
>>>>
>>>>    err = grub_relocator_alloc_chunk_addr (relocator, &ch,
>>>
>>> Ditto. I think that this could be passed as a grub_xen_boot() argument.
>>> Otherwise this appear from nowhere and reading/understanding is more
>>> difficult.
>>
>> Huh? How that? grub_xen_boot() can't have any parameters (see prototype
>> of grub_loader_set() which is used with grub_xen_boot() as parameter).
>>
>> BTW: I didn't introduce this kind of usage of relocator.
> 
> OK, I have checked other loaders and it looks that they use relocator in
> the same way. Let's leave it as is. However, this does not waive my
> concerns expressed above in regards to other global variables.

See my reply to patch 4. I'll change my series.


Juergen

[PATCH v2 2/6] xen: factor out allocation of special pages into separate function

[Juergen Gross]

Do the allocation of special pages (start info, console and xenbus
ring buffers) in a separate function. This will allow to do the
allocation at different times of the boot preparations depending on
the features the kernel is supporting.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
 grub-core/loader/i386/xen.c | 50 +++++++++++++++++++++++++++++----------------
 1 file changed, 32 insertions(+), 18 deletions(-)

diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
index 42ed7c7..e48cc3f 100644
--- a/grub-core/loader/i386/xen.c
+++ b/grub-core/loader/i386/xen.c
@@ -54,6 +54,8 @@ static grub_uint64_t modules_target_start;
 static grub_size_t n_modules;
 static struct grub_relocator_xen_state state;
 static grub_xen_mfn_t *virt_mfn_list;
+static struct start_info *virt_start_info;
+static grub_xen_mfn_t console_pfn;
 
 #define PAGE_SIZE 4096
 #define MAX_MODULES (PAGE_SIZE / sizeof (struct xen_multiboot_mod_list))
@@ -232,43 +234,51 @@ grub_xen_p2m_alloc (void)
 }
 
 static grub_err_t
-grub_xen_boot (void)
+grub_xen_special_alloc (void)
 {
   grub_relocator_chunk_t ch;
   grub_err_t err;
-  struct start_info *nst;
-  grub_uint64_t nr_info_pages;
-  grub_uint64_t nr_pages, nr_pt_pages, nr_need_pages;
-  struct gnttab_set_version gnttab_setver;
-  grub_size_t i;
-
-  if (grub_xen_n_allocated_shared_pages)
-    return grub_error (GRUB_ERR_BUG, "active grants");
-
-  err = grub_xen_p2m_alloc ();
-  if (err)
-    return err;
 
   err = grub_relocator_alloc_chunk_addr (relocator, &ch,
 					 max_addr, sizeof (next_start));
   if (err)
     return err;
   state.start_info = max_addr + xen_inf.virt_base;
-  nst = get_virtual_current_address (ch);
+  virt_start_info = get_virtual_current_address (ch);
   max_addr = ALIGN_UP (max_addr + sizeof (next_start), PAGE_SIZE);
+  console_pfn = max_addr >> PAGE_SHIFT;
+  max_addr += 2 * PAGE_SIZE;
 
   next_start.nr_pages = grub_xen_start_page_addr->nr_pages;
   grub_memcpy (next_start.magic, grub_xen_start_page_addr->magic,
 	       sizeof (next_start.magic));
+  next_start.shared_info = grub_xen_start_page_addr->shared_info;
   next_start.store_mfn = grub_xen_start_page_addr->store_mfn;
   next_start.store_evtchn = grub_xen_start_page_addr->store_evtchn;
   next_start.console.domU = grub_xen_start_page_addr->console.domU;
-  next_start.shared_info = grub_xen_start_page_addr->shared_info;
 
-  err = set_mfns (max_addr >> PAGE_SHIFT);
+  return GRUB_ERR_NONE;
+}
+
+static grub_err_t
+grub_xen_boot (void)
+{
+  grub_relocator_chunk_t ch;
+  grub_err_t err;
+  grub_uint64_t nr_info_pages;
+  grub_uint64_t nr_pages, nr_pt_pages, nr_need_pages;
+  struct gnttab_set_version gnttab_setver;
+  grub_size_t i;
+
+  if (grub_xen_n_allocated_shared_pages)
+    return grub_error (GRUB_ERR_BUG, "active grants");
+
+  err = grub_xen_p2m_alloc ();
+  if (err)
+    return err;
+  err = grub_xen_special_alloc ();
   if (err)
     return err;
-  max_addr += 2 * PAGE_SIZE;
 
   next_start.pt_base = max_addr + xen_inf.virt_base;
   state.paging_start = max_addr >> PAGE_SHIFT;
@@ -297,6 +307,10 @@ grub_xen_boot (void)
   if (err)
     return err;
 
+  err = set_mfns (console_pfn);
+  if (err)
+    return err;
+
   generate_page_table (get_virtual_current_address (ch),
 		       max_addr >> PAGE_SHIFT, nr_pages,
 		       xen_inf.virt_base, virt_mfn_list);
@@ -308,7 +322,7 @@ grub_xen_boot (void)
   next_start.nr_pt_frames = nr_pt_pages;
   state.paging_size = nr_pt_pages;
 
-  *nst = next_start;
+  *virt_start_info = next_start;
 
   grub_memset (&gnttab_setver, 0, sizeof (gnttab_setver));
 
-- 
2.6.2

Re: [PATCH v2 2/6] xen: factor out allocation of special pages into separate function

[Daniel Kiper]

On Thu, Feb 11, 2016 at 08:53:22AM +0100, Juergen Gross wrote:
> Do the allocation of special pages (start info, console and xenbus
> ring buffers) in a separate function. This will allow to do the
> allocation at different times of the boot preparations depending on
> the features the kernel is supporting.
>
> Signed-off-by: Juergen Gross <jgross@suse.com>
> ---
>  grub-core/loader/i386/xen.c | 50 +++++++++++++++++++++++++++++----------------
>  1 file changed, 32 insertions(+), 18 deletions(-)
>
> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
> index 42ed7c7..e48cc3f 100644
> --- a/grub-core/loader/i386/xen.c
> +++ b/grub-core/loader/i386/xen.c
> @@ -54,6 +54,8 @@ static grub_uint64_t modules_target_start;
>  static grub_size_t n_modules;
>  static struct grub_relocator_xen_state state;
>  static grub_xen_mfn_t *virt_mfn_list;
> +static struct start_info *virt_start_info;
> +static grub_xen_mfn_t console_pfn;

Same as in patch #1.

Daniel

Re: [PATCH v2 2/6] xen: factor out allocation of special pages into separate function

[Juergen Gross]

On 11/02/16 13:21, Daniel Kiper wrote:
> On Thu, Feb 11, 2016 at 08:53:22AM +0100, Juergen Gross wrote:
>> Do the allocation of special pages (start info, console and xenbus
>> ring buffers) in a separate function. This will allow to do the
>> allocation at different times of the boot preparations depending on
>> the features the kernel is supporting.
>>
>> Signed-off-by: Juergen Gross <jgross@suse.com>
>> ---
>>  grub-core/loader/i386/xen.c | 50 +++++++++++++++++++++++++++++----------------
>>  1 file changed, 32 insertions(+), 18 deletions(-)
>>
>> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
>> index 42ed7c7..e48cc3f 100644
>> --- a/grub-core/loader/i386/xen.c
>> +++ b/grub-core/loader/i386/xen.c
>> @@ -54,6 +54,8 @@ static grub_uint64_t modules_target_start;
>>  static grub_size_t n_modules;
>>  static struct grub_relocator_xen_state state;
>>  static grub_xen_mfn_t *virt_mfn_list;
>> +static struct start_info *virt_start_info;
>> +static grub_xen_mfn_t console_pfn;
> 
> Same as in patch #1.

Same answer. :-)


Juergen

[PATCH v2 3/6] xen: factor out allocation of page tables into separate function

[Juergen Gross]

Do the allocation of page tables in a separate function. This will
allow to do the allocation at different times of the boot preparations
depending on the features the kernel is supporting.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
 grub-core/loader/i386/xen.c | 82 ++++++++++++++++++++++++++++-----------------
 1 file changed, 51 insertions(+), 31 deletions(-)

diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
index e48cc3f..65cec27 100644
--- a/grub-core/loader/i386/xen.c
+++ b/grub-core/loader/i386/xen.c
@@ -56,6 +56,9 @@ static struct grub_relocator_xen_state state;
 static grub_xen_mfn_t *virt_mfn_list;
 static struct start_info *virt_start_info;
 static grub_xen_mfn_t console_pfn;
+static grub_uint64_t *virt_pgtable;
+static grub_uint64_t pgtbl_start;
+static grub_uint64_t pgtbl_end;
 
 #define PAGE_SIZE 4096
 #define MAX_MODULES (PAGE_SIZE / sizeof (struct xen_multiboot_mod_list))
@@ -106,17 +109,17 @@ get_pgtable_size (grub_uint64_t total_pages, grub_uint64_t virt_base)
 
 static void
 generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
-		     grub_uint64_t total_pages, grub_uint64_t virt_base,
-		     grub_xen_mfn_t *mfn_list)
+		     grub_uint64_t paging_end, grub_uint64_t total_pages,
+		     grub_uint64_t virt_base, grub_xen_mfn_t *mfn_list)
 {
   if (!virt_base)
-    total_pages++;
+    paging_end++;
 
   grub_uint64_t lx[NUMBER_OF_LEVELS], lxs[NUMBER_OF_LEVELS];
   grub_uint64_t nlx, nls, sz = 0;
   int l;
 
-  nlx = total_pages;
+  nlx = paging_end;
   nls = virt_base >> PAGE_SHIFT;
   for (l = 0; l < NUMBER_OF_LEVELS; l++)
     {
@@ -160,7 +163,7 @@ generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
   if (pr)
     pg += POINTERS_PER_PAGE;
 
-  for (j = 0; j < total_pages; j++)
+  for (j = 0; j < paging_end; j++)
     {
       if (j >= paging_start && j < lp)
 	pg[j + lxs[0]] = page2offset (mfn_list[j]) | 5;
@@ -261,24 +264,12 @@ grub_xen_special_alloc (void)
 }
 
 static grub_err_t
-grub_xen_boot (void)
+grub_xen_pt_alloc (void)
 {
   grub_relocator_chunk_t ch;
   grub_err_t err;
   grub_uint64_t nr_info_pages;
   grub_uint64_t nr_pages, nr_pt_pages, nr_need_pages;
-  struct gnttab_set_version gnttab_setver;
-  grub_size_t i;
-
-  if (grub_xen_n_allocated_shared_pages)
-    return grub_error (GRUB_ERR_BUG, "active grants");
-
-  err = grub_xen_p2m_alloc ();
-  if (err)
-    return err;
-  err = grub_xen_special_alloc ();
-  if (err)
-    return err;
 
   next_start.pt_base = max_addr + xen_inf.virt_base;
   state.paging_start = max_addr >> PAGE_SHIFT;
@@ -298,30 +289,59 @@ grub_xen_boot (void)
       nr_pages = nr_need_pages;
     }
 
-  grub_dprintf ("xen", "bootstrap domain %llx+%llx\n",
-		(unsigned long long) xen_inf.virt_base,
-		(unsigned long long) page2offset (nr_pages));
-
   err = grub_relocator_alloc_chunk_addr (relocator, &ch,
 					 max_addr, page2offset (nr_pt_pages));
   if (err)
     return err;
 
+  virt_pgtable = get_virtual_current_address (ch);
+  pgtbl_start = max_addr >> PAGE_SHIFT;
+  max_addr += page2offset (nr_pt_pages);
+  state.stack = max_addr + STACK_SIZE + xen_inf.virt_base;
+  state.paging_size = nr_pt_pages;
+  next_start.nr_pt_frames = nr_pt_pages;
+  max_addr = page2offset (nr_pages);
+  pgtbl_end = nr_pages;
+
+  return GRUB_ERR_NONE;
+}
+
+static grub_err_t
+grub_xen_boot (void)
+{
+  grub_err_t err;
+  grub_uint64_t nr_pages;
+  struct gnttab_set_version gnttab_setver;
+  grub_size_t i;
+
+  if (grub_xen_n_allocated_shared_pages)
+    return grub_error (GRUB_ERR_BUG, "active grants");
+
+  err = grub_xen_p2m_alloc ();
+  if (err)
+    return err;
+  err = grub_xen_special_alloc ();
+  if (err)
+    return err;
+  err = grub_xen_pt_alloc ();
+  if (err)
+    return err;
+
   err = set_mfns (console_pfn);
   if (err)
     return err;
 
-  generate_page_table (get_virtual_current_address (ch),
-		       max_addr >> PAGE_SHIFT, nr_pages,
+  nr_pages = max_addr >> PAGE_SHIFT;
+
+  grub_dprintf ("xen", "bootstrap domain %llx+%llx\n",
+		(unsigned long long) xen_inf.virt_base,
+		(unsigned long long) page2offset (nr_pages));
+
+  generate_page_table (virt_pgtable, pgtbl_start, pgtbl_end, nr_pages,
 		       xen_inf.virt_base, virt_mfn_list);
 
-  max_addr += page2offset (nr_pt_pages);
-  state.stack = max_addr + STACK_SIZE + xen_inf.virt_base;
   state.entry_point = xen_inf.entry_point;
 
-  next_start.nr_pt_frames = nr_pt_pages;
-  state.paging_size = nr_pt_pages;
-
   *virt_start_info = next_start;
 
   grub_memset (&gnttab_setver, 0, sizeof (gnttab_setver));
@@ -335,8 +355,8 @@ grub_xen_boot (void)
   return grub_relocator_xen_boot (relocator, state, nr_pages,
 				  xen_inf.virt_base <
 				  PAGE_SIZE ? page2offset (nr_pages) : 0,
-				  nr_pages - 1,
-				  page2offset (nr_pages - 1) +
+				  pgtbl_end - 1,
+				  page2offset (pgtbl_end - 1) +
 				  xen_inf.virt_base);
 }
 
-- 
2.6.2

Re: [PATCH v2 3/6] xen: factor out allocation of page tables into separate function

[Daniel Kiper]

On Thu, Feb 11, 2016 at 08:53:23AM +0100, Juergen Gross wrote:
> Do the allocation of page tables in a separate function. This will
> allow to do the allocation at different times of the boot preparations
> depending on the features the kernel is supporting.
>
> Signed-off-by: Juergen Gross <jgross@suse.com>
> ---
>  grub-core/loader/i386/xen.c | 82 ++++++++++++++++++++++++++++-----------------
>  1 file changed, 51 insertions(+), 31 deletions(-)
>
> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
> index e48cc3f..65cec27 100644
> --- a/grub-core/loader/i386/xen.c
> +++ b/grub-core/loader/i386/xen.c
> @@ -56,6 +56,9 @@ static struct grub_relocator_xen_state state;
>  static grub_xen_mfn_t *virt_mfn_list;
>  static struct start_info *virt_start_info;
>  static grub_xen_mfn_t console_pfn;
> +static grub_uint64_t *virt_pgtable;
> +static grub_uint64_t pgtbl_start;
> +static grub_uint64_t pgtbl_end;

Same as in patches #1 and #2.

>  #define PAGE_SIZE 4096
>  #define MAX_MODULES (PAGE_SIZE / sizeof (struct xen_multiboot_mod_list))
> @@ -106,17 +109,17 @@ get_pgtable_size (grub_uint64_t total_pages, grub_uint64_t virt_base)
>
>  static void
>  generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
> -		     grub_uint64_t total_pages, grub_uint64_t virt_base,
> -		     grub_xen_mfn_t *mfn_list)
> +		     grub_uint64_t paging_end, grub_uint64_t total_pages,
> +		     grub_uint64_t virt_base, grub_xen_mfn_t *mfn_list)
>  {
>    if (!virt_base)
> -    total_pages++;
> +    paging_end++;
>
>    grub_uint64_t lx[NUMBER_OF_LEVELS], lxs[NUMBER_OF_LEVELS];
>    grub_uint64_t nlx, nls, sz = 0;
>    int l;
>
> -  nlx = total_pages;
> +  nlx = paging_end;
>    nls = virt_base >> PAGE_SHIFT;
>    for (l = 0; l < NUMBER_OF_LEVELS; l++)
>      {
> @@ -160,7 +163,7 @@ generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
>    if (pr)
>      pg += POINTERS_PER_PAGE;
>
> -  for (j = 0; j < total_pages; j++)
> +  for (j = 0; j < paging_end; j++)
>      {
>        if (j >= paging_start && j < lp)
>  	pg[j + lxs[0]] = page2offset (mfn_list[j]) | 5;
> @@ -261,24 +264,12 @@ grub_xen_special_alloc (void)
>  }
>
>  static grub_err_t
> -grub_xen_boot (void)
> +grub_xen_pt_alloc (void)
>  {
>    grub_relocator_chunk_t ch;
>    grub_err_t err;
>    grub_uint64_t nr_info_pages;
>    grub_uint64_t nr_pages, nr_pt_pages, nr_need_pages;
> -  struct gnttab_set_version gnttab_setver;
> -  grub_size_t i;
> -
> -  if (grub_xen_n_allocated_shared_pages)
> -    return grub_error (GRUB_ERR_BUG, "active grants");
> -
> -  err = grub_xen_p2m_alloc ();
> -  if (err)
> -    return err;
> -  err = grub_xen_special_alloc ();
> -  if (err)
> -    return err;
>
>    next_start.pt_base = max_addr + xen_inf.virt_base;
>    state.paging_start = max_addr >> PAGE_SHIFT;
> @@ -298,30 +289,59 @@ grub_xen_boot (void)
>        nr_pages = nr_need_pages;
>      }
>
> -  grub_dprintf ("xen", "bootstrap domain %llx+%llx\n",
> -		(unsigned long long) xen_inf.virt_base,
> -		(unsigned long long) page2offset (nr_pages));
> -
>    err = grub_relocator_alloc_chunk_addr (relocator, &ch,
>  					 max_addr, page2offset (nr_pt_pages));
>    if (err)
>      return err;
>
> +  virt_pgtable = get_virtual_current_address (ch);
> +  pgtbl_start = max_addr >> PAGE_SHIFT;
> +  max_addr += page2offset (nr_pt_pages);
> +  state.stack = max_addr + STACK_SIZE + xen_inf.virt_base;
> +  state.paging_size = nr_pt_pages;
> +  next_start.nr_pt_frames = nr_pt_pages;
> +  max_addr = page2offset (nr_pages);
> +  pgtbl_end = nr_pages;
> +
> +  return GRUB_ERR_NONE;
> +}
> +
> +static grub_err_t
> +grub_xen_boot (void)
> +{
> +  grub_err_t err;
> +  grub_uint64_t nr_pages;
> +  struct gnttab_set_version gnttab_setver;
> +  grub_size_t i;
> +
> +  if (grub_xen_n_allocated_shared_pages)
> +    return grub_error (GRUB_ERR_BUG, "active grants");
> +
> +  err = grub_xen_p2m_alloc ();
> +  if (err)
> +    return err;
> +  err = grub_xen_special_alloc ();
> +  if (err)
> +    return err;
> +  err = grub_xen_pt_alloc ();
> +  if (err)
> +    return err;

Should not we free memory allocated by grub_xen_p2m_alloc() and
grub_xen_special_alloc() if grub_xen_pt_alloc() failed?

Daniel

Re: [PATCH v2 3/6] xen: factor out allocation of page tables into separate function

[Juergen Gross]

On 11/02/16 13:27, Daniel Kiper wrote:
> On Thu, Feb 11, 2016 at 08:53:23AM +0100, Juergen Gross wrote:
>> Do the allocation of page tables in a separate function. This will
>> allow to do the allocation at different times of the boot preparations
>> depending on the features the kernel is supporting.
>>
>> Signed-off-by: Juergen Gross <jgross@suse.com>
>> ---
>>  grub-core/loader/i386/xen.c | 82 ++++++++++++++++++++++++++++-----------------
>>  1 file changed, 51 insertions(+), 31 deletions(-)
>>
>> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
>> index e48cc3f..65cec27 100644
>> --- a/grub-core/loader/i386/xen.c
>> +++ b/grub-core/loader/i386/xen.c
>> @@ -56,6 +56,9 @@ static struct grub_relocator_xen_state state;
>>  static grub_xen_mfn_t *virt_mfn_list;
>>  static struct start_info *virt_start_info;
>>  static grub_xen_mfn_t console_pfn;
>> +static grub_uint64_t *virt_pgtable;
>> +static grub_uint64_t pgtbl_start;
>> +static grub_uint64_t pgtbl_end;
> 
> Same as in patches #1 and #2.

Yep.

> 
>>  #define PAGE_SIZE 4096
>>  #define MAX_MODULES (PAGE_SIZE / sizeof (struct xen_multiboot_mod_list))
>> @@ -106,17 +109,17 @@ get_pgtable_size (grub_uint64_t total_pages, grub_uint64_t virt_base)
>>
>>  static void
>>  generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
>> -		     grub_uint64_t total_pages, grub_uint64_t virt_base,
>> -		     grub_xen_mfn_t *mfn_list)
>> +		     grub_uint64_t paging_end, grub_uint64_t total_pages,
>> +		     grub_uint64_t virt_base, grub_xen_mfn_t *mfn_list)
>>  {
>>    if (!virt_base)
>> -    total_pages++;
>> +    paging_end++;
>>
>>    grub_uint64_t lx[NUMBER_OF_LEVELS], lxs[NUMBER_OF_LEVELS];
>>    grub_uint64_t nlx, nls, sz = 0;
>>    int l;
>>
>> -  nlx = total_pages;
>> +  nlx = paging_end;
>>    nls = virt_base >> PAGE_SHIFT;
>>    for (l = 0; l < NUMBER_OF_LEVELS; l++)
>>      {
>> @@ -160,7 +163,7 @@ generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
>>    if (pr)
>>      pg += POINTERS_PER_PAGE;
>>
>> -  for (j = 0; j < total_pages; j++)
>> +  for (j = 0; j < paging_end; j++)
>>      {
>>        if (j >= paging_start && j < lp)
>>  	pg[j + lxs[0]] = page2offset (mfn_list[j]) | 5;
>> @@ -261,24 +264,12 @@ grub_xen_special_alloc (void)
>>  }
>>
>>  static grub_err_t
>> -grub_xen_boot (void)
>> +grub_xen_pt_alloc (void)
>>  {
>>    grub_relocator_chunk_t ch;
>>    grub_err_t err;
>>    grub_uint64_t nr_info_pages;
>>    grub_uint64_t nr_pages, nr_pt_pages, nr_need_pages;
>> -  struct gnttab_set_version gnttab_setver;
>> -  grub_size_t i;
>> -
>> -  if (grub_xen_n_allocated_shared_pages)
>> -    return grub_error (GRUB_ERR_BUG, "active grants");
>> -
>> -  err = grub_xen_p2m_alloc ();
>> -  if (err)
>> -    return err;
>> -  err = grub_xen_special_alloc ();
>> -  if (err)
>> -    return err;
>>
>>    next_start.pt_base = max_addr + xen_inf.virt_base;
>>    state.paging_start = max_addr >> PAGE_SHIFT;
>> @@ -298,30 +289,59 @@ grub_xen_boot (void)
>>        nr_pages = nr_need_pages;
>>      }
>>
>> -  grub_dprintf ("xen", "bootstrap domain %llx+%llx\n",
>> -		(unsigned long long) xen_inf.virt_base,
>> -		(unsigned long long) page2offset (nr_pages));
>> -
>>    err = grub_relocator_alloc_chunk_addr (relocator, &ch,
>>  					 max_addr, page2offset (nr_pt_pages));
>>    if (err)
>>      return err;
>>
>> +  virt_pgtable = get_virtual_current_address (ch);
>> +  pgtbl_start = max_addr >> PAGE_SHIFT;
>> +  max_addr += page2offset (nr_pt_pages);
>> +  state.stack = max_addr + STACK_SIZE + xen_inf.virt_base;
>> +  state.paging_size = nr_pt_pages;
>> +  next_start.nr_pt_frames = nr_pt_pages;
>> +  max_addr = page2offset (nr_pages);
>> +  pgtbl_end = nr_pages;
>> +
>> +  return GRUB_ERR_NONE;
>> +}
>> +
>> +static grub_err_t
>> +grub_xen_boot (void)
>> +{
>> +  grub_err_t err;
>> +  grub_uint64_t nr_pages;
>> +  struct gnttab_set_version gnttab_setver;
>> +  grub_size_t i;
>> +
>> +  if (grub_xen_n_allocated_shared_pages)
>> +    return grub_error (GRUB_ERR_BUG, "active grants");
>> +
>> +  err = grub_xen_p2m_alloc ();
>> +  if (err)
>> +    return err;
>> +  err = grub_xen_special_alloc ();
>> +  if (err)
>> +    return err;
>> +  err = grub_xen_pt_alloc ();
>> +  if (err)
>> +    return err;
> 
> Should not we free memory allocated by grub_xen_p2m_alloc() and
> grub_xen_special_alloc() if grub_xen_pt_alloc() failed?

Hmm, why? If I don't miss anything freeing memory in case of error isn't
done anywhere (at least not in this source file).

Juergen

Re: [PATCH v2 3/6] xen: factor out allocation of page tables into separate function

[Daniel Kiper]

On Thu, Feb 11, 2016 at 01:53:51PM +0100, Juergen Gross wrote:
> On 11/02/16 13:27, Daniel Kiper wrote:
> > On Thu, Feb 11, 2016 at 08:53:23AM +0100, Juergen Gross wrote:
> >> Do the allocation of page tables in a separate function. This will
> >> allow to do the allocation at different times of the boot preparations
> >> depending on the features the kernel is supporting.
> >>
> >> Signed-off-by: Juergen Gross <jgross@suse.com>
> >> ---
> >>  grub-core/loader/i386/xen.c | 82 ++++++++++++++++++++++++++++-----------------
> >>  1 file changed, 51 insertions(+), 31 deletions(-)
> >>
> >> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
> >> index e48cc3f..65cec27 100644
> >> --- a/grub-core/loader/i386/xen.c
> >> +++ b/grub-core/loader/i386/xen.c
> >> @@ -56,6 +56,9 @@ static struct grub_relocator_xen_state state;
> >>  static grub_xen_mfn_t *virt_mfn_list;
> >>  static struct start_info *virt_start_info;
> >>  static grub_xen_mfn_t console_pfn;
> >> +static grub_uint64_t *virt_pgtable;
> >> +static grub_uint64_t pgtbl_start;
> >> +static grub_uint64_t pgtbl_end;
> >
> > Same as in patches #1 and #2.
>
> Yep.
>
> >
> >>  #define PAGE_SIZE 4096
> >>  #define MAX_MODULES (PAGE_SIZE / sizeof (struct xen_multiboot_mod_list))
> >> @@ -106,17 +109,17 @@ get_pgtable_size (grub_uint64_t total_pages, grub_uint64_t virt_base)
> >>
> >>  static void
> >>  generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
> >> -		     grub_uint64_t total_pages, grub_uint64_t virt_base,
> >> -		     grub_xen_mfn_t *mfn_list)
> >> +		     grub_uint64_t paging_end, grub_uint64_t total_pages,
> >> +		     grub_uint64_t virt_base, grub_xen_mfn_t *mfn_list)
> >>  {
> >>    if (!virt_base)
> >> -    total_pages++;
> >> +    paging_end++;
> >>
> >>    grub_uint64_t lx[NUMBER_OF_LEVELS], lxs[NUMBER_OF_LEVELS];
> >>    grub_uint64_t nlx, nls, sz = 0;
> >>    int l;
> >>
> >> -  nlx = total_pages;
> >> +  nlx = paging_end;
> >>    nls = virt_base >> PAGE_SHIFT;
> >>    for (l = 0; l < NUMBER_OF_LEVELS; l++)
> >>      {
> >> @@ -160,7 +163,7 @@ generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
> >>    if (pr)
> >>      pg += POINTERS_PER_PAGE;
> >>
> >> -  for (j = 0; j < total_pages; j++)
> >> +  for (j = 0; j < paging_end; j++)
> >>      {
> >>        if (j >= paging_start && j < lp)
> >>  	pg[j + lxs[0]] = page2offset (mfn_list[j]) | 5;
> >> @@ -261,24 +264,12 @@ grub_xen_special_alloc (void)
> >>  }
> >>
> >>  static grub_err_t
> >> -grub_xen_boot (void)
> >> +grub_xen_pt_alloc (void)
> >>  {
> >>    grub_relocator_chunk_t ch;
> >>    grub_err_t err;
> >>    grub_uint64_t nr_info_pages;
> >>    grub_uint64_t nr_pages, nr_pt_pages, nr_need_pages;
> >> -  struct gnttab_set_version gnttab_setver;
> >> -  grub_size_t i;
> >> -
> >> -  if (grub_xen_n_allocated_shared_pages)
> >> -    return grub_error (GRUB_ERR_BUG, "active grants");
> >> -
> >> -  err = grub_xen_p2m_alloc ();
> >> -  if (err)
> >> -    return err;
> >> -  err = grub_xen_special_alloc ();
> >> -  if (err)
> >> -    return err;
> >>
> >>    next_start.pt_base = max_addr + xen_inf.virt_base;
> >>    state.paging_start = max_addr >> PAGE_SHIFT;
> >> @@ -298,30 +289,59 @@ grub_xen_boot (void)
> >>        nr_pages = nr_need_pages;
> >>      }
> >>
> >> -  grub_dprintf ("xen", "bootstrap domain %llx+%llx\n",
> >> -		(unsigned long long) xen_inf.virt_base,
> >> -		(unsigned long long) page2offset (nr_pages));
> >> -
> >>    err = grub_relocator_alloc_chunk_addr (relocator, &ch,
> >>  					 max_addr, page2offset (nr_pt_pages));
> >>    if (err)
> >>      return err;
> >>
> >> +  virt_pgtable = get_virtual_current_address (ch);
> >> +  pgtbl_start = max_addr >> PAGE_SHIFT;
> >> +  max_addr += page2offset (nr_pt_pages);
> >> +  state.stack = max_addr + STACK_SIZE + xen_inf.virt_base;
> >> +  state.paging_size = nr_pt_pages;
> >> +  next_start.nr_pt_frames = nr_pt_pages;
> >> +  max_addr = page2offset (nr_pages);
> >> +  pgtbl_end = nr_pages;
> >> +
> >> +  return GRUB_ERR_NONE;
> >> +}
> >> +
> >> +static grub_err_t
> >> +grub_xen_boot (void)
> >> +{
> >> +  grub_err_t err;
> >> +  grub_uint64_t nr_pages;
> >> +  struct gnttab_set_version gnttab_setver;
> >> +  grub_size_t i;
> >> +
> >> +  if (grub_xen_n_allocated_shared_pages)
> >> +    return grub_error (GRUB_ERR_BUG, "active grants");
> >> +
> >> +  err = grub_xen_p2m_alloc ();
> >> +  if (err)
> >> +    return err;
> >> +  err = grub_xen_special_alloc ();
> >> +  if (err)
> >> +    return err;
> >> +  err = grub_xen_pt_alloc ();
> >> +  if (err)
> >> +    return err;
> >
> > Should not we free memory allocated by grub_xen_p2m_alloc() and
> > grub_xen_special_alloc() if grub_xen_pt_alloc() failed?
>
> Hmm, why? If I don't miss anything freeing memory in case of error isn't
> done anywhere (at least not in this source file).

It seems that memory is freed by higher level functions when you
try to use the same loader or other one. Otherwise it would be clear
memory leak. Could you double check it?

Daniel

Re: [PATCH v2 3/6] xen: factor out allocation of page tables into separate function

[Juergen Gross]

On 11/02/16 18:14, Daniel Kiper wrote:
> On Thu, Feb 11, 2016 at 01:53:51PM +0100, Juergen Gross wrote:
>> On 11/02/16 13:27, Daniel Kiper wrote:
>>> On Thu, Feb 11, 2016 at 08:53:23AM +0100, Juergen Gross wrote:
>>>> Do the allocation of page tables in a separate function. This will
>>>> allow to do the allocation at different times of the boot preparations
>>>> depending on the features the kernel is supporting.
>>>>
>>>> Signed-off-by: Juergen Gross <jgross@suse.com>
>>>> ---
>>>>  grub-core/loader/i386/xen.c | 82 ++++++++++++++++++++++++++++-----------------
>>>>  1 file changed, 51 insertions(+), 31 deletions(-)
>>>>
>>>> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
>>>> index e48cc3f..65cec27 100644
>>>> --- a/grub-core/loader/i386/xen.c
>>>> +++ b/grub-core/loader/i386/xen.c
>>>> @@ -56,6 +56,9 @@ static struct grub_relocator_xen_state state;
>>>>  static grub_xen_mfn_t *virt_mfn_list;
>>>>  static struct start_info *virt_start_info;
>>>>  static grub_xen_mfn_t console_pfn;
>>>> +static grub_uint64_t *virt_pgtable;
>>>> +static grub_uint64_t pgtbl_start;
>>>> +static grub_uint64_t pgtbl_end;
>>>
>>> Same as in patches #1 and #2.
>>
>> Yep.
>>
>>>
>>>>  #define PAGE_SIZE 4096
>>>>  #define MAX_MODULES (PAGE_SIZE / sizeof (struct xen_multiboot_mod_list))
>>>> @@ -106,17 +109,17 @@ get_pgtable_size (grub_uint64_t total_pages, grub_uint64_t virt_base)
>>>>
>>>>  static void
>>>>  generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
>>>> -		     grub_uint64_t total_pages, grub_uint64_t virt_base,
>>>> -		     grub_xen_mfn_t *mfn_list)
>>>> +		     grub_uint64_t paging_end, grub_uint64_t total_pages,
>>>> +		     grub_uint64_t virt_base, grub_xen_mfn_t *mfn_list)
>>>>  {
>>>>    if (!virt_base)
>>>> -    total_pages++;
>>>> +    paging_end++;
>>>>
>>>>    grub_uint64_t lx[NUMBER_OF_LEVELS], lxs[NUMBER_OF_LEVELS];
>>>>    grub_uint64_t nlx, nls, sz = 0;
>>>>    int l;
>>>>
>>>> -  nlx = total_pages;
>>>> +  nlx = paging_end;
>>>>    nls = virt_base >> PAGE_SHIFT;
>>>>    for (l = 0; l < NUMBER_OF_LEVELS; l++)
>>>>      {
>>>> @@ -160,7 +163,7 @@ generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
>>>>    if (pr)
>>>>      pg += POINTERS_PER_PAGE;
>>>>
>>>> -  for (j = 0; j < total_pages; j++)
>>>> +  for (j = 0; j < paging_end; j++)
>>>>      {
>>>>        if (j >= paging_start && j < lp)
>>>>  	pg[j + lxs[0]] = page2offset (mfn_list[j]) | 5;
>>>> @@ -261,24 +264,12 @@ grub_xen_special_alloc (void)
>>>>  }
>>>>
>>>>  static grub_err_t
>>>> -grub_xen_boot (void)
>>>> +grub_xen_pt_alloc (void)
>>>>  {
>>>>    grub_relocator_chunk_t ch;
>>>>    grub_err_t err;
>>>>    grub_uint64_t nr_info_pages;
>>>>    grub_uint64_t nr_pages, nr_pt_pages, nr_need_pages;
>>>> -  struct gnttab_set_version gnttab_setver;
>>>> -  grub_size_t i;
>>>> -
>>>> -  if (grub_xen_n_allocated_shared_pages)
>>>> -    return grub_error (GRUB_ERR_BUG, "active grants");
>>>> -
>>>> -  err = grub_xen_p2m_alloc ();
>>>> -  if (err)
>>>> -    return err;
>>>> -  err = grub_xen_special_alloc ();
>>>> -  if (err)
>>>> -    return err;
>>>>
>>>>    next_start.pt_base = max_addr + xen_inf.virt_base;
>>>>    state.paging_start = max_addr >> PAGE_SHIFT;
>>>> @@ -298,30 +289,59 @@ grub_xen_boot (void)
>>>>        nr_pages = nr_need_pages;
>>>>      }
>>>>
>>>> -  grub_dprintf ("xen", "bootstrap domain %llx+%llx\n",
>>>> -		(unsigned long long) xen_inf.virt_base,
>>>> -		(unsigned long long) page2offset (nr_pages));
>>>> -
>>>>    err = grub_relocator_alloc_chunk_addr (relocator, &ch,
>>>>  					 max_addr, page2offset (nr_pt_pages));
>>>>    if (err)
>>>>      return err;
>>>>
>>>> +  virt_pgtable = get_virtual_current_address (ch);
>>>> +  pgtbl_start = max_addr >> PAGE_SHIFT;
>>>> +  max_addr += page2offset (nr_pt_pages);
>>>> +  state.stack = max_addr + STACK_SIZE + xen_inf.virt_base;
>>>> +  state.paging_size = nr_pt_pages;
>>>> +  next_start.nr_pt_frames = nr_pt_pages;
>>>> +  max_addr = page2offset (nr_pages);
>>>> +  pgtbl_end = nr_pages;
>>>> +
>>>> +  return GRUB_ERR_NONE;
>>>> +}
>>>> +
>>>> +static grub_err_t
>>>> +grub_xen_boot (void)
>>>> +{
>>>> +  grub_err_t err;
>>>> +  grub_uint64_t nr_pages;
>>>> +  struct gnttab_set_version gnttab_setver;
>>>> +  grub_size_t i;
>>>> +
>>>> +  if (grub_xen_n_allocated_shared_pages)
>>>> +    return grub_error (GRUB_ERR_BUG, "active grants");
>>>> +
>>>> +  err = grub_xen_p2m_alloc ();
>>>> +  if (err)
>>>> +    return err;
>>>> +  err = grub_xen_special_alloc ();
>>>> +  if (err)
>>>> +    return err;
>>>> +  err = grub_xen_pt_alloc ();
>>>> +  if (err)
>>>> +    return err;
>>>
>>> Should not we free memory allocated by grub_xen_p2m_alloc() and
>>> grub_xen_special_alloc() if grub_xen_pt_alloc() failed?
>>
>> Hmm, why? If I don't miss anything freeing memory in case of error isn't
>> done anywhere (at least not in this source file).
> 
> It seems that memory is freed by higher level functions when you
> try to use the same loader or other one. Otherwise it would be clear
> memory leak. Could you double check it?

Yes, you are right. Other loaders are calling grub_relocator_unload() to
achieve this before allocating a new relocator or when unloading the
module.


Juergen

Re: [PATCH v2 3/6] xen: factor out allocation of page tables into separate function

[Vladimir 'φ-coder/phcoder' Serbinenko]

[-- Attachment #1: Type: text/plain, Size: 4938 bytes --]

On 11.02.2016 13:53, Juergen Gross wrote:
> On 11/02/16 13:27, Daniel Kiper wrote:
>> On Thu, Feb 11, 2016 at 08:53:23AM +0100, Juergen Gross wrote:
>>> Do the allocation of page tables in a separate function. This will
>>> allow to do the allocation at different times of the boot preparations
>>> depending on the features the kernel is supporting.
>>>
>>> Signed-off-by: Juergen Gross <jgross@suse.com>
>>> ---
>>>  grub-core/loader/i386/xen.c | 82 ++++++++++++++++++++++++++++-----------------
>>>  1 file changed, 51 insertions(+), 31 deletions(-)
>>>
>>> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
>>> index e48cc3f..65cec27 100644
>>> --- a/grub-core/loader/i386/xen.c
>>> +++ b/grub-core/loader/i386/xen.c
>>> @@ -56,6 +56,9 @@ static struct grub_relocator_xen_state state;
>>>  static grub_xen_mfn_t *virt_mfn_list;
>>>  static struct start_info *virt_start_info;
>>>  static grub_xen_mfn_t console_pfn;
>>> +static grub_uint64_t *virt_pgtable;
>>> +static grub_uint64_t pgtbl_start;
>>> +static grub_uint64_t pgtbl_end;
>>
>> Same as in patches #1 and #2.
> 
> Yep.
> 
>>
>>>  #define PAGE_SIZE 4096
>>>  #define MAX_MODULES (PAGE_SIZE / sizeof (struct xen_multiboot_mod_list))
>>> @@ -106,17 +109,17 @@ get_pgtable_size (grub_uint64_t total_pages, grub_uint64_t virt_base)
>>>
>>>  static void
>>>  generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
>>> -		     grub_uint64_t total_pages, grub_uint64_t virt_base,
>>> -		     grub_xen_mfn_t *mfn_list)
>>> +		     grub_uint64_t paging_end, grub_uint64_t total_pages,
>>> +		     grub_uint64_t virt_base, grub_xen_mfn_t *mfn_list)
>>>  {
>>>    if (!virt_base)
>>> -    total_pages++;
>>> +    paging_end++;
>>>
>>>    grub_uint64_t lx[NUMBER_OF_LEVELS], lxs[NUMBER_OF_LEVELS];
>>>    grub_uint64_t nlx, nls, sz = 0;
>>>    int l;
>>>
>>> -  nlx = total_pages;
>>> +  nlx = paging_end;
>>>    nls = virt_base >> PAGE_SHIFT;
>>>    for (l = 0; l < NUMBER_OF_LEVELS; l++)
>>>      {
>>> @@ -160,7 +163,7 @@ generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
>>>    if (pr)
>>>      pg += POINTERS_PER_PAGE;
>>>
>>> -  for (j = 0; j < total_pages; j++)
>>> +  for (j = 0; j < paging_end; j++)
>>>      {
>>>        if (j >= paging_start && j < lp)
>>>  	pg[j + lxs[0]] = page2offset (mfn_list[j]) | 5;
>>> @@ -261,24 +264,12 @@ grub_xen_special_alloc (void)
>>>  }
>>>
>>>  static grub_err_t
>>> -grub_xen_boot (void)
>>> +grub_xen_pt_alloc (void)
>>>  {
>>>    grub_relocator_chunk_t ch;
>>>    grub_err_t err;
>>>    grub_uint64_t nr_info_pages;
>>>    grub_uint64_t nr_pages, nr_pt_pages, nr_need_pages;
>>> -  struct gnttab_set_version gnttab_setver;
>>> -  grub_size_t i;
>>> -
>>> -  if (grub_xen_n_allocated_shared_pages)
>>> -    return grub_error (GRUB_ERR_BUG, "active grants");
>>> -
>>> -  err = grub_xen_p2m_alloc ();
>>> -  if (err)
>>> -    return err;
>>> -  err = grub_xen_special_alloc ();
>>> -  if (err)
>>> -    return err;
>>>
>>>    next_start.pt_base = max_addr + xen_inf.virt_base;
>>>    state.paging_start = max_addr >> PAGE_SHIFT;
>>> @@ -298,30 +289,59 @@ grub_xen_boot (void)
>>>        nr_pages = nr_need_pages;
>>>      }
>>>
>>> -  grub_dprintf ("xen", "bootstrap domain %llx+%llx\n",
>>> -		(unsigned long long) xen_inf.virt_base,
>>> -		(unsigned long long) page2offset (nr_pages));
>>> -
>>>    err = grub_relocator_alloc_chunk_addr (relocator, &ch,
>>>  					 max_addr, page2offset (nr_pt_pages));
>>>    if (err)
>>>      return err;
>>>
>>> +  virt_pgtable = get_virtual_current_address (ch);
>>> +  pgtbl_start = max_addr >> PAGE_SHIFT;
>>> +  max_addr += page2offset (nr_pt_pages);
>>> +  state.stack = max_addr + STACK_SIZE + xen_inf.virt_base;
>>> +  state.paging_size = nr_pt_pages;
>>> +  next_start.nr_pt_frames = nr_pt_pages;
>>> +  max_addr = page2offset (nr_pages);
>>> +  pgtbl_end = nr_pages;
>>> +
>>> +  return GRUB_ERR_NONE;
>>> +}
>>> +
>>> +static grub_err_t
>>> +grub_xen_boot (void)
>>> +{
>>> +  grub_err_t err;
>>> +  grub_uint64_t nr_pages;
>>> +  struct gnttab_set_version gnttab_setver;
>>> +  grub_size_t i;
>>> +
>>> +  if (grub_xen_n_allocated_shared_pages)
>>> +    return grub_error (GRUB_ERR_BUG, "active grants");
>>> +
>>> +  err = grub_xen_p2m_alloc ();
>>> +  if (err)
>>> +    return err;
>>> +  err = grub_xen_special_alloc ();
>>> +  if (err)
>>> +    return err;
>>> +  err = grub_xen_pt_alloc ();
>>> +  if (err)
>>> +    return err;
>>
>> Should not we free memory allocated by grub_xen_p2m_alloc() and
>> grub_xen_special_alloc() if grub_xen_pt_alloc() failed?
> 
> Hmm, why? If I don't miss anything freeing memory in case of error isn't
> done anywhere (at least not in this source file).
> 
If we don't it's a bug and not an excuse to continue doing bad things
> Juergen
> 
> .
> 



[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 213 bytes --]

[PATCH v2 4/6] xen: add capability to load initrd outside of initial mapping

[Juergen Gross]

Modern pvops linux kernels support an initrd not covered by the initial
mapping. This capability is flagged by an elf-note.

In case the elf-note is set by the kernel don't place the initrd into
the initial mapping. This will allow to load larger initrds and/or
support domains with larger memory, as the initial mapping is limited
to 2GB and it is containing the p2m list.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
 grub-core/loader/i386/xen.c        | 56 ++++++++++++++++++++++++++++++--------
 grub-core/loader/i386/xen_fileXX.c |  3 ++
 include/grub/xen_file.h            |  1 +
 3 files changed, 49 insertions(+), 11 deletions(-)

diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
index 65cec27..0f41048 100644
--- a/grub-core/loader/i386/xen.c
+++ b/grub-core/loader/i386/xen.c
@@ -307,15 +307,14 @@ grub_xen_pt_alloc (void)
 }
 
 static grub_err_t
-grub_xen_boot (void)
+grub_xen_alloc_end (void)
 {
   grub_err_t err;
-  grub_uint64_t nr_pages;
-  struct gnttab_set_version gnttab_setver;
-  grub_size_t i;
+  static int called = 0;
 
-  if (grub_xen_n_allocated_shared_pages)
-    return grub_error (GRUB_ERR_BUG, "active grants");
+  if (called)
+    return GRUB_ERR_NONE;
+  called = 1;
 
   err = grub_xen_p2m_alloc ();
   if (err)
@@ -327,6 +326,24 @@ grub_xen_boot (void)
   if (err)
     return err;
 
+  return GRUB_ERR_NONE;
+}
+
+static grub_err_t
+grub_xen_boot (void)
+{
+  grub_err_t err;
+  grub_uint64_t nr_pages;
+  struct gnttab_set_version gnttab_setver;
+  grub_size_t i;
+
+  if (grub_xen_n_allocated_shared_pages)
+    return grub_error (GRUB_ERR_BUG, "active grants");
+
+  err = grub_xen_alloc_end ();
+  if (err)
+    return err;
+
   err = set_mfns (console_pfn);
   if (err)
     return err;
@@ -587,6 +604,13 @@ grub_cmd_initrd (grub_command_t cmd __attribute__ ((unused)),
       goto fail;
     }
 
+  if (xen_inf.unmapped_initrd)
+    {
+      err = grub_xen_alloc_end ();
+      if (err)
+        goto fail;
+    }
+
   if (grub_initrd_init (argc, argv, &initrd_ctx))
     goto fail;
 
@@ -603,13 +627,22 @@ grub_cmd_initrd (grub_command_t cmd __attribute__ ((unused)),
 	goto fail;
     }
 
-  next_start.mod_start = max_addr + xen_inf.virt_base;
-  next_start.mod_len = size;
-
-  max_addr = ALIGN_UP (max_addr + size, PAGE_SIZE);
+  if (xen_inf.unmapped_initrd)
+    {
+      next_start.flags |= SIF_MOD_START_PFN;
+      next_start.mod_start = max_addr >> PAGE_SHIFT;
+      next_start.mod_len = size;
+    }
+  else
+    {
+      next_start.mod_start = max_addr + xen_inf.virt_base;
+      next_start.mod_len = size;
+    }
 
   grub_dprintf ("xen", "Initrd, addr=0x%x, size=0x%x\n",
-		(unsigned) next_start.mod_start, (unsigned) size);
+		(unsigned) (max_addr + xen_inf.virt_base), (unsigned) size);
+
+  max_addr = ALIGN_UP (max_addr + size, PAGE_SIZE);
 
 fail:
   grub_initrd_close (&initrd_ctx);
@@ -660,6 +693,7 @@ grub_cmd_module (grub_command_t cmd __attribute__ ((unused)),
 
   if (!xen_module_info_page)
     {
+      xen_inf.unmapped_initrd = 0;
       n_modules = 0;
       max_addr = ALIGN_UP (max_addr, PAGE_SIZE);
       modules_target_start = max_addr;
diff --git a/grub-core/loader/i386/xen_fileXX.c b/grub-core/loader/i386/xen_fileXX.c
index 1ba5649..69fccd2 100644
--- a/grub-core/loader/i386/xen_fileXX.c
+++ b/grub-core/loader/i386/xen_fileXX.c
@@ -253,6 +253,9 @@ parse_note (grub_elf_t elf, struct grub_xen_file_info *xi,
 					  descsz == 2 ? 2 : 3) == 0)
 	    xi->arch = GRUB_XEN_FILE_I386;
 	  break;
+	case 16:
+	  xi->unmapped_initrd = !!grub_le_to_cpu32(*(grub_uint32_t *) desc);
+	  break;
 	default:
 	  grub_dprintf ("xen", "unknown note type %d\n", nh->n_type);
 	  break;
diff --git a/include/grub/xen_file.h b/include/grub/xen_file.h
index 4b2ccba..ed749fa 100644
--- a/include/grub/xen_file.h
+++ b/include/grub/xen_file.h
@@ -36,6 +36,7 @@ struct grub_xen_file_info
   int has_note;
   int has_xen_guest;
   int extended_cr3;
+  int unmapped_initrd;
   enum
   {
     GRUB_XEN_FILE_I386 = 1,
-- 
2.6.2

Re: [PATCH v2 4/6] xen: add capability to load initrd outside of initial mapping

[Daniel Kiper]

On Thu, Feb 11, 2016 at 08:53:24AM +0100, Juergen Gross wrote:
> Modern pvops linux kernels support an initrd not covered by the initial
> mapping. This capability is flagged by an elf-note.
>
> In case the elf-note is set by the kernel don't place the initrd into
> the initial mapping. This will allow to load larger initrds and/or
> support domains with larger memory, as the initial mapping is limited
> to 2GB and it is containing the p2m list.
>
> Signed-off-by: Juergen Gross <jgross@suse.com>
> ---
>  grub-core/loader/i386/xen.c        | 56 ++++++++++++++++++++++++++++++--------
>  grub-core/loader/i386/xen_fileXX.c |  3 ++
>  include/grub/xen_file.h            |  1 +
>  3 files changed, 49 insertions(+), 11 deletions(-)
>
> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
> index 65cec27..0f41048 100644
> --- a/grub-core/loader/i386/xen.c
> +++ b/grub-core/loader/i386/xen.c
> @@ -307,15 +307,14 @@ grub_xen_pt_alloc (void)
>  }
>
>  static grub_err_t
> -grub_xen_boot (void)
> +grub_xen_alloc_end (void)
>  {
>    grub_err_t err;
> -  grub_uint64_t nr_pages;
> -  struct gnttab_set_version gnttab_setver;
> -  grub_size_t i;
> +  static int called = 0;
>
> -  if (grub_xen_n_allocated_shared_pages)
> -    return grub_error (GRUB_ERR_BUG, "active grants");
> +  if (called)
> +    return GRUB_ERR_NONE;

Why?

> +  called = 1;
>
>    err = grub_xen_p2m_alloc ();
>    if (err)
> @@ -327,6 +326,24 @@ grub_xen_boot (void)
>    if (err)
>      return err;
>
> +  return GRUB_ERR_NONE;
> +}
> +
> +static grub_err_t
> +grub_xen_boot (void)
> +{
> +  grub_err_t err;
> +  grub_uint64_t nr_pages;
> +  struct gnttab_set_version gnttab_setver;
> +  grub_size_t i;
> +
> +  if (grub_xen_n_allocated_shared_pages)
> +    return grub_error (GRUB_ERR_BUG, "active grants");

Why?

> +  err = grub_xen_alloc_end ();
> +  if (err)
> +    return err;
> +
>    err = set_mfns (console_pfn);
>    if (err)
>      return err;
> @@ -587,6 +604,13 @@ grub_cmd_initrd (grub_command_t cmd __attribute__ ((unused)),
>        goto fail;
>      }
>
> +  if (xen_inf.unmapped_initrd)
> +    {
> +      err = grub_xen_alloc_end ();
> +      if (err)
> +        goto fail;
> +    }
> +
>    if (grub_initrd_init (argc, argv, &initrd_ctx))
>      goto fail;
>
> @@ -603,13 +627,22 @@ grub_cmd_initrd (grub_command_t cmd __attribute__ ((unused)),
>  	goto fail;
>      }
>
> -  next_start.mod_start = max_addr + xen_inf.virt_base;
> -  next_start.mod_len = size;
> -
> -  max_addr = ALIGN_UP (max_addr + size, PAGE_SIZE);
> +  if (xen_inf.unmapped_initrd)
> +    {
> +      next_start.flags |= SIF_MOD_START_PFN;
> +      next_start.mod_start = max_addr >> PAGE_SHIFT;
> +      next_start.mod_len = size;
> +    }
> +  else
> +    {
> +      next_start.mod_start = max_addr + xen_inf.virt_base;
> +      next_start.mod_len = size;
> +    }
>
>    grub_dprintf ("xen", "Initrd, addr=0x%x, size=0x%x\n",
> -		(unsigned) next_start.mod_start, (unsigned) size);
> +		(unsigned) (max_addr + xen_inf.virt_base), (unsigned) size);
> +
> +  max_addr = ALIGN_UP (max_addr + size, PAGE_SIZE);
>
>  fail:
>    grub_initrd_close (&initrd_ctx);
> @@ -660,6 +693,7 @@ grub_cmd_module (grub_command_t cmd __attribute__ ((unused)),
>
>    if (!xen_module_info_page)
>      {
> +      xen_inf.unmapped_initrd = 0;
>        n_modules = 0;
>        max_addr = ALIGN_UP (max_addr, PAGE_SIZE);
>        modules_target_start = max_addr;
> diff --git a/grub-core/loader/i386/xen_fileXX.c b/grub-core/loader/i386/xen_fileXX.c
> index 1ba5649..69fccd2 100644
> --- a/grub-core/loader/i386/xen_fileXX.c
> +++ b/grub-core/loader/i386/xen_fileXX.c
> @@ -253,6 +253,9 @@ parse_note (grub_elf_t elf, struct grub_xen_file_info *xi,
>  					  descsz == 2 ? 2 : 3) == 0)
>  	    xi->arch = GRUB_XEN_FILE_I386;
>  	  break;
> +	case 16:

Could you define this a constant and use it here?

Daniel

Re: [PATCH v2 4/6] xen: add capability to load initrd outside of initial mapping

[Juergen Gross]

On 11/02/16 13:33, Daniel Kiper wrote:
> On Thu, Feb 11, 2016 at 08:53:24AM +0100, Juergen Gross wrote:
>> Modern pvops linux kernels support an initrd not covered by the initial
>> mapping. This capability is flagged by an elf-note.
>>
>> In case the elf-note is set by the kernel don't place the initrd into
>> the initial mapping. This will allow to load larger initrds and/or
>> support domains with larger memory, as the initial mapping is limited
>> to 2GB and it is containing the p2m list.
>>
>> Signed-off-by: Juergen Gross <jgross@suse.com>
>> ---
>>  grub-core/loader/i386/xen.c        | 56 ++++++++++++++++++++++++++++++--------
>>  grub-core/loader/i386/xen_fileXX.c |  3 ++
>>  include/grub/xen_file.h            |  1 +
>>  3 files changed, 49 insertions(+), 11 deletions(-)
>>
>> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
>> index 65cec27..0f41048 100644
>> --- a/grub-core/loader/i386/xen.c
>> +++ b/grub-core/loader/i386/xen.c
>> @@ -307,15 +307,14 @@ grub_xen_pt_alloc (void)
>>  }
>>
>>  static grub_err_t
>> -grub_xen_boot (void)
>> +grub_xen_alloc_end (void)
>>  {
>>    grub_err_t err;
>> -  grub_uint64_t nr_pages;
>> -  struct gnttab_set_version gnttab_setver;
>> -  grub_size_t i;
>> +  static int called = 0;
>>
>> -  if (grub_xen_n_allocated_shared_pages)
>> -    return grub_error (GRUB_ERR_BUG, "active grants");
>> +  if (called)
>> +    return GRUB_ERR_NONE;
> 
> Why?

Did you look at the function? grub_xen_alloc_end() (some parts of the
original grub_xen_boot()) is new and may be called multiple times. It
was much easier to just call it and let it do what must be done only
at the first time called.

> 
>> +  called = 1;
>>
>>    err = grub_xen_p2m_alloc ();
>>    if (err)
>> @@ -327,6 +326,24 @@ grub_xen_boot (void)
>>    if (err)
>>      return err;
>>
>> +  return GRUB_ERR_NONE;
>> +}
>> +
>> +static grub_err_t
>> +grub_xen_boot (void)
>> +{
>> +  grub_err_t err;
>> +  grub_uint64_t nr_pages;
>> +  struct gnttab_set_version gnttab_setver;
>> +  grub_size_t i;
>> +
>> +  if (grub_xen_n_allocated_shared_pages)
>> +    return grub_error (GRUB_ERR_BUG, "active grants");
> 
> Why?

That's how it has been before. git just decided to generate the diff
that way.

> 
>> +  err = grub_xen_alloc_end ();
>> +  if (err)
>> +    return err;
>> +
>>    err = set_mfns (console_pfn);
>>    if (err)
>>      return err;
>> @@ -587,6 +604,13 @@ grub_cmd_initrd (grub_command_t cmd __attribute__ ((unused)),
>>        goto fail;
>>      }
>>
>> +  if (xen_inf.unmapped_initrd)
>> +    {
>> +      err = grub_xen_alloc_end ();
>> +      if (err)
>> +        goto fail;
>> +    }
>> +
>>    if (grub_initrd_init (argc, argv, &initrd_ctx))
>>      goto fail;
>>
>> @@ -603,13 +627,22 @@ grub_cmd_initrd (grub_command_t cmd __attribute__ ((unused)),
>>  	goto fail;
>>      }
>>
>> -  next_start.mod_start = max_addr + xen_inf.virt_base;
>> -  next_start.mod_len = size;
>> -
>> -  max_addr = ALIGN_UP (max_addr + size, PAGE_SIZE);
>> +  if (xen_inf.unmapped_initrd)
>> +    {
>> +      next_start.flags |= SIF_MOD_START_PFN;
>> +      next_start.mod_start = max_addr >> PAGE_SHIFT;
>> +      next_start.mod_len = size;
>> +    }
>> +  else
>> +    {
>> +      next_start.mod_start = max_addr + xen_inf.virt_base;
>> +      next_start.mod_len = size;
>> +    }
>>
>>    grub_dprintf ("xen", "Initrd, addr=0x%x, size=0x%x\n",
>> -		(unsigned) next_start.mod_start, (unsigned) size);
>> +		(unsigned) (max_addr + xen_inf.virt_base), (unsigned) size);
>> +
>> +  max_addr = ALIGN_UP (max_addr + size, PAGE_SIZE);
>>
>>  fail:
>>    grub_initrd_close (&initrd_ctx);
>> @@ -660,6 +693,7 @@ grub_cmd_module (grub_command_t cmd __attribute__ ((unused)),
>>
>>    if (!xen_module_info_page)
>>      {
>> +      xen_inf.unmapped_initrd = 0;
>>        n_modules = 0;
>>        max_addr = ALIGN_UP (max_addr, PAGE_SIZE);
>>        modules_target_start = max_addr;
>> diff --git a/grub-core/loader/i386/xen_fileXX.c b/grub-core/loader/i386/xen_fileXX.c
>> index 1ba5649..69fccd2 100644
>> --- a/grub-core/loader/i386/xen_fileXX.c
>> +++ b/grub-core/loader/i386/xen_fileXX.c
>> @@ -253,6 +253,9 @@ parse_note (grub_elf_t elf, struct grub_xen_file_info *xi,
>>  					  descsz == 2 ? 2 : 3) == 0)
>>  	    xi->arch = GRUB_XEN_FILE_I386;
>>  	  break;
>> +	case 16:
> 
> Could you define this a constant and use it here?

This would be the only case with a constant. All others are numeric
as well.


Juergen

Re: [PATCH v2 4/6] xen: add capability to load initrd outside of initial mapping

[Daniel Kiper]

On Thu, Feb 11, 2016 at 03:13:40PM +0100, Juergen Gross wrote:
> On 11/02/16 13:33, Daniel Kiper wrote:
> > On Thu, Feb 11, 2016 at 08:53:24AM +0100, Juergen Gross wrote:
> >> Modern pvops linux kernels support an initrd not covered by the initial
> >> mapping. This capability is flagged by an elf-note.
> >>
> >> In case the elf-note is set by the kernel don't place the initrd into
> >> the initial mapping. This will allow to load larger initrds and/or
> >> support domains with larger memory, as the initial mapping is limited
> >> to 2GB and it is containing the p2m list.
> >>
> >> Signed-off-by: Juergen Gross <jgross@suse.com>
> >> ---
> >>  grub-core/loader/i386/xen.c        | 56 ++++++++++++++++++++++++++++++--------
> >>  grub-core/loader/i386/xen_fileXX.c |  3 ++
> >>  include/grub/xen_file.h            |  1 +
> >>  3 files changed, 49 insertions(+), 11 deletions(-)
> >>
> >> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
> >> index 65cec27..0f41048 100644
> >> --- a/grub-core/loader/i386/xen.c
> >> +++ b/grub-core/loader/i386/xen.c
> >> @@ -307,15 +307,14 @@ grub_xen_pt_alloc (void)
> >>  }
> >>
> >>  static grub_err_t
> >> -grub_xen_boot (void)
> >> +grub_xen_alloc_end (void)
> >>  {
> >>    grub_err_t err;
> >> -  grub_uint64_t nr_pages;
> >> -  struct gnttab_set_version gnttab_setver;
> >> -  grub_size_t i;
> >> +  static int called = 0;
> >>
> >> -  if (grub_xen_n_allocated_shared_pages)
> >> -    return grub_error (GRUB_ERR_BUG, "active grants");
> >> +  if (called)
> >> +    return GRUB_ERR_NONE;
> >
> > Why?
>
> Did you look at the function? grub_xen_alloc_end() (some parts of the
> original grub_xen_boot()) is new and may be called multiple times. It
> was much easier to just call it and let it do what must be done only
> at the first time called.

OK, but this means that you can use this loader only once. If you make
a mistake in boot command line then you cannot fix it by calling this
loader again because grub_xen_alloc_end() will fail. Am I right?

> >> +  called = 1;
> >>
> >>    err = grub_xen_p2m_alloc ();
> >>    if (err)
> >> @@ -327,6 +326,24 @@ grub_xen_boot (void)
> >>    if (err)
> >>      return err;
> >>
> >> +  return GRUB_ERR_NONE;
> >> +}
> >> +
> >> +static grub_err_t
> >> +grub_xen_boot (void)
> >> +{
> >> +  grub_err_t err;
> >> +  grub_uint64_t nr_pages;
> >> +  struct gnttab_set_version gnttab_setver;
> >> +  grub_size_t i;
> >> +
> >> +  if (grub_xen_n_allocated_shared_pages)
> >> +    return grub_error (GRUB_ERR_BUG, "active grants");
> >
> > Why?
>
> That's how it has been before. git just decided to generate the diff
> that way.

Could you try --patience git option?

> >> +  err = grub_xen_alloc_end ();
> >> +  if (err)
> >> +    return err;
> >> +
> >>    err = set_mfns (console_pfn);
> >>    if (err)
> >>      return err;
> >> @@ -587,6 +604,13 @@ grub_cmd_initrd (grub_command_t cmd __attribute__ ((unused)),
> >>        goto fail;
> >>      }
> >>
> >> +  if (xen_inf.unmapped_initrd)
> >> +    {
> >> +      err = grub_xen_alloc_end ();
> >> +      if (err)
> >> +        goto fail;
> >> +    }
> >> +
> >>    if (grub_initrd_init (argc, argv, &initrd_ctx))
> >>      goto fail;
> >>
> >> @@ -603,13 +627,22 @@ grub_cmd_initrd (grub_command_t cmd __attribute__ ((unused)),
> >>  	goto fail;
> >>      }
> >>
> >> -  next_start.mod_start = max_addr + xen_inf.virt_base;
> >> -  next_start.mod_len = size;
> >> -
> >> -  max_addr = ALIGN_UP (max_addr + size, PAGE_SIZE);
> >> +  if (xen_inf.unmapped_initrd)
> >> +    {
> >> +      next_start.flags |= SIF_MOD_START_PFN;
> >> +      next_start.mod_start = max_addr >> PAGE_SHIFT;
> >> +      next_start.mod_len = size;
> >> +    }
> >> +  else
> >> +    {
> >> +      next_start.mod_start = max_addr + xen_inf.virt_base;
> >> +      next_start.mod_len = size;
> >> +    }
> >>
> >>    grub_dprintf ("xen", "Initrd, addr=0x%x, size=0x%x\n",
> >> -		(unsigned) next_start.mod_start, (unsigned) size);
> >> +		(unsigned) (max_addr + xen_inf.virt_base), (unsigned) size);
> >> +
> >> +  max_addr = ALIGN_UP (max_addr + size, PAGE_SIZE);
> >>
> >>  fail:
> >>    grub_initrd_close (&initrd_ctx);
> >> @@ -660,6 +693,7 @@ grub_cmd_module (grub_command_t cmd __attribute__ ((unused)),
> >>
> >>    if (!xen_module_info_page)
> >>      {
> >> +      xen_inf.unmapped_initrd = 0;
> >>        n_modules = 0;
> >>        max_addr = ALIGN_UP (max_addr, PAGE_SIZE);
> >>        modules_target_start = max_addr;
> >> diff --git a/grub-core/loader/i386/xen_fileXX.c b/grub-core/loader/i386/xen_fileXX.c
> >> index 1ba5649..69fccd2 100644
> >> --- a/grub-core/loader/i386/xen_fileXX.c
> >> +++ b/grub-core/loader/i386/xen_fileXX.c
> >> @@ -253,6 +253,9 @@ parse_note (grub_elf_t elf, struct grub_xen_file_info *xi,
> >>  					  descsz == 2 ? 2 : 3) == 0)
> >>  	    xi->arch = GRUB_XEN_FILE_I386;
> >>  	  break;
> >> +	case 16:
> >
> > Could you define this a constant and use it here?
>
> This would be the only case with a constant. All others are numeric
> as well.

Ugh... So, please, please fix it at first. It is not an excuse
if somebody before you made a mistake (let's call it in that way).
Or at least put comments what these values mean. However, I prefer
constants. Constants with comments (at least in header file) would
be perfect.

Daniel

Re: [PATCH v2 4/6] xen: add capability to load initrd outside of initial mapping

[Juergen Gross]

On 11/02/16 18:25, Daniel Kiper wrote:
> On Thu, Feb 11, 2016 at 03:13:40PM +0100, Juergen Gross wrote:
>> On 11/02/16 13:33, Daniel Kiper wrote:
>>> On Thu, Feb 11, 2016 at 08:53:24AM +0100, Juergen Gross wrote:
>>>> Modern pvops linux kernels support an initrd not covered by the initial
>>>> mapping. This capability is flagged by an elf-note.
>>>>
>>>> In case the elf-note is set by the kernel don't place the initrd into
>>>> the initial mapping. This will allow to load larger initrds and/or
>>>> support domains with larger memory, as the initial mapping is limited
>>>> to 2GB and it is containing the p2m list.
>>>>
>>>> Signed-off-by: Juergen Gross <jgross@suse.com>
>>>> ---
>>>>  grub-core/loader/i386/xen.c        | 56 ++++++++++++++++++++++++++++++--------
>>>>  grub-core/loader/i386/xen_fileXX.c |  3 ++
>>>>  include/grub/xen_file.h            |  1 +
>>>>  3 files changed, 49 insertions(+), 11 deletions(-)
>>>>
>>>> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
>>>> index 65cec27..0f41048 100644
>>>> --- a/grub-core/loader/i386/xen.c
>>>> +++ b/grub-core/loader/i386/xen.c
>>>> @@ -307,15 +307,14 @@ grub_xen_pt_alloc (void)
>>>>  }
>>>>
>>>>  static grub_err_t
>>>> -grub_xen_boot (void)
>>>> +grub_xen_alloc_end (void)
>>>>  {
>>>>    grub_err_t err;
>>>> -  grub_uint64_t nr_pages;
>>>> -  struct gnttab_set_version gnttab_setver;
>>>> -  grub_size_t i;
>>>> +  static int called = 0;
>>>>
>>>> -  if (grub_xen_n_allocated_shared_pages)
>>>> -    return grub_error (GRUB_ERR_BUG, "active grants");
>>>> +  if (called)
>>>> +    return GRUB_ERR_NONE;
>>>
>>> Why?
>>
>> Did you look at the function? grub_xen_alloc_end() (some parts of the
>> original grub_xen_boot()) is new and may be called multiple times. It
>> was much easier to just call it and let it do what must be done only
>> at the first time called.
> 
> OK, but this means that you can use this loader only once. If you make
> a mistake in boot command line then you cannot fix it by calling this
> loader again because grub_xen_alloc_end() will fail. Am I right?

Yes, you are. I'll need to introduce some kind of state reset function
which will re-enable grub_xen_alloc_end() and free all memory allocated.
This function should be called on module unload and before initializing
the relocator (this is how the other loaders are doing it).

> 
>>>> +  called = 1;
>>>>
>>>>    err = grub_xen_p2m_alloc ();
>>>>    if (err)
>>>> @@ -327,6 +326,24 @@ grub_xen_boot (void)
>>>>    if (err)
>>>>      return err;
>>>>
>>>> +  return GRUB_ERR_NONE;
>>>> +}
>>>> +
>>>> +static grub_err_t
>>>> +grub_xen_boot (void)
>>>> +{
>>>> +  grub_err_t err;
>>>> +  grub_uint64_t nr_pages;
>>>> +  struct gnttab_set_version gnttab_setver;
>>>> +  grub_size_t i;
>>>> +
>>>> +  if (grub_xen_n_allocated_shared_pages)
>>>> +    return grub_error (GRUB_ERR_BUG, "active grants");
>>>
>>> Why?
>>
>> That's how it has been before. git just decided to generate the diff
>> that way.
> 
> Could you try --patience git option?

I think I can. :-)

> 
>>>> +  err = grub_xen_alloc_end ();
>>>> +  if (err)
>>>> +    return err;
>>>> +
>>>>    err = set_mfns (console_pfn);
>>>>    if (err)
>>>>      return err;
>>>> @@ -587,6 +604,13 @@ grub_cmd_initrd (grub_command_t cmd __attribute__ ((unused)),
>>>>        goto fail;
>>>>      }
>>>>
>>>> +  if (xen_inf.unmapped_initrd)
>>>> +    {
>>>> +      err = grub_xen_alloc_end ();
>>>> +      if (err)
>>>> +        goto fail;
>>>> +    }
>>>> +
>>>>    if (grub_initrd_init (argc, argv, &initrd_ctx))
>>>>      goto fail;
>>>>
>>>> @@ -603,13 +627,22 @@ grub_cmd_initrd (grub_command_t cmd __attribute__ ((unused)),
>>>>  	goto fail;
>>>>      }
>>>>
>>>> -  next_start.mod_start = max_addr + xen_inf.virt_base;
>>>> -  next_start.mod_len = size;
>>>> -
>>>> -  max_addr = ALIGN_UP (max_addr + size, PAGE_SIZE);
>>>> +  if (xen_inf.unmapped_initrd)
>>>> +    {
>>>> +      next_start.flags |= SIF_MOD_START_PFN;
>>>> +      next_start.mod_start = max_addr >> PAGE_SHIFT;
>>>> +      next_start.mod_len = size;
>>>> +    }
>>>> +  else
>>>> +    {
>>>> +      next_start.mod_start = max_addr + xen_inf.virt_base;
>>>> +      next_start.mod_len = size;
>>>> +    }
>>>>
>>>>    grub_dprintf ("xen", "Initrd, addr=0x%x, size=0x%x\n",
>>>> -		(unsigned) next_start.mod_start, (unsigned) size);
>>>> +		(unsigned) (max_addr + xen_inf.virt_base), (unsigned) size);
>>>> +
>>>> +  max_addr = ALIGN_UP (max_addr + size, PAGE_SIZE);
>>>>
>>>>  fail:
>>>>    grub_initrd_close (&initrd_ctx);
>>>> @@ -660,6 +693,7 @@ grub_cmd_module (grub_command_t cmd __attribute__ ((unused)),
>>>>
>>>>    if (!xen_module_info_page)
>>>>      {
>>>> +      xen_inf.unmapped_initrd = 0;
>>>>        n_modules = 0;
>>>>        max_addr = ALIGN_UP (max_addr, PAGE_SIZE);
>>>>        modules_target_start = max_addr;
>>>> diff --git a/grub-core/loader/i386/xen_fileXX.c b/grub-core/loader/i386/xen_fileXX.c
>>>> index 1ba5649..69fccd2 100644
>>>> --- a/grub-core/loader/i386/xen_fileXX.c
>>>> +++ b/grub-core/loader/i386/xen_fileXX.c
>>>> @@ -253,6 +253,9 @@ parse_note (grub_elf_t elf, struct grub_xen_file_info *xi,
>>>>  					  descsz == 2 ? 2 : 3) == 0)
>>>>  	    xi->arch = GRUB_XEN_FILE_I386;
>>>>  	  break;
>>>> +	case 16:
>>>
>>> Could you define this a constant and use it here?
>>
>> This would be the only case with a constant. All others are numeric
>> as well.
> 
> Ugh... So, please, please fix it at first. It is not an excuse
> if somebody before you made a mistake (let's call it in that way).
> Or at least put comments what these values mean. However, I prefer
> constants. Constants with comments (at least in header file) would
> be perfect.

Okay, let me do some cleanup work on the xen loader:

- add the possibility to call it multiple times (state reset, free the
  allocated memory)
- merge all necessary global variables into one state structure, use
  local variables where possible
- introduce lots of constants instead of using numerical values all over
  the code

The first two items should go in before my series. The third is more a
question of style, so it might block the complete series if I put it
early in the series and such a change isn't regarded to be positive.
Would you be okay if I put the constant introduction at the end of the
series? This would result in readable code (at least I hope so) in case
it is accepted and wouldn't block the functionality in case it is
rejected. In case there is an early okay from the maintainer(s), I can
do the constant introduction early, too, of course.


Juergen

Re: [PATCH v2 4/6] xen: add capability to load initrd outside of initial mapping

[Daniel Kiper]

Hey Juergen,

On Fri, Feb 12, 2016 at 07:25:02AM +0100, Juergen Gross wrote:

[...]

> Okay, let me do some cleanup work on the xen loader:
>
> - add the possibility to call it multiple times (state reset, free the
>   allocated memory)
> - merge all necessary global variables into one state structure, use
>   local variables where possible
> - introduce lots of constants instead of using numerical values all over
>   the code

Make sense for me.

> The first two items should go in before my series. The third is more a
> question of style, so it might block the complete series if I put it
> early in the series and such a change isn't regarded to be positive.
> Would you be okay if I put the constant introduction at the end of the
> series? This would result in readable code (at least I hope so) in case

OK, let's do that in that way.

> it is accepted and wouldn't block the functionality in case it is
> rejected. In case there is an early okay from the maintainer(s), I can
> do the constant introduction early, too, of course.

Granted!

Daniel

Re: [PATCH v2 4/6] xen: add capability to load initrd outside of initial mapping

[Vladimir 'φ-coder/phcoder' Serbinenko]

[-- Attachment #1: Type: text/plain, Size: 2473 bytes --]

On 11.02.2016 15:13, Juergen Gross wrote:
> On 11/02/16 13:33, Daniel Kiper wrote:
>> On Thu, Feb 11, 2016 at 08:53:24AM +0100, Juergen Gross wrote:
>>> Modern pvops linux kernels support an initrd not covered by the initial
>>> mapping. This capability is flagged by an elf-note.
>>>
>>> In case the elf-note is set by the kernel don't place the initrd into
>>> the initial mapping. This will allow to load larger initrds and/or
>>> support domains with larger memory, as the initial mapping is limited
>>> to 2GB and it is containing the p2m list.
>>>
>>> Signed-off-by: Juergen Gross <jgross@suse.com>
>>> ---
>>>  grub-core/loader/i386/xen.c        | 56
++++++++++++++++++++++++++++++--------
>>>  grub-core/loader/i386/xen_fileXX.c |  3 ++
>>>  include/grub/xen_file.h            |  1 +
>>>  3 files changed, 49 insertions(+), 11 deletions(-)
>>>
>>> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
>>> index 65cec27..0f41048 100644
>>> --- a/grub-core/loader/i386/xen.c
>>> +++ b/grub-core/loader/i386/xen.c
>>> @@ -307,15 +307,14 @@ grub_xen_pt_alloc (void)
>>>  }
>>>
>>>  static grub_err_t
>>> -grub_xen_boot (void)
>>> +grub_xen_alloc_end (void)
>>>  {
>>>    grub_err_t err;
>>> -  grub_uint64_t nr_pages;
>>> -  struct gnttab_set_version gnttab_setver;
>>> -  grub_size_t i;
>>> +  static int called = 0;
>>>
>>> -  if (grub_xen_n_allocated_shared_pages)
>>> -    return grub_error (GRUB_ERR_BUG, "active grants");
>>> +  if (called)
>>> +    return GRUB_ERR_NONE;
>>
>> Why?
>
> Did you look at the function? grub_xen_alloc_end() (some parts of the
> original grub_xen_boot()) is new and may be called multiple times. It
> was much easier to just call it and let it do what must be done only
> at the first time called.
>
What if a boot fails and then fallback kernel is loaded?
>>> +  if (grub_xen_n_allocated_shared_pages)
>>> +    return grub_error (GRUB_ERR_BUG, "active grants");
>>
>> Why?
>
> That's how it has been before. git just decided to generate the diff
> that way.
>
This is also needed to avoid passing control when some drivers are still
active
>>> +	case 16:
>>
>> Could you define this a constant and use it here?
>
> This would be the only case with a constant. All others are numeric
> as well.
>
I'm ok with not insisisting on using constants given current state but
in general constants are preferable (yes, xen code isn't always clean)





[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 213 bytes --]

Re: [PATCH v2 4/6] xen: add capability to load initrd outside of initial mapping

[Juergen Gross]

On 12/02/16 13:24, Vladimir 'φ-coder/phcoder' Serbinenko wrote:
> On 11.02.2016 15:13, Juergen Gross wrote:
>> On 11/02/16 13:33, Daniel Kiper wrote:
>>> On Thu, Feb 11, 2016 at 08:53:24AM +0100, Juergen Gross wrote:
>>>> Modern pvops linux kernels support an initrd not covered by
>>>> the initial mapping. This capability is flagged by an
>>>> elf-note.
>>>> 
>>>> In case the elf-note is set by the kernel don't place the
>>>> initrd into the initial mapping. This will allow to load
>>>> larger initrds and/or support domains with larger memory, as
>>>> the initial mapping is limited to 2GB and it is containing
>>>> the p2m list.
>>>> 
>>>> Signed-off-by: Juergen Gross <jgross@suse.com> --- 
>>>> grub-core/loader/i386/xen.c        | 56
> ++++++++++++++++++++++++++++++--------
>>>> grub-core/loader/i386/xen_fileXX.c |  3 ++ 
>>>> include/grub/xen_file.h            |  1 + 3 files changed, 49
>>>> insertions(+), 11 deletions(-)
>>>> 
>>>> diff --git a/grub-core/loader/i386/xen.c
>>>> b/grub-core/loader/i386/xen.c index 65cec27..0f41048 100644 
>>>> --- a/grub-core/loader/i386/xen.c +++
>>>> b/grub-core/loader/i386/xen.c @@ -307,15 +307,14 @@
>>>> grub_xen_pt_alloc (void) }
>>>> 
>>>> static grub_err_t -grub_xen_boot (void) +grub_xen_alloc_end
>>>> (void) { grub_err_t err; -  grub_uint64_t nr_pages; -  struct
>>>> gnttab_set_version gnttab_setver; -  grub_size_t i; +  static
>>>> int called = 0;
>>>> 
>>>> -  if (grub_xen_n_allocated_shared_pages) -    return
>>>> grub_error (GRUB_ERR_BUG, "active grants"); +  if (called) +
>>>> return GRUB_ERR_NONE;
>>> 
>>> Why?
>> 
>> Did you look at the function? grub_xen_alloc_end() (some parts of
>> the original grub_xen_boot()) is new and may be called multiple
>> times. It was much easier to just call it and let it do what must
>> be done only at the first time called.
>> 
> What if a boot fails and then fallback kernel is loaded?

As already stated before: I'll add a patch handling freeing of memory
in a reset function which will reset the "called" flag as well.

>>>> +  if (grub_xen_n_allocated_shared_pages) +    return
>>>> grub_error (GRUB_ERR_BUG, "active grants");
>>> 
>>> Why?
>> 
>> That's how it has been before. git just decided to generate the
>> diff that way.
>> 
> This is also needed to avoid passing control when some drivers are
> still active
>>>> +	case 16:
>>> 
>>> Could you define this a constant and use it here?
>> 
>> This would be the only case with a constant. All others are
>> numeric as well.
>> 
> I'm ok with not insisisting on using constants given current state
> but in general constants are preferable (yes, xen code isn't always
> clean)

Okay, thanks for the confirmation. I'll add a patch (or better: some
patches) which will clean up the xen code by introducing (lots of)
constants.


Juergen

[PATCH v2 5/6] xen: modify page table construction

[Juergen Gross]

Modify the page table construction to allow multiple virtual regions
to be mapped. This is done as preparation for removing the p2m list
from the initial kernel mapping in order to support huge pv domains.

This allows a cleaner approach for mapping the relocator page by
using this capability.

The interface to the assembler level of the relocator has to be changed
in order to be able to process multiple page table areas.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
 grub-core/lib/i386/xen/relocator.S   |  47 +++---
 grub-core/lib/x86_64/xen/relocator.S |  44 +++--
 grub-core/lib/xen/relocator.c        |  22 ++-
 grub-core/loader/i386/xen.c          | 313 +++++++++++++++++++++++------------
 include/grub/xen/relocator.h         |   6 +-
 5 files changed, 277 insertions(+), 155 deletions(-)

diff --git a/grub-core/lib/i386/xen/relocator.S b/grub-core/lib/i386/xen/relocator.S
index 694a54c..c23b405 100644
--- a/grub-core/lib/i386/xen/relocator.S
+++ b/grub-core/lib/i386/xen/relocator.S
@@ -50,41 +50,45 @@ VARIABLE(grub_relocator_xen_remapper_map_high)
 	jmp *%ebx
 
 LOCAL(cont):
-	xorl	%eax, %eax
-	movl	%eax, %ebp
+	/* mov imm32, %eax */
+	.byte	0xb8
+VARIABLE(grub_relocator_xen_paging_areas_addr)
+	.long	0
+	movl	%eax, %ebx
 1:
-
+	movl	0(%ebx), %ebp
+	movl	4(%ebx), %ecx
+	testl	%ecx, %ecx
+	jz	3f
+	addl	$8, %ebx
+	movl	%ebx, %esp
+
+2:
+	movl	%ecx, %edi
 	/* mov imm32, %eax */
 	.byte	0xb8
 VARIABLE(grub_relocator_xen_mfn_list)
 	.long	0
-	movl	%eax, %edi
-	movl	%ebp, %eax
-	movl    0(%edi, %eax, 4), %ecx
-
-	/* mov imm32, %ebx */
-	.byte	0xbb
-VARIABLE(grub_relocator_xen_paging_start)
-	.long	0
-	shll	$12, %eax
-	addl	%eax, %ebx
+	movl    0(%eax, %ebp, 4), %ecx
+	movl	%ebp, %ebx
+	shll	$12, %ebx
 	movl    %ecx, %edx
 	shll    $12,  %ecx
 	shrl    $20,  %edx
 	orl     $5, %ecx
 	movl    $2, %esi
 	movl    $__HYPERVISOR_update_va_mapping, %eax
-	int     $0x82
+	int     $0x82	/* parameters: eax, ebx, ecx, edx, esi */
 
 	incl	%ebp
-	/* mov imm32, %ecx */
-	.byte	0xb9
-VARIABLE(grub_relocator_xen_paging_size)
-	.long	0
-	cmpl	%ebp, %ecx
+	movl	%edi, %ecx
+
+	loop	2b
 
-	ja	1b
+	mov	%esp, %ebx
+	jmp	1b
 
+3:
 	/* mov imm32, %ebx */
 	.byte	0xbb
 VARIABLE(grub_relocator_xen_mmu_op_addr)
@@ -102,6 +106,9 @@ VARIABLE(grub_relocator_xen_remap_continue)
 
 	jmp *%eax
 
+VARIABLE(grub_relocator_xen_paging_areas)
+	.long	0, 0, 0, 0, 0, 0, 0, 0
+
 VARIABLE(grub_relocator_xen_mmu_op)
 	.space 256
 
diff --git a/grub-core/lib/x86_64/xen/relocator.S b/grub-core/lib/x86_64/xen/relocator.S
index 92e9e72..dbb90c7 100644
--- a/grub-core/lib/x86_64/xen/relocator.S
+++ b/grub-core/lib/x86_64/xen/relocator.S
@@ -50,31 +50,24 @@ VARIABLE(grub_relocator_xen_remapper_map)
 
 LOCAL(cont):
 	
-	/* mov imm64, %rcx */
-	.byte 	0x48
-	.byte	0xb9
-VARIABLE(grub_relocator_xen_paging_size)
-	.quad	0
-
-	/* mov imm64, %rax */
-	.byte 	0x48
-	.byte	0xb8
-VARIABLE(grub_relocator_xen_paging_start)
-	.quad	0
-
-	movq	%rax, %r12
-
 	/* mov imm64, %rax */
 	.byte 	0x48
 	.byte	0xb8
 VARIABLE(grub_relocator_xen_mfn_list)
 	.quad	0
 
-	movq	%rax, %rsi
+	movq	%rax, %rbx
+	leaq	EXT_C(grub_relocator_xen_paging_areas) (%rip), %r8
+
 1:
+	movq	0(%r8), %r12
+	movq	8(%r8), %rcx
+	testq	%rcx, %rcx
+	jz	3f
+2:
 	movq	%r12, %rdi
-	movq    %rsi, %rbx
-	movq    0(%rsi), %rsi
+	shlq	$12, %rdi
+	movq    (%rbx, %r12, 8), %rsi
 	shlq    $12,  %rsi
 	orq     $5, %rsi
 	movq    $2, %rdx
@@ -83,13 +76,15 @@ VARIABLE(grub_relocator_xen_mfn_list)
 	syscall
 
 	movq    %r9, %rcx
-	addq    $8, %rbx
-	addq    $4096, %r12
-	movq    %rbx, %rsi
+	incq	%r12
+
+	loop 2b
 
-	loop 1b
+	addq	$16, %r8
+	jmp	1b
 
-	leaq   LOCAL(mmu_op) (%rip), %rdi
+3:
+	leaq   EXT_C(grub_relocator_xen_mmu_op) (%rip), %rdi
 	movq   $3, %rsi
 	movq   $0, %rdx
 	movq   $0x7FF0, %r10
@@ -104,7 +99,10 @@ VARIABLE(grub_relocator_xen_remap_continue)
 
 	jmp *%rax
 
-LOCAL(mmu_op):
+VARIABLE(grub_relocator_xen_paging_areas)
+	/* array of start, size pairs, size 0 is end marker */
+	.quad	0, 0, 0, 0, 0, 0, 0, 0
+
 VARIABLE(grub_relocator_xen_mmu_op)
 	.space 256
 
diff --git a/grub-core/lib/xen/relocator.c b/grub-core/lib/xen/relocator.c
index 8f427d3..bc29055 100644
--- a/grub-core/lib/xen/relocator.c
+++ b/grub-core/lib/xen/relocator.c
@@ -36,15 +36,15 @@ extern grub_uint8_t grub_relocator_xen_remap_end;
 extern grub_xen_reg_t grub_relocator_xen_stack;
 extern grub_xen_reg_t grub_relocator_xen_start_info;
 extern grub_xen_reg_t grub_relocator_xen_entry_point;
-extern grub_xen_reg_t grub_relocator_xen_paging_start;
-extern grub_xen_reg_t grub_relocator_xen_paging_size;
 extern grub_xen_reg_t grub_relocator_xen_remapper_virt;
 extern grub_xen_reg_t grub_relocator_xen_remapper_virt2;
 extern grub_xen_reg_t grub_relocator_xen_remapper_map;
 extern grub_xen_reg_t grub_relocator_xen_mfn_list;
+extern grub_xen_reg_t grub_relocator_xen_paging_areas[2 * XEN_MAX_MAPPINGS];
 extern grub_xen_reg_t grub_relocator_xen_remap_continue;
 #ifdef __i386__
 extern grub_xen_reg_t grub_relocator_xen_mmu_op_addr;
+extern grub_xen_reg_t grub_relocator_xen_paging_areas_addr;
 extern grub_xen_reg_t grub_relocator_xen_remapper_map_high;
 #endif
 extern mmuext_op_t grub_relocator_xen_mmu_op[3];
@@ -61,6 +61,7 @@ grub_relocator_xen_boot (struct grub_relocator *rel,
 {
   grub_err_t err;
   void *relst;
+  int i;
   grub_relocator_chunk_t ch, ch_tramp;
   grub_xen_mfn_t *mfn_list =
     (grub_xen_mfn_t *) grub_xen_start_page_addr->mfn_list;
@@ -77,8 +78,11 @@ grub_relocator_xen_boot (struct grub_relocator *rel,
   grub_relocator_xen_stack = state.stack;
   grub_relocator_xen_start_info = state.start_info;
   grub_relocator_xen_entry_point = state.entry_point;
-  grub_relocator_xen_paging_start = state.paging_start << 12;
-  grub_relocator_xen_paging_size = state.paging_size;
+  for (i = 0; i < XEN_MAX_MAPPINGS; i++)
+    {
+      grub_relocator_xen_paging_areas[2 * i] = state.paging_start[i];
+      grub_relocator_xen_paging_areas[2 * i + 1] = state.paging_size[i];
+    }
   grub_relocator_xen_remapper_virt = remapper_virt;
   grub_relocator_xen_remapper_virt2 = remapper_virt;
   grub_relocator_xen_remap_continue = trampoline_virt;
@@ -88,10 +92,12 @@ grub_relocator_xen_boot (struct grub_relocator *rel,
   grub_relocator_xen_remapper_map_high = (mfn_list[remapper_pfn] >> 20);
   grub_relocator_xen_mmu_op_addr = (char *) &grub_relocator_xen_mmu_op
     - (char *) &grub_relocator_xen_remap_start + remapper_virt;
+  grub_relocator_xen_paging_areas_addr =
+    (char *) &grub_relocator_xen_paging_areas
+    - (char *) &grub_relocator_xen_remap_start + remapper_virt;
 #endif
 
-  grub_relocator_xen_mfn_list = state.mfn_list
-    + state.paging_start * sizeof (grub_addr_t);
+  grub_relocator_xen_mfn_list = state.mfn_list;
 
   grub_memset (grub_relocator_xen_mmu_op, 0,
 	       sizeof (grub_relocator_xen_mmu_op));
@@ -100,9 +106,9 @@ grub_relocator_xen_boot (struct grub_relocator *rel,
 #else
   grub_relocator_xen_mmu_op[0].cmd = MMUEXT_PIN_L4_TABLE;
 #endif
-  grub_relocator_xen_mmu_op[0].arg1.mfn = mfn_list[state.paging_start];
+  grub_relocator_xen_mmu_op[0].arg1.mfn = mfn_list[state.paging_start[0]];
   grub_relocator_xen_mmu_op[1].cmd = MMUEXT_NEW_BASEPTR;
-  grub_relocator_xen_mmu_op[1].arg1.mfn = mfn_list[state.paging_start];
+  grub_relocator_xen_mmu_op[1].arg1.mfn = mfn_list[state.paging_start[0]];
   grub_relocator_xen_mmu_op[2].cmd = MMUEXT_UNPIN_TABLE;
   grub_relocator_xen_mmu_op[2].arg1.mfn =
     mfn_list[grub_xen_start_page_addr->pt_base >> 12];
diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
index 0f41048..5e10420 100644
--- a/grub-core/loader/i386/xen.c
+++ b/grub-core/loader/i386/xen.c
@@ -42,6 +42,29 @@
 
 GRUB_MOD_LICENSE ("GPLv3+");
 
+#ifdef __x86_64__
+#define NUMBER_OF_LEVELS 4
+#define INTERMEDIATE_OR 7
+#define VIRT_MASK 0x0000ffffffffffffULL
+#else
+#define NUMBER_OF_LEVELS 3
+#define INTERMEDIATE_OR 3
+#define VIRT_MASK 0x00000000ffffffffULL
+#endif
+
+struct grub_xen_mapping_lvl {
+  grub_uint64_t virt_start;
+  grub_uint64_t virt_end;
+  grub_uint64_t pfn_start;
+  grub_uint64_t n_pt_pages;
+};
+
+struct grub_xen_mapping {
+  grub_uint64_t *where;
+  struct grub_xen_mapping_lvl area;
+  struct grub_xen_mapping_lvl lvls[NUMBER_OF_LEVELS];
+};
+
 static struct grub_relocator *relocator = NULL;
 static grub_uint64_t max_addr;
 static grub_dl_t my_mod;
@@ -56,9 +79,10 @@ static struct grub_relocator_xen_state state;
 static grub_xen_mfn_t *virt_mfn_list;
 static struct start_info *virt_start_info;
 static grub_xen_mfn_t console_pfn;
-static grub_uint64_t *virt_pgtable;
-static grub_uint64_t pgtbl_start;
 static grub_uint64_t pgtbl_end;
+static struct grub_xen_mapping mappings[XEN_MAX_MAPPINGS];
+static int n_mappings;
+static struct grub_xen_mapping *map_reloc;
 
 #define PAGE_SIZE 4096
 #define MAX_MODULES (PAGE_SIZE / sizeof (struct xen_multiboot_mod_list))
@@ -75,100 +99,162 @@ page2offset (grub_uint64_t page)
   return page << PAGE_SHIFT;
 }
 
-#ifdef __x86_64__
-#define NUMBER_OF_LEVELS 4
-#define INTERMEDIATE_OR 7
-#else
-#define NUMBER_OF_LEVELS 3
-#define INTERMEDIATE_OR 3
-#endif
-
-static grub_uint64_t
-get_pgtable_size (grub_uint64_t total_pages, grub_uint64_t virt_base)
+static grub_err_t
+get_pgtable_size (grub_uint64_t from, grub_uint64_t to, grub_uint64_t pfn)
 {
-  if (!virt_base)
-    total_pages++;
-  grub_uint64_t ret = 0;
-  grub_uint64_t ll = total_pages;
-  int i;
-  for (i = 0; i < NUMBER_OF_LEVELS; i++)
-    {
-      ll = (ll + POINTERS_PER_PAGE - 1) >> LOG_POINTERS_PER_PAGE;
-      /* PAE wants all 4 root directories present.  */
-#ifdef __i386__
-      if (i == 1)
-	ll = 4;
-#endif
-      ret += ll;
-    }
-  for (i = 1; i < NUMBER_OF_LEVELS; i++)
-    if (virt_base >> (PAGE_SHIFT + i * LOG_POINTERS_PER_PAGE))
-      ret++;
-  return ret;
-}
+  struct grub_xen_mapping *map, *map_cmp;
+  grub_uint64_t mask, bits;
+  int i, m;
 
-static void
-generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
-		     grub_uint64_t paging_end, grub_uint64_t total_pages,
-		     grub_uint64_t virt_base, grub_xen_mfn_t *mfn_list)
-{
-  if (!virt_base)
-    paging_end++;
+  if (n_mappings == XEN_MAX_MAPPINGS)
+    return grub_error (GRUB_ERR_BUG, "too many mapped areas");
+
+  grub_dprintf ("xen", "get_pgtable_size %d from=%llx, to=%llx, pfn=%llx\n",
+		n_mappings, (unsigned long long) from, (unsigned long long) to,
+		(unsigned long long) pfn);
 
-  grub_uint64_t lx[NUMBER_OF_LEVELS], lxs[NUMBER_OF_LEVELS];
-  grub_uint64_t nlx, nls, sz = 0;
-  int l;
+  map = mappings + n_mappings;
+  grub_memset (map, 0, sizeof (*map));
 
-  nlx = paging_end;
-  nls = virt_base >> PAGE_SHIFT;
-  for (l = 0; l < NUMBER_OF_LEVELS; l++)
+  map->area.virt_start = from & VIRT_MASK;
+  map->area.virt_end = (to - 1) & VIRT_MASK;
+  map->area.n_pt_pages = 0;
+
+  for (i = NUMBER_OF_LEVELS - 1; i >= 0; i--)
     {
-      nlx = (nlx + POINTERS_PER_PAGE - 1) >> LOG_POINTERS_PER_PAGE;
-      /* PAE wants all 4 root directories present.  */
+      map->lvls[i].pfn_start = pfn + map->area.n_pt_pages;
+      if (i == NUMBER_OF_LEVELS - 1)
+	{
+	  if (n_mappings == 0)
+	    {
+	      map->lvls[i].virt_start = 0;
+	      map->lvls[i].virt_end = VIRT_MASK;
+	      map->lvls[i].n_pt_pages = 1;
+	      map->area.n_pt_pages++;
+	    }
+	  continue;
+	}
+
+      bits = PAGE_SHIFT + (i + 1) * LOG_POINTERS_PER_PAGE;
+      mask = (1ULL << bits) - 1;
+      map->lvls[i].virt_start = map->area.virt_start & ~mask;
+      map->lvls[i].virt_end = map->area.virt_end | mask;
 #ifdef __i386__
-      if (l == 1)
-	nlx = 4;
+      /* PAE wants last root directory present. */
+      if (i == 1 && to <= 0xc0000000 && n_mappings == 0)
+	map->lvls[i].virt_end = VIRT_MASK;
 #endif
-      lx[l] = nlx;
-      sz += lx[l];
-      lxs[l] = nls & (POINTERS_PER_PAGE - 1);
-      if (nls && l != 0)
-	sz++;
-      nls >>= LOG_POINTERS_PER_PAGE;
+      for (m = 0; m < n_mappings; m++)
+	{
+	  map_cmp = mappings + m;
+	  if (map_cmp->lvls[i].virt_start == map_cmp->lvls[i].virt_end)
+	    continue;
+	  if (map->lvls[i].virt_start >= map_cmp->lvls[i].virt_start &&
+	      map->lvls[i].virt_end <= map_cmp->lvls[i].virt_end)
+	   {
+	     map->lvls[i].virt_start = 0;
+	     map->lvls[i].virt_end = 0;
+	     break;
+	   }
+	   if (map->lvls[i].virt_start >= map_cmp->lvls[i].virt_start &&
+	       map->lvls[i].virt_start <= map_cmp->lvls[i].virt_end)
+	     map->lvls[i].virt_start = map_cmp->lvls[i].virt_end + 1;
+	   if (map->lvls[i].virt_end >= map_cmp->lvls[i].virt_start &&
+	       map->lvls[i].virt_end <= map_cmp->lvls[i].virt_end)
+	     map->lvls[i].virt_end = map_cmp->lvls[i].virt_start - 1;
+	}
+      if (map->lvls[i].virt_start < map->lvls[i].virt_end)
+	map->lvls[i].n_pt_pages =
+	  ((map->lvls[i].virt_end - map->lvls[i].virt_start) >> bits) + 1;
+      map->area.n_pt_pages += map->lvls[i].n_pt_pages;
+      grub_dprintf ("xen", "get_pgtable_size level %d: virt %llx-%llx %d pts\n",
+		    i, (unsigned long long)  map->lvls[i].virt_start,
+		    (unsigned long long)  map->lvls[i].virt_end,
+		    (int) map->lvls[i].n_pt_pages);
     }
 
-  grub_uint64_t lp;
-  grub_uint64_t j;
-  grub_uint64_t *pg = (grub_uint64_t *) where;
-  int pr = 0;
+  grub_dprintf ("xen", "get_pgtable_size return: %d page tables\n",
+		(int) map->area.n_pt_pages);
+
+  state.paging_start[n_mappings] = pfn;
+  state.paging_size[n_mappings] = map->area.n_pt_pages;
+
+  return GRUB_ERR_NONE;
+}
+
+static grub_uint64_t *
+get_pg_table_virt (int mapping, int level)
+{
+  grub_uint64_t pfn;
+  struct grub_xen_mapping *map;
+
+  map = mappings + mapping;
+  pfn = map->lvls[level].pfn_start - map->lvls[NUMBER_OF_LEVELS - 1].pfn_start;
+  return map->where + pfn * POINTERS_PER_PAGE;
+}
 
-  grub_memset (pg, 0, sz * PAGE_SIZE);
+static grub_uint64_t
+get_pg_table_prot (int level, grub_uint64_t pfn)
+{
+  int m;
+  grub_uint64_t pfn_s, pfn_e;
 
-  lp = paging_start + lx[NUMBER_OF_LEVELS - 1];
-  for (l = NUMBER_OF_LEVELS - 1; l >= 1; l--)
+  if (level > 0)
+    return INTERMEDIATE_OR;
+  for (m = 0; m < n_mappings; m++)
     {
-      if (lxs[l] || pr)
-	pg[0] = page2offset (mfn_list[lp++]) | INTERMEDIATE_OR;
-      if (pr)
-	pg += POINTERS_PER_PAGE;
-      for (j = 0; j < lx[l - 1]; j++)
-	pg[j + lxs[l]] = page2offset (mfn_list[lp++]) | INTERMEDIATE_OR;
-      pg += lx[l] * POINTERS_PER_PAGE;
-      if (lxs[l])
-	pr = 1;
+      pfn_s = mappings[m].lvls[NUMBER_OF_LEVELS - 1].pfn_start;
+      pfn_e = mappings[m].area.n_pt_pages + pfn_s;
+      if (pfn >= pfn_s && pfn < pfn_e)
+	return 5;
     }
+  return 7;
+}
+
+static void
+generate_page_table (grub_xen_mfn_t *mfn_list)
+{
+  int l, m1, m2;
+  long p, p_s, p_e;
+  grub_uint64_t start, end, pfn;
+  grub_uint64_t *pg;
+  struct grub_xen_mapping_lvl *lvl;
 
-  if (lxs[0] || pr)
-    pg[0] = page2offset (mfn_list[total_pages]) | 5;
-  if (pr)
-    pg += POINTERS_PER_PAGE;
+  for (m1 = 0; m1 < n_mappings; m1++)
+    grub_memset (mappings[m1].where, 0,
+		 mappings[m1].area.n_pt_pages * PAGE_SIZE);
 
-  for (j = 0; j < paging_end; j++)
+  for (l = NUMBER_OF_LEVELS - 1; l >= 0; l--)
     {
-      if (j >= paging_start && j < lp)
-	pg[j + lxs[0]] = page2offset (mfn_list[j]) | 5;
-      else
-	pg[j + lxs[0]] = page2offset (mfn_list[j]) | 7;
+      for (m1 = 0; m1 < n_mappings; m1++)
+	{
+	  start = mappings[m1].lvls[l].virt_start;
+	  end = mappings[m1].lvls[l].virt_end;
+	  pg = get_pg_table_virt(m1, l);
+	  for (m2 = 0; m2 < n_mappings; m2++)
+	    {
+	      lvl = (l > 0) ? mappings[m2].lvls + l - 1 : &mappings[m2].area;
+	      if (l > 0 && lvl->n_pt_pages == 0)
+		continue;
+	      if (lvl->virt_start >= end || lvl->virt_end <= start)
+		continue;
+	      p_s = (grub_max (start, lvl->virt_start) - start) >>
+		    (PAGE_SHIFT + l * LOG_POINTERS_PER_PAGE);
+	      p_e = (grub_min (end, lvl->virt_end) - start) >>
+		    (PAGE_SHIFT + l * LOG_POINTERS_PER_PAGE);
+	      pfn = ((grub_max (start, lvl->virt_start) - lvl->virt_start) >>
+		     (PAGE_SHIFT + l * LOG_POINTERS_PER_PAGE)) + lvl->pfn_start;
+	      grub_dprintf ("xen", "write page table entries level %d pg %p "
+			    "mapping %d/%d index %lx-%lx pfn %llx\n",
+			    l, pg, m1, m2, p_s, p_e, (unsigned long long) pfn);
+	      for (p = p_s; p <= p_e; p++)
+		{
+		  pg[p] = page2offset (mfn_list[pfn]) |
+			  get_pg_table_prot (l, pfn);
+		  pfn++;
+		}
+	    }
+	}
     }
 }
 
@@ -269,39 +355,62 @@ grub_xen_pt_alloc (void)
   grub_relocator_chunk_t ch;
   grub_err_t err;
   grub_uint64_t nr_info_pages;
-  grub_uint64_t nr_pages, nr_pt_pages, nr_need_pages;
+  grub_uint64_t nr_need_pages;
+  grub_uint64_t try_virt_end;
+  struct grub_xen_mapping *map;
 
-  next_start.pt_base = max_addr + xen_inf.virt_base;
-  state.paging_start = max_addr >> PAGE_SHIFT;
+  map = mappings + n_mappings;
+  map_reloc = map + 1;
 
+  next_start.pt_base = max_addr + xen_inf.virt_base;
   nr_info_pages = max_addr >> PAGE_SHIFT;
-  nr_pages = nr_info_pages;
+  nr_need_pages = nr_info_pages;
 
   while (1)
     {
-      nr_pages = ALIGN_UP (nr_pages, (ALIGN_SIZE >> PAGE_SHIFT));
-      nr_pt_pages = get_pgtable_size (nr_pages, xen_inf.virt_base);
-      nr_need_pages =
-	nr_info_pages + nr_pt_pages +
-	((ADDITIONAL_SIZE + STACK_SIZE) >> PAGE_SHIFT);
-      if (nr_pages >= nr_need_pages)
+      try_virt_end = ALIGN_UP (xen_inf.virt_base + page2offset (nr_need_pages) +
+			       ADDITIONAL_SIZE + STACK_SIZE, ALIGN_SIZE);
+      if (!xen_inf.virt_base)
+	try_virt_end += PAGE_SIZE;
+
+      err = get_pgtable_size (xen_inf.virt_base, try_virt_end, nr_info_pages);
+      if (err)
+	return err;
+      n_mappings++;
+
+      /* Map the relocator page either at virtual 0 or after end of area. */
+      nr_need_pages = nr_info_pages + map->area.n_pt_pages;
+      if (xen_inf.virt_base)
+	err = get_pgtable_size (0, PAGE_SIZE, nr_need_pages);
+      else
+	err = get_pgtable_size (try_virt_end - PAGE_SIZE, try_virt_end,
+				nr_need_pages);
+      if (err)
+	return err;
+      nr_need_pages += map_reloc->area.n_pt_pages;
+
+      if (xen_inf.virt_base + page2offset (nr_need_pages) <= try_virt_end)
 	break;
-      nr_pages = nr_need_pages;
+
+      n_mappings--;
     }
 
-  err = grub_relocator_alloc_chunk_addr (relocator, &ch,
-					 max_addr, page2offset (nr_pt_pages));
+  n_mappings++;
+  nr_need_pages = map->area.n_pt_pages + map_reloc->area.n_pt_pages;
+  err = grub_relocator_alloc_chunk_addr (relocator, &ch, max_addr,
+					 page2offset (nr_need_pages));
   if (err)
     return err;
 
-  virt_pgtable = get_virtual_current_address (ch);
-  pgtbl_start = max_addr >> PAGE_SHIFT;
-  max_addr += page2offset (nr_pt_pages);
+  map->where = get_virtual_current_address (ch);
+  map->area.pfn_start = 0;
+  max_addr += page2offset (nr_need_pages);
   state.stack = max_addr + STACK_SIZE + xen_inf.virt_base;
-  state.paging_size = nr_pt_pages;
-  next_start.nr_pt_frames = nr_pt_pages;
-  max_addr = page2offset (nr_pages);
-  pgtbl_end = nr_pages;
+  next_start.nr_pt_frames = nr_need_pages;
+  max_addr = try_virt_end - xen_inf.virt_base;
+  pgtbl_end = max_addr >> PAGE_SHIFT;
+  map_reloc->where = (grub_uint64_t *) ((char *) map->where +
+					page2offset (map->area.n_pt_pages));
 
   return GRUB_ERR_NONE;
 }
@@ -354,8 +463,8 @@ grub_xen_boot (void)
 		(unsigned long long) xen_inf.virt_base,
 		(unsigned long long) page2offset (nr_pages));
 
-  generate_page_table (virt_pgtable, pgtbl_start, pgtbl_end, nr_pages,
-		       xen_inf.virt_base, virt_mfn_list);
+  map_reloc->area.pfn_start = nr_pages;
+  generate_page_table (virt_mfn_list);
 
   state.entry_point = xen_inf.entry_point;
 
diff --git a/include/grub/xen/relocator.h b/include/grub/xen/relocator.h
index ae45dce..35a0ad9 100644
--- a/include/grub/xen/relocator.h
+++ b/include/grub/xen/relocator.h
@@ -23,11 +23,13 @@
 #include <grub/err.h>
 #include <grub/relocator.h>
 
+#define XEN_MAX_MAPPINGS 3
+
 struct grub_relocator_xen_state
 {
   grub_addr_t start_info;
-  grub_addr_t paging_start;
-  grub_addr_t paging_size;
+  grub_addr_t paging_start[XEN_MAX_MAPPINGS];
+  grub_addr_t paging_size[XEN_MAX_MAPPINGS];
   grub_addr_t mfn_list;
   grub_addr_t stack;
   grub_addr_t entry_point;
-- 
2.6.2

Re: [PATCH v2 5/6] xen: modify page table construction

[Daniel Kiper]

On Thu, Feb 11, 2016 at 08:53:25AM +0100, Juergen Gross wrote:
> Modify the page table construction to allow multiple virtual regions
> to be mapped. This is done as preparation for removing the p2m list
> from the initial kernel mapping in order to support huge pv domains.
>
> This allows a cleaner approach for mapping the relocator page by
> using this capability.
>
> The interface to the assembler level of the relocator has to be changed
> in order to be able to process multiple page table areas.

I hope that older kernels could be loaded as is and everything works as expected.

> Signed-off-by: Juergen Gross <jgross@suse.com>
> ---
>  grub-core/lib/i386/xen/relocator.S   |  47 +++---
>  grub-core/lib/x86_64/xen/relocator.S |  44 +++--
>  grub-core/lib/xen/relocator.c        |  22 ++-
>  grub-core/loader/i386/xen.c          | 313 +++++++++++++++++++++++------------
>  include/grub/xen/relocator.h         |   6 +-
>  5 files changed, 277 insertions(+), 155 deletions(-)
>
> diff --git a/grub-core/lib/i386/xen/relocator.S b/grub-core/lib/i386/xen/relocator.S
> index 694a54c..c23b405 100644
> --- a/grub-core/lib/i386/xen/relocator.S
> +++ b/grub-core/lib/i386/xen/relocator.S
> @@ -50,41 +50,45 @@ VARIABLE(grub_relocator_xen_remapper_map_high)
>  	jmp *%ebx
>
>  LOCAL(cont):
> -	xorl	%eax, %eax
> -	movl	%eax, %ebp
> +	/* mov imm32, %eax */
> +	.byte	0xb8
> +VARIABLE(grub_relocator_xen_paging_areas_addr)
> +	.long	0
> +	movl	%eax, %ebx
>  1:
> -
> +	movl	0(%ebx), %ebp
> +	movl	4(%ebx), %ecx

Oh... No, please use constants not plain numbers and
explain in comments what is going on. Otherwise it
is unreadable.

> +	testl	%ecx, %ecx
> +	jz	3f
> +	addl	$8, %ebx
> +	movl	%ebx, %esp
> +
> +2:
> +	movl	%ecx, %edi
>  	/* mov imm32, %eax */
>  	.byte	0xb8
>  VARIABLE(grub_relocator_xen_mfn_list)
>  	.long	0
> -	movl	%eax, %edi
> -	movl	%ebp, %eax
> -	movl    0(%edi, %eax, 4), %ecx
> -
> -	/* mov imm32, %ebx */
> -	.byte	0xbb
> -VARIABLE(grub_relocator_xen_paging_start)
> -	.long	0
> -	shll	$12, %eax
> -	addl	%eax, %ebx
> +	movl    0(%eax, %ebp, 4), %ecx
> +	movl	%ebp, %ebx
> +	shll	$12, %ebx

Ditto.

>  	movl    %ecx, %edx
>  	shll    $12,  %ecx
>  	shrl    $20,  %edx
>  	orl     $5, %ecx
>  	movl    $2, %esi
>  	movl    $__HYPERVISOR_update_va_mapping, %eax
> -	int     $0x82
> +	int     $0x82	/* parameters: eax, ebx, ecx, edx, esi */

Please use more verbose comments.

>
>  	incl	%ebp
> -	/* mov imm32, %ecx */
> -	.byte	0xb9
> -VARIABLE(grub_relocator_xen_paging_size)
> -	.long	0
> -	cmpl	%ebp, %ecx
> +	movl	%edi, %ecx
> +
> +	loop	2b
>
> -	ja	1b
> +	mov	%esp, %ebx
> +	jmp	1b
>
> +3:
>  	/* mov imm32, %ebx */
>  	.byte	0xbb
>  VARIABLE(grub_relocator_xen_mmu_op_addr)
> @@ -102,6 +106,9 @@ VARIABLE(grub_relocator_xen_remap_continue)
>
>  	jmp *%eax
>
> +VARIABLE(grub_relocator_xen_paging_areas)
> +	.long	0, 0, 0, 0, 0, 0, 0, 0
> +
>  VARIABLE(grub_relocator_xen_mmu_op)
>  	.space 256
>
> diff --git a/grub-core/lib/x86_64/xen/relocator.S b/grub-core/lib/x86_64/xen/relocator.S
> index 92e9e72..dbb90c7 100644
> --- a/grub-core/lib/x86_64/xen/relocator.S
> +++ b/grub-core/lib/x86_64/xen/relocator.S

Is to possible to split this patch to i386 and x86-64 stuff patches?

> @@ -50,31 +50,24 @@ VARIABLE(grub_relocator_xen_remapper_map)
>
>  LOCAL(cont):
>
> -	/* mov imm64, %rcx */
> -	.byte 	0x48
> -	.byte	0xb9
> -VARIABLE(grub_relocator_xen_paging_size)
> -	.quad	0
> -
> -	/* mov imm64, %rax */
> -	.byte 	0x48
> -	.byte	0xb8
> -VARIABLE(grub_relocator_xen_paging_start)
> -	.quad	0
> -
> -	movq	%rax, %r12
> -
>  	/* mov imm64, %rax */
>  	.byte 	0x48
>  	.byte	0xb8
>  VARIABLE(grub_relocator_xen_mfn_list)
>  	.quad	0
>
> -	movq	%rax, %rsi
> +	movq	%rax, %rbx
> +	leaq	EXT_C(grub_relocator_xen_paging_areas) (%rip), %r8
> +
>  1:
> +	movq	0(%r8), %r12
> +	movq	8(%r8), %rcx

Ditto.

> +	testq	%rcx, %rcx
> +	jz	3f
> +2:
>  	movq	%r12, %rdi
> -	movq    %rsi, %rbx
> -	movq    0(%rsi), %rsi
> +	shlq	$12, %rdi
> +	movq    (%rbx, %r12, 8), %rsi

Ditto.

>  	shlq    $12,  %rsi
>  	orq     $5, %rsi
>  	movq    $2, %rdx
> @@ -83,13 +76,15 @@ VARIABLE(grub_relocator_xen_mfn_list)
>  	syscall
>
>  	movq    %r9, %rcx
> -	addq    $8, %rbx
> -	addq    $4096, %r12
> -	movq    %rbx, %rsi
> +	incq	%r12
> +
> +	loop 2b
>
> -	loop 1b
> +	addq	$16, %r8

Ditto.

> +	jmp	1b
>
> -	leaq   LOCAL(mmu_op) (%rip), %rdi
> +3:
> +	leaq   EXT_C(grub_relocator_xen_mmu_op) (%rip), %rdi
>  	movq   $3, %rsi
>  	movq   $0, %rdx
>  	movq   $0x7FF0, %r10

Ugh... Please fix this too. Probably in separate patch.

> @@ -104,7 +99,10 @@ VARIABLE(grub_relocator_xen_remap_continue)
>
>  	jmp *%rax
>
> -LOCAL(mmu_op):
> +VARIABLE(grub_relocator_xen_paging_areas)
> +	/* array of start, size pairs, size 0 is end marker */
> +	.quad	0, 0, 0, 0, 0, 0, 0, 0
> +
>  VARIABLE(grub_relocator_xen_mmu_op)
>  	.space 256
>
> diff --git a/grub-core/lib/xen/relocator.c b/grub-core/lib/xen/relocator.c
> index 8f427d3..bc29055 100644
> --- a/grub-core/lib/xen/relocator.c
> +++ b/grub-core/lib/xen/relocator.c
> @@ -36,15 +36,15 @@ extern grub_uint8_t grub_relocator_xen_remap_end;
>  extern grub_xen_reg_t grub_relocator_xen_stack;
>  extern grub_xen_reg_t grub_relocator_xen_start_info;
>  extern grub_xen_reg_t grub_relocator_xen_entry_point;
> -extern grub_xen_reg_t grub_relocator_xen_paging_start;
> -extern grub_xen_reg_t grub_relocator_xen_paging_size;
>  extern grub_xen_reg_t grub_relocator_xen_remapper_virt;
>  extern grub_xen_reg_t grub_relocator_xen_remapper_virt2;
>  extern grub_xen_reg_t grub_relocator_xen_remapper_map;
>  extern grub_xen_reg_t grub_relocator_xen_mfn_list;
> +extern grub_xen_reg_t grub_relocator_xen_paging_areas[2 * XEN_MAX_MAPPINGS];
>  extern grub_xen_reg_t grub_relocator_xen_remap_continue;
>  #ifdef __i386__
>  extern grub_xen_reg_t grub_relocator_xen_mmu_op_addr;
> +extern grub_xen_reg_t grub_relocator_xen_paging_areas_addr;
>  extern grub_xen_reg_t grub_relocator_xen_remapper_map_high;
>  #endif
>  extern mmuext_op_t grub_relocator_xen_mmu_op[3];
> @@ -61,6 +61,7 @@ grub_relocator_xen_boot (struct grub_relocator *rel,
>  {
>    grub_err_t err;
>    void *relst;
> +  int i;
>    grub_relocator_chunk_t ch, ch_tramp;
>    grub_xen_mfn_t *mfn_list =
>      (grub_xen_mfn_t *) grub_xen_start_page_addr->mfn_list;
> @@ -77,8 +78,11 @@ grub_relocator_xen_boot (struct grub_relocator *rel,
>    grub_relocator_xen_stack = state.stack;
>    grub_relocator_xen_start_info = state.start_info;
>    grub_relocator_xen_entry_point = state.entry_point;
> -  grub_relocator_xen_paging_start = state.paging_start << 12;
> -  grub_relocator_xen_paging_size = state.paging_size;
> +  for (i = 0; i < XEN_MAX_MAPPINGS; i++)
> +    {
> +      grub_relocator_xen_paging_areas[2 * i] = state.paging_start[i];
> +      grub_relocator_xen_paging_areas[2 * i + 1] = state.paging_size[i];

Why 2 and 1? Constants?

> +    }
>    grub_relocator_xen_remapper_virt = remapper_virt;
>    grub_relocator_xen_remapper_virt2 = remapper_virt;
>    grub_relocator_xen_remap_continue = trampoline_virt;
> @@ -88,10 +92,12 @@ grub_relocator_xen_boot (struct grub_relocator *rel,
>    grub_relocator_xen_remapper_map_high = (mfn_list[remapper_pfn] >> 20);
>    grub_relocator_xen_mmu_op_addr = (char *) &grub_relocator_xen_mmu_op
>      - (char *) &grub_relocator_xen_remap_start + remapper_virt;
> +  grub_relocator_xen_paging_areas_addr =
> +    (char *) &grub_relocator_xen_paging_areas
> +    - (char *) &grub_relocator_xen_remap_start + remapper_virt;
>  #endif
>
> -  grub_relocator_xen_mfn_list = state.mfn_list
> -    + state.paging_start * sizeof (grub_addr_t);
> +  grub_relocator_xen_mfn_list = state.mfn_list;
>
>    grub_memset (grub_relocator_xen_mmu_op, 0,
>  	       sizeof (grub_relocator_xen_mmu_op));
> @@ -100,9 +106,9 @@ grub_relocator_xen_boot (struct grub_relocator *rel,
>  #else
>    grub_relocator_xen_mmu_op[0].cmd = MMUEXT_PIN_L4_TABLE;
>  #endif
> -  grub_relocator_xen_mmu_op[0].arg1.mfn = mfn_list[state.paging_start];
> +  grub_relocator_xen_mmu_op[0].arg1.mfn = mfn_list[state.paging_start[0]];
>    grub_relocator_xen_mmu_op[1].cmd = MMUEXT_NEW_BASEPTR;
> -  grub_relocator_xen_mmu_op[1].arg1.mfn = mfn_list[state.paging_start];
> +  grub_relocator_xen_mmu_op[1].arg1.mfn = mfn_list[state.paging_start[0]];
>    grub_relocator_xen_mmu_op[2].cmd = MMUEXT_UNPIN_TABLE;
>    grub_relocator_xen_mmu_op[2].arg1.mfn =
>      mfn_list[grub_xen_start_page_addr->pt_base >> 12];
> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
> index 0f41048..5e10420 100644
> --- a/grub-core/loader/i386/xen.c
> +++ b/grub-core/loader/i386/xen.c
> @@ -42,6 +42,29 @@
>
>  GRUB_MOD_LICENSE ("GPLv3+");
>
> +#ifdef __x86_64__
> +#define NUMBER_OF_LEVELS 4
> +#define INTERMEDIATE_OR 7
> +#define VIRT_MASK 0x0000ffffffffffffULL
> +#else
> +#define NUMBER_OF_LEVELS 3
> +#define INTERMEDIATE_OR 3
> +#define VIRT_MASK 0x00000000ffffffffULL

Long expected constants... Nice!

> +#endif
> +
> +struct grub_xen_mapping_lvl {
> +  grub_uint64_t virt_start;
> +  grub_uint64_t virt_end;
> +  grub_uint64_t pfn_start;
> +  grub_uint64_t n_pt_pages;
> +};
> +
> +struct grub_xen_mapping {
> +  grub_uint64_t *where;
> +  struct grub_xen_mapping_lvl area;
> +  struct grub_xen_mapping_lvl lvls[NUMBER_OF_LEVELS];
> +};
> +
>  static struct grub_relocator *relocator = NULL;
>  static grub_uint64_t max_addr;
>  static grub_dl_t my_mod;
> @@ -56,9 +79,10 @@ static struct grub_relocator_xen_state state;
>  static grub_xen_mfn_t *virt_mfn_list;
>  static struct start_info *virt_start_info;
>  static grub_xen_mfn_t console_pfn;
> -static grub_uint64_t *virt_pgtable;
> -static grub_uint64_t pgtbl_start;
>  static grub_uint64_t pgtbl_end;
> +static struct grub_xen_mapping mappings[XEN_MAX_MAPPINGS];
> +static int n_mappings;
> +static struct grub_xen_mapping *map_reloc;
>
>  #define PAGE_SIZE 4096
>  #define MAX_MODULES (PAGE_SIZE / sizeof (struct xen_multiboot_mod_list))
> @@ -75,100 +99,162 @@ page2offset (grub_uint64_t page)
>    return page << PAGE_SHIFT;
>  }
>
> -#ifdef __x86_64__
> -#define NUMBER_OF_LEVELS 4
> -#define INTERMEDIATE_OR 7
> -#else
> -#define NUMBER_OF_LEVELS 3
> -#define INTERMEDIATE_OR 3
> -#endif
> -
> -static grub_uint64_t
> -get_pgtable_size (grub_uint64_t total_pages, grub_uint64_t virt_base)
> +static grub_err_t
> +get_pgtable_size (grub_uint64_t from, grub_uint64_t to, grub_uint64_t pfn)
>  {
> -  if (!virt_base)
> -    total_pages++;
> -  grub_uint64_t ret = 0;
> -  grub_uint64_t ll = total_pages;
> -  int i;
> -  for (i = 0; i < NUMBER_OF_LEVELS; i++)
> -    {
> -      ll = (ll + POINTERS_PER_PAGE - 1) >> LOG_POINTERS_PER_PAGE;
> -      /* PAE wants all 4 root directories present.  */
> -#ifdef __i386__
> -      if (i == 1)
> -	ll = 4;
> -#endif
> -      ret += ll;
> -    }
> -  for (i = 1; i < NUMBER_OF_LEVELS; i++)
> -    if (virt_base >> (PAGE_SHIFT + i * LOG_POINTERS_PER_PAGE))
> -      ret++;
> -  return ret;
> -}
> +  struct grub_xen_mapping *map, *map_cmp;
> +  grub_uint64_t mask, bits;
> +  int i, m;
>
> -static void
> -generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
> -		     grub_uint64_t paging_end, grub_uint64_t total_pages,
> -		     grub_uint64_t virt_base, grub_xen_mfn_t *mfn_list)
> -{
> -  if (!virt_base)
> -    paging_end++;
> +  if (n_mappings == XEN_MAX_MAPPINGS)
> +    return grub_error (GRUB_ERR_BUG, "too many mapped areas");
> +
> +  grub_dprintf ("xen", "get_pgtable_size %d from=%llx, to=%llx, pfn=%llx\n",
> +		n_mappings, (unsigned long long) from, (unsigned long long) to,
> +		(unsigned long long) pfn);
>
> -  grub_uint64_t lx[NUMBER_OF_LEVELS], lxs[NUMBER_OF_LEVELS];
> -  grub_uint64_t nlx, nls, sz = 0;
> -  int l;
> +  map = mappings + n_mappings;
> +  grub_memset (map, 0, sizeof (*map));
>
> -  nlx = paging_end;
> -  nls = virt_base >> PAGE_SHIFT;
> -  for (l = 0; l < NUMBER_OF_LEVELS; l++)
> +  map->area.virt_start = from & VIRT_MASK;
> +  map->area.virt_end = (to - 1) & VIRT_MASK;
> +  map->area.n_pt_pages = 0;
> +
> +  for (i = NUMBER_OF_LEVELS - 1; i >= 0; i--)
>      {
> -      nlx = (nlx + POINTERS_PER_PAGE - 1) >> LOG_POINTERS_PER_PAGE;
> -      /* PAE wants all 4 root directories present.  */
> +      map->lvls[i].pfn_start = pfn + map->area.n_pt_pages;
> +      if (i == NUMBER_OF_LEVELS - 1)
> +	{
> +	  if (n_mappings == 0)
> +	    {
> +	      map->lvls[i].virt_start = 0;
> +	      map->lvls[i].virt_end = VIRT_MASK;
> +	      map->lvls[i].n_pt_pages = 1;
> +	      map->area.n_pt_pages++;
> +	    }
> +	  continue;
> +	}
> +
> +      bits = PAGE_SHIFT + (i + 1) * LOG_POINTERS_PER_PAGE;
> +      mask = (1ULL << bits) - 1;
> +      map->lvls[i].virt_start = map->area.virt_start & ~mask;
> +      map->lvls[i].virt_end = map->area.virt_end | mask;
>  #ifdef __i386__
> -      if (l == 1)
> -	nlx = 4;
> +      /* PAE wants last root directory present. */
> +      if (i == 1 && to <= 0xc0000000 && n_mappings == 0)

Ditto.

> +	map->lvls[i].virt_end = VIRT_MASK;
>  #endif
> -      lx[l] = nlx;
> -      sz += lx[l];
> -      lxs[l] = nls & (POINTERS_PER_PAGE - 1);
> -      if (nls && l != 0)
> -	sz++;
> -      nls >>= LOG_POINTERS_PER_PAGE;
> +      for (m = 0; m < n_mappings; m++)
> +	{
> +	  map_cmp = mappings + m;
> +	  if (map_cmp->lvls[i].virt_start == map_cmp->lvls[i].virt_end)
> +	    continue;
> +	  if (map->lvls[i].virt_start >= map_cmp->lvls[i].virt_start &&
> +	      map->lvls[i].virt_end <= map_cmp->lvls[i].virt_end)
> +	   {
> +	     map->lvls[i].virt_start = 0;
> +	     map->lvls[i].virt_end = 0;
> +	     break;
> +	   }
> +	   if (map->lvls[i].virt_start >= map_cmp->lvls[i].virt_start &&
> +	       map->lvls[i].virt_start <= map_cmp->lvls[i].virt_end)
> +	     map->lvls[i].virt_start = map_cmp->lvls[i].virt_end + 1;
> +	   if (map->lvls[i].virt_end >= map_cmp->lvls[i].virt_start &&
> +	       map->lvls[i].virt_end <= map_cmp->lvls[i].virt_end)
> +	     map->lvls[i].virt_end = map_cmp->lvls[i].virt_start - 1;
> +	}
> +      if (map->lvls[i].virt_start < map->lvls[i].virt_end)
> +	map->lvls[i].n_pt_pages =
> +	  ((map->lvls[i].virt_end - map->lvls[i].virt_start) >> bits) + 1;
> +      map->area.n_pt_pages += map->lvls[i].n_pt_pages;
> +      grub_dprintf ("xen", "get_pgtable_size level %d: virt %llx-%llx %d pts\n",
> +		    i, (unsigned long long)  map->lvls[i].virt_start,
> +		    (unsigned long long)  map->lvls[i].virt_end,
> +		    (int) map->lvls[i].n_pt_pages);
>      }
>
> -  grub_uint64_t lp;
> -  grub_uint64_t j;
> -  grub_uint64_t *pg = (grub_uint64_t *) where;
> -  int pr = 0;
> +  grub_dprintf ("xen", "get_pgtable_size return: %d page tables\n",
> +		(int) map->area.n_pt_pages);
> +
> +  state.paging_start[n_mappings] = pfn;
> +  state.paging_size[n_mappings] = map->area.n_pt_pages;
> +
> +  return GRUB_ERR_NONE;
> +}
> +
> +static grub_uint64_t *
> +get_pg_table_virt (int mapping, int level)
> +{
> +  grub_uint64_t pfn;
> +  struct grub_xen_mapping *map;
> +
> +  map = mappings + mapping;
> +  pfn = map->lvls[level].pfn_start - map->lvls[NUMBER_OF_LEVELS - 1].pfn_start;
> +  return map->where + pfn * POINTERS_PER_PAGE;
> +}
>
> -  grub_memset (pg, 0, sz * PAGE_SIZE);
> +static grub_uint64_t
> +get_pg_table_prot (int level, grub_uint64_t pfn)
> +{
> +  int m;
> +  grub_uint64_t pfn_s, pfn_e;
>
> -  lp = paging_start + lx[NUMBER_OF_LEVELS - 1];
> -  for (l = NUMBER_OF_LEVELS - 1; l >= 1; l--)
> +  if (level > 0)
> +    return INTERMEDIATE_OR;
> +  for (m = 0; m < n_mappings; m++)
>      {
> -      if (lxs[l] || pr)
> -	pg[0] = page2offset (mfn_list[lp++]) | INTERMEDIATE_OR;
> -      if (pr)
> -	pg += POINTERS_PER_PAGE;
> -      for (j = 0; j < lx[l - 1]; j++)
> -	pg[j + lxs[l]] = page2offset (mfn_list[lp++]) | INTERMEDIATE_OR;
> -      pg += lx[l] * POINTERS_PER_PAGE;
> -      if (lxs[l])
> -	pr = 1;
> +      pfn_s = mappings[m].lvls[NUMBER_OF_LEVELS - 1].pfn_start;
> +      pfn_e = mappings[m].area.n_pt_pages + pfn_s;
> +      if (pfn >= pfn_s && pfn < pfn_e)
> +	return 5;
>      }
> +  return 7;

What 7 and 5 means?

I am exhausted. Please fix above stuff and I will review this patch
again after fixes.

Daniel

Re: [PATCH v2 5/6] xen: modify page table construction

[Juergen Gross]

On 11/02/16 13:47, Daniel Kiper wrote:
> On Thu, Feb 11, 2016 at 08:53:25AM +0100, Juergen Gross wrote:
>> Modify the page table construction to allow multiple virtual regions
>> to be mapped. This is done as preparation for removing the p2m list
>> from the initial kernel mapping in order to support huge pv domains.
>>
>> This allows a cleaner approach for mapping the relocator page by
>> using this capability.
>>
>> The interface to the assembler level of the relocator has to be changed
>> in order to be able to process multiple page table areas.
> 
> I hope that older kernels could be loaded as is and everything works as expected.

See Patch 0/6. I have tested old kernels, too.

> 
>> Signed-off-by: Juergen Gross <jgross@suse.com>
>> ---
>>  grub-core/lib/i386/xen/relocator.S   |  47 +++---
>>  grub-core/lib/x86_64/xen/relocator.S |  44 +++--
>>  grub-core/lib/xen/relocator.c        |  22 ++-
>>  grub-core/loader/i386/xen.c          | 313 +++++++++++++++++++++++------------
>>  include/grub/xen/relocator.h         |   6 +-
>>  5 files changed, 277 insertions(+), 155 deletions(-)
>>
>> diff --git a/grub-core/lib/i386/xen/relocator.S b/grub-core/lib/i386/xen/relocator.S
>> index 694a54c..c23b405 100644
>> --- a/grub-core/lib/i386/xen/relocator.S
>> +++ b/grub-core/lib/i386/xen/relocator.S
>> @@ -50,41 +50,45 @@ VARIABLE(grub_relocator_xen_remapper_map_high)
>>  	jmp *%ebx
>>
>>  LOCAL(cont):
>> -	xorl	%eax, %eax
>> -	movl	%eax, %ebp
>> +	/* mov imm32, %eax */
>> +	.byte	0xb8
>> +VARIABLE(grub_relocator_xen_paging_areas_addr)
>> +	.long	0
>> +	movl	%eax, %ebx
>>  1:
>> -
>> +	movl	0(%ebx), %ebp
>> +	movl	4(%ebx), %ecx
> 
> Oh... No, please use constants not plain numbers and
> explain in comments what is going on. Otherwise it
> is unreadable.

Hmm, yes, some comments wouldn't hurt. :-)
Regarding constants: do you really think I should introduce their
first usage in this file with my patch?

> 
>> +	testl	%ecx, %ecx
>> +	jz	3f
>> +	addl	$8, %ebx
>> +	movl	%ebx, %esp
>> +
>> +2:
>> +	movl	%ecx, %edi
>>  	/* mov imm32, %eax */
>>  	.byte	0xb8
>>  VARIABLE(grub_relocator_xen_mfn_list)
>>  	.long	0
>> -	movl	%eax, %edi
>> -	movl	%ebp, %eax
>> -	movl    0(%edi, %eax, 4), %ecx
>> -
>> -	/* mov imm32, %ebx */
>> -	.byte	0xbb
>> -VARIABLE(grub_relocator_xen_paging_start)
>> -	.long	0
>> -	shll	$12, %eax
>> -	addl	%eax, %ebx
>> +	movl    0(%eax, %ebp, 4), %ecx
>> +	movl	%ebp, %ebx
>> +	shll	$12, %ebx
> 
> Ditto.

Look at the removed line above: I just switched register usage.

> 
>>  	movl    %ecx, %edx
>>  	shll    $12,  %ecx
>>  	shrl    $20,  %edx
>>  	orl     $5, %ecx
>>  	movl    $2, %esi
>>  	movl    $__HYPERVISOR_update_va_mapping, %eax
>> -	int     $0x82
>> +	int     $0x82	/* parameters: eax, ebx, ecx, edx, esi */
> 
> Please use more verbose comments.

:-)

> 
>>
>>  	incl	%ebp
>> -	/* mov imm32, %ecx */
>> -	.byte	0xb9
>> -VARIABLE(grub_relocator_xen_paging_size)
>> -	.long	0
>> -	cmpl	%ebp, %ecx
>> +	movl	%edi, %ecx
>> +
>> +	loop	2b
>>
>> -	ja	1b
>> +	mov	%esp, %ebx
>> +	jmp	1b
>>
>> +3:
>>  	/* mov imm32, %ebx */
>>  	.byte	0xbb
>>  VARIABLE(grub_relocator_xen_mmu_op_addr)
>> @@ -102,6 +106,9 @@ VARIABLE(grub_relocator_xen_remap_continue)
>>
>>  	jmp *%eax
>>
>> +VARIABLE(grub_relocator_xen_paging_areas)
>> +	.long	0, 0, 0, 0, 0, 0, 0, 0
>> +
>>  VARIABLE(grub_relocator_xen_mmu_op)
>>  	.space 256
>>
>> diff --git a/grub-core/lib/x86_64/xen/relocator.S b/grub-core/lib/x86_64/xen/relocator.S
>> index 92e9e72..dbb90c7 100644
>> --- a/grub-core/lib/x86_64/xen/relocator.S
>> +++ b/grub-core/lib/x86_64/xen/relocator.S
> 
> Is to possible to split this patch to i386 and x86-64 stuff patches?

I don't think so. The C-part is common and assembler sources must both
match.

> 
>> @@ -50,31 +50,24 @@ VARIABLE(grub_relocator_xen_remapper_map)
>>
>>  LOCAL(cont):
>>
>> -	/* mov imm64, %rcx */
>> -	.byte 	0x48
>> -	.byte	0xb9
>> -VARIABLE(grub_relocator_xen_paging_size)
>> -	.quad	0
>> -
>> -	/* mov imm64, %rax */
>> -	.byte 	0x48
>> -	.byte	0xb8
>> -VARIABLE(grub_relocator_xen_paging_start)
>> -	.quad	0
>> -
>> -	movq	%rax, %r12
>> -
>>  	/* mov imm64, %rax */
>>  	.byte 	0x48
>>  	.byte	0xb8
>>  VARIABLE(grub_relocator_xen_mfn_list)
>>  	.quad	0
>>
>> -	movq	%rax, %rsi
>> +	movq	%rax, %rbx
>> +	leaq	EXT_C(grub_relocator_xen_paging_areas) (%rip), %r8
>> +
>>  1:
>> +	movq	0(%r8), %r12
>> +	movq	8(%r8), %rcx
> 
> Ditto.
> 
>> +	testq	%rcx, %rcx
>> +	jz	3f
>> +2:
>>  	movq	%r12, %rdi
>> -	movq    %rsi, %rbx
>> -	movq    0(%rsi), %rsi
>> +	shlq	$12, %rdi
>> +	movq    (%rbx, %r12, 8), %rsi
> 
> Ditto.
> 
>>  	shlq    $12,  %rsi
>>  	orq     $5, %rsi
>>  	movq    $2, %rdx
>> @@ -83,13 +76,15 @@ VARIABLE(grub_relocator_xen_mfn_list)
>>  	syscall
>>
>>  	movq    %r9, %rcx
>> -	addq    $8, %rbx
>> -	addq    $4096, %r12
>> -	movq    %rbx, %rsi
>> +	incq	%r12
>> +
>> +	loop 2b
>>
>> -	loop 1b
>> +	addq	$16, %r8
> 
> Ditto.
> 
>> +	jmp	1b
>>
>> -	leaq   LOCAL(mmu_op) (%rip), %rdi
>> +3:
>> +	leaq   EXT_C(grub_relocator_xen_mmu_op) (%rip), %rdi
>>  	movq   $3, %rsi
>>  	movq   $0, %rdx
>>  	movq   $0x7FF0, %r10
> 
> Ugh... Please fix this too. Probably in separate patch.

I don't mind cleaning the assembler sources by using more comments
and constants. I just don't think this should be part of this series.

> 
>> @@ -104,7 +99,10 @@ VARIABLE(grub_relocator_xen_remap_continue)
>>
>>  	jmp *%rax
>>
>> -LOCAL(mmu_op):
>> +VARIABLE(grub_relocator_xen_paging_areas)
>> +	/* array of start, size pairs, size 0 is end marker */
>> +	.quad	0, 0, 0, 0, 0, 0, 0, 0
>> +
>>  VARIABLE(grub_relocator_xen_mmu_op)
>>  	.space 256
>>
>> diff --git a/grub-core/lib/xen/relocator.c b/grub-core/lib/xen/relocator.c
>> index 8f427d3..bc29055 100644
>> --- a/grub-core/lib/xen/relocator.c
>> +++ b/grub-core/lib/xen/relocator.c
>> @@ -36,15 +36,15 @@ extern grub_uint8_t grub_relocator_xen_remap_end;
>>  extern grub_xen_reg_t grub_relocator_xen_stack;
>>  extern grub_xen_reg_t grub_relocator_xen_start_info;
>>  extern grub_xen_reg_t grub_relocator_xen_entry_point;
>> -extern grub_xen_reg_t grub_relocator_xen_paging_start;
>> -extern grub_xen_reg_t grub_relocator_xen_paging_size;
>>  extern grub_xen_reg_t grub_relocator_xen_remapper_virt;
>>  extern grub_xen_reg_t grub_relocator_xen_remapper_virt2;
>>  extern grub_xen_reg_t grub_relocator_xen_remapper_map;
>>  extern grub_xen_reg_t grub_relocator_xen_mfn_list;
>> +extern grub_xen_reg_t grub_relocator_xen_paging_areas[2 * XEN_MAX_MAPPINGS];
>>  extern grub_xen_reg_t grub_relocator_xen_remap_continue;
>>  #ifdef __i386__
>>  extern grub_xen_reg_t grub_relocator_xen_mmu_op_addr;
>> +extern grub_xen_reg_t grub_relocator_xen_paging_areas_addr;
>>  extern grub_xen_reg_t grub_relocator_xen_remapper_map_high;
>>  #endif
>>  extern mmuext_op_t grub_relocator_xen_mmu_op[3];
>> @@ -61,6 +61,7 @@ grub_relocator_xen_boot (struct grub_relocator *rel,
>>  {
>>    grub_err_t err;
>>    void *relst;
>> +  int i;
>>    grub_relocator_chunk_t ch, ch_tramp;
>>    grub_xen_mfn_t *mfn_list =
>>      (grub_xen_mfn_t *) grub_xen_start_page_addr->mfn_list;
>> @@ -77,8 +78,11 @@ grub_relocator_xen_boot (struct grub_relocator *rel,
>>    grub_relocator_xen_stack = state.stack;
>>    grub_relocator_xen_start_info = state.start_info;
>>    grub_relocator_xen_entry_point = state.entry_point;
>> -  grub_relocator_xen_paging_start = state.paging_start << 12;
>> -  grub_relocator_xen_paging_size = state.paging_size;
>> +  for (i = 0; i < XEN_MAX_MAPPINGS; i++)
>> +    {
>> +      grub_relocator_xen_paging_areas[2 * i] = state.paging_start[i];
>> +      grub_relocator_xen_paging_areas[2 * i + 1] = state.paging_size[i];
> 
> Why 2 and 1? Constants?

I guess I should probably define grub_relocator_xen_paging_areas as
an array of struct { grub_xen_reg_t start; grub_xen_reg_t size; };

> 
>> +    }
>>    grub_relocator_xen_remapper_virt = remapper_virt;
>>    grub_relocator_xen_remapper_virt2 = remapper_virt;
>>    grub_relocator_xen_remap_continue = trampoline_virt;
>> @@ -88,10 +92,12 @@ grub_relocator_xen_boot (struct grub_relocator *rel,
>>    grub_relocator_xen_remapper_map_high = (mfn_list[remapper_pfn] >> 20);
>>    grub_relocator_xen_mmu_op_addr = (char *) &grub_relocator_xen_mmu_op
>>      - (char *) &grub_relocator_xen_remap_start + remapper_virt;
>> +  grub_relocator_xen_paging_areas_addr =
>> +    (char *) &grub_relocator_xen_paging_areas
>> +    - (char *) &grub_relocator_xen_remap_start + remapper_virt;
>>  #endif
>>
>> -  grub_relocator_xen_mfn_list = state.mfn_list
>> -    + state.paging_start * sizeof (grub_addr_t);
>> +  grub_relocator_xen_mfn_list = state.mfn_list;
>>
>>    grub_memset (grub_relocator_xen_mmu_op, 0,
>>  	       sizeof (grub_relocator_xen_mmu_op));
>> @@ -100,9 +106,9 @@ grub_relocator_xen_boot (struct grub_relocator *rel,
>>  #else
>>    grub_relocator_xen_mmu_op[0].cmd = MMUEXT_PIN_L4_TABLE;
>>  #endif
>> -  grub_relocator_xen_mmu_op[0].arg1.mfn = mfn_list[state.paging_start];
>> +  grub_relocator_xen_mmu_op[0].arg1.mfn = mfn_list[state.paging_start[0]];
>>    grub_relocator_xen_mmu_op[1].cmd = MMUEXT_NEW_BASEPTR;
>> -  grub_relocator_xen_mmu_op[1].arg1.mfn = mfn_list[state.paging_start];
>> +  grub_relocator_xen_mmu_op[1].arg1.mfn = mfn_list[state.paging_start[0]];
>>    grub_relocator_xen_mmu_op[2].cmd = MMUEXT_UNPIN_TABLE;
>>    grub_relocator_xen_mmu_op[2].arg1.mfn =
>>      mfn_list[grub_xen_start_page_addr->pt_base >> 12];
>> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
>> index 0f41048..5e10420 100644
>> --- a/grub-core/loader/i386/xen.c
>> +++ b/grub-core/loader/i386/xen.c
>> @@ -42,6 +42,29 @@
>>
>>  GRUB_MOD_LICENSE ("GPLv3+");
>>
>> +#ifdef __x86_64__
>> +#define NUMBER_OF_LEVELS 4
>> +#define INTERMEDIATE_OR 7
>> +#define VIRT_MASK 0x0000ffffffffffffULL
>> +#else
>> +#define NUMBER_OF_LEVELS 3
>> +#define INTERMEDIATE_OR 3
>> +#define VIRT_MASK 0x00000000ffffffffULL
> 
> Long expected constants... Nice!

Just to please you. :-)
They were just moved up some lines as they are used in the new
structure definitions.

Using constants was unavoidable here. ;-)

> 
>> +#endif
>> +
>> +struct grub_xen_mapping_lvl {
>> +  grub_uint64_t virt_start;
>> +  grub_uint64_t virt_end;
>> +  grub_uint64_t pfn_start;
>> +  grub_uint64_t n_pt_pages;
>> +};
>> +
>> +struct grub_xen_mapping {
>> +  grub_uint64_t *where;
>> +  struct grub_xen_mapping_lvl area;
>> +  struct grub_xen_mapping_lvl lvls[NUMBER_OF_LEVELS];
>> +};
>> +
>>  static struct grub_relocator *relocator = NULL;
>>  static grub_uint64_t max_addr;
>>  static grub_dl_t my_mod;
>> @@ -56,9 +79,10 @@ static struct grub_relocator_xen_state state;
>>  static grub_xen_mfn_t *virt_mfn_list;
>>  static struct start_info *virt_start_info;
>>  static grub_xen_mfn_t console_pfn;
>> -static grub_uint64_t *virt_pgtable;
>> -static grub_uint64_t pgtbl_start;
>>  static grub_uint64_t pgtbl_end;
>> +static struct grub_xen_mapping mappings[XEN_MAX_MAPPINGS];
>> +static int n_mappings;
>> +static struct grub_xen_mapping *map_reloc;
>>
>>  #define PAGE_SIZE 4096
>>  #define MAX_MODULES (PAGE_SIZE / sizeof (struct xen_multiboot_mod_list))
>> @@ -75,100 +99,162 @@ page2offset (grub_uint64_t page)
>>    return page << PAGE_SHIFT;
>>  }
>>
>> -#ifdef __x86_64__
>> -#define NUMBER_OF_LEVELS 4
>> -#define INTERMEDIATE_OR 7
>> -#else
>> -#define NUMBER_OF_LEVELS 3
>> -#define INTERMEDIATE_OR 3
>> -#endif
>> -
>> -static grub_uint64_t
>> -get_pgtable_size (grub_uint64_t total_pages, grub_uint64_t virt_base)
>> +static grub_err_t
>> +get_pgtable_size (grub_uint64_t from, grub_uint64_t to, grub_uint64_t pfn)
>>  {
>> -  if (!virt_base)
>> -    total_pages++;
>> -  grub_uint64_t ret = 0;
>> -  grub_uint64_t ll = total_pages;
>> -  int i;
>> -  for (i = 0; i < NUMBER_OF_LEVELS; i++)
>> -    {
>> -      ll = (ll + POINTERS_PER_PAGE - 1) >> LOG_POINTERS_PER_PAGE;
>> -      /* PAE wants all 4 root directories present.  */
>> -#ifdef __i386__
>> -      if (i == 1)
>> -	ll = 4;
>> -#endif
>> -      ret += ll;
>> -    }
>> -  for (i = 1; i < NUMBER_OF_LEVELS; i++)
>> -    if (virt_base >> (PAGE_SHIFT + i * LOG_POINTERS_PER_PAGE))
>> -      ret++;
>> -  return ret;
>> -}
>> +  struct grub_xen_mapping *map, *map_cmp;
>> +  grub_uint64_t mask, bits;
>> +  int i, m;
>>
>> -static void
>> -generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
>> -		     grub_uint64_t paging_end, grub_uint64_t total_pages,
>> -		     grub_uint64_t virt_base, grub_xen_mfn_t *mfn_list)
>> -{
>> -  if (!virt_base)
>> -    paging_end++;
>> +  if (n_mappings == XEN_MAX_MAPPINGS)
>> +    return grub_error (GRUB_ERR_BUG, "too many mapped areas");
>> +
>> +  grub_dprintf ("xen", "get_pgtable_size %d from=%llx, to=%llx, pfn=%llx\n",
>> +		n_mappings, (unsigned long long) from, (unsigned long long) to,
>> +		(unsigned long long) pfn);
>>
>> -  grub_uint64_t lx[NUMBER_OF_LEVELS], lxs[NUMBER_OF_LEVELS];
>> -  grub_uint64_t nlx, nls, sz = 0;
>> -  int l;
>> +  map = mappings + n_mappings;
>> +  grub_memset (map, 0, sizeof (*map));
>>
>> -  nlx = paging_end;
>> -  nls = virt_base >> PAGE_SHIFT;
>> -  for (l = 0; l < NUMBER_OF_LEVELS; l++)
>> +  map->area.virt_start = from & VIRT_MASK;
>> +  map->area.virt_end = (to - 1) & VIRT_MASK;
>> +  map->area.n_pt_pages = 0;
>> +
>> +  for (i = NUMBER_OF_LEVELS - 1; i >= 0; i--)
>>      {
>> -      nlx = (nlx + POINTERS_PER_PAGE - 1) >> LOG_POINTERS_PER_PAGE;
>> -      /* PAE wants all 4 root directories present.  */
>> +      map->lvls[i].pfn_start = pfn + map->area.n_pt_pages;
>> +      if (i == NUMBER_OF_LEVELS - 1)
>> +	{
>> +	  if (n_mappings == 0)
>> +	    {
>> +	      map->lvls[i].virt_start = 0;
>> +	      map->lvls[i].virt_end = VIRT_MASK;
>> +	      map->lvls[i].n_pt_pages = 1;
>> +	      map->area.n_pt_pages++;
>> +	    }
>> +	  continue;
>> +	}
>> +
>> +      bits = PAGE_SHIFT + (i + 1) * LOG_POINTERS_PER_PAGE;
>> +      mask = (1ULL << bits) - 1;
>> +      map->lvls[i].virt_start = map->area.virt_start & ~mask;
>> +      map->lvls[i].virt_end = map->area.virt_end | mask;
>>  #ifdef __i386__
>> -      if (l == 1)
>> -	nlx = 4;
>> +      /* PAE wants last root directory present. */
>> +      if (i == 1 && to <= 0xc0000000 && n_mappings == 0)
> 
> Ditto.
> 
>> +	map->lvls[i].virt_end = VIRT_MASK;
>>  #endif
>> -      lx[l] = nlx;
>> -      sz += lx[l];
>> -      lxs[l] = nls & (POINTERS_PER_PAGE - 1);
>> -      if (nls && l != 0)
>> -	sz++;
>> -      nls >>= LOG_POINTERS_PER_PAGE;
>> +      for (m = 0; m < n_mappings; m++)
>> +	{
>> +	  map_cmp = mappings + m;
>> +	  if (map_cmp->lvls[i].virt_start == map_cmp->lvls[i].virt_end)
>> +	    continue;
>> +	  if (map->lvls[i].virt_start >= map_cmp->lvls[i].virt_start &&
>> +	      map->lvls[i].virt_end <= map_cmp->lvls[i].virt_end)
>> +	   {
>> +	     map->lvls[i].virt_start = 0;
>> +	     map->lvls[i].virt_end = 0;
>> +	     break;
>> +	   }
>> +	   if (map->lvls[i].virt_start >= map_cmp->lvls[i].virt_start &&
>> +	       map->lvls[i].virt_start <= map_cmp->lvls[i].virt_end)
>> +	     map->lvls[i].virt_start = map_cmp->lvls[i].virt_end + 1;
>> +	   if (map->lvls[i].virt_end >= map_cmp->lvls[i].virt_start &&
>> +	       map->lvls[i].virt_end <= map_cmp->lvls[i].virt_end)
>> +	     map->lvls[i].virt_end = map_cmp->lvls[i].virt_start - 1;
>> +	}
>> +      if (map->lvls[i].virt_start < map->lvls[i].virt_end)
>> +	map->lvls[i].n_pt_pages =
>> +	  ((map->lvls[i].virt_end - map->lvls[i].virt_start) >> bits) + 1;
>> +      map->area.n_pt_pages += map->lvls[i].n_pt_pages;
>> +      grub_dprintf ("xen", "get_pgtable_size level %d: virt %llx-%llx %d pts\n",
>> +		    i, (unsigned long long)  map->lvls[i].virt_start,
>> +		    (unsigned long long)  map->lvls[i].virt_end,
>> +		    (int) map->lvls[i].n_pt_pages);
>>      }
>>
>> -  grub_uint64_t lp;
>> -  grub_uint64_t j;
>> -  grub_uint64_t *pg = (grub_uint64_t *) where;
>> -  int pr = 0;
>> +  grub_dprintf ("xen", "get_pgtable_size return: %d page tables\n",
>> +		(int) map->area.n_pt_pages);
>> +
>> +  state.paging_start[n_mappings] = pfn;
>> +  state.paging_size[n_mappings] = map->area.n_pt_pages;
>> +
>> +  return GRUB_ERR_NONE;
>> +}
>> +
>> +static grub_uint64_t *
>> +get_pg_table_virt (int mapping, int level)
>> +{
>> +  grub_uint64_t pfn;
>> +  struct grub_xen_mapping *map;
>> +
>> +  map = mappings + mapping;
>> +  pfn = map->lvls[level].pfn_start - map->lvls[NUMBER_OF_LEVELS - 1].pfn_start;
>> +  return map->where + pfn * POINTERS_PER_PAGE;
>> +}
>>
>> -  grub_memset (pg, 0, sz * PAGE_SIZE);
>> +static grub_uint64_t
>> +get_pg_table_prot (int level, grub_uint64_t pfn)
>> +{
>> +  int m;
>> +  grub_uint64_t pfn_s, pfn_e;
>>
>> -  lp = paging_start + lx[NUMBER_OF_LEVELS - 1];
>> -  for (l = NUMBER_OF_LEVELS - 1; l >= 1; l--)
>> +  if (level > 0)
>> +    return INTERMEDIATE_OR;
>> +  for (m = 0; m < n_mappings; m++)
>>      {
>> -      if (lxs[l] || pr)
>> -	pg[0] = page2offset (mfn_list[lp++]) | INTERMEDIATE_OR;
>> -      if (pr)
>> -	pg += POINTERS_PER_PAGE;
>> -      for (j = 0; j < lx[l - 1]; j++)
>> -	pg[j + lxs[l]] = page2offset (mfn_list[lp++]) | INTERMEDIATE_OR;
>> -      pg += lx[l] * POINTERS_PER_PAGE;
>> -      if (lxs[l])
>> -	pr = 1;
>> +      pfn_s = mappings[m].lvls[NUMBER_OF_LEVELS - 1].pfn_start;
>> +      pfn_e = mappings[m].area.n_pt_pages + pfn_s;
>> +      if (pfn >= pfn_s && pfn < pfn_e)
>> +	return 5;
>>      }
>> +  return 7;
> 
> What 7 and 5 means?

Page table protection bits.

> I am exhausted. Please fix above stuff and I will review this patch
> again after fixes.

I'd really like to have the maintainer's opinion on usage of constants
vs. pure numbers. Up to now I have the impression that constants are
used only to abstract i386 vs. x86-64. I wouldn't mind changing that,
but I don't like modifying all my patches which are then rejected due to
that change.

Thanks for doing the review,


Juergen

Re: [PATCH v2 5/6] xen: modify page table construction

[Daniel Kiper]

On Thu, Feb 11, 2016 at 03:35:45PM +0100, Juergen Gross wrote:
> On 11/02/16 13:47, Daniel Kiper wrote:
> > On Thu, Feb 11, 2016 at 08:53:25AM +0100, Juergen Gross wrote:
> >> Modify the page table construction to allow multiple virtual regions
> >> to be mapped. This is done as preparation for removing the p2m list
> >> from the initial kernel mapping in order to support huge pv domains.
> >>
> >> This allows a cleaner approach for mapping the relocator page by
> >> using this capability.
> >>
> >> The interface to the assembler level of the relocator has to be changed
> >> in order to be able to process multiple page table areas.
> >
> > I hope that older kernels could be loaded as is and everything works as expected.
>
> See Patch 0/6. I have tested old kernels, too.

That is great! I just wanted to be sure.

> >> Signed-off-by: Juergen Gross <jgross@suse.com>
> >> ---
> >>  grub-core/lib/i386/xen/relocator.S   |  47 +++---
> >>  grub-core/lib/x86_64/xen/relocator.S |  44 +++--
> >>  grub-core/lib/xen/relocator.c        |  22 ++-
> >>  grub-core/loader/i386/xen.c          | 313 +++++++++++++++++++++++------------
> >>  include/grub/xen/relocator.h         |   6 +-
> >>  5 files changed, 277 insertions(+), 155 deletions(-)
> >>
> >> diff --git a/grub-core/lib/i386/xen/relocator.S b/grub-core/lib/i386/xen/relocator.S
> >> index 694a54c..c23b405 100644
> >> --- a/grub-core/lib/i386/xen/relocator.S
> >> +++ b/grub-core/lib/i386/xen/relocator.S
> >> @@ -50,41 +50,45 @@ VARIABLE(grub_relocator_xen_remapper_map_high)
> >>  	jmp *%ebx
> >>
> >>  LOCAL(cont):
> >> -	xorl	%eax, %eax
> >> -	movl	%eax, %ebp
> >> +	/* mov imm32, %eax */
> >> +	.byte	0xb8
> >> +VARIABLE(grub_relocator_xen_paging_areas_addr)
> >> +	.long	0
> >> +	movl	%eax, %ebx
> >>  1:
> >> -
> >> +	movl	0(%ebx), %ebp
> >> +	movl	4(%ebx), %ecx
> >
> > Oh... No, please use constants not plain numbers and
> > explain in comments what is going on. Otherwise it
> > is unreadable.
>
> Hmm, yes, some comments wouldn't hurt. :-)
> Regarding constants: do you really think I should introduce their
> first usage in this file with my patch?

I would not afraid that. At least in code you change/touch. If you
are not sure please ask maintainer about that.

> >> +	testl	%ecx, %ecx
> >> +	jz	3f
> >> +	addl	$8, %ebx
> >> +	movl	%ebx, %esp
> >> +
> >> +2:
> >> +	movl	%ecx, %edi
> >>  	/* mov imm32, %eax */
> >>  	.byte	0xb8
> >>  VARIABLE(grub_relocator_xen_mfn_list)
> >>  	.long	0
> >> -	movl	%eax, %edi
> >> -	movl	%ebp, %eax
> >> -	movl    0(%edi, %eax, 4), %ecx
> >> -
> >> -	/* mov imm32, %ebx */
> >> -	.byte	0xbb
> >> -VARIABLE(grub_relocator_xen_paging_start)
> >> -	.long	0
> >> -	shll	$12, %eax
> >> -	addl	%eax, %ebx
> >> +	movl    0(%eax, %ebp, 4), %ecx
> >> +	movl	%ebp, %ebx
> >> +	shll	$12, %ebx
> >
> > Ditto.
>
> Look at the removed line above: I just switched register usage.

It does not hurt to change number to constant here too.

> >>  	movl    %ecx, %edx
> >>  	shll    $12,  %ecx
> >>  	shrl    $20,  %edx
> >>  	orl     $5, %ecx
> >>  	movl    $2, %esi
> >>  	movl    $__HYPERVISOR_update_va_mapping, %eax
> >> -	int     $0x82
> >> +	int     $0x82	/* parameters: eax, ebx, ecx, edx, esi */
> >
> > Please use more verbose comments.
>
> :-)
>
> >
> >>
> >>  	incl	%ebp
> >> -	/* mov imm32, %ecx */
> >> -	.byte	0xb9
> >> -VARIABLE(grub_relocator_xen_paging_size)
> >> -	.long	0
> >> -	cmpl	%ebp, %ecx
> >> +	movl	%edi, %ecx
> >> +
> >> +	loop	2b
> >>
> >> -	ja	1b
> >> +	mov	%esp, %ebx
> >> +	jmp	1b
> >>
> >> +3:
> >>  	/* mov imm32, %ebx */
> >>  	.byte	0xbb
> >>  VARIABLE(grub_relocator_xen_mmu_op_addr)
> >> @@ -102,6 +106,9 @@ VARIABLE(grub_relocator_xen_remap_continue)
> >>
> >>  	jmp *%eax
> >>
> >> +VARIABLE(grub_relocator_xen_paging_areas)
> >> +	.long	0, 0, 0, 0, 0, 0, 0, 0
> >> +
> >>  VARIABLE(grub_relocator_xen_mmu_op)
> >>  	.space 256
> >>
> >> diff --git a/grub-core/lib/x86_64/xen/relocator.S b/grub-core/lib/x86_64/xen/relocator.S
> >> index 92e9e72..dbb90c7 100644
> >> --- a/grub-core/lib/x86_64/xen/relocator.S
> >> +++ b/grub-core/lib/x86_64/xen/relocator.S
> >
> > Is to possible to split this patch to i386 and x86-64 stuff patches?
>
> I don't think so. The C-part is common and assembler sources must both
> match.

Yep, I am aware of that. However, I was able to that in my patch series.
So, maybe it is possible here. Could you try it?

> >> @@ -50,31 +50,24 @@ VARIABLE(grub_relocator_xen_remapper_map)
> >>
> >>  LOCAL(cont):
> >>
> >> -	/* mov imm64, %rcx */
> >> -	.byte 	0x48
> >> -	.byte	0xb9
> >> -VARIABLE(grub_relocator_xen_paging_size)
> >> -	.quad	0
> >> -
> >> -	/* mov imm64, %rax */
> >> -	.byte 	0x48
> >> -	.byte	0xb8
> >> -VARIABLE(grub_relocator_xen_paging_start)
> >> -	.quad	0
> >> -
> >> -	movq	%rax, %r12
> >> -
> >>  	/* mov imm64, %rax */
> >>  	.byte 	0x48
> >>  	.byte	0xb8
> >>  VARIABLE(grub_relocator_xen_mfn_list)
> >>  	.quad	0
> >>
> >> -	movq	%rax, %rsi
> >> +	movq	%rax, %rbx
> >> +	leaq	EXT_C(grub_relocator_xen_paging_areas) (%rip), %r8
> >> +
> >>  1:
> >> +	movq	0(%r8), %r12
> >> +	movq	8(%r8), %rcx
> >
> > Ditto.
> >
> >> +	testq	%rcx, %rcx
> >> +	jz	3f
> >> +2:
> >>  	movq	%r12, %rdi
> >> -	movq    %rsi, %rbx
> >> -	movq    0(%rsi), %rsi
> >> +	shlq	$12, %rdi
> >> +	movq    (%rbx, %r12, 8), %rsi
> >
> > Ditto.
> >
> >>  	shlq    $12,  %rsi
> >>  	orq     $5, %rsi
> >>  	movq    $2, %rdx
> >> @@ -83,13 +76,15 @@ VARIABLE(grub_relocator_xen_mfn_list)
> >>  	syscall
> >>
> >>  	movq    %r9, %rcx
> >> -	addq    $8, %rbx
> >> -	addq    $4096, %r12
> >> -	movq    %rbx, %rsi
> >> +	incq	%r12
> >> +
> >> +	loop 2b
> >>
> >> -	loop 1b
> >> +	addq	$16, %r8
> >
> > Ditto.
> >
> >> +	jmp	1b
> >>
> >> -	leaq   LOCAL(mmu_op) (%rip), %rdi
> >> +3:
> >> +	leaq   EXT_C(grub_relocator_xen_mmu_op) (%rip), %rdi
> >>  	movq   $3, %rsi
> >>  	movq   $0, %rdx
> >>  	movq   $0x7FF0, %r10
> >
> > Ugh... Please fix this too. Probably in separate patch.
>
> I don't mind cleaning the assembler sources by using more comments
> and constants. I just don't think this should be part of this series.

I think that you can safely fix that issue in code you touch. I do not
insist on fixing rest of the code in the same patch. However, it would
be nice if you remove such worst/obvious warts at least at the end of
this patch series.

> >> @@ -104,7 +99,10 @@ VARIABLE(grub_relocator_xen_remap_continue)
> >>
> >>  	jmp *%rax
> >>
> >> -LOCAL(mmu_op):
> >> +VARIABLE(grub_relocator_xen_paging_areas)
> >> +	/* array of start, size pairs, size 0 is end marker */
> >> +	.quad	0, 0, 0, 0, 0, 0, 0, 0
> >> +
> >>  VARIABLE(grub_relocator_xen_mmu_op)
> >>  	.space 256
> >>
> >> diff --git a/grub-core/lib/xen/relocator.c b/grub-core/lib/xen/relocator.c
> >> index 8f427d3..bc29055 100644
> >> --- a/grub-core/lib/xen/relocator.c
> >> +++ b/grub-core/lib/xen/relocator.c
> >> @@ -36,15 +36,15 @@ extern grub_uint8_t grub_relocator_xen_remap_end;
> >>  extern grub_xen_reg_t grub_relocator_xen_stack;
> >>  extern grub_xen_reg_t grub_relocator_xen_start_info;
> >>  extern grub_xen_reg_t grub_relocator_xen_entry_point;
> >> -extern grub_xen_reg_t grub_relocator_xen_paging_start;
> >> -extern grub_xen_reg_t grub_relocator_xen_paging_size;
> >>  extern grub_xen_reg_t grub_relocator_xen_remapper_virt;
> >>  extern grub_xen_reg_t grub_relocator_xen_remapper_virt2;
> >>  extern grub_xen_reg_t grub_relocator_xen_remapper_map;
> >>  extern grub_xen_reg_t grub_relocator_xen_mfn_list;
> >> +extern grub_xen_reg_t grub_relocator_xen_paging_areas[2 * XEN_MAX_MAPPINGS];
> >>  extern grub_xen_reg_t grub_relocator_xen_remap_continue;
> >>  #ifdef __i386__
> >>  extern grub_xen_reg_t grub_relocator_xen_mmu_op_addr;
> >> +extern grub_xen_reg_t grub_relocator_xen_paging_areas_addr;
> >>  extern grub_xen_reg_t grub_relocator_xen_remapper_map_high;
> >>  #endif
> >>  extern mmuext_op_t grub_relocator_xen_mmu_op[3];
> >> @@ -61,6 +61,7 @@ grub_relocator_xen_boot (struct grub_relocator *rel,
> >>  {
> >>    grub_err_t err;
> >>    void *relst;
> >> +  int i;
> >>    grub_relocator_chunk_t ch, ch_tramp;
> >>    grub_xen_mfn_t *mfn_list =
> >>      (grub_xen_mfn_t *) grub_xen_start_page_addr->mfn_list;
> >> @@ -77,8 +78,11 @@ grub_relocator_xen_boot (struct grub_relocator *rel,
> >>    grub_relocator_xen_stack = state.stack;
> >>    grub_relocator_xen_start_info = state.start_info;
> >>    grub_relocator_xen_entry_point = state.entry_point;
> >> -  grub_relocator_xen_paging_start = state.paging_start << 12;
> >> -  grub_relocator_xen_paging_size = state.paging_size;
> >> +  for (i = 0; i < XEN_MAX_MAPPINGS; i++)
> >> +    {
> >> +      grub_relocator_xen_paging_areas[2 * i] = state.paging_start[i];
> >> +      grub_relocator_xen_paging_areas[2 * i + 1] = state.paging_size[i];
> >
> > Why 2 and 1? Constants?
>
> I guess I should probably define grub_relocator_xen_paging_areas as
> an array of struct { grub_xen_reg_t start; grub_xen_reg_t size; };

Make sense for me.

> >> +    }
> >>    grub_relocator_xen_remapper_virt = remapper_virt;
> >>    grub_relocator_xen_remapper_virt2 = remapper_virt;
> >>    grub_relocator_xen_remap_continue = trampoline_virt;
> >> @@ -88,10 +92,12 @@ grub_relocator_xen_boot (struct grub_relocator *rel,
> >>    grub_relocator_xen_remapper_map_high = (mfn_list[remapper_pfn] >> 20);
> >>    grub_relocator_xen_mmu_op_addr = (char *) &grub_relocator_xen_mmu_op
> >>      - (char *) &grub_relocator_xen_remap_start + remapper_virt;
> >> +  grub_relocator_xen_paging_areas_addr =
> >> +    (char *) &grub_relocator_xen_paging_areas
> >> +    - (char *) &grub_relocator_xen_remap_start + remapper_virt;
> >>  #endif
> >>
> >> -  grub_relocator_xen_mfn_list = state.mfn_list
> >> -    + state.paging_start * sizeof (grub_addr_t);
> >> +  grub_relocator_xen_mfn_list = state.mfn_list;
> >>
> >>    grub_memset (grub_relocator_xen_mmu_op, 0,
> >>  	       sizeof (grub_relocator_xen_mmu_op));
> >> @@ -100,9 +106,9 @@ grub_relocator_xen_boot (struct grub_relocator *rel,
> >>  #else
> >>    grub_relocator_xen_mmu_op[0].cmd = MMUEXT_PIN_L4_TABLE;
> >>  #endif
> >> -  grub_relocator_xen_mmu_op[0].arg1.mfn = mfn_list[state.paging_start];
> >> +  grub_relocator_xen_mmu_op[0].arg1.mfn = mfn_list[state.paging_start[0]];
> >>    grub_relocator_xen_mmu_op[1].cmd = MMUEXT_NEW_BASEPTR;
> >> -  grub_relocator_xen_mmu_op[1].arg1.mfn = mfn_list[state.paging_start];
> >> +  grub_relocator_xen_mmu_op[1].arg1.mfn = mfn_list[state.paging_start[0]];
> >>    grub_relocator_xen_mmu_op[2].cmd = MMUEXT_UNPIN_TABLE;
> >>    grub_relocator_xen_mmu_op[2].arg1.mfn =
> >>      mfn_list[grub_xen_start_page_addr->pt_base >> 12];
> >> diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
> >> index 0f41048..5e10420 100644
> >> --- a/grub-core/loader/i386/xen.c
> >> +++ b/grub-core/loader/i386/xen.c
> >> @@ -42,6 +42,29 @@
> >>
> >>  GRUB_MOD_LICENSE ("GPLv3+");
> >>
> >> +#ifdef __x86_64__
> >> +#define NUMBER_OF_LEVELS 4
> >> +#define INTERMEDIATE_OR 7
> >> +#define VIRT_MASK 0x0000ffffffffffffULL
> >> +#else
> >> +#define NUMBER_OF_LEVELS 3
> >> +#define INTERMEDIATE_OR 3
> >> +#define VIRT_MASK 0x00000000ffffffffULL
> >
> > Long expected constants... Nice!
>
> Just to please you. :-)
> They were just moved up some lines as they are used in the new
> structure definitions.
>
> Using constants was unavoidable here. ;-)

Thanks a lot! :-)))

> >> +#endif
> >> +
> >> +struct grub_xen_mapping_lvl {
> >> +  grub_uint64_t virt_start;
> >> +  grub_uint64_t virt_end;
> >> +  grub_uint64_t pfn_start;
> >> +  grub_uint64_t n_pt_pages;
> >> +};
> >> +
> >> +struct grub_xen_mapping {
> >> +  grub_uint64_t *where;
> >> +  struct grub_xen_mapping_lvl area;
> >> +  struct grub_xen_mapping_lvl lvls[NUMBER_OF_LEVELS];
> >> +};
> >> +
> >>  static struct grub_relocator *relocator = NULL;
> >>  static grub_uint64_t max_addr;
> >>  static grub_dl_t my_mod;
> >> @@ -56,9 +79,10 @@ static struct grub_relocator_xen_state state;
> >>  static grub_xen_mfn_t *virt_mfn_list;
> >>  static struct start_info *virt_start_info;
> >>  static grub_xen_mfn_t console_pfn;
> >> -static grub_uint64_t *virt_pgtable;
> >> -static grub_uint64_t pgtbl_start;
> >>  static grub_uint64_t pgtbl_end;
> >> +static struct grub_xen_mapping mappings[XEN_MAX_MAPPINGS];
> >> +static int n_mappings;
> >> +static struct grub_xen_mapping *map_reloc;
> >>
> >>  #define PAGE_SIZE 4096
> >>  #define MAX_MODULES (PAGE_SIZE / sizeof (struct xen_multiboot_mod_list))
> >> @@ -75,100 +99,162 @@ page2offset (grub_uint64_t page)
> >>    return page << PAGE_SHIFT;
> >>  }
> >>
> >> -#ifdef __x86_64__
> >> -#define NUMBER_OF_LEVELS 4
> >> -#define INTERMEDIATE_OR 7
> >> -#else
> >> -#define NUMBER_OF_LEVELS 3
> >> -#define INTERMEDIATE_OR 3
> >> -#endif
> >> -
> >> -static grub_uint64_t
> >> -get_pgtable_size (grub_uint64_t total_pages, grub_uint64_t virt_base)
> >> +static grub_err_t
> >> +get_pgtable_size (grub_uint64_t from, grub_uint64_t to, grub_uint64_t pfn)
> >>  {
> >> -  if (!virt_base)
> >> -    total_pages++;
> >> -  grub_uint64_t ret = 0;
> >> -  grub_uint64_t ll = total_pages;
> >> -  int i;
> >> -  for (i = 0; i < NUMBER_OF_LEVELS; i++)
> >> -    {
> >> -      ll = (ll + POINTERS_PER_PAGE - 1) >> LOG_POINTERS_PER_PAGE;
> >> -      /* PAE wants all 4 root directories present.  */
> >> -#ifdef __i386__
> >> -      if (i == 1)
> >> -	ll = 4;
> >> -#endif
> >> -      ret += ll;
> >> -    }
> >> -  for (i = 1; i < NUMBER_OF_LEVELS; i++)
> >> -    if (virt_base >> (PAGE_SHIFT + i * LOG_POINTERS_PER_PAGE))
> >> -      ret++;
> >> -  return ret;
> >> -}
> >> +  struct grub_xen_mapping *map, *map_cmp;
> >> +  grub_uint64_t mask, bits;
> >> +  int i, m;
> >>
> >> -static void
> >> -generate_page_table (grub_uint64_t *where, grub_uint64_t paging_start,
> >> -		     grub_uint64_t paging_end, grub_uint64_t total_pages,
> >> -		     grub_uint64_t virt_base, grub_xen_mfn_t *mfn_list)
> >> -{
> >> -  if (!virt_base)
> >> -    paging_end++;
> >> +  if (n_mappings == XEN_MAX_MAPPINGS)
> >> +    return grub_error (GRUB_ERR_BUG, "too many mapped areas");
> >> +
> >> +  grub_dprintf ("xen", "get_pgtable_size %d from=%llx, to=%llx, pfn=%llx\n",
> >> +		n_mappings, (unsigned long long) from, (unsigned long long) to,
> >> +		(unsigned long long) pfn);
> >>
> >> -  grub_uint64_t lx[NUMBER_OF_LEVELS], lxs[NUMBER_OF_LEVELS];
> >> -  grub_uint64_t nlx, nls, sz = 0;
> >> -  int l;
> >> +  map = mappings + n_mappings;
> >> +  grub_memset (map, 0, sizeof (*map));
> >>
> >> -  nlx = paging_end;
> >> -  nls = virt_base >> PAGE_SHIFT;
> >> -  for (l = 0; l < NUMBER_OF_LEVELS; l++)
> >> +  map->area.virt_start = from & VIRT_MASK;
> >> +  map->area.virt_end = (to - 1) & VIRT_MASK;
> >> +  map->area.n_pt_pages = 0;
> >> +
> >> +  for (i = NUMBER_OF_LEVELS - 1; i >= 0; i--)
> >>      {
> >> -      nlx = (nlx + POINTERS_PER_PAGE - 1) >> LOG_POINTERS_PER_PAGE;
> >> -      /* PAE wants all 4 root directories present.  */
> >> +      map->lvls[i].pfn_start = pfn + map->area.n_pt_pages;
> >> +      if (i == NUMBER_OF_LEVELS - 1)
> >> +	{
> >> +	  if (n_mappings == 0)
> >> +	    {
> >> +	      map->lvls[i].virt_start = 0;
> >> +	      map->lvls[i].virt_end = VIRT_MASK;
> >> +	      map->lvls[i].n_pt_pages = 1;
> >> +	      map->area.n_pt_pages++;
> >> +	    }
> >> +	  continue;
> >> +	}
> >> +
> >> +      bits = PAGE_SHIFT + (i + 1) * LOG_POINTERS_PER_PAGE;
> >> +      mask = (1ULL << bits) - 1;
> >> +      map->lvls[i].virt_start = map->area.virt_start & ~mask;
> >> +      map->lvls[i].virt_end = map->area.virt_end | mask;
> >>  #ifdef __i386__
> >> -      if (l == 1)
> >> -	nlx = 4;
> >> +      /* PAE wants last root directory present. */
> >> +      if (i == 1 && to <= 0xc0000000 && n_mappings == 0)
> >
> > Ditto.
> >
> >> +	map->lvls[i].virt_end = VIRT_MASK;
> >>  #endif
> >> -      lx[l] = nlx;
> >> -      sz += lx[l];
> >> -      lxs[l] = nls & (POINTERS_PER_PAGE - 1);
> >> -      if (nls && l != 0)
> >> -	sz++;
> >> -      nls >>= LOG_POINTERS_PER_PAGE;
> >> +      for (m = 0; m < n_mappings; m++)
> >> +	{
> >> +	  map_cmp = mappings + m;
> >> +	  if (map_cmp->lvls[i].virt_start == map_cmp->lvls[i].virt_end)
> >> +	    continue;
> >> +	  if (map->lvls[i].virt_start >= map_cmp->lvls[i].virt_start &&
> >> +	      map->lvls[i].virt_end <= map_cmp->lvls[i].virt_end)
> >> +	   {
> >> +	     map->lvls[i].virt_start = 0;
> >> +	     map->lvls[i].virt_end = 0;
> >> +	     break;
> >> +	   }
> >> +	   if (map->lvls[i].virt_start >= map_cmp->lvls[i].virt_start &&
> >> +	       map->lvls[i].virt_start <= map_cmp->lvls[i].virt_end)
> >> +	     map->lvls[i].virt_start = map_cmp->lvls[i].virt_end + 1;
> >> +	   if (map->lvls[i].virt_end >= map_cmp->lvls[i].virt_start &&
> >> +	       map->lvls[i].virt_end <= map_cmp->lvls[i].virt_end)
> >> +	     map->lvls[i].virt_end = map_cmp->lvls[i].virt_start - 1;
> >> +	}
> >> +      if (map->lvls[i].virt_start < map->lvls[i].virt_end)
> >> +	map->lvls[i].n_pt_pages =
> >> +	  ((map->lvls[i].virt_end - map->lvls[i].virt_start) >> bits) + 1;
> >> +      map->area.n_pt_pages += map->lvls[i].n_pt_pages;
> >> +      grub_dprintf ("xen", "get_pgtable_size level %d: virt %llx-%llx %d pts\n",
> >> +		    i, (unsigned long long)  map->lvls[i].virt_start,
> >> +		    (unsigned long long)  map->lvls[i].virt_end,
> >> +		    (int) map->lvls[i].n_pt_pages);
> >>      }
> >>
> >> -  grub_uint64_t lp;
> >> -  grub_uint64_t j;
> >> -  grub_uint64_t *pg = (grub_uint64_t *) where;
> >> -  int pr = 0;
> >> +  grub_dprintf ("xen", "get_pgtable_size return: %d page tables\n",
> >> +		(int) map->area.n_pt_pages);
> >> +
> >> +  state.paging_start[n_mappings] = pfn;
> >> +  state.paging_size[n_mappings] = map->area.n_pt_pages;
> >> +
> >> +  return GRUB_ERR_NONE;
> >> +}
> >> +
> >> +static grub_uint64_t *
> >> +get_pg_table_virt (int mapping, int level)
> >> +{
> >> +  grub_uint64_t pfn;
> >> +  struct grub_xen_mapping *map;
> >> +
> >> +  map = mappings + mapping;
> >> +  pfn = map->lvls[level].pfn_start - map->lvls[NUMBER_OF_LEVELS - 1].pfn_start;
> >> +  return map->where + pfn * POINTERS_PER_PAGE;
> >> +}
> >>
> >> -  grub_memset (pg, 0, sz * PAGE_SIZE);
> >> +static grub_uint64_t
> >> +get_pg_table_prot (int level, grub_uint64_t pfn)
> >> +{
> >> +  int m;
> >> +  grub_uint64_t pfn_s, pfn_e;
> >>
> >> -  lp = paging_start + lx[NUMBER_OF_LEVELS - 1];
> >> -  for (l = NUMBER_OF_LEVELS - 1; l >= 1; l--)
> >> +  if (level > 0)
> >> +    return INTERMEDIATE_OR;
> >> +  for (m = 0; m < n_mappings; m++)
> >>      {
> >> -      if (lxs[l] || pr)
> >> -	pg[0] = page2offset (mfn_list[lp++]) | INTERMEDIATE_OR;
> >> -      if (pr)
> >> -	pg += POINTERS_PER_PAGE;
> >> -      for (j = 0; j < lx[l - 1]; j++)
> >> -	pg[j + lxs[l]] = page2offset (mfn_list[lp++]) | INTERMEDIATE_OR;
> >> -      pg += lx[l] * POINTERS_PER_PAGE;
> >> -      if (lxs[l])
> >> -	pr = 1;
> >> +      pfn_s = mappings[m].lvls[NUMBER_OF_LEVELS - 1].pfn_start;
> >> +      pfn_e = mappings[m].area.n_pt_pages + pfn_s;
> >> +      if (pfn >= pfn_s && pfn < pfn_e)
> >> +	return 5;
> >>      }
> >> +  return 7;
> >
> > What 7 and 5 means?
>
> Page table protection bits.

Constants or at least comments please. Yep, I am boring. I know.
That is my style.

> > I am exhausted. Please fix above stuff and I will review this patch
> > again after fixes.
>
> I'd really like to have the maintainer's opinion on usage of constants
> vs. pure numbers. Up to now I have the impression that constants are
> used only to abstract i386 vs. x86-64. I wouldn't mind changing that,
> but I don't like modifying all my patches which are then rejected due to
> that change.

OK, let's wait for maintainer opinion. However, I feel that using
constants ease code reading. Though it does not mean that I think
that every number/string should be changed to constant. We should
find a proper balance and if plain numbers are used then there
should be at least one line comment what this number means.

> Thanks for doing the review,

You are welcome.

Daniel

[PATCH v2 6/6] xen: add capability to load p2m list outside of kernel mapping

[Juergen Gross]

Modern pvops linux kernels support a p2m list not covered by the
kernel mapping. This capability is flagged by an elf-note specifying
the virtual address the kernel is expecting the p2m list to be mapped
to.

In case the elf-note is set by the kernel don't place the p2m list
into the kernel mapping, but map it to the given address. This will
allow to support domains with larger memory, as the kernel mapping is
limited to 2GB and a domain with huge memory in the TB range will have
a p2m list larger than this.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
 grub-core/loader/i386/xen.c        | 50 ++++++++++++++++++++++++++++++++------
 grub-core/loader/i386/xen_fileXX.c |  4 +++
 include/grub/xen_file.h            |  2 ++
 3 files changed, 48 insertions(+), 8 deletions(-)

diff --git a/grub-core/loader/i386/xen.c b/grub-core/loader/i386/xen.c
index 5e10420..9ddc6c2 100644
--- a/grub-core/loader/i386/xen.c
+++ b/grub-core/loader/i386/xen.c
@@ -305,19 +305,44 @@ static grub_err_t
 grub_xen_p2m_alloc (void)
 {
   grub_relocator_chunk_t ch;
-  grub_size_t p2msize;
+  grub_size_t p2msize, p2malloc;
   grub_err_t err;
+  struct grub_xen_mapping *map;
+
+  map = mappings + n_mappings;
+  p2msize = ALIGN_UP (sizeof (grub_xen_mfn_t) *
+		      grub_xen_start_page_addr->nr_pages, PAGE_SIZE);
+  if (xen_inf.has_p2m_base)
+    {
+      err = get_pgtable_size (xen_inf.p2m_base, xen_inf.p2m_base + p2msize,
+			      (max_addr + p2msize) >> PAGE_SHIFT);
+      if (err)
+	return err;
+
+      map->area.pfn_start = max_addr >> PAGE_SHIFT;
+      p2malloc = p2msize + page2offset (map->area.n_pt_pages);
+      n_mappings++;
+      next_start.mfn_list = xen_inf.p2m_base;
+      next_start.first_p2m_pfn = map->area.pfn_start;
+      next_start.nr_p2m_frames = p2malloc >> PAGE_SHIFT;
+    }
+  else
+    {
+      next_start.mfn_list = max_addr + xen_inf.virt_base;
+      p2malloc = p2msize;
+    }
 
   state.mfn_list = max_addr;
-  next_start.mfn_list = max_addr + xen_inf.virt_base;
-  p2msize = sizeof (grub_xen_mfn_t) * grub_xen_start_page_addr->nr_pages;
-  err = grub_relocator_alloc_chunk_addr (relocator, &ch, max_addr, p2msize);
+  err = grub_relocator_alloc_chunk_addr (relocator, &ch, max_addr, p2malloc);
   if (err)
     return err;
   virt_mfn_list = get_virtual_current_address (ch);
+  if (xen_inf.has_p2m_base)
+    map->where = (grub_uint64_t *) virt_mfn_list +
+		 p2msize / sizeof (grub_uint64_t);
   grub_memcpy (virt_mfn_list,
 	       (void *) grub_xen_start_page_addr->mfn_list, p2msize);
-  max_addr = ALIGN_UP (max_addr + p2msize, PAGE_SIZE);
+  max_addr += p2malloc;
 
   return GRUB_ERR_NONE;
 }
@@ -425,9 +450,12 @@ grub_xen_alloc_end (void)
     return GRUB_ERR_NONE;
   called = 1;
 
-  err = grub_xen_p2m_alloc ();
-  if (err)
-    return err;
+  if (!xen_inf.has_p2m_base)
+    {
+      err = grub_xen_p2m_alloc ();
+      if (err)
+	return err;
+    }
   err = grub_xen_special_alloc ();
   if (err)
     return err;
@@ -452,6 +480,12 @@ grub_xen_boot (void)
   err = grub_xen_alloc_end ();
   if (err)
     return err;
+  if (xen_inf.has_p2m_base)
+    {
+      err = grub_xen_p2m_alloc ();
+      if (err)
+	return err;
+    }
 
   err = set_mfns (console_pfn);
   if (err)
diff --git a/grub-core/loader/i386/xen_fileXX.c b/grub-core/loader/i386/xen_fileXX.c
index 69fccd2..8d01adb 100644
--- a/grub-core/loader/i386/xen_fileXX.c
+++ b/grub-core/loader/i386/xen_fileXX.c
@@ -253,6 +253,10 @@ parse_note (grub_elf_t elf, struct grub_xen_file_info *xi,
 					  descsz == 2 ? 2 : 3) == 0)
 	    xi->arch = GRUB_XEN_FILE_I386;
 	  break;
+	case 15:
+	  xi->p2m_base = grub_le_to_cpu_addr (*(Elf_Addr *) desc);
+	  xi->has_p2m_base = 1;
+	  break;
 	case 16:
 	  xi->unmapped_initrd = !!grub_le_to_cpu32(*(grub_uint32_t *) desc);
 	  break;
diff --git a/include/grub/xen_file.h b/include/grub/xen_file.h
index ed749fa..6587999 100644
--- a/include/grub/xen_file.h
+++ b/include/grub/xen_file.h
@@ -32,9 +32,11 @@ struct grub_xen_file_info
   grub_uint64_t entry_point;
   grub_uint64_t hypercall_page;
   grub_uint64_t paddr_offset;
+  grub_uint64_t p2m_base;
   int has_hypercall_page;
   int has_note;
   int has_xen_guest;
+  int has_p2m_base;
   int extended_cr3;
   int unmapped_initrd;
   enum
-- 
2.6.2