PAE patches / hypercall interface changes

PAE patches / hypercall interface changes

[Gerd Knorr]

  Hi,

New PAE patchset (#13) has been uploaded to 
http://dl.bytesex.org/patches/

The hypercall interface has been updated to pass around
64bit-wide page table entries now.

The xen patch is here:

==============================[ cut here ]==============================
--- xen.orig/include/public/xen.h	2005-06-08 11:39:05.000000000 +0200
+++ xen/include/public/xen.h	2005-06-08 12:08:24.000000000 +0200
@@ -270,8 +270,8 @@ typedef u16 domid_t;
  */
 typedef struct
 {
-    memory_t ptr;       /* Machine address of PTE. */
-    memory_t val;       /* New contents of PTE.    */
+    u64 ptr;       /* Machine address of PTE. */
+    u64 val;       /* New contents of PTE.    */
 } mmu_update_t;
 
 /*
--- xen.orig/arch/x86/mm.c	2005-06-08 11:42:11.000000000 +0200
+++ xen/arch/x86/mm.c	2005-06-08 12:12:13.000000000 +0200
@@ -2023,7 +2023,8 @@ int do_mmu_update(
             }
 
             va = map_domain_page_with_cache(mfn, &mapcache);
-            va = (void *)((unsigned long)va + (req.ptr & ~PAGE_MASK));
+            va = (void *)((unsigned long)va +
+                          (unsigned long)(req.ptr & ~PAGE_MASK));
             page = &frame_table[mfn];
 
             switch ( (type_info = page->u.inuse.type_info) & PGT_type_mask )
@@ -2167,7 +2168,7 @@ int do_mmu_update(
             break;
 
         default:
-            MEM_LOG("Invalid page update command %lx", req.ptr);
+            MEM_LOG("Invalid page update command %x", cmd);
             break;
         }
 
@@ -2254,11 +2255,10 @@ int update_grant_va_mapping(unsigned lon
 }
 
 
-int do_update_va_mapping(unsigned long va,
-                         unsigned long val32,
+int do_update_va_mapping(unsigned long va, u64 val64,
                          unsigned long flags)
 {
-    l1_pgentry_t   val = l1e_from_intpte(val32);
+    l1_pgentry_t   val = l1e_from_intpte(val64);
     struct vcpu   *v   = current;
     struct domain *d   = v->domain;
     unsigned int   cpu = v->processor;
@@ -2352,8 +2352,7 @@ int do_update_va_mapping(unsigned long v
     return rc;
 }
 
-int do_update_va_mapping_otherdomain(unsigned long va,
-                                     unsigned long val32,
+int do_update_va_mapping_otherdomain(unsigned long va, u64 val64,
                                      unsigned long flags,
                                      domid_t domid)
 {
@@ -2371,7 +2370,7 @@ int do_update_va_mapping_otherdomain(uns
         return -ESRCH;
     }
 
-    rc = do_update_va_mapping(va, val32, flags);
+    rc = do_update_va_mapping(va, val64, flags);
 
     return rc;
 }
==============================[ cut here ]==============================

xenlinux 2.6.x needs these changes (also take care that xenlinux
uses the updated xen/include/public/xen.h from the patch above).

==============================[ cut here ]==============================
--- linux-2.6.11.orig/include/asm-xen/asm-i386/hypercall.h	2005-06-08 12:39:16.000000000 +0200
+++ linux-2.6.11/include/asm-xen/asm-i386/hypercall.h	2005-06-08 13:01:04.000000000 +0200
@@ -371,13 +371,19 @@ HYPERVISOR_update_va_mapping(
     unsigned long va, pte_t new_val, unsigned long flags)
 {
     int ret;
-    unsigned long ign1, ign2, ign3;
+    unsigned long ign1, ign2, ign3, ign4;
 
     __asm__ __volatile__ (
         TRAP_INSTR
-        : "=a" (ret), "=b" (ign1), "=c" (ign2), "=d" (ign3)
+        : "=a" (ret), "=b" (ign1), "=c" (ign2), "=d" (ign3), "=S" (ign4)
 	: "0" (__HYPERVISOR_update_va_mapping), 
-          "1" (va), "2" ((new_val).pte_low), "3" (flags)
+          "1" (va), "2" ((new_val).pte_low),
+#ifdef CONFIG_X86_PAE
+	  "3" ((new_val).pte_high),
+#else
+	  "3" (0),
+#endif
+	  "4" (flags)
 	: "memory" );
 
     if ( unlikely(ret < 0) )
@@ -473,13 +479,20 @@ HYPERVISOR_update_va_mapping_otherdomain
     unsigned long va, pte_t new_val, unsigned long flags, domid_t domid)
 {
     int ret;
-    unsigned long ign1, ign2, ign3, ign4;
+    unsigned long ign1, ign2, ign3, ign4, ign5;
 
     __asm__ __volatile__ (
         TRAP_INSTR
-        : "=a" (ret), "=b" (ign1), "=c" (ign2), "=d" (ign3), "=S" (ign4)
+        : "=a" (ret), "=b" (ign1), "=c" (ign2), "=d" (ign3),
+	  "=S" (ign4), "=D" (ign5)
 	: "0" (__HYPERVISOR_update_va_mapping_otherdomain),
-          "1" (va), "2" ((new_val).pte_low), "3" (flags), "4" (domid) :
+          "1" (va), "2" ((new_val).pte_low),
+#ifdef CONFIG_X86_PAE
+	  "3" ((new_val).pte_high),
+#else
+	  "3" (0),
+#endif
+	  "4" (flags), "5" (domid) :
         "memory" );
     
     return ret;
==============================[ cut here ]==============================

Comments?  Domain0 boots fine.  Other guest OS'es are not
updated yet, tools also not updated yet.


The recent PAE cleanups by Keir which added a new sanity check
prevent the current xenlinux kernel from booting in PAE mode.
I've disabled that for now to make it boot again:

==============================[ cut here ]==============================
--- xen.orig/arch/x86/mm.c	2005-06-08 16:25:03.000000000 +0200
+++ xen/arch/x86/mm.c	2005-06-08 16:25:44.000000000 +0200
@@ -716,8 +716,8 @@ static int create_pae_xen_mappings(l3_pg
     BUG_ON((page->u.inuse.type_info & PGT_count_mask) == 0);
     if ( (page->u.inuse.type_info & PGT_count_mask) != 1 )
     {
-        MEM_LOG("PAE L3 3rd slot is shared");
-        return 0;
+        MEM_LOG("WARNING: PAE L3 3rd slot is shared");
+//        return 0;
     }
 
     /* Xen private mappings. */
==============================[ cut here ]==============================

The xenlinux kernel does that exactly once at boot time, when
initializing swapper_pg_dir.  Happens in arch/xen/i386/mm/init.c:

static void __init pagetable_init (void)
{
        unsigned long vaddr;
        pgd_t *pgd_base = swapper_pg_dir;
        pgd_t *old_pgd = (pgd_t *)xen_start_info.pt_base;
[ ... ]
        memcpy(pgd_base, old_pgd, PTRS_PER_PGD_NO_HV*sizeof(pgd_t));
        make_page_readonly(pgd_base);
        xen_pgd_pin(__pa(pgd_base));      <= here it blows up
        load_cr3(pgd_base);
        xen_pgd_unpin(__pa(old_pgd));
        make_page_writable(old_pgd);
[ ... ]
}

What this check is needed for?  Is there some reason why xen
must do that?  Or is this just to catch bugs in the guest OS
kernel?  

  Gerd

Re: PAE patches / hypercall interface changes

[Christopher S. Aker]

> New PAE patchset (#13) has been uploaded to
> http://dl.bytesex.org/patches/

  Booting 'Xen (xen.gz) (2.6.11.11) LVM'

kernel /xen.gz dom0_mem=140000 com1=115200,8n1
   [Multiboot-elf, <0x100000:0x601ac:0x20e54>, shtab=0x181078, entry=0x100000]
module /vmlinuz-2.6.11.11-xen0 root=/dev/ram0 lvm2root=/dev/vg1/root elevator=cfq
ramdisk_size=10240 console=ttyS0
   [Multiboot-module @ 0x182000, 0x3b95d0 bytes]
module /initrd-lvm2.gz
   [Multiboot-module @ 0x53c000, 0x43e000 bytes]

 __  __            _____  ___         _                _
 \ \/ /___ _ __   |___ / / _ \     __| | _____   _____| |
  \  // _ \ '_ \    |_ \| | | |__ / _` |/ _ \ \ / / _ \ |
  /  \  __/ | | |  ___) | |_| |__| (_| |  __/\ V /  __/ |
 /_/\_\___|_| |_| |____(_)___/    \__,_|\___| \_/ \___|_|

 http://www.cl.cam.ac.uk/netos/xen
 University of Cambridge Computer Laboratory

 Xen version 3.0-devel (root@linode.com) (gcc version 3.4.3 20041212 (Red Hat
3.4.3-9.EL4)) Wed Jun  8 12:42:10 EDT 2005
 Latest ChangeSet: 2005/06/08 16:13:43 1.1701 42a70b27-Oq9klHVoCm0PlmeoehF8g

(XEN) Physical RAM map:
(XEN)  0000000000000000 - 000000000009b800 (usable)
(XEN)  000000000009b800 - 00000000000a0000 (reserved)
(XEN)  00000000000e4000 - 0000000000100000 (reserved)
(XEN)  0000000000100000 - 00000000bff70000 (usable)
(XEN)  00000000bff70000 - 00000000bff78000 (ACPI data)
(XEN)  00000000bff78000 - 00000000bff80000 (ACPI NVS)
(XEN)  00000000bff80000 - 00000000c0000000 (reserved)
(XEN)  00000000e0000000 - 00000000f0000000 (reserved)
(XEN)  00000000fec00000 - 00000000fec10000 (reserved)
(XEN)  00000000fee00000 - 00000000fee01000 (reserved)
(XEN)  00000000ff800000 - 00000000ffc00000 (reserved)
(XEN)  00000000fffffc00 - 0000000100000000 (reserved)
(XEN)  0000000100000000 - 00000001c0000000 (usable)
(XEN) System RAM: 6143MB (6290476kB)

****************************************
Panic on CPU0:
Not enough memory for frame table
****************************************

Reboot in five seconds...
(XEN) Unknown interrupt
[hangs]

-Chris

Re: PAE patches / hypercall interface changes

[Gerd Knorr]

> (XEN) System RAM: 6143MB (6290476kB)
> 
> ****************************************
> Panic on CPU0:
> Not enough memory for frame table
> ****************************************

Ok, so using memory above 4 GB doesn't work yet ;)

I'll have a look tomorrow.  Reversing the pae-memlimit patch
should make it work for now, but will also limit the memory to
4 GB again ...

  Gerd

-- 
-mm seems unusually stable at present.
	-- akpm about 2.6.12-rc3-mm3

Re: PAE patches / hypercall interface changes

[Gerd Knorr]

Gerd Knorr <kraxel@suse.de> writes:

> > (XEN) System RAM: 6143MB (6290476kB)
> > Not enough memory for frame table
> 
> I'll have a look tomorrow.

Here we go.

Boot allocator uses "unsigned long" for physical addresses and thus
doesn't work in PAE mode.  Fix below.  That patch could make it work
(or at least fail later in boot ;)

cheers,

  Gerd

==============================[ cut here ]==============================
--- xen.orig/common/page_alloc.c	2005-06-08 16:25:02.000000000 +0200
+++ xen/common/page_alloc.c	2005-06-09 11:37:29.000000000 +0200
@@ -148,7 +148,7 @@ unsigned long init_boot_allocator(unsign
     return bitmap_start + bitmap_size;
 }
 
-void init_boot_pages(unsigned long ps, unsigned long pe)
+void init_boot_pages(physaddr_t ps, physaddr_t pe)
 {
     unsigned long bad_pfn;
     char *p;
--- xen.orig/include/xen/mm.h	2005-06-08 16:25:02.000000000 +0200
+++ xen/include/xen/mm.h	2005-06-09 11:38:41.000000000 +0200
@@ -11,7 +11,7 @@ struct pfn_info;
 
 /* Boot-time allocator. Turns into generic allocator after bootstrap. */
 unsigned long init_boot_allocator(unsigned long bitmap_start);
-void init_boot_pages(unsigned long ps, unsigned long pe);
+void init_boot_pages(physaddr_t ps, physaddr_t pe);
 unsigned long alloc_boot_pages(unsigned long size, unsigned long align);
 void end_boot_allocator(void);

Re: PAE patches / hypercall interface changes

[Christopher S. Aker]

> Boot allocator uses "unsigned long" for physical addresses and thus
> doesn't work in PAE mode.  Fix below.  That patch could make it work
> (or at least fail later in boot ;)

Applied the changes -- same thing happens:

 Xen version 3.0-devel (root@linode.com) (gcc version 3.4.3 20041212 (Red Hat
3.4.3-9.EL4)) Fri Jun 10 22:45:37 EDT 2005
 Latest ChangeSet: 2005/06/08 16:13:43 1.1701 42a70b27-Oq9klHVoCm0PlmeoehF8g

(XEN) Physical RAM map:
(XEN)  0000000000000000 - 000000000009b800 (usable)
(XEN)  000000000009b800 - 00000000000a0000 (reserved)
(XEN)  00000000000e4000 - 0000000000100000 (reserved)
(XEN)  0000000000100000 - 00000000bff70000 (usable)
(XEN)  00000000bff70000 - 00000000bff78000 (ACPI data)
(XEN)  00000000bff78000 - 00000000bff80000 (ACPI NVS)
(XEN)  00000000bff80000 - 00000000c0000000 (reserved)
(XEN)  00000000e0000000 - 00000000f0000000 (reserved)
(XEN)  00000000fec00000 - 00000000fec10000 (reserved)
(XEN)  00000000fee00000 - 00000000fee01000 (reserved)
(XEN)  00000000ff800000 - 00000000ffc00000 (reserved)
(XEN)  00000000fffffc00 - 0000000100000000 (reserved)
(XEN)  0000000100000000 - 00000001c0000000 (usable)
(XEN) System RAM: 6143MB (6290476kB)

****************************************
Panic on CPU0:
Not enough memory for frame table
****************************************

-Chris