[PATCH 00/16] xenpaging changes for xen-unstable

[PATCH 00/16] xenpaging changes for xen-unstable

[Olaf Hering]

Here are some changes for xenpaging in xen-unstable.

Its just for review and comments.

In a previous series for 4.0 I had a retry loop for the XENMEM_* commands.
This could be done in a different way to prevent changes to existing kernel
drivers in the guest. The changes to guest_physmap_add_entry are not needed
with the update for machine_to_phys_mapping handling.
But the patch to handle guest_remove_page does not work as expected, the guest
will hang. Any help to get this fixed without retry loops is appreciated.

Olaf

[PATCH 01/16] xenpaging: whitespace fixes after addition of __get_paged_frame

[Olaf Hering]

[-- Attachment #1: xen-unstable.xenpaging.get_paged_frame.patch --]
[-- Type: text/plain, Size: 3216 bytes --]

Cleanup whitespace which was introduced in patch that added __get_paged_frame.
Convert tabs to spaces to follow the coding style in this file.

Signed-off-by: Olaf Hering <olaf@aepfle.de>

---
 xen/common/grant_table.c |   26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

--- xen-unstable.hg-4.1.22344.orig/xen/common/grant_table.c
+++ xen-unstable.hg-4.1.22344/xen/common/grant_table.c
@@ -147,13 +147,13 @@ static int __get_paged_frame(unsigned lo
     struct p2m_domain *p2m;
     p2m_type_t p2mt;
     mfn_t mfn;
-    
+
     p2m = p2m_get_hostp2m(rd);
     if ( readonly )
         mfn = gfn_to_mfn(p2m, gfn, &p2mt);
     else
         mfn = gfn_to_mfn_unshare(p2m, gfn, &p2mt, 1);
-    
+
     if ( p2m_is_valid(p2mt) ) {
         *frame = mfn_x(mfn);
         if ( p2m_is_paged(p2mt) )
@@ -167,7 +167,7 @@ static int __get_paged_frame(unsigned lo
 #else
     *frame = readonly ? gmfn_to_mfn(rd, gfn) : gfn_to_mfn_private(rd, gfn);
 #endif
-    
+
     return rc;
 }
 
@@ -565,7 +565,7 @@ __gnttab_map_grant_ref(
             gfn = sha1 ? sha1->frame : sha2->full_page.frame;
             rc = __get_paged_frame(gfn, &frame, !!(op->flags & GNTMAP_readonly), rd);
             if ( rc != GNTST_okay )
-	         goto unlock_out;
+                goto unlock_out;
             act->gfn = gfn;
             act->domid = ld->domain_id;
             act->frame = frame;
@@ -1852,7 +1852,7 @@ __acquire_grant_for_copy(
             gfn = sha1->frame;
             rc = __get_paged_frame(gfn, &grant_frame, readonly, rd);
             if ( rc != GNTST_okay )
-		goto unlock_out;
+                goto unlock_out;
             act->gfn = gfn;
             is_sub_page = 0;
             trans_page_off = 0;
@@ -1864,7 +1864,7 @@ __acquire_grant_for_copy(
             gfn = sha2->full_page.frame;
             rc = __get_paged_frame(gfn, &grant_frame, readonly, rd);
             if ( rc != GNTST_okay )
-		    goto unlock_out;
+                goto unlock_out;
             act->gfn = gfn;
             is_sub_page = 0;
             trans_page_off = 0;
@@ -1876,7 +1876,7 @@ __acquire_grant_for_copy(
             gfn = sha2->sub_page.frame;
             rc = __get_paged_frame(gfn, &grant_frame, readonly, rd);
             if ( rc != GNTST_okay )
-		    goto unlock_out;
+                goto unlock_out;
             act->gfn = gfn;
             is_sub_page = 1;
             trans_page_off = sha2->sub_page.page_off;
@@ -1973,9 +1973,9 @@ __gnttab_copy(
     else
     {
 #ifdef CONFIG_X86
-	rc = __get_paged_frame(op->source.u.gmfn, &s_frame, 1, sd);
-	if ( rc != GNTST_okay )
-		goto error_out;
+        rc = __get_paged_frame(op->source.u.gmfn, &s_frame, 1, sd);
+        if ( rc != GNTST_okay )
+            goto error_out;
 #else
         s_frame = gmfn_to_mfn(sd, op->source.u.gmfn);        
 #endif
@@ -2012,9 +2012,9 @@ __gnttab_copy(
     else
     {
 #ifdef CONFIG_X86
-	rc = __get_paged_frame(op->dest.u.gmfn, &d_frame, 0, dd);
-	if ( rc != GNTST_okay )
-		goto error_out;
+        rc = __get_paged_frame(op->dest.u.gmfn, &d_frame, 0, dd);
+        if ( rc != GNTST_okay )
+            goto error_out;
 #else
         d_frame = gmfn_to_mfn(dd, op->dest.u.gmfn);
 #endif

Re: [PATCH 01/16] xenpaging: whitespace fixes after addition of __get_paged_frame

[Ian Jackson]

Olaf Hering writes ("[Xen-devel] [PATCH 01/16] xenpaging: whitespace fixes after addition of __get_paged_frame"):
> Cleanup whitespace which was introduced in patch that added __get_paged_frame.
> Convert tabs to spaces to follow the coding style in this file.

If we approve of patches to fix whitespace and intending infelicities,
then I think this patch should be applied right away.

So on that condition:

Acked-by: Ian Jackson <ian.jackson@eu.citrix.com>

Ian.

[PATCH 02/16] xenpaging: break endless loop during inital page-out with large pagefiles

[Olaf Hering]

[-- Attachment #1: xen-unstable.xenpaging.policy_linear.patch --]
[-- Type: text/plain, Size: 3879 bytes --]

To allow the starting for xenpaging right after 'xm start XYZ', I
specified a pagefile size equal to the guest memory size in the hope to
catch more errors where the paged-out state of a p2mt is not checked.

While doing that, xenpaging got into an endless loop because some pages
cant be paged out right away. Now the policy reports an error if the gfn
number wraps.

Signed-off-by: Olaf Hering <olaf@aepfle.de>
Already-Acked-by: Patrick Colp <pjcolp@cs.ubc.ca>
Already-Acked-by: Keir Fraser <keir.fraser@citrix.com>

---
 tools/xenpaging/policy_default.c |   35 ++++++++++++++++++++++++++++-------
 tools/xenpaging/xenpaging.c      |    7 +++++--
 2 files changed, 33 insertions(+), 9 deletions(-)

--- xen-unstable.hg-4.1.22344.orig/tools/xenpaging/policy_default.c
+++ xen-unstable.hg-4.1.22344/tools/xenpaging/policy_default.c
@@ -30,8 +30,12 @@
 
 
 static unsigned long mru[MRU_SIZE];
-static unsigned int i_mru = 0;
+static unsigned int i_mru;
 static unsigned long *bitmap;
+static unsigned long *unconsumed;
+static unsigned long current_gfn;
+static unsigned long bitmap_size;
+static unsigned long max_pages;
 
 
 int policy_init(xenpaging_t *paging)
@@ -43,6 +47,14 @@ int policy_init(xenpaging_t *paging)
     rc = alloc_bitmap(&bitmap, paging->bitmap_size);
     if ( rc != 0 )
         goto out;
+    /* Allocate bitmap to track unusable pages */
+    rc = alloc_bitmap(&unconsumed, paging->bitmap_size);
+    if ( rc != 0 )
+        goto out;
+
+    /* record bitmap_size */
+    bitmap_size = paging->bitmap_size;
+    max_pages = paging->domain_info->max_pages;
 
     /* Initialise MRU list of paged in pages */
     for ( i = 0; i < MRU_SIZE; i++ )
@@ -51,8 +63,6 @@ int policy_init(xenpaging_t *paging)
     /* Don't page out page 0 */
     set_bit(0, bitmap);
 
-    rc = 0;
-
  out:
     return rc;
 }
@@ -61,17 +71,27 @@ int policy_choose_victim(xc_interface *x
                          xenpaging_t *paging, domid_t domain_id,
                          xenpaging_victim_t *victim)
 {
+    unsigned long wrap = current_gfn;
     ASSERT(victim != NULL);
 
     /* Domain to pick on */
     victim->domain_id = domain_id;
-    
+
     do
     {
-        /* Randomly choose a gfn to evict */
-        victim->gfn = rand() % paging->domain_info->max_pages;
+        current_gfn++;
+        if ( current_gfn >= max_pages )
+            current_gfn = 0;
+        if ( wrap == current_gfn )
+        {
+            victim->gfn = INVALID_MFN;
+            return -ENOSPC;
+        }
     }
-    while ( test_bit(victim->gfn, bitmap) );
+    while ( test_bit(current_gfn, bitmap) || test_bit(current_gfn, unconsumed) );
+
+    set_bit(current_gfn, unconsumed);
+    victim->gfn = current_gfn;
 
     return 0;
 }
@@ -79,6 +99,7 @@ int policy_choose_victim(xc_interface *x
 void policy_notify_paged_out(domid_t domain_id, unsigned long gfn)
 {
     set_bit(gfn, bitmap);
+    clear_bit(gfn, unconsumed);
 }
 
 void policy_notify_paged_in(domid_t domain_id, unsigned long gfn)
--- xen-unstable.hg-4.1.22344.orig/tools/xenpaging/xenpaging.c
+++ xen-unstable.hg-4.1.22344/tools/xenpaging/xenpaging.c
@@ -446,7 +446,8 @@ static int evict_victim(xc_interface *xc
         ret = policy_choose_victim(xch, paging, domain_id, victim);
         if ( ret != 0 )
         {
-            ERROR("Error choosing victim");
+            if ( ret != -ENOSPC )
+                ERROR("Error choosing victim");
             goto out;
         }
 
@@ -525,7 +526,9 @@ int main(int argc, char *argv[])
     memset(victims, 0, sizeof(xenpaging_victim_t) * num_pages);
     for ( i = 0; i < num_pages; i++ )
     {
-        evict_victim(xch, paging, domain_id, &victims[i], fd, i);
+        rc = evict_victim(xch, paging, domain_id, &victims[i], fd, i);
+        if ( rc == -ENOSPC )
+            break;
         if ( i % 100 == 0 )
             DPRINTF("%d pages evicted\n", i);
     }

[PATCH 03/16] xenpaging: Open paging file only if xenpaging_init() succeeds

[Olaf Hering]

[-- Attachment #1: xen-unstable.xenpaging.pagefile.patch --]
[-- Type: text/plain, Size: 1408 bytes --]

Open paging file only if xenpaging_init() succeeds. It can fail if the host
does not support the required virtualization features such as EPT or if
xenpaging was already started for this domain_id.

Signed-off-by: Olaf Hering <olaf@aepfle.de>
Already-Acked-by: Patrick Colp <pjcolp@cs.ubc.ca>
Already-Acked-by: Keir Fraser <keir.fraser@citrix.com>

---
 tools/xenpaging/xenpaging.c |   18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

--- xen-unstable.hg-4.1.22344.orig/tools/xenpaging/xenpaging.c
+++ xen-unstable.hg-4.1.22344/tools/xenpaging/xenpaging.c
@@ -502,15 +502,6 @@ int main(int argc, char *argv[])
 
     victims = calloc(num_pages, sizeof(xenpaging_victim_t));
 
-    /* Open file */
-    sprintf(filename, "page_cache_%d", domain_id);
-    fd = open(filename, open_flags, open_mode);
-    if ( fd < 0 )
-    {
-        perror("failed to open file");
-        return -1;
-    }
-
     /* Seed random-number generator */
     srand(time(NULL));
 
@@ -522,6 +513,15 @@ int main(int argc, char *argv[])
         goto out;
     }
 
+    /* Open file */
+    sprintf(filename, "page_cache_%d", domain_id);
+    fd = open(filename, open_flags, open_mode);
+    if ( fd < 0 )
+    {
+        perror("failed to open file");
+        return -1;
+    }
+
     /* Evict pages */
     memset(victims, 0, sizeof(xenpaging_victim_t) * num_pages);
     for ( i = 0; i < num_pages; i++ )

[PATCH 04/16] xenpaging: allow only one xenpaging binary per guest

[Olaf Hering]

[-- Attachment #1: xen-unstable.xenpaging.xenpaging_init.patch --]
[-- Type: text/plain, Size: 1792 bytes --]

Make sure only one xenpaging binary is active per domain.
Print info when the host lacks the required features for xenpaging.

Signed-off-by: Olaf Hering <olaf@aepfle.de>
Already-Acked-by: Patrick Colp <pjcolp@cs.ubc.ca>
Already-Acked-by: Keir Fraser <keir.fraser@citrix.com>

---
v2: use perror for default case

 tools/xenpaging/xenpaging.c |   12 +++++++++++-
 xen/arch/x86/mm/mem_event.c |    7 +++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

--- xen-unstable.hg-4.1.22344.orig/tools/xenpaging/xenpaging.c
+++ xen-unstable.hg-4.1.22344/tools/xenpaging/xenpaging.c
@@ -123,7 +123,17 @@ xenpaging_t *xenpaging_init(xc_interface
                              paging->mem_event.ring_page);
     if ( rc != 0 )
     {
-        ERROR("Error initialising shared page");
+        switch ( errno ) {
+            case EBUSY:
+                ERROR("xenpaging is (or was) active on this domain");
+                break;
+            case ENODEV:
+                ERROR("EPT not supported for this guest");
+                break;
+            default:
+                perror("Error initialising shared page");
+                break;
+        }
         goto err;
     }
 
--- xen-unstable.hg-4.1.22344.orig/xen/arch/x86/mm/mem_event.c
+++ xen-unstable.hg-4.1.22344/xen/arch/x86/mm/mem_event.c
@@ -214,6 +214,13 @@ int mem_event_domctl(struct domain *d, x
             mfn_t ring_mfn;
             mfn_t shared_mfn;
 
+            /* Only one xenpaging at a time. If xenpaging crashed,
+             * the cache is in an undefined state and so is the guest
+             */
+            rc = -EBUSY;
+            if ( d->mem_event.ring_page )
+                break;
+
             /* Currently only EPT is supported */
             rc = -ENODEV;
             if ( !(hap_enabled(d) &&

[PATCH 05/16] xenpaging: update machine_to_phys_mapping during page-in and page-out

[Olaf Hering]

[-- Attachment #1: xen-unstable.xenpaging.machine_to_phys_mapping.patch --]
[-- Type: text/plain, Size: 1978 bytes --]

The machine_to_phys_mapping array needs updating during page-out.
If a page is gone, a call to get_gpfn_from_mfn will still return the old
gfn for an already paged-out page.  This happens when the entire guest
ram is paged-out before xen_vga_populate_vram() runs.  Then
XENMEM_populate_physmap is called with gfn 0xff000.  A new page is
allocated with alloc_domheap_pages.  This new page does not have a gfn
yet.  However, in guest_physmap_add_entry() the passed mfn maps still to
an old gfn.  This old gfn is paged-out and has no mfn anymore.  As a
result, the ASSERT() triggers because p2m_is_ram() is true for
p2m_ram_paging* types.

If the machine_to_phys_mapping array is updated properly, both loops in
guest_physmap_add_entry() turn into no-ops for the new page and the
mfn/gfn mapping will be done at the end of the function.


The same thing needs to happen dring a page-in.
If XENMEM_add_to_physmap is used with XENMAPSPACE_gmfn,
get_gpfn_from_mfn() will return an appearently valid gfn.  As a result,
guest_physmap_remove_page() is called.  The ASSERT in p2m_remove_page
triggers because the passed mfn does not match the old mfn for the
passed gfn.

Signed-off-by: Olaf Hering <olaf@aepfle.de>

---
 xen/arch/x86/mm/p2m.c |    2 ++
 1 file changed, 2 insertions(+)

--- xen-unstable.hg-4.1.22344.orig/xen/arch/x86/mm/p2m.c
+++ xen-unstable.hg-4.1.22344/xen/arch/x86/mm/p2m.c
@@ -2742,6 +2742,7 @@ int p2m_mem_paging_evict(struct p2m_doma
     /* Remove mapping from p2m table */
     p2m_lock(p2m);
     set_p2m_entry(p2m, gfn, _mfn(PAGING_MFN), 0, p2m_ram_paged);
+    set_gpfn_from_mfn(mfn_x(mfn), INVALID_M2P_ENTRY);
     p2m_unlock(p2m);
 
     /* Put the page back so it gets freed */
@@ -2820,6 +2821,7 @@ void p2m_mem_paging_resume(struct p2m_do
     mfn = gfn_to_mfn(p2m, rsp.gfn, &p2mt);
     p2m_lock(p2m);
     set_p2m_entry(p2m, rsp.gfn, mfn, 0, p2m_ram_rw);
+    set_gpfn_from_mfn(mfn_x(mfn), rsp.gfn);
     p2m_unlock(p2m);
 
     /* Unpause domain */

Re: [PATCH 05/16] xenpaging: update machine_to_phys_mapping during page-in and page-out

[Olaf Hering]

The machine_to_phys_mapping array needs updating during page-out.
If a page is gone, a call to get_gpfn_from_mfn will still return the old
gfn for an already paged-out page.  This happens when the entire guest
ram is paged-out before xen_vga_populate_vram() runs.  Then
XENMEM_populate_physmap is called with gfn 0xff000.  A new page is
allocated with alloc_domheap_pages.  This new page does not have a gfn
yet.  However, in guest_physmap_add_entry() the passed mfn maps still to
an old gfn.  This old gfn is paged-out and has no mfn anymore.  As a
result, the ASSERT() triggers because p2m_is_ram() is true for
p2m_ram_paging* types.

If the machine_to_phys_mapping array is updated properly, both loops in
guest_physmap_add_entry() turn into no-ops for the new page and the
mfn/gfn mapping will be done at the end of the function.


The same thing needs to happen dring a page-in.
If XENMEM_add_to_physmap is used with XENMAPSPACE_gmfn,
get_gpfn_from_mfn() will return an appearently valid gfn.  As a result,
guest_physmap_remove_page() is called.  The ASSERT in p2m_remove_page
triggers because the passed mfn does not match the old mfn for the
passed gfn.

Signed-off-by: Olaf Hering <olaf@aepfle.de>

---
v2:
  call set_gpfn_from_mfn only if mfn is valid

 xen/arch/x86/mm/p2m.c |    3 +++
 1 file changed, 3 insertions(+)

--- xen-unstable.hg-4.1.22353.orig/xen/arch/x86/mm/p2m.c
+++ xen-unstable.hg-4.1.22353/xen/arch/x86/mm/p2m.c
@@ -2742,6 +2742,7 @@ int p2m_mem_paging_evict(struct p2m_doma
     /* Remove mapping from p2m table */
     p2m_lock(p2m);
     set_p2m_entry(p2m, gfn, _mfn(PAGING_MFN), 0, p2m_ram_paged);
+    set_gpfn_from_mfn(mfn_x(mfn), INVALID_M2P_ENTRY);
     p2m_unlock(p2m);
 
     /* Put the page back so it gets freed */
@@ -2820,6 +2821,8 @@ void p2m_mem_paging_resume(struct p2m_do
     mfn = gfn_to_mfn(p2m, rsp.gfn, &p2mt);
     p2m_lock(p2m);
     set_p2m_entry(p2m, rsp.gfn, mfn, 0, p2m_ram_rw);
+    if ( mfn_valid(mfn) )
+        set_gpfn_from_mfn(mfn_x(mfn), rsp.gfn);
     p2m_unlock(p2m);
 
     /* Unpause domain */

[PATCH 06/16] xenpaging: drop paged pages in guest_remove_page

[Olaf Hering]

[-- Attachment #1: xen-unstable.xenpaging.guest_remove_page.patch --]
[-- Type: text/plain, Size: 7852 bytes --]

Simply drop paged-pages in guest_remove_page(), and notify xenpaging to
drop reference to the gfn.

This version does not work for some reasons.
The guest hangs after pages got dropped.



Signed-off-by: Olaf Hering <olaf@aepfle.de>
---
 tools/xenpaging/xenpaging.c    |   10 ++++++++
 xen/arch/x86/mm/p2m.c          |   51 +++++++++++++++++++++++++++++++++--------
 xen/common/memory.c            |    8 ++++++
 xen/include/asm-x86/p2m.h      |    4 +++
 xen/include/public/mem_event.h |    1 
 5 files changed, 65 insertions(+), 9 deletions(-)

--- xen-unstable.hg-4.1.22344.orig/tools/xenpaging/xenpaging.c
+++ xen-unstable.hg-4.1.22344/tools/xenpaging/xenpaging.c
@@ -586,6 +586,15 @@ int main(int argc, char *argv[])
                     goto out;
                 }
                 
+                if ( req.flags & MEM_EVENT_FLAG_DROP_PAGE )
+                {
+                    DPRINTF("Dropping page %"PRIx64" p2m %x\n", req.gfn, req.p2mt);
+
+                    /* Notify policy of page being dropped */
+                    policy_notify_paged_in(paging->mem_event.domain_id, req.gfn);
+                }
+                else
+                {
                 /* Populate the page */
                 rc = xenpaging_populate_page(xch, paging, &req.gfn, fd, i);
                 if ( rc != 0 )
@@ -606,6 +615,7 @@ int main(int argc, char *argv[])
                     ERROR("Error resuming page");
                     goto out;
                 }
+                }
 
                 /* Evict a new page to replace the one we just paged in */
                 evict_victim(xch, paging, domain_id, &victims[i], fd, i);
--- xen-unstable.hg-4.1.22344.orig/xen/arch/x86/mm/p2m.c
+++ xen-unstable.hg-4.1.22344/xen/arch/x86/mm/p2m.c
@@ -1984,11 +1984,11 @@ static void audit_p2m(struct p2m_domain
         {
             mpbad++;
             P2M_PRINTK("map mismatch mfn %#lx -> gfn %#lx -> mfn %#lx"
-                       " (-> gfn %#lx)\n",
+                       " (-> gfn %#lx) p2mt %x\n",
                        mfn, gfn, mfn_x(p2mfn),
                        (mfn_valid(p2mfn)
                         ? get_gpfn_from_mfn(mfn_x(p2mfn))
-                        : -1u));
+                        : -1u), type);
             /* This m2p entry is stale: the domain has another frame in
              * this physical slot.  No great disaster, but for neatness,
              * blow away the m2p entry. */
@@ -2001,12 +2001,12 @@ static void audit_p2m(struct p2m_domain
             if ( lp2mfn != mfn_x(p2mfn) )
             {
                 P2M_PRINTK("linear mismatch gfn %#lx -> mfn %#lx "
-                           "(!= mfn %#lx)\n", gfn, lp2mfn, mfn_x(p2mfn));
+                           "(!= mfn %#lx) p2mt %x\n", gfn, lp2mfn, mfn_x(p2mfn), type);
             }
         }
 
         // P2M_PRINTK("OK: mfn=%#lx, gfn=%#lx, p2mfn=%#lx, lp2mfn=%#lx\n",
-        //                mfn, gfn, p2mfn, lp2mfn);
+        //                mfn, gfn, mfn_x(p2mfn), lp2mfn);
     }
 
     spin_unlock(&d->page_alloc_lock);
@@ -2132,11 +2132,9 @@ static void audit_p2m(struct p2m_domain
                              !p2m_is_shared(type) )
                         {
                             pmbad++;
-                            printk("mismatch: gfn %#lx -> mfn %#lx"
-                                   " -> gfn %#lx\n", gfn, mfn, m2pfn);
                             P2M_PRINTK("mismatch: gfn %#lx -> mfn %#lx"
-                                       " -> gfn %#lx\n", gfn, mfn, m2pfn);
-                            BUG();
+                                       " -> gfn %#lx (p2mt %x)\n", gfn, mfn, m2pfn, type);
+                        //    BUG();
                         }
                     }
                     unmap_domain_page(l1e);
@@ -2168,7 +2166,7 @@ static void audit_p2m(struct p2m_domain
     //P2M_PRINTK("p2m audit complete\n");
     //if ( orphans_i | orphans_d | mpbad | pmbad )
     //    P2M_PRINTK("p2m audit found %lu orphans (%lu inval %lu debug)\n",
-    //                   orphans_i + orphans_d, orphans_i, orphans_d,
+    //                   orphans_i + orphans_d, orphans_i, orphans_d);
     if ( mpbad | pmbad )
         P2M_PRINTK("p2m audit found %lu odd p2m, %lu bad m2p entries\n",
                    pmbad, mpbad);
@@ -2195,6 +2193,8 @@ p2m_remove_page(struct p2m_domain *p2m,
 
     P2M_DEBUG("removing gfn=%#lx mfn=%#lx\n", gfn, mfn);
 
+    if (mfn_valid(_mfn(mfn)))
+    {
     for ( i = 0; i < (1UL << page_order); i++ )
     {
         mfn_return = p2m->get_entry(p2m, gfn + i, &t, p2m_query);
@@ -2203,6 +2203,9 @@ p2m_remove_page(struct p2m_domain *p2m,
         ASSERT( !p2m_is_valid(t) || mfn + i == mfn_x(mfn_return) );
     }
     set_p2m_entry(p2m, gfn, _mfn(INVALID_MFN), page_order, p2m_invalid);
+    }
+    else
+    P2M_DEBUG("set_p2m_entry %x\n", set_p2m_entry(p2m, gfn, _mfn(INVALID_MFN), page_order, p2m_invalid));
 }
 
 void
@@ -2751,6 +2754,36 @@ int p2m_mem_paging_evict(struct p2m_doma
     return 0;
 }
 
+void p2m_mem_paging_drop_page(struct p2m_domain *p2m, unsigned long gfn)
+{
+    struct vcpu *v = current;
+    mem_event_request_t req;
+    p2m_type_t p2mt;
+    struct domain *d = p2m->domain;
+
+    memset(&req, 0, sizeof(req));
+
+    /* Check that there's space on the ring for this request */
+    if ( mem_event_check_ring(d) )
+        return;
+
+    gfn_to_mfn(p2m, gfn, &p2mt);
+    /* Pause domain */
+    if ( 0 && v->domain->domain_id == d->domain_id )
+    {
+        vcpu_pause_nosync(v);
+        req.flags |= MEM_EVENT_FLAG_VCPU_PAUSED;
+    }
+
+    /* Send request to pager */
+    req.flags |= MEM_EVENT_FLAG_DROP_PAGE;
+    req.gfn = gfn;
+    req.p2mt = p2mt;
+    req.vcpu_id = v->vcpu_id;
+
+    mem_event_put_request(d, &req);
+}
+
 void p2m_mem_paging_populate(struct p2m_domain *p2m, unsigned long gfn)
 {
     struct vcpu *v = current;
--- xen-unstable.hg-4.1.22344.orig/xen/common/memory.c
+++ xen-unstable.hg-4.1.22344/xen/common/memory.c
@@ -162,6 +162,14 @@ int guest_remove_page(struct domain *d,
 
 #ifdef CONFIG_X86
     mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(d), gmfn, &p2mt)); 
+    if ( unlikely(p2m_is_paging(p2mt)) )
+    {
+        gdprintk(XENLOG_INFO, "Domain %u invalidate paged gfn %lx/%x\n",
+                d->domain_id, gmfn, p2mt);
+        guest_physmap_remove_page(d, gmfn, mfn, 0);
+        p2m_mem_paging_drop_page(p2m_get_hostp2m(d), gmfn);
+        return 1;
+    }
 #else
     mfn = gmfn_to_mfn(d, gmfn);
 #endif
--- xen-unstable.hg-4.1.22344.orig/xen/include/asm-x86/p2m.h
+++ xen-unstable.hg-4.1.22344/xen/include/asm-x86/p2m.h
@@ -480,6 +480,8 @@ int set_shared_p2m_entry(struct p2m_doma
 int p2m_mem_paging_nominate(struct p2m_domain *p2m, unsigned long gfn);
 /* Evict a frame */
 int p2m_mem_paging_evict(struct p2m_domain *p2m, unsigned long gfn);
+/* Tell xenpaging to drop a paged out frame */
+void p2m_mem_paging_drop_page(struct p2m_domain *p2m, unsigned long gfn);
 /* Start populating a paged out frame */
 void p2m_mem_paging_populate(struct p2m_domain *p2m, unsigned long gfn);
 /* Prepare the p2m for paging a frame in */
@@ -487,6 +489,8 @@ int p2m_mem_paging_prep(struct p2m_domai
 /* Resume normal operation (in case a domain was paused) */
 void p2m_mem_paging_resume(struct p2m_domain *p2m);
 #else
+static inline void p2m_mem_paging_drop_page(struct p2m_domain *p2m, unsigned long gfn)
+{ }
 static inline void p2m_mem_paging_populate(struct p2m_domain *p2m, unsigned long gfn)
 { }
 #endif
--- xen-unstable.hg-4.1.22344.orig/xen/include/public/mem_event.h
+++ xen-unstable.hg-4.1.22344/xen/include/public/mem_event.h
@@ -37,6 +37,7 @@
 #define MEM_EVENT_FLAG_VCPU_PAUSED  (1 << 0)
 #define MEM_EVENT_FLAG_DOM_PAUSED   (1 << 1)
 #define MEM_EVENT_FLAG_OUT_OF_MEM   (1 << 2)
+#define MEM_EVENT_FLAG_DROP_PAGE    (1 << 3)
 
 
 typedef struct mem_event_shared_page {

Re: [PATCH 06/16] xenpaging: drop paged pages in guest_remove_page

[Olaf Hering]

Simply drop paged-pages in guest_remove_page(), and notify xenpaging to
drop reference to the gfn.

Signed-off-by: Olaf Hering <olaf@aepfle.de>

---
v2:
 resume dropped page to unpause vcpus

 tools/xenpaging/xenpaging.c    |   17 ++++++++++----
 xen/arch/x86/mm/p2m.c          |   47 ++++++++++++++++++++++++++++++++++-------
 xen/common/memory.c            |    6 +++++
 xen/include/asm-x86/p2m.h      |    4 +++
 xen/include/public/mem_event.h |    1 
 5 files changed, 63 insertions(+), 12 deletions(-)

--- xen-unstable.hg-4.1.22353.orig/tools/xenpaging/xenpaging.c
+++ xen-unstable.hg-4.1.22353/tools/xenpaging/xenpaging.c
@@ -586,12 +586,19 @@ int main(int argc, char *argv[])
                     goto out;
                 }
                 
-                /* Populate the page */
-                rc = xenpaging_populate_page(xch, paging, &req.gfn, fd, i);
-                if ( rc != 0 )
+                if ( req.flags & MEM_EVENT_FLAG_DROP_PAGE )
                 {
-                    ERROR("Error populating page");
-                    goto out;
+                    DPRINTF("Dropping page %"PRIx64" p2mt %x\n", req.gfn, req.p2mt);
+                }
+                else
+                {
+                    /* Populate the page */
+                    rc = xenpaging_populate_page(xch, paging, &req.gfn, fd, i);
+                    if ( rc != 0 )
+                    {
+                        ERROR("Error populating page");
+                        goto out;
+                    }
                 }
 
                 /* Prepare the response */
--- xen-unstable.hg-4.1.22353.orig/xen/arch/x86/mm/p2m.c
+++ xen-unstable.hg-4.1.22353/xen/arch/x86/mm/p2m.c
@@ -2195,12 +2195,15 @@ p2m_remove_page(struct p2m_domain *p2m,
 
     P2M_DEBUG("removing gfn=%#lx mfn=%#lx\n", gfn, mfn);
 
-    for ( i = 0; i < (1UL << page_order); i++ )
+    if ( mfn_valid(_mfn(mfn)) )
     {
-        mfn_return = p2m->get_entry(p2m, gfn + i, &t, p2m_query);
-        if ( !p2m_is_grant(t) )
-            set_gpfn_from_mfn(mfn+i, INVALID_M2P_ENTRY);
-        ASSERT( !p2m_is_valid(t) || mfn + i == mfn_x(mfn_return) );
+        for ( i = 0; i < (1UL << page_order); i++ )
+        {
+            mfn_return = p2m->get_entry(p2m, gfn + i, &t, p2m_query);
+            if ( !p2m_is_grant(t) )
+                set_gpfn_from_mfn(mfn+i, INVALID_M2P_ENTRY);
+            ASSERT( !p2m_is_valid(t) || mfn + i == mfn_x(mfn_return) );
+        }
     }
     set_p2m_entry(p2m, gfn, _mfn(INVALID_MFN), page_order, p2m_invalid);
 }
@@ -2751,6 +2754,36 @@ int p2m_mem_paging_evict(struct p2m_doma
     return 0;
 }
 
+void p2m_mem_paging_drop_page(struct p2m_domain *p2m, unsigned long gfn)
+{
+    struct vcpu *v = current;
+    mem_event_request_t req;
+    p2m_type_t p2mt;
+    struct domain *d = p2m->domain;
+
+    memset(&req, 0, sizeof(req));
+
+    /* Check that there's space on the ring for this request */
+    if ( mem_event_check_ring(d) )
+        return;
+
+    gfn_to_mfn(p2m, gfn, &p2mt);
+    /* Pause domain */
+    if ( v->domain->domain_id == d->domain_id )
+    {
+        vcpu_pause_nosync(v);
+        req.flags |= MEM_EVENT_FLAG_VCPU_PAUSED;
+    }
+
+    /* Send request to pager */
+    req.flags |= MEM_EVENT_FLAG_DROP_PAGE;
+    req.gfn = gfn;
+    req.p2mt = p2mt;
+    req.vcpu_id = v->vcpu_id;
+
+    mem_event_put_request(d, &req);
+}
+
 void p2m_mem_paging_populate(struct p2m_domain *p2m, unsigned long gfn)
 {
     struct vcpu *v = current;
--- xen-unstable.hg-4.1.22353.orig/xen/common/memory.c
+++ xen-unstable.hg-4.1.22353/xen/common/memory.c
@@ -162,6 +162,12 @@ int guest_remove_page(struct domain *d,
 
 #ifdef CONFIG_X86
     mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(d), gmfn, &p2mt)); 
+    if ( unlikely(p2m_is_paging(p2mt)) )
+    {
+        guest_physmap_remove_page(d, gmfn, mfn, 0);
+        p2m_mem_paging_drop_page(p2m_get_hostp2m(d), gmfn);
+        return 1;
+    }
 #else
     mfn = gmfn_to_mfn(d, gmfn);
 #endif
--- xen-unstable.hg-4.1.22353.orig/xen/include/asm-x86/p2m.h
+++ xen-unstable.hg-4.1.22353/xen/include/asm-x86/p2m.h
@@ -480,6 +480,8 @@ int set_shared_p2m_entry(struct p2m_doma
 int p2m_mem_paging_nominate(struct p2m_domain *p2m, unsigned long gfn);
 /* Evict a frame */
 int p2m_mem_paging_evict(struct p2m_domain *p2m, unsigned long gfn);
+/* Tell xenpaging to drop a paged out frame */
+void p2m_mem_paging_drop_page(struct p2m_domain *p2m, unsigned long gfn);
 /* Start populating a paged out frame */
 void p2m_mem_paging_populate(struct p2m_domain *p2m, unsigned long gfn);
 /* Prepare the p2m for paging a frame in */
@@ -487,6 +489,8 @@ int p2m_mem_paging_prep(struct p2m_domai
 /* Resume normal operation (in case a domain was paused) */
 void p2m_mem_paging_resume(struct p2m_domain *p2m);
 #else
+static inline void p2m_mem_paging_drop_page(struct p2m_domain *p2m, unsigned long gfn)
+{ }
 static inline void p2m_mem_paging_populate(struct p2m_domain *p2m, unsigned long gfn)
 { }
 #endif
--- xen-unstable.hg-4.1.22353.orig/xen/include/public/mem_event.h
+++ xen-unstable.hg-4.1.22353/xen/include/public/mem_event.h
@@ -37,6 +37,7 @@
 #define MEM_EVENT_FLAG_VCPU_PAUSED  (1 << 0)
 #define MEM_EVENT_FLAG_DOM_PAUSED   (1 << 1)
 #define MEM_EVENT_FLAG_OUT_OF_MEM   (1 << 2)
+#define MEM_EVENT_FLAG_DROP_PAGE    (1 << 3)
 
 
 typedef struct mem_event_shared_page {

[PATCH 07/16] xenpaging: populate only paged-out pages

[Olaf Hering]

[-- Attachment #1: xen-unstable.xenpaging.populate_only_if_paged.patch --]
[-- Type: text/plain, Size: 4160 bytes --]

populdate a paged-out page only once to reduce pressure in the ringbuffer.
Several cpus may still request a page at once. xenpaging can handle this.

But: maybe this will miss pages in paging-out state?

Signed-off-by: Olaf Hering <olaf@aepfle.de>

---
 xen/arch/x86/hvm/emulate.c       |    3 ++-
 xen/arch/x86/hvm/hvm.c           |   15 ++++++++++-----
 xen/arch/x86/mm/guest_walk.c     |    3 ++-
 xen/arch/x86/mm/hap/guest_walk.c |    6 ++++--
 4 files changed, 18 insertions(+), 9 deletions(-)

--- xen-unstable.hg-4.1.22344.orig/xen/arch/x86/hvm/emulate.c
+++ xen-unstable.hg-4.1.22344/xen/arch/x86/hvm/emulate.c
@@ -66,7 +66,8 @@ static int hvmemul_do_io(
     ram_mfn = gfn_to_mfn_unshare(p2m, ram_gfn, &p2mt, 0);
     if ( p2m_is_paging(p2mt) )
     {
-        p2m_mem_paging_populate(p2m, ram_gfn);
+        if ( p2m_is_paged(p2mt) )
+            p2m_mem_paging_populate(p2m, ram_gfn);
         return X86EMUL_RETRY;
     }
     if ( p2m_is_shared(p2mt) )
--- xen-unstable.hg-4.1.22344.orig/xen/arch/x86/hvm/hvm.c
+++ xen-unstable.hg-4.1.22344/xen/arch/x86/hvm/hvm.c
@@ -345,7 +345,8 @@ static int hvm_set_ioreq_page(
         return -EINVAL;
     if ( p2m_is_paging(p2mt) )
     {
-        p2m_mem_paging_populate(p2m, gmfn);
+        if ( p2m_is_paged(p2mt) )
+            p2m_mem_paging_populate(p2m, gmfn);
         return -ENOENT;
     }
     if ( p2m_is_shared(p2mt) )
@@ -1369,7 +1370,8 @@ static void *__hvm_map_guest_frame(unsig
         return NULL;
     if ( p2m_is_paging(p2mt) )
     {
-        p2m_mem_paging_populate(p2m, gfn);
+        if ( p2m_is_paged(p2mt) )
+            p2m_mem_paging_populate(p2m, gfn);
         return NULL;
     }
 
@@ -1811,7 +1813,8 @@ static enum hvm_copy_result __hvm_copy(
 
         if ( p2m_is_paging(p2mt) )
         {
-            p2m_mem_paging_populate(p2m, gfn);
+            if ( p2m_is_paged(p2mt) )
+                p2m_mem_paging_populate(p2m, gfn);
             return HVMCOPY_gfn_paged_out;
         }
         if ( p2m_is_shared(p2mt) )
@@ -3118,7 +3121,8 @@ long do_hvm_op(unsigned long op, XEN_GUE
             mfn_t mfn = gfn_to_mfn(p2m, pfn, &t);
             if ( p2m_is_paging(t) )
             {
-                p2m_mem_paging_populate(p2m, pfn);
+                if ( p2m_is_paged(t) )
+                    p2m_mem_paging_populate(p2m, pfn);
 
                 rc = -EINVAL;
                 goto param_fail3;
@@ -3184,7 +3188,8 @@ long do_hvm_op(unsigned long op, XEN_GUE
             mfn = gfn_to_mfn_unshare(p2m, pfn, &t, 0);
             if ( p2m_is_paging(t) )
             {
-                p2m_mem_paging_populate(p2m, pfn);
+                if ( p2m_is_paged(t) )
+                    p2m_mem_paging_populate(p2m, pfn);
 
                 rc = -EINVAL;
                 goto param_fail4;
--- xen-unstable.hg-4.1.22344.orig/xen/arch/x86/mm/guest_walk.c
+++ xen-unstable.hg-4.1.22344/xen/arch/x86/mm/guest_walk.c
@@ -96,7 +96,8 @@ static inline void *map_domain_gfn(struc
     *mfn = gfn_to_mfn_unshare(p2m, gfn_x(gfn), p2mt, 0);
     if ( p2m_is_paging(*p2mt) )
     {
-        p2m_mem_paging_populate(p2m, gfn_x(gfn));
+        if ( p2m_is_paged(*p2mt) )
+            p2m_mem_paging_populate(p2m, gfn_x(gfn));
 
         *rc = _PAGE_PAGED;
         return NULL;
--- xen-unstable.hg-4.1.22344.orig/xen/arch/x86/mm/hap/guest_walk.c
+++ xen-unstable.hg-4.1.22344/xen/arch/x86/mm/hap/guest_walk.c
@@ -50,7 +50,8 @@ unsigned long hap_gva_to_gfn(GUEST_PAGIN
     top_mfn = gfn_to_mfn_unshare(p2m, cr3 >> PAGE_SHIFT, &p2mt, 0);
     if ( p2m_is_paging(p2mt) )
     {
-        p2m_mem_paging_populate(p2m, cr3 >> PAGE_SHIFT);
+        if ( p2m_is_paged(p2mt) )
+            p2m_mem_paging_populate(p2m, cr3 >> PAGE_SHIFT);
 
         pfec[0] = PFEC_page_paged;
         return INVALID_GFN;
@@ -82,7 +83,8 @@ unsigned long hap_gva_to_gfn(GUEST_PAGIN
         gfn_to_mfn_unshare(p2m, gfn_x(gfn), &p2mt, 0);
         if ( p2m_is_paging(p2mt) )
         {
-            p2m_mem_paging_populate(p2m, gfn_x(gfn));
+            if ( p2m_is_paged(p2mt) )
+                p2m_mem_paging_populate(p2m, gfn_x(gfn));
 
             pfec[0] = PFEC_page_paged;
             return INVALID_GFN;

Re: [PATCH 07/16] xenpaging: populate only paged-out pages

[Ian Jackson]

Olaf Hering writes ("[Xen-devel] [PATCH 07/16] xenpaging: populate only paged-out pages"):
> populdate a paged-out page only once to reduce pressure in the ringbuffer.
> Several cpus may still request a page at once. xenpaging can handle this.
>
> But: maybe this will miss pages in paging-out state?

I can't answer your second question, but surely this pattern:

> -        p2m_mem_paging_populate(p2m, ram_gfn);
> +        if ( p2m_is_paged(p2mt) )
> +            p2m_mem_paging_populate(p2m, ram_gfn);

would better be done inside p2m_mem_paging_populate ?  Unless there
are some calls to _populate which should skip the check ?

Ian.

Re: [PATCH 07/16] xenpaging: populate only paged-out pages

[Olaf Hering]

On Wed, Nov 03, Ian Jackson wrote:

> Olaf Hering writes ("[Xen-devel] [PATCH 07/16] xenpaging: populate only paged-out pages"):
> > populdate a paged-out page only once to reduce pressure in the ringbuffer.
> > Several cpus may still request a page at once. xenpaging can handle this.
> >
> > But: maybe this will miss pages in paging-out state?
> 
> I can't answer your second question, but surely this pattern:
> 
> > -        p2m_mem_paging_populate(p2m, ram_gfn);
> > +        if ( p2m_is_paged(p2mt) )
> > +            p2m_mem_paging_populate(p2m, ram_gfn);
> 
> would better be done inside p2m_mem_paging_populate ?  Unless there
> are some calls to _populate which should skip the check ?

Ian,

thanks for bringing this up. After revisiting this patch and the reasons
for it, two different things need changing.

One thing is that p2m_mem_paging_populate needs to be called
unconditionally. The reason is that the vcpu may need stopping and
resuming until the page content is usable again.
A change for grant_table.c:__get_paged_frame is required.

The other thing is that p2m_mem_paging_populate must only invalidate the
mfn if either the page is still in the process of being paged out, or if
the page is already in the process of being paged in. The latter does
not happen right now. The result is that the page may have state
p2m_ram_paging_in and a new valid mfn. Then that new mfn is invalidated,
the gfn content in the xenpaging process was already overwritten and
there is no wakeup of the vcpu. This situation is not recoverable.
p2m.c:p2m_mem_paging_populate needs a change to no call set_p2m_entry if
the p2mt state is already p2m_ram_paging_in

I will send two new patches for grant_table.c and p2m.c

Olaf

PS:

Below is a custom xenalyze debug log for the this. It translates to:
At some point gfn 3bdb7 was paged out, and later requested on vcpu 0. It
got a new mfn 1120b9 in p2m_mem_paging_prep on vcpu 5.
Later vcpu 3 called p2m_mem_paging_populate again, and it reset the new
mfn 1120b9. The xenpaging binary noticed that gfn 3bdb7 was already
populated and therefore could not prep/resume this gfn.
However, the first p2m_mem_paging_resume call ran into the new invalid
mfn and could not set the gfn state back to p2m_ram_rw. I have a change
to check the mfn in p2m_mem_paging_resume.
If p2m_mem_paging_populate had checked the p2mt properly,
p2m_mem_paging_resume would have succeed and everything could proceed.

19699656: 2073.683567637 -x--|--- d0v3 olh p2m_mem_paging_nominate gfn 3bdb7 mfn f5b7 flags 0 p2mt 1 ret 0
19699705: 2073.683601641 -x--|--- d0v3 olh p2m_mem_paging_evict gfn 3bdb7 mfn f5b7 flags 41 p2mt 9 ret 0
24430211: 2107.837821159 x-|||--| d0v0 olh p2m_mem_paging_populate_trace gfn 3bdb7 mfn ffffffff p2mt a line 159
24430212: 2107.837822145 x-|||--| d0v0 olh p2m_mem_paging_populate gfn 3bdb7 mfn ffffffff flags 20 p2mt a vcpu 0
24434493: 2107.845357003 |-x--||| d0v5 olh p2m_mem_paging_prep gfn 3bdb7 mfn 1120b9 ret 0
24434542: 2107.845409334 |-|--||x d0v3 olh p2m_mem_paging_populate_trace gfn 3bdb7 mfn 1120b9 p2mt b line 159
24434544: 2107.845411875 |-|--||x d0v3 olh p2m_mem_paging_populate gfn 3bdb7 mfn 1120b9 flags 20 p2mt b vcpu 3
24435286: 2107.856054450 |||--||x d0v3 olh p2m_mem_paging_populate_trace gfn 3bdb7 mfn ffffffff p2mt c line 159
24435287: 2107.856056334 |||--||x d0v3 olh p2m_mem_paging_populate gfn 3bdb7 mfn ffffffff flags 20 p2mt c vcpu 3
24435461: 2107.860575628 --x--||- d0v5 olh p2m_mem_paging_resume gfn 3bdb7 mfn ffffffff flags 0 p2mt c
24439829: 2107.864952416 -|--|||x d0v3 olh p2m_mem_paging_populate_trace gfn 3bdb7 mfn ffffffff p2mt c line 159
24439840: 2107.864955042 -|--|||x d0v3 olh p2m_mem_paging_populate gfn 3bdb7 mfn ffffffff flags 20 p2mt c vcpu 3
24449379: 2107.873837210 --|---|x d0v3 olh p2m_mem_paging_populate_trace gfn 3bdb7 mfn ffffffff p2mt c line 159
24449389: 2107.873839448 --|----x d0v3 olh p2m_mem_paging_populate gfn 3bdb7 mfn ffffffff flags 20 p2mt c vcpu 3
24455031: 2107.882722451 ---|-|-x d0v3 olh p2m_mem_paging_populate_trace gfn 3bdb7 mfn ffffffff p2mt c line 159
24455033: 2107.882724440 ---|-|-x d0v3 olh p2m_mem_paging_populate gfn 3bdb7 mfn ffffffff flags 20 p2mt c vcpu 3
24462725: 2107.896056767 -|---|-x d0v3 olh p2m_mem_paging_populate_trace gfn 3bdb7 mfn ffffffff p2mt c line 159
24462730: 2107.896059777 -|---|-x d0v3 olh p2m_mem_paging_populate gfn 3bdb7 mfn ffffffff flags 20 p2mt c vcpu 3
...

Re: [PATCH 07/16] xenpaging: populate only paged-out pages

[Olaf Hering]

On Tue, Nov 09, Olaf Hering wrote:

> The other thing is that p2m_mem_paging_populate must only invalidate the
> mfn if either the page is still in the process of being paged out, or if
> the page is already in the process of being paged in. The latter does
> not happen right now. The result is that the page may have state
> p2m_ram_paging_in and a new valid mfn. Then that new mfn is invalidated,
> the gfn content in the xenpaging process was already overwritten and
> there is no wakeup of the vcpu. This situation is not recoverable.
> p2m.c:p2m_mem_paging_populate needs a change to no call set_p2m_entry if
> the p2mt state is already p2m_ram_paging_in

And even this is not perfect.
p2m_mem_paging_populate() could be called again even if the p2m state was
just set to p2m_ram_rw in p2m_mem_paging_resume(). In this case the thread who calls
p2m_mem_paging_populate() cant make any progress anymore.
So in the end, p2m_mem_paging_populate() can only invalidate the mfn if
the p2mt happens to be p2m_ram_paged.

Olaf

[PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME

[Olaf Hering]

[-- Attachment #1: xen-unstable.xenpaging.MINIMUM_RESTART_TIME.patch --]
[-- Type: text/plain, Size: 612 bytes --]

Allow runlevel 6 as default runlevel.

Signed-off-by: Olaf Hering <olaf@aepfle.de>

---
 tools/python/xen/xend/XendConstants.py |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

--- xen-unstable.hg-4.1.22344.orig/tools/python/xen/xend/XendConstants.py
+++ xen-unstable.hg-4.1.22344/tools/python/xen/xend/XendConstants.py
@@ -92,7 +92,7 @@ DOM_STATES_OLD = [
 SHUTDOWN_TIMEOUT = (60.0 * 5)
 
 """Minimum time between domain restarts in seconds."""
-MINIMUM_RESTART_TIME = 60
+MINIMUM_RESTART_TIME = 5
 
 RESTART_IN_PROGRESS = 'xend/restart_in_progress'
 DUMPCORE_IN_PROGRESS = 'xend/dumpcore_in_progress'

Re: [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME

[Ian Jackson]

Olaf Hering writes ("[Xen-devel] [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME"):
> Allow runlevel 6 as default runlevel.
...
> -MINIMUM_RESTART_TIME = 60
> +MINIMUM_RESTART_TIME = 5

I don't understand what this change is doing in your xenpaging series;
nor does setting runlevel 6 as the default runlevel seem very
sensible other than perhaps for some kind of stress testing.

Perhaps this patch snuck into the series by mistake ?

Ian.

Re: [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME

[Olaf Hering]

On Wed, Nov 03, Ian Jackson wrote:

> Olaf Hering writes ("[Xen-devel] [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME"):
> > Allow runlevel 6 as default runlevel.
> ...
> > -MINIMUM_RESTART_TIME = 60
> > +MINIMUM_RESTART_TIME = 5
> 
> I don't understand what this change is doing in your xenpaging series;
> nor does setting runlevel 6 as the default runlevel seem very
> sensible other than perhaps for some kind of stress testing.
> 
> Perhaps this patch snuck into the series by mistake ?

Its there intentional in my series, just to get the startup code tested
automatically. The runtime code seems to work ok now that the granttable
part got fixed.

Last time I tried to get this variable into the vm config file, it did
not work as expected. Appearently its a global variable, not per guest.


And its value was changed from 20 (I think) to 60 without a good
changelog entry, not too long ago. For my testing, even 20 would have
been to long. If I use 'reboot' quickly in grub, guest runtime is around
7 seconds I think.


Too bad, this variable is only a bandaid because it doesnt know any
context of the guest. And it cant know any context. When is a guest in a
good or bad state? Should it panic, or crash? Its not easy to detect
a bad condition.

A better change would be to allow an override in the guest config file.


Olaf

Re: [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME

[Ian Jackson]

Olaf Hering writes ("Re: [Xen-devel] [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME"):
> Too bad, this variable is only a bandaid because it doesnt know any
> context of the guest. And it cant know any context. When is a guest in a
> good or bad state? Should it panic, or crash? Its not easy to detect
> a bad condition.

Absolutely.  Also some guests may reboot rapidly the first time (or
two) eg due to fsck, and thereafter be fine.  It ought to be more
sophisticated.

> A better change would be to allow an override in the guest config file.

Yes :-).

Ian.

Re: [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME

[Xavier Beaudouin]

Hi :)

Le 3 nov. 2010 à 17:55, Ian Jackson a écrit :

> Olaf Hering writes ("Re: [Xen-devel] [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME"):
>> Too bad, this variable is only a bandaid because it doesnt know any
>> context of the guest. And it cant know any context. When is a guest in a
>> good or bad state? Should it panic, or crash? Its not easy to detect
>> a bad condition.
> 
> Absolutely.  Also some guests may reboot rapidly the first time (or
> two) eg due to fsck, and thereafter be fine.  It ought to be more
> sophisticated.

Or even more, this is needed for example for Windows to register theirselves to KMS. Sometimes I need to add a sleep() to avoid this going into action.

>> A better change would be to allow an override in the guest config file.
> 
> Yes :-).


I'd really like this too :)

Cheers,
Xavier

--
Xavier Beaudouin - xb@soprive.net - http://www.soprive.net/
So Privé - Le premier acteur dédié au cloud computing en France
GPG Fingerprints : A6B2 D563 F93B A3AF C08A  CBAC 6BC6 79EB DCC9 9867

Re: [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME

[Ian Jackson]

Xavier Beaudouin writes ("Re: [Xen-devel] [PATCH 08/16] xenpaging: reduce MINIMUM_RESTART_TIME"):
> Or even more, this is needed for example for Windows to register
> theirselves to KMS. Sometimes I need to add a sleep() to avoid this
> going into action.

Also, I notice that xl does not yet have any machinery for detecting
infinite guest reboots.  I suggest that we take this opportunity to
invent a new mechanism for that, and it could be configurable.

xl could record the times of the last N restarts in an array, quite
easily, and spot when there have been more than N restarts in time T
(for default values of N=5 and T= fifteen minutes, perhaps).

Ian.

[PATCH 09/16] xenpaging: start xenpaging via config option

[Olaf Hering]

[-- Attachment #1: xen-unstable.xenpaging.autostart.patch --]
[-- Type: text/plain, Size: 9452 bytes --]

Start xenpaging via config option.

TODO: add config option for pagefile directory
TODO: add libxl support
TODO: parse config values like 42K, 42M, 42G, 42%

Signed-off-by: Olaf Hering <olaf@aepfle.de>

---
 tools/examples/xmexample.hvm            |    3 +
 tools/python/README.XendConfig          |    1 
 tools/python/README.sxpcfg              |    1 
 tools/python/xen/xend/XendConfig.py     |    3 +
 tools/python/xen/xend/XendDomainInfo.py |    6 ++
 tools/python/xen/xend/image.py          |   87 ++++++++++++++++++++++++++++++++
 tools/python/xen/xm/create.py           |    5 +
 tools/python/xen/xm/xenapi_create.py    |    1 
 8 files changed, 107 insertions(+)

--- xen-unstable.hg-4.1.22344.orig/tools/examples/xmexample.hvm
+++ xen-unstable.hg-4.1.22344/tools/examples/xmexample.hvm
@@ -127,6 +127,9 @@ disk = [ 'file:/var/images/min-el3-i386.
 # Device Model to be used
 device_model = 'qemu-dm'
 
+# xenpaging, number of pages
+xenpaging = 42
+
 #-----------------------------------------------------------------------------
 # boot on floppy (a), hard disk (c), Network (n) or CD-ROM (d) 
 # default: hard disk, cd-rom, floppy
--- xen-unstable.hg-4.1.22344.orig/tools/python/README.XendConfig
+++ xen-unstable.hg-4.1.22344/tools/python/README.XendConfig
@@ -120,6 +120,7 @@ otherConfig
                                 image.vncdisplay
                                 image.vncunused
                                 image.hvm.device_model
+                                image.hvm.xenpaging
                                 image.hvm.display
                                 image.hvm.xauthority
                                 image.hvm.vncconsole
--- xen-unstable.hg-4.1.22344.orig/tools/python/README.sxpcfg
+++ xen-unstable.hg-4.1.22344/tools/python/README.sxpcfg
@@ -51,6 +51,7 @@ image
   - vncunused
   (HVM)
   - device_model
+  - xenpaging
   - display
   - xauthority
   - vncconsole
--- xen-unstable.hg-4.1.22344.orig/tools/python/xen/xend/XendConfig.py
+++ xen-unstable.hg-4.1.22344/tools/python/xen/xend/XendConfig.py
@@ -147,6 +147,7 @@ XENAPI_PLATFORM_CFG_TYPES = {
     'apic': int,
     'boot': str,
     'device_model': str,
+    'xenpaging': int,
     'loader': str,
     'display' : str,
     'fda': str,
@@ -508,6 +509,8 @@ class XendConfig(dict):
             self['platform']['nomigrate'] = 0
 
         if self.is_hvm():
+            if 'xenpaging' not in self['platform']:
+                self['platform']['xenpaging'] = None
             if 'timer_mode' not in self['platform']:
                 self['platform']['timer_mode'] = 1
             if 'viridian' not in self['platform']:
--- xen-unstable.hg-4.1.22344.orig/tools/python/xen/xend/XendDomainInfo.py
+++ xen-unstable.hg-4.1.22344/tools/python/xen/xend/XendDomainInfo.py
@@ -2390,6 +2390,7 @@ class XendDomainInfo:
 
         if self.image:
             self.image.createDeviceModel()
+            self.image.createXenPaging()
 
         #if have pass-through devs, need the virtual pci slots info from qemu
         self.pci_device_configure_boot()
@@ -2402,6 +2403,11 @@ class XendDomainInfo:
                 self.image.destroyDeviceModel()
             except Exception, e:
                 log.exception("Device model destroy failed %s" % str(e))
+            try:
+                log.debug("stopping xenpaging")
+                self.image.destroyXenPaging()
+            except Exception, e:
+                log.exception("stopping xenpaging failed %s" % str(e))
         else:
             log.debug("No device model")
 
--- xen-unstable.hg-4.1.22344.orig/tools/python/xen/xend/image.py
+++ xen-unstable.hg-4.1.22344/tools/python/xen/xend/image.py
@@ -122,12 +122,14 @@ class ImageHandler:
         self.vm.permissionsVm("image/cmdline", { 'dom': self.vm.getDomid(), 'read': True } )
 
         self.device_model = vmConfig['platform'].get('device_model')
+        self.xenpaging = vmConfig['platform'].get('xenpaging')
 
         self.display = vmConfig['platform'].get('display')
         self.xauthority = vmConfig['platform'].get('xauthority')
         self.vncconsole = int(vmConfig['platform'].get('vncconsole', 0))
         self.dmargs = self.parseDeviceModelArgs(vmConfig)
         self.pid = None
+        self.xenpaging_pid = None
         rtc_timeoffset = int(vmConfig['platform'].get('rtc_timeoffset', 0))
         if int(vmConfig['platform'].get('localtime', 0)):
             if time.localtime(time.time())[8]:
@@ -392,6 +394,91 @@ class ImageHandler:
         sentinel_fifos_inuse[sentinel_path_fifo] = 1
         self.sentinel_path_fifo = sentinel_path_fifo
 
+    def createXenPaging(self):
+        if self.xenpaging is None:
+            return
+        if self.xenpaging == 0:
+            return
+        if self.xenpaging_pid:
+            return
+        xenpaging_bin = auxbin.pathTo("xenpaging")
+        args = [xenpaging_bin]
+        args = args + ([ "%d" % self.vm.getDomid()])
+        args = args + ([ "%s" % self.xenpaging])
+        env = dict(os.environ)
+        self.xenpaging_logfile = "/var/log/xen/xenpaging-%s.log" %  str(self.vm.info['name_label'])
+        logfile_mode = os.O_WRONLY|os.O_CREAT|os.O_APPEND|os.O_TRUNC
+        null = os.open("/dev/null", os.O_RDONLY)
+        logfd = os.open(self.xenpaging_logfile, logfile_mode, 0644)
+        sys.stderr.flush()
+        contract = osdep.prefork("%s:%d" % (self.vm.getName(), self.vm.getDomid()))
+        xenpaging_pid = os.fork()
+        if xenpaging_pid == 0: #child
+            try:
+                xenpaging_dir = "/var/lib/xen/xenpaging"
+                osdep.postfork(contract)
+                os.dup2(null, 0)
+                os.dup2(logfd, 1)
+                os.dup2(logfd, 2)
+                try:
+                    os.mkdir(xenpaging_dir)
+                except:
+                    log.info("mkdir %s failed" % xenpaging_dir)
+                    pass
+                try:
+                    os.chdir(xenpaging_dir)
+                except:
+                    log.warn("chdir %s failed" % xenpaging_dir)
+                try:
+                    log.info("starting %s" % args)
+                    os.execve(xenpaging_bin, args, env)
+                except Exception, e:
+                    print >>sys.stderr, (
+                        'failed to execute xenpaging: %s: %s' %
+                        xenpaging_bin, utils.exception_string(e))
+                    os._exit(126)
+            except Exception, e:
+                log.warn("staring xenpaging in %s failed" % xenpaging_dir)
+                os._exit(127)
+        else:
+            osdep.postfork(contract, abandon=True)
+            self.xenpaging_pid = xenpaging_pid
+            os.close(null)
+            os.close(logfd)
+
+    def destroyXenPaging(self):
+        if self.xenpaging is None:
+            return
+        if self.xenpaging_pid:
+            try:
+                os.kill(self.xenpaging_pid, signal.SIGHUP)
+            except OSError, exn:
+                log.exception(exn)
+            for i in xrange(100):
+                try:
+                    (p, rv) = os.waitpid(self.xenpaging_pid, os.WNOHANG)
+                    if p == self.xenpaging_pid:
+                        break
+                except OSError:
+                    # This is expected if Xend has been restarted within
+                    # the life of this domain.  In this case, we can kill
+                    # the process, but we can't wait for it because it's
+                    # not our child. We continue this loop, and after it is
+                    # terminated make really sure the process is going away
+                    # (SIGKILL).
+                    pass
+                time.sleep(0.1)
+            else:
+                log.warning("xenpaging %d took more than 10s "
+                            "to terminate: sending SIGKILL" % self.xenpaging_pid)
+                try:
+                    os.kill(self.xenpaging_pid, signal.SIGKILL)
+                    os.waitpid(self.xenpaging_pid, 0)
+                except OSError:
+                    # This happens if the process doesn't exist.
+                    pass
+        self.xenpaging_pid = None
+
     def createDeviceModel(self, restore = False):
         if self.device_model is None:
             return
--- xen-unstable.hg-4.1.22344.orig/tools/python/xen/xm/create.py
+++ xen-unstable.hg-4.1.22344/tools/python/xen/xm/create.py
@@ -491,6 +491,10 @@ gopts.var('nfs_root', val="PATH",
           fn=set_value, default=None,
           use="Set the path of the root NFS directory.")
 
+gopts.var('xenpaging', val='NUM',
+          fn=set_int, default=None,
+          use="Number of pages to swap.")
+
 gopts.var('device_model', val='FILE',
           fn=set_value, default=None,
           use="Path to device model program.")
@@ -1076,6 +1080,7 @@ def configure_hvm(config_image, vals):
     args = [ 'acpi', 'apic',
              'boot',
              'cpuid', 'cpuid_check',
+             'xenpaging',
              'device_model', 'display',
              'fda', 'fdb',
              'gfx_passthru', 'guest_os_type',
--- xen-unstable.hg-4.1.22344.orig/tools/python/xen/xm/xenapi_create.py
+++ xen-unstable.hg-4.1.22344/tools/python/xen/xm/xenapi_create.py
@@ -1085,6 +1085,7 @@ class sxp2xml:
             'acpi',
             'apic',
             'boot',
+            'xenpaging',
             'device_model',
             'loader',
             'fda',

[PATCH 10/16] xenpaging: add signal handling

[Olaf Hering]

[-- Attachment #1: xen-unstable.xenpaging.signal_handling.patch --]
[-- Type: text/plain, Size: 4167 bytes --]

Leave paging loop if xenpaging gets a signal.
Remove paging file on exit.

Signed-off-by: Olaf Hering <olaf@aepfle.de>

---
 tools/xenpaging/xenpaging.c |   39 +++++++++++++++++++++++++++++++--------
 1 file changed, 31 insertions(+), 8 deletions(-)

--- xen-unstable.hg-4.1.22344.orig/tools/xenpaging/xenpaging.c
+++ xen-unstable.hg-4.1.22344/tools/xenpaging/xenpaging.c
@@ -25,6 +25,7 @@
 #include <stdlib.h>
 #include <stdarg.h>
 #include <time.h>
+#include <signal.h>
 #include <xc_private.h>
 
 #include <xen/mem_event.h>
@@ -43,6 +44,11 @@
 #define DPRINTF(...) ((void)0)
 #endif
 
+static int interrupted;
+static void close_handler(int sig)
+{
+    interrupted = sig;
+}
 
 static void *init_page(void)
 {
@@ -248,7 +254,6 @@ int xenpaging_teardown(xc_interface *xch
     if ( rc != 0 )
     {
         ERROR("Error tearing down domain paging in xen");
-        goto err;
     }
 
     /* Unbind VIRQ */
@@ -256,7 +261,6 @@ int xenpaging_teardown(xc_interface *xch
     if ( rc != 0 )
     {
         ERROR("Error unbinding event port");
-        goto err;
     }
     paging->mem_event.port = -1;
 
@@ -265,7 +269,6 @@ int xenpaging_teardown(xc_interface *xch
     if ( rc != 0 )
     {
         ERROR("Error closing event channel");
-        goto err;
     }
     paging->mem_event.xce_handle = -1;
     
@@ -274,7 +277,6 @@ int xenpaging_teardown(xc_interface *xch
     if ( rc != 0 )
     {
         ERROR("Error closing connection to xen");
-        goto err;
     }
     paging->xc_handle = NULL;
 
@@ -380,7 +382,7 @@ int xenpaging_evict_page(xc_interface *x
     return ret;
 }
 
-int xenpaging_resume_page(xenpaging_t *paging, mem_event_response_t *rsp)
+static int xenpaging_resume_page(xenpaging_t *paging, mem_event_response_t *rsp)
 {
     int ret;
 
@@ -461,6 +463,11 @@ static int evict_victim(xc_interface *xc
             goto out;
         }
 
+        if ( interrupted )
+        {
+            ret = -EINTR;
+            goto out;
+        }
         ret = xc_mem_paging_nominate(paging->xc_handle,
                                      paging->mem_event.domain_id, victim->gfn);
         if ( ret == 0 )
@@ -485,6 +492,7 @@ static int evict_victim(xc_interface *xc
 
 int main(int argc, char *argv[])
 {
+    struct sigaction act;
     domid_t domain_id;
     int num_pages;
     xenpaging_t *paging;
@@ -520,7 +528,7 @@ int main(int argc, char *argv[])
     if ( paging == NULL )
     {
         ERROR("Error initialising paging");
-        goto out;
+        return 1;
     }
 
     /* Open file */
@@ -529,9 +537,18 @@ int main(int argc, char *argv[])
     if ( fd < 0 )
     {
         perror("failed to open file");
-        return -1;
+        return 2;
     }
 
+    /* ensure that if we get a signal, we'll do cleanup, then exit */
+    act.sa_handler = close_handler;
+    act.sa_flags = 0;
+    sigemptyset(&act.sa_mask);
+    sigaction(SIGHUP,  &act, NULL);
+    sigaction(SIGTERM, &act, NULL);
+    sigaction(SIGINT,  &act, NULL);
+    sigaction(SIGALRM, &act, NULL);
+
     /* Evict pages */
     memset(victims, 0, sizeof(xenpaging_victim_t) * num_pages);
     for ( i = 0; i < num_pages; i++ )
@@ -539,6 +556,8 @@ int main(int argc, char *argv[])
         rc = evict_victim(xch, paging, domain_id, &victims[i], fd, i);
         if ( rc == -ENOSPC )
             break;
+        if ( rc == -EINTR )
+            break;
         if ( i % 100 == 0 )
             DPRINTF("%d pages evicted\n", i);
     }
@@ -546,7 +565,7 @@ int main(int argc, char *argv[])
     DPRINTF("pages evicted\n");
 
     /* Swap pages in and out */
-    while ( 1 )
+    while ( !interrupted )
     {
         /* Wait for Xen to signal that a page needs paged in */
         rc = xc_wait_for_event_or_timeout(xch, paging->mem_event.xce_handle, 100);
@@ -647,8 +666,11 @@ int main(int argc, char *argv[])
             }
         }
     }
+    DPRINTF("xenpaging got signal %d\n", interrupted);
 
  out:
+    unlink(filename);
+    close(fd);
     free(victims);
 
     /* Tear down domain paging */
@@ -661,6 +683,7 @@ int main(int argc, char *argv[])
 
     xc_interface_close(xch);
 
+    DPRINTF("xenpaging exit code %d\n", rc);
     return rc;
 }

[PATCH 11/16] xenpaging: increase recently used pages from 4MB to 64MB

[Olaf Hering]

[-- Attachment #1: xen-unstable.xenpaging.MRU_SIZE.patch --]
[-- Type: text/plain, Size: 575 bytes --]

Increase recently used pages from 4MB to 64MB.
Keeping more pages in memory allows the guest to make more progress if the
paging file spans the entire guest memory.

Signed-off-by: Olaf Hering <olaf@aepfle.de>

---
 tools/xenpaging/policy_default.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

--- xen-unstable.hg-4.1.22344.orig/tools/xenpaging/policy_default.c
+++ xen-unstable.hg-4.1.22344/tools/xenpaging/policy_default.c
@@ -26,7 +26,7 @@
 #include "policy.h"
 
 
-#define MRU_SIZE 1024
+#define MRU_SIZE (1024 * 16)
 
 
 static unsigned long mru[MRU_SIZE];

[PATCH 12/16] xenpaging: print info when free request slots drop below 3

[Olaf Hering]

[-- Attachment #1: xen-unstable.xenpaging.mem_event_check_ring-free_requests.patch --]
[-- Type: text/plain, Size: 842 bytes --]

Add debugging aid to free request slots in the ring buffer.
It should not happen that the ring gets full, print info anyway if it happens. 

Signed-off-by: Olaf Hering <olaf@aepfle.de>

---
 xen/arch/x86/mm/mem_event.c |    5 +++++
 1 file changed, 5 insertions(+)

--- xen-unstable.hg-4.1.22344.orig/xen/arch/x86/mm/mem_event.c
+++ xen-unstable.hg-4.1.22344/xen/arch/x86/mm/mem_event.c
@@ -152,6 +152,11 @@ int mem_event_check_ring(struct domain *
     mem_event_ring_lock(d);
 
     free_requests = RING_FREE_REQUESTS(&d->mem_event.front_ring);
+    if ( unlikely(free_requests < 3) )
+    {
+        gdprintk(XENLOG_INFO, "free request slots: %d\n", free_requests);
+        WARN_ON(free_requests == 0);
+    }
     ring_full = free_requests < MEM_EVENT_RING_THRESHOLD;
 
     if ( (curr->domain->domain_id == d->domain_id) && ring_full )

[PATCH 13/16] xenpaging: prevent page-out of first 16MB

[Olaf Hering]

[-- Attachment #1: xen-unstable.xenpaging.blacklist.patch --]
[-- Type: text/plain, Size: 814 bytes --]

This is more a workaround than a bugfix:
Don't page out first 16MB of memory.
When the BIOS does its initialization process and xenpaging removes pages,
crashes will occour due to lack of support of xenpaging.

Signed-off-by: Olaf Hering <olaf@aepfle.de>

---
 tools/xenpaging/policy_default.c |    5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

--- xen-unstable.hg-4.1.22344.orig/tools/xenpaging/policy_default.c
+++ xen-unstable.hg-4.1.22344/tools/xenpaging/policy_default.c
@@ -60,8 +60,9 @@ int policy_init(xenpaging_t *paging)
     for ( i = 0; i < MRU_SIZE; i++ )
         mru[i] = INVALID_MFN;
 
-    /* Don't page out page 0 */
-    set_bit(0, bitmap);
+    /* Don't page out first 16MB */
+    for ( i = 0; i < ((16*1024*1024)/4096); i++ )
+        set_bit(i, bitmap);
 
  out:
     return rc;

[PATCH 14/16] xenpaging: add dynamic startup delay for xenpaging

[Olaf Hering]

[-- Attachment #1: xen-unstable.xenpaging.autostart_delay.patch --]
[-- Type: text/plain, Size: 4588 bytes --]

This is a debug helper. Since the xenpaging support is still fragile, run
xenpaging at different stages in the bootprocess. Different delays will trigger
more bugs. This implementation starts without delay for 5 reboots, then
increments the delay by 0.1 seconds It uses xenstore for presistant storage of
delay values

TODO: find the correct place to remove the xenstore directory when the guest is shutdown or crashed

Signed-off-by: Olaf Hering <olaf@aepfle.de>

---
 tools/python/xen/xend/image.py |   32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

--- xen-unstable.hg-4.1.22344.orig/tools/python/xen/xend/image.py
+++ xen-unstable.hg-4.1.22344/tools/python/xen/xend/image.py
@@ -123,6 +123,18 @@ class ImageHandler:
 
         self.device_model = vmConfig['platform'].get('device_model')
         self.xenpaging = vmConfig['platform'].get('xenpaging')
+        self.xenpaging_delay = xstransact.Read("/local/domain/0/xenpaging/%s/xenpaging_delay" % self.vm.info['name_label'])
+        if self.xenpaging_delay == None:
+            log.warn("XXX creating /local/domain/0/xenpaging/%s" % self.vm.info['name_label'])
+            xstransact.Mkdir("/local/domain/0/xenpaging/%s" % self.vm.info['name_label'])
+            xstransact.Store("/local/domain/0/xenpaging/%s" % self.vm.info['name_label'], ('xenpaging_delay', '7.0'))
+            xstransact.Store("/local/domain/0/xenpaging/%s" % self.vm.info['name_label'], ('xenpaging_delay_inc', '0.1'))
+            xstransact.Store("/local/domain/0/xenpaging/%s" % self.vm.info['name_label'], ('xenpaging_delay_use', '5'))
+            xstransact.Store("/local/domain/0/xenpaging/%s" % self.vm.info['name_label'], ('xenpaging_delay_used', '0'))
+        self.xenpaging_delay = float(xstransact.Read("/local/domain/0/xenpaging/%s/xenpaging_delay" % self.vm.info['name_label']))
+        self.xenpaging_delay_inc = float(xstransact.Read("/local/domain/0/xenpaging/%s/xenpaging_delay_inc" % self.vm.info['name_label']))
+        self.xenpaging_delay_use = int(xstransact.Read("/local/domain/0/xenpaging/%s/xenpaging_delay_use" % self.vm.info['name_label']))
+        self.xenpaging_delay_used = int(xstransact.Read("/local/domain/0/xenpaging/%s/xenpaging_delay_used" % self.vm.info['name_label']))
 
         self.display = vmConfig['platform'].get('display')
         self.xauthority = vmConfig['platform'].get('xauthority')
@@ -401,6 +413,17 @@ class ImageHandler:
             return
         if self.xenpaging_pid:
             return
+        if self.xenpaging_delay_used < self.xenpaging_delay_use:
+            self.xenpaging_delay_used += 1
+        else:
+            self.xenpaging_delay_used = 0
+            self.xenpaging_delay += self.xenpaging_delay_inc
+        log.info("delay_used %s" % self.xenpaging_delay_used)
+        log.info("delay_use %s" % self.xenpaging_delay_use)
+        log.info("delay %s" % self.xenpaging_delay)
+        log.info("delay_inc %s" % self.xenpaging_delay_inc)
+        xstransact.Store("/local/domain/0/xenpaging/%s" % self.vm.info['name_label'], ('xenpaging_delay', self.xenpaging_delay))
+        xstransact.Store("/local/domain/0/xenpaging/%s" % self.vm.info['name_label'], ('xenpaging_delay_used', self.xenpaging_delay_used))
         xenpaging_bin = auxbin.pathTo("xenpaging")
         args = [xenpaging_bin]
         args = args + ([ "%d" % self.vm.getDomid()])
@@ -430,6 +453,9 @@ class ImageHandler:
                 except:
                     log.warn("chdir %s failed" % xenpaging_dir)
                 try:
+                    if self.xenpaging_delay != 0.0:
+                        log.info("delaying xenpaging startup %s seconds ..." % self.xenpaging_delay)
+                        time.sleep(self.xenpaging_delay)
                     log.info("starting %s" % args)
                     os.execve(xenpaging_bin, args, env)
                 except Exception, e:
@@ -445,10 +471,16 @@ class ImageHandler:
             self.xenpaging_pid = xenpaging_pid
             os.close(null)
             os.close(logfd)
+            if self.xenpaging_delay == 0.0:
+                log.warn("waiting for xenpaging ...")
+                time.sleep(22)
+                log.warn("waiting for xenpaging done.")
 
     def destroyXenPaging(self):
         if self.xenpaging is None:
             return
+        # FIXME find correct place for guest shutdown or crash
+        #xstransact.Remove("/local/domain/0/xenpaging/%s" % self.vm.info['name_label'])
         if self.xenpaging_pid:
             try:
                 os.kill(self.xenpaging_pid, signal.SIGHUP)

[PATCH 15/16] xenpaging: print p2mt for already paged-in pages

[Olaf Hering]

[-- Attachment #1: xen-unstable.xenpaging.page_already_populated.patch --]
[-- Type: text/plain, Size: 824 bytes --]

Add more debug output, print p2mt for pages which were requested more than once.

Signed-off-by: Olaf Hering <olaf@aepfle.de>

---
 tools/xenpaging/xenpaging.c |    2 ++
 1 file changed, 2 insertions(+)

--- xen-unstable.hg-4.1.22344.orig/tools/xenpaging/xenpaging.c
+++ xen-unstable.hg-4.1.22344/tools/xenpaging/xenpaging.c
@@ -642,8 +642,10 @@ int main(int argc, char *argv[])
             else
             {
                 DPRINTF("page already populated (domain = %d; vcpu = %d;"
+                        " p2mt = %x;"
                         " gfn = %"PRIx64"; paused = %"PRId64")\n",
                         paging->mem_event.domain_id, req.vcpu_id,
+                        req.p2mt,
                         req.gfn, req.flags & MEM_EVENT_FLAG_VCPU_PAUSED);
 
                 /* Tell Xen to resume the vcpu */

[PATCH 16/16] xenpaging: (sparse) documenation

[Olaf Hering]

[-- Attachment #1: xen-unstable.xenpaging.doc.patch --]
[-- Type: text/plain, Size: 1775 bytes --]

Write up some sparse documentation about xenpaging usage.

Signed-off-by: Olaf Hering <olaf@aepfle.de>

---
 docs/misc/xenpaging.txt |   48 ++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

--- /dev/null
+++ xen-unstable.hg-4.1.22344/docs/misc/xenpaging.txt
@@ -0,0 +1,48 @@
+Warning:
+
+The xenpaging code is new and not fully debugged.
+Usage of xenpaging can crash Xen or cause severe data corruption in the
+guest memory and its filesystems!
+
+Description:
+
+xenpaging writes memory pages of a given guest to a file and moves the
+pages back to the pool of available memory.  Once the guests wants to
+access the paged-out memory, the page is read from disk and placed into
+memory.  This allows the sum of all running guests to use more memory
+than physically available on the host.
+
+Usage:
+
+Once the guest is running, run xenpaging with the guest_id and the
+number of pages to page-out:
+
+  chdir /var/lib/xen/xenpaging
+  xenpaging <guest_id>  <number_of_pages>
+
+To obtain the guest_id, run 'xm list'.
+xenpaging will write the pagefile to the current directory.
+Example with 128MB pagefile on guest 1:
+
+  xenpaging 1 32768
+
+Caution: stopping xenpaging manually will cause the guest to stall or
+crash because the paged-out memory is not written back into the guest!
+
+After a reboot of a guest, its guest_id changes, the current xenpaging
+binary has no target anymore. To automate restarting of xenpaging after
+guest reboot, specify the number if pages in the guest configuration
+file /etc/xen/vm/<guest_name>:
+
+xenpaging=32768
+
+Redo the guest with 'xm create /etc/xen/vm/<guest_name>' to activate the
+changes.
+
+
+Todo:
+- implement stopping of xenpaging
+- implement/test live migration
+
+
+# vim: tw=72

Re: [PATCH 00/16] xenpaging changes for xen-unstable

[Olaf Hering]

On Tue, Nov 02, Olaf Hering wrote:

> But the patch to handle guest_remove_page does not work as expected, the guest
> will hang. Any help to get this fixed without retry loops is appreciated.

If the vcpu is stopped and later resumed, similar to a
p2m_mem_paging_populate -> p2m_mem_paging_prep -> p2m_mem_paging_resume
cycle, the guest will not stall. I will send an updated patch.

Olaf

[PATCH 17/16] xenpaging: notify policy only on resume

[Olaf Hering]

If a page is requested more than once, the policy is also notified more
than once about the page-in. However, a page-in happens only once. Any
further resume will only unpause the other vcpu. The multiple notify
will put the page into the mru list multiple times and it will unlock
other already resumed pages too early. In the worst case, a page that
was just resumed can be evicted right away, causing a deadlock in the
guest.  

Signed-off-by: Olaf Hering <olaf@aepfle.de>

---
This happend since some time already, today I found the cause.

 tools/xenpaging/xenpaging.c |    9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

--- xen-unstable.hg-4.1.22353.orig/tools/xenpaging/xenpaging.c
+++ xen-unstable.hg-4.1.22353/tools/xenpaging/xenpaging.c
@@ -382,7 +382,7 @@ int xenpaging_evict_page(xc_interface *x
     return ret;
 }
 
-static int xenpaging_resume_page(xenpaging_t *paging, mem_event_response_t *rsp)
+static int xenpaging_resume_page(xenpaging_t *paging, mem_event_response_t *rsp, int notify_policy)
 {
     int ret;
 
@@ -392,7 +392,8 @@ static int xenpaging_resume_page(xenpagi
         goto out;
 
     /* Notify policy of page being paged in */
-    policy_notify_paged_in(paging->mem_event.domain_id, rsp->gfn);
+    if ( notify_policy )
+        policy_notify_paged_in(paging->mem_event.domain_id, rsp->gfn);
 
     /* Tell Xen page is ready */
     ret = xc_mem_paging_resume(paging->xc_handle, paging->mem_event.domain_id,
@@ -626,7 +627,7 @@ int main(int argc, char *argv[])
                 rsp.vcpu_id = req.vcpu_id;
                 rsp.flags = req.flags;
 
-                rc = xenpaging_resume_page(paging, &rsp);
+                rc = xenpaging_resume_page(paging, &rsp, 1);
                 if ( rc != 0 )
                 {
                     ERROR("Error resuming page");
@@ -655,7 +656,7 @@ int main(int argc, char *argv[])
                     rsp.vcpu_id = req.vcpu_id;
                     rsp.flags = req.flags;
 
-                    rc = xenpaging_resume_page(paging, &rsp);
+                    rc = xenpaging_resume_page(paging, &rsp, 0);
                     if ( rc != 0 )
                     {
                         ERROR("Error resuming");