Re: [PATCH 06/16] xenpaging: drop paged pages in guest_remove_page

[Olaf Hering <olaf@aepfle.de>]

Simply drop paged-pages in guest_remove_page(), and notify xenpaging to
drop reference to the gfn.

Signed-off-by: Olaf Hering <olaf@aepfle.de>

---
v2:
 resume dropped page to unpause vcpus

 tools/xenpaging/xenpaging.c    |   17 ++++++++++----
 xen/arch/x86/mm/p2m.c          |   47 ++++++++++++++++++++++++++++++++++-------
 xen/common/memory.c            |    6 +++++
 xen/include/asm-x86/p2m.h      |    4 +++
 xen/include/public/mem_event.h |    1 
 5 files changed, 63 insertions(+), 12 deletions(-)

--- xen-unstable.hg-4.1.22353.orig/tools/xenpaging/xenpaging.c
+++ xen-unstable.hg-4.1.22353/tools/xenpaging/xenpaging.c
@@ -586,12 +586,19 @@ int main(int argc, char *argv[])
                     goto out;
                 }
                 
-                /* Populate the page */
-                rc = xenpaging_populate_page(xch, paging, &req.gfn, fd, i);
-                if ( rc != 0 )
+                if ( req.flags & MEM_EVENT_FLAG_DROP_PAGE )
                 {
-                    ERROR("Error populating page");
-                    goto out;
+                    DPRINTF("Dropping page %"PRIx64" p2mt %x\n", req.gfn, req.p2mt);
+                }
+                else
+                {
+                    /* Populate the page */
+                    rc = xenpaging_populate_page(xch, paging, &req.gfn, fd, i);
+                    if ( rc != 0 )
+                    {
+                        ERROR("Error populating page");
+                        goto out;
+                    }
                 }
 
                 /* Prepare the response */
--- xen-unstable.hg-4.1.22353.orig/xen/arch/x86/mm/p2m.c
+++ xen-unstable.hg-4.1.22353/xen/arch/x86/mm/p2m.c
@@ -2195,12 +2195,15 @@ p2m_remove_page(struct p2m_domain *p2m,
 
     P2M_DEBUG("removing gfn=%#lx mfn=%#lx\n", gfn, mfn);
 
-    for ( i = 0; i < (1UL << page_order); i++ )
+    if ( mfn_valid(_mfn(mfn)) )
     {
-        mfn_return = p2m->get_entry(p2m, gfn + i, &t, p2m_query);
-        if ( !p2m_is_grant(t) )
-            set_gpfn_from_mfn(mfn+i, INVALID_M2P_ENTRY);
-        ASSERT( !p2m_is_valid(t) || mfn + i == mfn_x(mfn_return) );
+        for ( i = 0; i < (1UL << page_order); i++ )
+        {
+            mfn_return = p2m->get_entry(p2m, gfn + i, &t, p2m_query);
+            if ( !p2m_is_grant(t) )
+                set_gpfn_from_mfn(mfn+i, INVALID_M2P_ENTRY);
+            ASSERT( !p2m_is_valid(t) || mfn + i == mfn_x(mfn_return) );
+        }
     }
     set_p2m_entry(p2m, gfn, _mfn(INVALID_MFN), page_order, p2m_invalid);
 }
@@ -2751,6 +2754,36 @@ int p2m_mem_paging_evict(struct p2m_doma
     return 0;
 }
 
+void p2m_mem_paging_drop_page(struct p2m_domain *p2m, unsigned long gfn)
+{
+    struct vcpu *v = current;
+    mem_event_request_t req;
+    p2m_type_t p2mt;
+    struct domain *d = p2m->domain;
+
+    memset(&req, 0, sizeof(req));
+
+    /* Check that there's space on the ring for this request */
+    if ( mem_event_check_ring(d) )
+        return;
+
+    gfn_to_mfn(p2m, gfn, &p2mt);
+    /* Pause domain */
+    if ( v->domain->domain_id == d->domain_id )
+    {
+        vcpu_pause_nosync(v);
+        req.flags |= MEM_EVENT_FLAG_VCPU_PAUSED;
+    }
+
+    /* Send request to pager */
+    req.flags |= MEM_EVENT_FLAG_DROP_PAGE;
+    req.gfn = gfn;
+    req.p2mt = p2mt;
+    req.vcpu_id = v->vcpu_id;
+
+    mem_event_put_request(d, &req);
+}
+
 void p2m_mem_paging_populate(struct p2m_domain *p2m, unsigned long gfn)
 {
     struct vcpu *v = current;
--- xen-unstable.hg-4.1.22353.orig/xen/common/memory.c
+++ xen-unstable.hg-4.1.22353/xen/common/memory.c
@@ -162,6 +162,12 @@ int guest_remove_page(struct domain *d,
 
 #ifdef CONFIG_X86
     mfn = mfn_x(gfn_to_mfn(p2m_get_hostp2m(d), gmfn, &p2mt)); 
+    if ( unlikely(p2m_is_paging(p2mt)) )
+    {
+        guest_physmap_remove_page(d, gmfn, mfn, 0);
+        p2m_mem_paging_drop_page(p2m_get_hostp2m(d), gmfn);
+        return 1;
+    }
 #else
     mfn = gmfn_to_mfn(d, gmfn);
 #endif
--- xen-unstable.hg-4.1.22353.orig/xen/include/asm-x86/p2m.h
+++ xen-unstable.hg-4.1.22353/xen/include/asm-x86/p2m.h
@@ -480,6 +480,8 @@ int set_shared_p2m_entry(struct p2m_doma
 int p2m_mem_paging_nominate(struct p2m_domain *p2m, unsigned long gfn);
 /* Evict a frame */
 int p2m_mem_paging_evict(struct p2m_domain *p2m, unsigned long gfn);
+/* Tell xenpaging to drop a paged out frame */
+void p2m_mem_paging_drop_page(struct p2m_domain *p2m, unsigned long gfn);
 /* Start populating a paged out frame */
 void p2m_mem_paging_populate(struct p2m_domain *p2m, unsigned long gfn);
 /* Prepare the p2m for paging a frame in */
@@ -487,6 +489,8 @@ int p2m_mem_paging_prep(struct p2m_domai
 /* Resume normal operation (in case a domain was paused) */
 void p2m_mem_paging_resume(struct p2m_domain *p2m);
 #else
+static inline void p2m_mem_paging_drop_page(struct p2m_domain *p2m, unsigned long gfn)
+{ }
 static inline void p2m_mem_paging_populate(struct p2m_domain *p2m, unsigned long gfn)
 { }
 #endif
--- xen-unstable.hg-4.1.22353.orig/xen/include/public/mem_event.h
+++ xen-unstable.hg-4.1.22353/xen/include/public/mem_event.h
@@ -37,6 +37,7 @@
 #define MEM_EVENT_FLAG_VCPU_PAUSED  (1 << 0)
 #define MEM_EVENT_FLAG_DOM_PAUSED   (1 << 1)
 #define MEM_EVENT_FLAG_OUT_OF_MEM   (1 << 2)
+#define MEM_EVENT_FLAG_DROP_PAGE    (1 << 3)
 
 
 typedef struct mem_event_shared_page {