[PATCH 0 of 7] guest checkpointing patches take 3

[PATCH 0 of 7] guest checkpointing patches take 3

[Brendan Cully]

Here's version 3 of the checkpointing code. It retains backward
compatibility with older kernels and tools by setting the return code
of the suspend hypercall to 1 in the parent domain (a case never seen
by older versions) and leaving it set to 0 in the child. It also adds
an untested 'unsuspend' hook that devices can use to roll back any
suspend prep work they may have done.

It still advertises checkpointability via a feature flag in
xenstore. I'm putting the patches out now because the elfnote approach
is a) mainly a cosmetic difference and b) likely to take a few rounds
of review for itself. In the meantime, I think these patches are
suitable for testing.

Thanks,
Brendan

[PATCH 1 of 7] Add resumedomain domctl to resume a domain after checkpoint

[Brendan Cully]

# HG changeset patch
# User Brendan Cully <brendan@cs.ubc.ca>
# Date 1168891235 28800
# Node ID fa960afcb0ac1f4938f173b9a2cd97d1f6d5775e
# Parent  fb46005e07564bd152621c2ebf2c737b4115dc83
Add resumedomain domctl to resume a domain after checkpoint.
Export resumedomain domctl to libxc, xend.

Signed-off-by: Brendan Cully <brendan@cs.ubc.ca>

diff -r fb46005e0756 -r fa960afcb0ac tools/libxc/xc_domain.c
--- a/tools/libxc/xc_domain.c	Mon Jan 15 12:25:04 2007 +0000
+++ b/tools/libxc/xc_domain.c	Mon Jan 15 12:00:35 2007 -0800
@@ -86,6 +86,16 @@ int xc_domain_shutdown(int xc_handle,
 
  out1:
     return ret;
+}
+
+
+int xc_domain_resume(int xc_handle,
+                      uint32_t domid)
+{
+    DECLARE_DOMCTL;
+    domctl.cmd = XEN_DOMCTL_resumedomain;
+    domctl.domain = (domid_t)domid;
+    return do_domctl(xc_handle, &domctl);
 }
 
 
diff -r fb46005e0756 -r fa960afcb0ac tools/libxc/xenctrl.h
--- a/tools/libxc/xenctrl.h	Mon Jan 15 12:25:04 2007 +0000
+++ b/tools/libxc/xenctrl.h	Mon Jan 15 12:00:35 2007 -0800
@@ -236,6 +236,18 @@ int xc_domain_destroy(int xc_handle,
 int xc_domain_destroy(int xc_handle,
                       uint32_t domid);
 
+
+/**
+ * This function resumes a suspended domain. The domain should have
+ * been previously suspended.
+ *
+ * @parm xc_handle a handle to an open hypervisor interface
+ * @parm domid the domain id to resume
+ * return 0 on success, -1 on failure
+ */
+int xc_domain_resume(int xc_handle,
+                      uint32_t domid);
+
 /**
  * This function will shutdown a domain. This is intended for use in
  * fully-virtualized domains where this operation is analogous to the
diff -r fb46005e0756 -r fa960afcb0ac tools/python/xen/lowlevel/xc/xc.c
--- a/tools/python/xen/lowlevel/xc/xc.c	Mon Jan 15 12:25:04 2007 +0000
+++ b/tools/python/xen/lowlevel/xc/xc.c	Mon Jan 15 12:00:35 2007 -0800
@@ -160,6 +160,10 @@ static PyObject *pyxc_domain_destroy(XcO
     return dom_op(self, args, xc_domain_destroy);
 }
 
+static PyObject *pyxc_domain_resume(XcObject *self, PyObject *args)
+{
+    return dom_op(self, args, xc_domain_resume);
+}
 
 static PyObject *pyxc_vcpu_setaffinity(XcObject *self,
                                        PyObject *args,
@@ -1027,6 +1031,13 @@ static PyMethodDef pyxc_methods[] = {
       METH_VARARGS, "\n"
       "Destroy a domain.\n"
       " dom [int]:    Identifier of domain to be destroyed.\n\n"
+      "Returns: [int] 0 on success; -1 on error.\n" },
+    
+    { "domain_resume", 
+      (PyCFunction)pyxc_domain_resume, 
+      METH_VARARGS, "\n"
+      "Resume execution of a suspended domain.\n"
+      " dom [int]: Identifier of domain to be resumed.\n\n"
       "Returns: [int] 0 on success; -1 on error.\n" },
 
     { "vcpu_setaffinity", 
diff -r fb46005e0756 -r fa960afcb0ac tools/python/xen/xend/XendDomainInfo.py
--- a/tools/python/xen/xend/XendDomainInfo.py	Mon Jan 15 12:25:04 2007 +0000
+++ b/tools/python/xen/xend/XendDomainInfo.py	Mon Jan 15 12:00:35 2007 -0800
@@ -1533,6 +1533,15 @@ class XendDomainInfo:
         self.cleanupDomain()
 
 
+    def resumeDomain(self):
+        log.debug("XendDomainInfo.resumeDomain(%s)", str(self.domid))
+
+        try:
+            if self.domid is not None:
+                xc.domain_resume(self.domid)
+        except:
+            log.exception("XendDomainInfo.resume: xc.domain_resume failed on domain %s." % (str(self.domid)))
+
     #
     # Channels for xenstore and console
     # 
diff -r fb46005e0756 -r fa960afcb0ac xen/common/domctl.c
--- a/xen/common/domctl.c	Mon Jan 15 12:25:04 2007 +0000
+++ b/xen/common/domctl.c	Mon Jan 15 12:00:35 2007 -0800
@@ -250,6 +250,31 @@ ret_t do_domctl(XEN_GUEST_HANDLE(xen_dom
     }
     break;
 
+    case XEN_DOMCTL_resumedomain:
+    {
+        struct domain *d = find_domain_by_id(op->domain);
+        struct vcpu *v;
+
+        ret = -ESRCH;
+        if ( d != NULL )
+        {
+            ret = -EINVAL;
+            printk("Resuming domain %d\n", op->domain);
+            if ( (d != current->domain) && (d->vcpu[0] != NULL) &&
+                 test_bit(_DOMF_shutdown, &d->domain_flags) )
+            {
+                clear_bit(_DOMF_shutdown, &d->domain_flags);
+
+                for_each_vcpu (d, v)
+                    vcpu_wake (v);
+
+                ret = 0;
+            }
+            put_domain(d);
+        }
+    }
+    break;
+
     case XEN_DOMCTL_createdomain:
     {
         struct domain *d;
diff -r fb46005e0756 -r fa960afcb0ac xen/include/public/domctl.h
--- a/xen/include/public/domctl.h	Mon Jan 15 12:25:04 2007 +0000
+++ b/xen/include/public/domctl.h	Mon Jan 15 12:00:35 2007 -0800
@@ -63,6 +63,7 @@ DEFINE_XEN_GUEST_HANDLE(xen_domctl_creat
 #define XEN_DOMCTL_destroydomain      2
 #define XEN_DOMCTL_pausedomain        3
 #define XEN_DOMCTL_unpausedomain      4
+#define XEN_DOMCTL_resumedomain      27
 
 #define XEN_DOMCTL_getdomaininfo      5
 struct xen_domctl_getdomaininfo {

[PATCH 2 of 7] Add XS_RESUME command; export it to xend

[Brendan Cully]

# HG changeset patch
# User Brendan Cully <brendan@cs.ubc.ca>
# Date 1168891287 28800
# Node ID 204aa85417bd0c618cfe6dd69088a9a443191196
# Parent  fa960afcb0ac1f4938f173b9a2cd97d1f6d5775e
Add XS_RESUME command; export it to xend.

This clears the shutdown flag for a domain in xenstore, allowing
subsequent shutdowns of the same domain to fire the appropriate
watches.

Signed-off-by: Brendan Cully <brendan@cs.ubc.ca>

diff -r fa960afcb0ac -r 204aa85417bd tools/python/xen/lowlevel/xs/xs.c
--- a/tools/python/xen/lowlevel/xs/xs.c	Mon Jan 15 12:00:35 2007 -0800
+++ b/tools/python/xen/lowlevel/xs/xs.c	Mon Jan 15 12:01:27 2007 -0800
@@ -618,6 +618,33 @@ static PyObject *xspy_introduce_domain(X
     return none(result);
 }
 
+#define xspy_resume_domain_doc "\n"                                \
+	"Tell xenstore to clear its shutdown flag for a domain.\n" \
+	"This ensures that a subsequent shutdown will fire the\n"  \
+	"appropriate watches.\n"                                   \
+	" dom [int]: domain id\n"			           \
+        "\n"						           \
+        "Returns None on success.\n"				   \
+        "Raises xen.lowlevel.xs.Error on error.\n"
+
+static PyObject *xspy_resume_domain(XsHandle *self, PyObject *args)
+{
+    uint32_t dom;
+
+    struct xs_handle *xh = xshandle(self);
+    bool result = 0;
+
+    if (!xh)
+        return NULL;
+    if (!PyArg_ParseTuple(args, "i", &dom))
+        return NULL;
+
+    Py_BEGIN_ALLOW_THREADS
+    result = xs_resume_domain(xh, dom);
+    Py_END_ALLOW_THREADS
+
+    return none(result);
+}
 
 #define xspy_release_domain_doc "\n"					\
 	"Tell xenstore to release its channel to a domain.\n"		\
@@ -789,6 +816,7 @@ static PyMethodDef xshandle_methods[] = 
     XSPY_METH(transaction_start, METH_NOARGS),
     XSPY_METH(transaction_end,   METH_VARARGS | METH_KEYWORDS),
     XSPY_METH(introduce_domain,  METH_VARARGS),
+    XSPY_METH(resume_domain,     METH_VARARGS),
     XSPY_METH(release_domain,    METH_VARARGS),
     XSPY_METH(close,             METH_NOARGS),
     XSPY_METH(get_domain_path,   METH_VARARGS),
diff -r fa960afcb0ac -r 204aa85417bd tools/python/xen/xend/XendDomainInfo.py
--- a/tools/python/xen/xend/XendDomainInfo.py	Mon Jan 15 12:00:35 2007 -0800
+++ b/tools/python/xen/xend/XendDomainInfo.py	Mon Jan 15 12:01:27 2007 -0800
@@ -45,7 +45,7 @@ from xen.xend.XendError import XendError
 from xen.xend.XendError import XendError, VmError
 from xen.xend.XendDevices import XendDevices
 from xen.xend.xenstore.xstransact import xstransact, complete
-from xen.xend.xenstore.xsutil import GetDomainPath, IntroduceDomain
+from xen.xend.xenstore.xsutil import GetDomainPath, IntroduceDomain, ResumeDomain
 from xen.xend.xenstore.xswatch import xswatch
 from xen.xend.XendConstants import *
 from xen.xend.XendAPIConstants import *
@@ -1539,6 +1539,7 @@ class XendDomainInfo:
         try:
             if self.domid is not None:
                 xc.domain_resume(self.domid)
+                ResumeDomain(self.domid)
         except:
             log.exception("XendDomainInfo.resume: xc.domain_resume failed on domain %s." % (str(self.domid)))
 
diff -r fa960afcb0ac -r 204aa85417bd tools/python/xen/xend/xenstore/xsutil.py
--- a/tools/python/xen/xend/xenstore/xsutil.py	Mon Jan 15 12:00:35 2007 -0800
+++ b/tools/python/xen/xend/xenstore/xsutil.py	Mon Jan 15 12:01:27 2007 -0800
@@ -24,3 +24,6 @@ def IntroduceDomain(domid, page, port):
 
 def GetDomainPath(domid):
     return xshandle().get_domain_path(domid)
+
+def ResumeDomain(domid):
+    return xshandle().resume_domain(domid)
diff -r fa960afcb0ac -r 204aa85417bd tools/xenstore/xenstored_core.c
--- a/tools/xenstore/xenstored_core.c	Mon Jan 15 12:00:35 2007 -0800
+++ b/tools/xenstore/xenstored_core.c	Mon Jan 15 12:01:27 2007 -0800
@@ -164,6 +164,7 @@ static char *sockmsg_string(enum xsd_soc
 	case XS_WATCH_EVENT: return "WATCH_EVENT";
 	case XS_ERROR: return "ERROR";
 	case XS_IS_DOMAIN_INTRODUCED: return "XS_IS_DOMAIN_INTRODUCED";
+	case XS_RESUME: return "RESUME";
 	default:
 		return "**UNKNOWN**";
 	}
@@ -1265,6 +1266,10 @@ static void process_message(struct conne
 
 	case XS_GET_DOMAIN_PATH:
 		do_get_domain_path(conn, onearg(in));
+		break;
+
+	case XS_RESUME:
+		do_resume(conn, onearg(in));
 		break;
 
 	default:
diff -r fa960afcb0ac -r 204aa85417bd tools/xenstore/xenstored_domain.c
--- a/tools/xenstore/xenstored_domain.c	Mon Jan 15 12:00:35 2007 -0800
+++ b/tools/xenstore/xenstored_domain.c	Mon Jan 15 12:01:27 2007 -0800
@@ -395,6 +395,43 @@ void do_release(struct connection *conn,
 	send_ack(conn, XS_RELEASE);
 }
 
+void do_resume(struct connection *conn, const char *domid_str)
+{
+	struct domain *domain;
+	unsigned int domid;
+
+	if (!domid_str) {
+		send_error(conn, EINVAL);
+		return;
+	}
+
+	domid = atoi(domid_str);
+	if (!domid) {
+		send_error(conn, EINVAL);
+		return;
+	}
+
+	if (conn->id != 0) {
+		send_error(conn, EACCES);
+		return;
+	}
+
+	domain = find_domain_by_domid(domid);
+	if (!domain) {
+		send_error(conn, ENOENT);
+		return;
+	}
+
+	if (!domain->conn) {
+		send_error(conn, EINVAL);
+		return;
+	}
+
+	domain->shutdown = 0;
+	
+	send_ack(conn, XS_RESUME);
+}
+
 void do_get_domain_path(struct connection *conn, const char *domid_str)
 {
 	char *path;
diff -r fa960afcb0ac -r 204aa85417bd tools/xenstore/xenstored_domain.h
--- a/tools/xenstore/xenstored_domain.h	Mon Jan 15 12:00:35 2007 -0800
+++ b/tools/xenstore/xenstored_domain.h	Mon Jan 15 12:01:27 2007 -0800
@@ -32,6 +32,9 @@ void do_release(struct connection *conn,
 void do_release(struct connection *conn, const char *domid_str);
 
 /* domid */
+void do_resume(struct connection *conn, const char *domid_str);
+
+/* domid */
 void do_get_domain_path(struct connection *conn, const char *domid_str);
 
 /* Returns the event channel handle */
diff -r fa960afcb0ac -r 204aa85417bd tools/xenstore/xs.c
--- a/tools/xenstore/xs.c	Mon Jan 15 12:00:35 2007 -0800
+++ b/tools/xenstore/xs.c	Mon Jan 15 12:01:27 2007 -0800
@@ -719,6 +719,12 @@ bool xs_release_domain(struct xs_handle 
 	return xs_bool(single_with_domid(h, XS_RELEASE, domid));
 }
 
+/* clear the shutdown bit for the given domain */
+bool xs_resume_domain(struct xs_handle *h, unsigned int domid)
+{
+	return xs_bool(single_with_domid(h, XS_RESUME, domid));
+}
+
 char *xs_get_domain_path(struct xs_handle *h, unsigned int domid)
 {
 	char domid_str[MAX_STRLEN(domid)];
diff -r fa960afcb0ac -r 204aa85417bd tools/xenstore/xs.h
--- a/tools/xenstore/xs.h	Mon Jan 15 12:00:35 2007 -0800
+++ b/tools/xenstore/xs.h	Mon Jan 15 12:01:27 2007 -0800
@@ -133,6 +133,11 @@ bool xs_introduce_domain(struct xs_handl
 			 unsigned int domid,
 			 unsigned long mfn,
                          unsigned int eventchn); 
+/* Resume a domain.
+ * Clear the shutdown flag for this domain in the store.
+ */
+bool xs_resume_domain(struct xs_handle *h, unsigned int domid);
+
 /* Release a domain.
  * Tells the store domain to release the memory page to the domain.
  */
diff -r fa960afcb0ac -r 204aa85417bd xen/include/public/io/xs_wire.h
--- a/xen/include/public/io/xs_wire.h	Mon Jan 15 12:00:35 2007 -0800
+++ b/xen/include/public/io/xs_wire.h	Mon Jan 15 12:01:27 2007 -0800
@@ -45,7 +45,8 @@ enum xsd_sockmsg_type
     XS_SET_PERMS,
     XS_WATCH_EVENT,
     XS_ERROR,
-    XS_IS_DOMAIN_INTRODUCED
+    XS_IS_DOMAIN_INTRODUCED,
+    XS_RESUME
 };
 
 #define XS_WRITE_NONE "NONE"

[PATCH 3 of 7] Make suspend return 1 when a domain is resumed

[Brendan Cully]

# HG changeset patch
# User Brendan Cully <brendan@cs.ubc.ca>
# Date 1168891374 28800
# Node ID 477813c50e5db59ae08d346288edf19eaf19dbcb
# Parent  204aa85417bd0c618cfe6dd69088a9a443191196
Make suspend return 1 when a domain is resumed.

Signed-off-by: Brendan Cully <brendan@cs.ubc.ca>

diff -r 204aa85417bd -r 477813c50e5d tools/libxc/xc_linux_save.c
--- a/tools/libxc/xc_linux_save.c	Mon Jan 15 12:01:27 2007 -0800
+++ b/tools/libxc/xc_linux_save.c	Mon Jan 15 12:02:54 2007 -0800
@@ -569,7 +569,18 @@ static xen_pfn_t *xc_map_m2p(int xc_hand
     return m2p;
 }
 
-
+static int xc_set_vcpucontext(int xc_handle, uint32_t dom,
+                              vcpu_guest_context_t *ctxt)
+{
+    DECLARE_DOMCTL;
+
+    domctl.cmd = XEN_DOMCTL_setvcpucontext;
+    domctl.domain = (domid_t)dom;
+    domctl.u.vcpucontext.vcpu = 0;
+    set_xen_guest_handle(domctl.u.vcpucontext.ctxt, ctxt);
+
+    return xc_domctl(xc_handle, &domctl);
+}
 
 int xc_linux_save(int xc_handle, int io_fd, uint32_t dom, uint32_t max_iters,
                   uint32_t max_factor, uint32_t flags, int (*suspend)(int))
@@ -1188,6 +1199,12 @@ int xc_linux_save(int xc_handle, int io_
         goto out;
     }
 
+    /* set hypercall return code to indicate domain is resuming */
+    ctxt.user_regs.eax = 1;
+    rc = xc_set_vcpucontext(xc_handle, dom, &ctxt);
+    if (rc)
+        ERROR("Error setting hypercall return code");
+
     /* Success! */
     rc = 0;

Re: [PATCH 3 of 7] Make suspend return 1 when a domain is resumed

[Keir Fraser]

On 15/1/07 19:05, "Brendan Cully" <brendan@cs.ubc.ca> wrote:

> Make suspend return 1 when a domain is resumed.
> 
> Signed-off-by: Brendan Cully <brendan@cs.ubc.ca>

We'll do this the other way round (0 on resume; 1 on
checkpoint/suspend-cancel) for backwards compat.

xc_set_vcpucontext() is a nice thing to have, but it belongs in xc_domain.c
and it would be great if all direct users of the domctl could be ported to
use it. I'd gladly take that patch. :-)

 -- Keir

Re: [PATCH 3 of 7] Make suspend return 1 when a domain is resumed

[Brendan Cully]

On Friday, 19 January 2007 at 15:32, Keir Fraser wrote:
> On 15/1/07 19:05, "Brendan Cully" <brendan@cs.ubc.ca> wrote:
> 
> > Make suspend return 1 when a domain is resumed.
> > 
> > Signed-off-by: Brendan Cully <brendan@cs.ubc.ca>
> 
> We'll do this the other way round (0 on resume; 1 on
> checkpoint/suspend-cancel) for backwards compat.

sorry, that was a bad comment. I'd been using "resume" to mean
suspend-cancel and "reconnect" to mean run in a new domain. This patch
is the backwards compatible version (it sets EAX to 1 in xc_linux_save
rather than xc_linux_restore).

> xc_set_vcpucontext() is a nice thing to have, but it belongs in xc_domain.c
> and it would be great if all direct users of the domctl could be ported to
> use it. I'd gladly take that patch. :-)

ok, I'll prepare that in a bit.

I poked around the elfnote route for advertising kernel features to
xend, and I have a feeling someone else should be designing the API
for that - I only want very little out of it and would probably be
shortsighted. For instance, it wasn't clear to me whether it'd be
better to have an xc function that returns almost raw elfnotes (an
array of (type, length, contents)) or only parsed, named elfnotes, or
only features.

Re: [PATCH 3 of 7] Make suspend return 1 when a domain is resumed

[Keir Fraser]

On 19/1/07 16:36, "Brendan Cully" <brendan@cs.ubc.ca> wrote:

> sorry, that was a bad comment. I'd been using "resume" to mean
> suspend-cancel and "reconnect" to mean run in a new domain. This patch
> is the backwards compatible version (it sets EAX to 1 in xc_linux_save
> rather than xc_linux_restore).

Oh, I see. Makes sense.

> I poked around the elfnote route for advertising kernel features to
> xend, and I have a feeling someone else should be designing the API
> for that - I only want very little out of it and would probably be
> shortsighted. For instance, it wasn't clear to me whether it'd be
> better to have an xc function that returns almost raw elfnotes (an
> array of (type, length, contents)) or only parsed, named elfnotes, or
> only features.

The xenctrl function that pulls out the elfnote info is going to need to
know about specific elfnotes, the type of contents, and have a sensible
strategy for stringifying them. I suggest (type, contents) pairs where type
is the elfnote type name, and contents is content-specific string. There are
lots of elfnotes so you only need worry about the elfnote you care about
(your new one) -- all I want is a framework where we can extract more
elfnotes as and when we need to.

 -- Keir

Re: [PATCH 3 of 7] Make suspend return 1 when a domain is resumed

[Brendan Cully]

[-- Attachment #1: Type: text/plain, Size: 679 bytes --]

On Friday, 19 January 2007 at 15:32, Keir Fraser wrote:
> On 15/1/07 19:05, "Brendan Cully" <brendan@cs.ubc.ca> wrote:
> 
> > Make suspend return 1 when a domain is resumed.
> > 
> > Signed-off-by: Brendan Cully <brendan@cs.ubc.ca>
> 
> We'll do this the other way round (0 on resume; 1 on
> checkpoint/suspend-cancel) for backwards compat.
> 
> xc_set_vcpucontext() is a nice thing to have, but it belongs in xc_domain.c
> and it would be great if all direct users of the domctl could be ported to
> use it. I'd gladly take that patch. :-)

What's even nicer is it's already been done. Here's a refresh of this
patch; I'll send along a cleanup for the other callers separately.

[-- Attachment #2: xc-save-suspend-rc.patch --]
[-- Type: text/plain, Size: 827 bytes --]

# HG changeset patch
# User Brendan Cully <brendan@cs.ubc.ca>
# Date 1169250376 28800
# Node ID fbe5ab1ecd2131818922929852fe50ec75dfc88b
# Parent  f89126a590738ad804463772230dde0b4a6640ba
Make suspend return 1 from suspend-cancel.

Signed-off-by: Brendan Cully <brendan@cs.ubc.ca>

diff -r f89126a59073 -r fbe5ab1ecd21 tools/libxc/xc_linux_save.c
--- a/tools/libxc/xc_linux_save.c	Fri Jan 19 16:11:31 2007 +0000
+++ b/tools/libxc/xc_linux_save.c	Fri Jan 19 15:46:16 2007 -0800
@@ -1181,6 +1181,12 @@ int xc_linux_save(int xc_handle, int io_
         goto out;
     }
 
+    /* set hypercall return code to indicate domain is resuming */
+    ctxt.user_regs.eax = 1;
+    rc = xc_vcpu_setcontext(xc_handle, dom, 0, &ctxt);
+    if (rc)
+        ERROR("Error setting hypercall return code");
+
     /* Success! */
     rc = 0;
 

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [PATCH 3 of 7] Make suspend return 1 when a domain is resumed

[Brendan Cully]

[-- Attachment #1: Type: text/plain, Size: 443 bytes --]

On Friday, 19 January 2007 at 15:32, Keir Fraser wrote:
> xc_set_vcpucontext() is a nice thing to have, but it belongs in xc_domain.c
> and it would be great if all direct users of the domctl could be ported to
> use it. I'd gladly take that patch. :-)

Here's the cleanup patch. lock_pages happens later now (just before
the domctl). If that matters, I think it's harmless to leave the
original call in, so that mlock would get called twice.

[-- Attachment #2: xc-use-vcpu_setcontext.patch --]
[-- Type: text/plain, Size: 3877 bytes --]

# HG changeset patch
# User Brendan Cully <brendan@cs.ubc.ca>
# Date 1169252138 28800
# Node ID 2048d2698d461070786a655e7d0d24f015b6cb8b
# Parent  f89126a590738ad804463772230dde0b4a6640ba
Use xc_vcpu_setcontext instead of the lowlevel domctl.

Signed-off-by: Brendan Cully <brendan@cs.ubc.ca>

diff -r f89126a59073 -r 2048d2698d46 tools/libxc/xc_hvm_build.c
--- a/tools/libxc/xc_hvm_build.c	Fri Jan 19 16:11:31 2007 +0000
+++ b/tools/libxc/xc_hvm_build.c	Fri Jan 19 16:15:38 2007 -0800
@@ -270,7 +270,6 @@ static int xc_hvm_build_internal(int xc_
                                  char *image,
                                  unsigned long image_size)
 {
-    struct xen_domctl launch_domctl;
     vcpu_guest_context_t ctxt;
     int rc;
 
@@ -287,20 +286,7 @@ static int xc_hvm_build_internal(int xc_
         goto error_out;
     }
 
-    if ( lock_pages(&ctxt, sizeof(ctxt) ) )
-    {
-        PERROR("%s: ctxt mlock failed", __func__);
-        goto error_out;
-    }
-
-    memset(&launch_domctl, 0, sizeof(launch_domctl));
-    launch_domctl.domain = (domid_t)domid;
-    launch_domctl.u.vcpucontext.vcpu   = 0;
-    set_xen_guest_handle(launch_domctl.u.vcpucontext.ctxt, &ctxt);
-    launch_domctl.cmd = XEN_DOMCTL_setvcpucontext;
-    rc = xc_domctl(xc_handle, &launch_domctl);
-
-    unlock_pages(&ctxt, sizeof(ctxt));
+    rc = xc_vcpu_setcontext(xc_handle, domid, 0, &ctxt);
 
     return rc;
 
diff -r f89126a59073 -r 2048d2698d46 tools/libxc/xc_linux_build.c
--- a/tools/libxc/xc_linux_build.c	Fri Jan 19 16:11:31 2007 +0000
+++ b/tools/libxc/xc_linux_build.c	Fri Jan 19 16:15:38 2007 -0800
@@ -1092,7 +1092,6 @@ static int xc_linux_build_internal(int x
                                    unsigned int console_evtchn,
                                    unsigned long *console_mfn)
 {
-    struct xen_domctl launch_domctl;
     DECLARE_DOMCTL;
     int rc;
     struct vcpu_guest_context st_ctxt, *ctxt = &st_ctxt;
@@ -1109,12 +1108,6 @@ static int xc_linux_build_internal(int x
     }
 
     memset(ctxt, 0, sizeof(*ctxt));
-
-    if ( lock_pages(ctxt, sizeof(*ctxt) ) )
-    {
-        PERROR("%s: ctxt lock failed", __func__);
-        return 1;
-    }
 
     domctl.cmd = XEN_DOMCTL_getdomaininfo;
     domctl.domain = (domid_t)domid;
@@ -1170,14 +1163,7 @@ static int xc_linux_build_internal(int x
     ctxt->kernel_sp = ctxt->user_regs.esp;
 #endif /* x86 */
 
-    memset(&launch_domctl, 0, sizeof(launch_domctl));
-
-    launch_domctl.domain = (domid_t)domid;
-    launch_domctl.u.vcpucontext.vcpu   = 0;
-    set_xen_guest_handle(launch_domctl.u.vcpucontext.ctxt, ctxt);
-
-    launch_domctl.cmd = XEN_DOMCTL_setvcpucontext;
-    rc = xc_domctl(xc_handle, &launch_domctl);
+    rc = xc_vcpu_setcontext(xc_handle, domid, 0, ctxt);
 
     return rc;
 
diff -r f89126a59073 -r 2048d2698d46 tools/libxc/xc_linux_restore.c
--- a/tools/libxc/xc_linux_restore.c	Fri Jan 19 16:11:31 2007 +0000
+++ b/tools/libxc/xc_linux_restore.c	Fri Jan 19 16:15:38 2007 -0800
@@ -195,12 +195,6 @@ int xc_linux_restore(int xc_handle, int 
     if(!get_platform_info(xc_handle, dom,
                           &max_mfn, &hvirt_start, &pt_levels)) {
         ERROR("Unable to get platform info.");
-        return 1;
-    }
-
-    if (lock_pages(&ctxt, sizeof(ctxt))) {
-        /* needed for build domctl, but might as well do early */
-        ERROR("Unable to lock ctxt");
         return 1;
     }
 
@@ -845,12 +839,7 @@ int xc_linux_restore(int xc_handle, int 
 
     DPRINTF("Domain ready to be built.\n");
 
-    domctl.cmd = XEN_DOMCTL_setvcpucontext;
-    domctl.domain = (domid_t)dom;
-    domctl.u.vcpucontext.vcpu   = 0;
-    set_xen_guest_handle(domctl.u.vcpucontext.ctxt, &ctxt);
-    rc = xc_domctl(xc_handle, &domctl);
-
+    rc = xc_vcpu_setcontext(xc_handle, dom, 0, &ctxt);
     if (rc != 0) {
         ERROR("Couldn't build the domain");
         goto out;

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

[PATCH 4 of 7] Make xen_suspend handle resume

[Brendan Cully]

# HG changeset patch
# User Brendan Cully <brendan@cs.ubc.ca>
# Date 1168891374 28800
# Node ID 5faad71162917237df8c8574f0186981b621a161
# Parent  477813c50e5db59ae08d346288edf19eaf19dbcb
Make xen_suspend handle resume.

Don't destroy xenstore watches on suspend, and only recreate them when
resuming in a new domain. Likewise, only invoke frontend device resume
code when in a new domain (the resume functions all tear down the
existing function and wait for the backend to negotiate a new one,
which does not happen in the source domain).

Signed-off-by: Brendan Cully <brendan@cs.ubc.ca>

diff -r 477813c50e5d -r 5faad7116291 linux-2.6-xen-sparse/drivers/xen/core/machine_reboot.c
--- a/linux-2.6-xen-sparse/drivers/xen/core/machine_reboot.c	Mon Jan 15 12:02:54 2007 -0800
+++ b/linux-2.6-xen-sparse/drivers/xen/core/machine_reboot.c	Mon Jan 15 12:02:54 2007 -0800
@@ -85,13 +85,20 @@ static void pre_suspend(void)
 		mfn_to_pfn(xen_start_info->console.domU.mfn);
 }
 
-static void post_suspend(void)
+static void post_suspend(int reconnect)
 {
 	int i, j, k, fpp;
 	extern unsigned long max_pfn;
 	extern unsigned long *pfn_to_mfn_frame_list_list;
 	extern unsigned long *pfn_to_mfn_frame_list[];
 
+	if (!reconnect) {
+		xen_start_info->store_mfn =
+			pfn_to_mfn(xen_start_info->store_mfn);
+		xen_start_info->console.domU.mfn =
+			pfn_to_mfn(xen_start_info->console.domU.mfn);
+	}
+	
 	set_fixmap(FIX_SHARED_INFO, xen_start_info->shared_info);
 
 	HYPERVISOR_shared_info = (shared_info_t *)fix_to_virt(FIX_SHARED_INFO);
@@ -120,7 +127,7 @@ static void post_suspend(void)
 #define switch_idle_mm()	((void)0)
 #define mm_pin_all()		((void)0)
 #define pre_suspend()		((void)0)
-#define post_suspend()		((void)0)
+#define post_suspend(x)		((void)0)
 
 #endif
 
@@ -158,16 +165,18 @@ int __xen_suspend(void)
 	pre_suspend();
 
 	/*
-	 * We'll stop somewhere inside this hypercall. When it returns,
-	 * we'll start resuming after the restore.
+	 * This hypercall returns 1 if suspend was cancelled or
+	 * the domain was merely checkpointed, and 0 if it is
+	 * resuming in a new domain.
 	 */
-	HYPERVISOR_suspend(virt_to_mfn(xen_start_info));
+	err = !HYPERVISOR_suspend(virt_to_mfn(xen_start_info));
 
-	post_suspend();
+	post_suspend(err);
 
 	gnttab_resume();
 
-	irq_resume();
+	if (err)
+		irq_resume();
 
 	time_resume();
 
@@ -175,9 +184,10 @@ int __xen_suspend(void)
 
 	local_irq_enable();
 
-	xencons_resume();
+	if (err)
+		xencons_resume();
 
-	xenbus_resume();
+	xenbus_resume(err);
 
 	smp_resume();
 
diff -r 477813c50e5d -r 5faad7116291 linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c
--- a/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c	Mon Jan 15 12:02:54 2007 -0800
+++ b/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c	Mon Jan 15 12:02:54 2007 -0800
@@ -727,11 +727,15 @@ void xenbus_suspend(void)
 }
 EXPORT_SYMBOL_GPL(xenbus_suspend);
 
-void xenbus_resume(void)
-{
-	xb_init_comms();
-	xs_resume();
-	bus_for_each_dev(&xenbus_frontend.bus, NULL, NULL, resume_dev);
+void xenbus_resume(int reconnect)
+{
+	if (reconnect)
+		xb_init_comms();
+	xs_resume(reconnect);
+
+	if (reconnect) {
+		bus_for_each_dev(&xenbus_frontend.bus, NULL, NULL, resume_dev);
+	}
 	xenbus_backend_resume(resume_dev);
 }
 EXPORT_SYMBOL_GPL(xenbus_resume);
diff -r 477813c50e5d -r 5faad7116291 linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_xs.c
--- a/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_xs.c	Mon Jan 15 12:02:54 2007 -0800
+++ b/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_xs.c	Mon Jan 15 12:02:54 2007 -0800
@@ -668,31 +668,23 @@ EXPORT_SYMBOL_GPL(unregister_xenbus_watc
 
 void xs_suspend(void)
 {
+	down_write(&xs_state.suspend_mutex);
+	mutex_lock(&xs_state.request_mutex);
+}
+
+void xs_resume(int reconnect)
+{
 	struct xenbus_watch *watch;
 	char token[sizeof(watch) * 2 + 1];
 
-	down_write(&xs_state.suspend_mutex);
-
-	/* No need for watches_lock: the suspend_mutex is sufficient. */
-	list_for_each_entry(watch, &watches, list) {
-		sprintf(token, "%lX", (long)watch);
-		xs_unwatch(watch->node, token);
-	}
-
-	mutex_lock(&xs_state.request_mutex);
-}
-
-void xs_resume(void)
-{
-	struct xenbus_watch *watch;
-	char token[sizeof(watch) * 2 + 1];
-
 	mutex_unlock(&xs_state.request_mutex);
 
-	/* No need for watches_lock: the suspend_mutex is sufficient. */
-	list_for_each_entry(watch, &watches, list) {
-		sprintf(token, "%lX", (long)watch);
-		xs_watch(watch->node, token);
+	if (reconnect) {
+		/* No need for watches_lock: the suspend_mutex is sufficient. */
+		list_for_each_entry(watch, &watches, list) {
+			sprintf(token, "%lX", (long)watch);
+			xs_watch(watch->node, token);
+		}
 	}
 
 	up_write(&xs_state.suspend_mutex);
diff -r 477813c50e5d -r 5faad7116291 linux-2.6-xen-sparse/include/xen/xenbus.h
--- a/linux-2.6-xen-sparse/include/xen/xenbus.h	Mon Jan 15 12:02:54 2007 -0800
+++ b/linux-2.6-xen-sparse/include/xen/xenbus.h	Mon Jan 15 12:02:54 2007 -0800
@@ -159,14 +159,14 @@ int register_xenbus_watch(struct xenbus_
 int register_xenbus_watch(struct xenbus_watch *watch);
 void unregister_xenbus_watch(struct xenbus_watch *watch);
 void xs_suspend(void);
-void xs_resume(void);
+void xs_resume(int reconnect);
 
 /* Used by xenbus_dev to borrow kernel's store connection. */
 void *xenbus_dev_request_and_reply(struct xsd_sockmsg *msg);
 
 /* Called from xen core code. */
 void xenbus_suspend(void);
-void xenbus_resume(void);
+void xenbus_resume(int reconnect);
 
 #define XENBUS_IS_ERR_READ(str) ({			\
 	if (!IS_ERR(str) && strlen(str) == 0) {		\

[PATCH 5 of 7] Add xm save -c/--checkpoint option

[Brendan Cully]

# HG changeset patch
# User Brendan Cully <brendan@cs.ubc.ca>
# Date 1168891374 28800
# Node ID dafb7b444c2ccf1d2ab9efb8c13d62e018713a85
# Parent  5faad71162917237df8c8574f0186981b621a161
Add xm save -c/--checkpoint option

xm save --checkpoint leaves the domain running after creating the
snapshot.

Signed-off-by: Brendan Cully <brendan@cs.ubc.ca>

diff -r 5faad7116291 -r dafb7b444c2c tools/python/xen/xend/XendCheckpoint.py
--- a/tools/python/xen/xend/XendCheckpoint.py	Mon Jan 15 12:02:54 2007 -0800
+++ b/tools/python/xen/xend/XendCheckpoint.py	Mon Jan 15 12:02:54 2007 -0800
@@ -51,7 +51,7 @@ def read_exact(fd, size, errmsg):
     return buf
 
 
-def save(fd, dominfo, network, live, dst):
+def save(fd, dominfo, network, live, dst, checkpoint=False):
     write_exact(fd, SIGNATURE, "could not write guest state file: signature")
 
     config = sxp.to_string(dominfo.sxpr())
@@ -96,7 +96,8 @@ def save(fd, dominfo, network, live, dst
 
         forkHelper(cmd, fd, saveInputHandler, False)
 
-        dominfo.destroyDomain()
+        if not checkpoint:
+            dominfo.destroyDomain()
         try:
             dominfo.setName(domain_name)
         except VmError:
@@ -105,6 +106,8 @@ def save(fd, dominfo, network, live, dst
             # persistent VM, we need the rename, and don't expect the
             # conflict.  This needs more thought.
             pass
+        if checkpoint:
+            dominfo.resumeDomain()
 
     except Exception, exn:
         log.exception("Save failed on domain %s (%s).", domain_name,
diff -r 5faad7116291 -r dafb7b444c2c tools/python/xen/xend/XendDomain.py
--- a/tools/python/xen/xend/XendDomain.py	Mon Jan 15 12:02:54 2007 -0800
+++ b/tools/python/xen/xend/XendDomain.py	Mon Jan 15 12:02:54 2007 -0800
@@ -1177,7 +1177,7 @@ class XendDomain:
         dominfo.testDeviceComplete()
         sock.close()
 
-    def domain_save(self, domid, dst):
+    def domain_save(self, domid, dst, checkpoint):
         """Start saving a domain to file.
 
         @param domid: Domain ID or Name
@@ -1198,8 +1198,8 @@ class XendDomain:
 
             fd = os.open(dst, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
             try:
-                # For now we don't support 'live checkpoint' 
-                XendCheckpoint.save(fd, dominfo, False, False, dst)
+                XendCheckpoint.save(fd, dominfo, False, False, dst,
+                                    checkpoint=checkpoint)
             finally:
                 os.close(fd)
         except OSError, ex:
diff -r 5faad7116291 -r dafb7b444c2c tools/python/xen/xm/main.py
--- a/tools/python/xen/xm/main.py	Mon Jan 15 12:02:54 2007 -0800
+++ b/tools/python/xen/xm/main.py	Mon Jan 15 12:02:54 2007 -0800
@@ -102,7 +102,7 @@ SUBCOMMAND_HELP = {
     'reboot'      : ('<Domain> [-wa]', 'Reboot a domain.'),
     'restore'     : ('<CheckpointFile> [-p]',
                      'Restore a domain from a saved state.'),
-    'save'        : ('<Domain> <CheckpointFile>',
+    'save'        : ('[-c] <Domain> <CheckpointFile>',
                      'Save a domain state to restore later.'),
     'shutdown'    : ('<Domain> [-waRH]', 'Shutdown a domain.'),
     'top'         : ('', 'Monitor a host and the domains in real time.'),
@@ -230,6 +230,9 @@ SUBCOMMAND_OPTIONS = {
     'resume': (
       ('-p', '--paused', 'Do not unpause domain after resuming it'),
     ),
+    'save': (
+      ('-c', '--checkpoint', 'Leave domain running after creating snapshot'),
+    ),
    'restore': (
       ('-p', '--paused', 'Do not unpause domain after restoring it'),
     ),
@@ -586,21 +589,37 @@ def xm_shell(args):
 #########################################################################
 
 def xm_save(args):
-    arg_check(args, "save", 2)
-
-    try:
-        dominfo = parse_doms_info(server.xend.domain(args[0]))
+    arg_check(args, "save", 2, 3)
+
+    try:
+        (options, params) = getopt.gnu_getopt(args, 'c', ['checkpoint'])
+    except getopt.GetoptError, opterr:
+        err(opterr)
+        sys.exit(1)
+
+    checkpoint = False
+    for (k, v) in options:
+        if k in ['-c', '--checkpoint']:
+            checkpoint = True
+
+    if len(params) != 2:
+        err("Wrong number of parameters")
+        usage('save')
+        sys.exit(1)
+
+    try:
+        dominfo = parse_doms_info(server.xend.domain(params[0]))
     except xmlrpclib.Fault, ex:
         raise ex
     
     domid = dominfo['domid']
-    savefile = os.path.abspath(args[1])
+    savefile = os.path.abspath(params[1])
 
     if not os.access(os.path.dirname(savefile), os.W_OK):
         err("xm save: Unable to create file %s" % savefile)
         sys.exit(1)
     
-    server.xend.domain.save(domid, savefile)
+    server.xend.domain.save(domid, savefile, checkpoint)
     
 def xm_restore(args):
     arg_check(args, "restore", 1, 2)

[PATCH 6 of 7] Advertise resumability feature in guest kernel

[Brendan Cully]

# HG changeset patch
# User Brendan Cully <brendan@cs.ubc.ca>
# Date 1168891374 28800
# Node ID 1a3b78919a43a68948775eb4077f5dab62054f7c
# Parent  dafb7b444c2ccf1d2ab9efb8c13d62e018713a85
Advertise resumability feature in guest kernel.
Disallow xm save -c unless the guest advertises that it is resumable.

Signed-off-by: Brendan Cully <brendan@cs.ubc.ca>

diff -r dafb7b444c2c -r 1a3b78919a43 linux-2.6-xen-sparse/drivers/xen/core/reboot.c
--- a/linux-2.6-xen-sparse/drivers/xen/core/reboot.c	Mon Jan 15 12:02:54 2007 -0800
+++ b/linux-2.6-xen-sparse/drivers/xen/core/reboot.c	Mon Jan 15 12:02:54 2007 -0800
@@ -204,6 +204,8 @@ static int setup_shutdown_watcher(struct
 	else
 		xenbus_write(XBT_NIL, "control", "feature-sysrq", "1");
 
+	xenbus_write(XBT_NIL, "control", "feature-resumable", "1");
+
 	return NOTIFY_DONE;
 }
 
diff -r dafb7b444c2c -r 1a3b78919a43 tools/python/xen/xend/XendDomain.py
--- a/tools/python/xen/xend/XendDomain.py	Mon Jan 15 12:02:54 2007 -0800
+++ b/tools/python/xen/xend/XendDomain.py	Mon Jan 15 12:02:54 2007 -0800
@@ -1196,6 +1196,10 @@ class XendDomain:
             if dominfo.getDomid() == DOM0_ID:
                 raise XendError("Cannot save privileged domain %i" % domid)
 
+            if checkpoint:
+                resumable = dominfo.readDom("control/feature-resumable")
+                if not resumable:
+                    raise XendError("Guest does not support checkpointing")
             fd = os.open(dst, os.O_WRONLY | os.O_CREAT | os.O_TRUNC)
             try:
                 XendCheckpoint.save(fd, dominfo, False, False, dst,

[PATCH 7 of 7] Add unsuspend hook for resuming devices in the checkpoint parent

[Brendan Cully]

# HG changeset patch
# User Brendan Cully <brendan@cs.ubc.ca>
# Date 1168891374 28800
# Node ID 70508fff24c52b7b16ea6c7be630a43cca45a5e1
# Parent  1a3b78919a43a68948775eb4077f5dab62054f7c
Add unsuspend hook for resuming devices in the checkpoint parent.

Nothing uses it at the moment.

Signed-off-by: Brendan Cully <brendan@cs.ubc.ca>

diff -r 1a3b78919a43 -r 70508fff24c5 linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c
--- a/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c	Mon Jan 15 12:02:54 2007 -0800
+++ b/linux-2.6-xen-sparse/drivers/xen/xenbus/xenbus_probe.c	Mon Jan 15 12:02:54 2007 -0800
@@ -672,6 +672,26 @@ static int suspend_dev(struct device *de
 	return 0;
 }
 
+static int unsuspend_dev(struct device *dev, void *data)
+{
+	int err = 0;
+	struct xenbus_driver *drv;
+	struct xenbus_device *xdev;
+
+	DPRINTK("");
+
+	if (dev->driver == NULL)
+		return 0;
+	drv = to_xenbus_driver(dev->driver);
+	xdev = container_of(dev, struct xenbus_device, dev);
+	if (drv->unsuspend)
+		err = drv->unsuspend(xdev);
+	if (err)
+		printk(KERN_WARNING
+		       "xenbus: unsuspend %s failed: %i\n", dev->bus_id, err);
+	return 0;
+}
+
 static int resume_dev(struct device *dev, void *data)
 {
 	int err;
@@ -735,6 +755,9 @@ void xenbus_resume(int reconnect)
 
 	if (reconnect) {
 		bus_for_each_dev(&xenbus_frontend.bus, NULL, NULL, resume_dev);
+	} else {
+		bus_for_each_dev(&xenbus_frontend.bus, NULL, NULL,
+				 unsuspend_dev);
 	}
 	xenbus_backend_resume(resume_dev);
 }
diff -r 1a3b78919a43 -r 70508fff24c5 linux-2.6-xen-sparse/include/xen/xenbus.h
--- a/linux-2.6-xen-sparse/include/xen/xenbus.h	Mon Jan 15 12:02:54 2007 -0800
+++ b/linux-2.6-xen-sparse/include/xen/xenbus.h	Mon Jan 15 12:02:54 2007 -0800
@@ -101,6 +101,7 @@ struct xenbus_driver {
 				 enum xenbus_state backend_state);
 	int (*remove)(struct xenbus_device *dev);
 	int (*suspend)(struct xenbus_device *dev);
+	int (*unsuspend)(struct xenbus_device *dev);
 	int (*resume)(struct xenbus_device *dev);
 	int (*uevent)(struct xenbus_device *, char **, int, char *, int);
 	struct device_driver driver;