[PATCH 3/3] [linux-2.6.18-xen.hg] delay xenbus init if xenbus master is a separate domain

[PATCH 3/3] [linux-2.6.18-xen.hg] delay xenbus init if xenbus master is a separate domain

[Alex Zeffertt]

[-- Attachment #1: Type: text/plain, Size: 1 bytes --]



[-- Attachment #2: xenbus_lazy_init --]
[-- Type: text/plain, Size: 12970 bytes --]

This patch modifies xenbus to lazily initialize. This is necessary when
xenstored is running in a seperate minios domain, since it can't connect as a
slave just yet.

Ian Campbell has looked at an earlier version of this patch, and seems
to generally approve.

Here's an explanation of the states:

It starts out in XENBUS_XSD_UNCOMMITTED.

As the master xenbus (the one local to xenstored), it will receive an
mmap from xenstore, putting it in XENBUS_XSD_LOCAL_INIT. This enables
the wake_waiting IRQ, which will put it in XENBUS_XSD_LOCAL_READY.

Alternatively, as a slave xenbus, it will receive an ioctl from the
xenstore domain builder, putting it in XENBUS_XSD_FOREIGN_INIT. This
enables the wake_waiting IRQ, which will put it in
XENBUS_XSD_FOREIGN_READY.

DomU's are immediately initialized to XENBUS_XSD_FOREIGN_READY.

TODO: The types in the IOCTL might not be correct.

Signed-off-by: Diego Ongaro <diego.ongaro@citrix.com>
Signed-off-by: Alex Zeffertt <alex.zeffertt@eu.citrix.com>
---

diff -r e68a58170585 drivers/xen/core/xen_sysfs.c
--- a/drivers/xen/core/xen_sysfs.c	Mon Mar 16 17:02:03 2009 +0000
+++ b/drivers/xen/core/xen_sysfs.c	Thu Mar 19 10:46:26 2009 +0000
@@ -16,6 +16,7 @@
 #include <xen/hypervisor_sysfs.h>
 #include <xen/xenbus.h>
 #include <xen/interface/kexec.h>
+#include "../xenbus/xenbus_comms.h"
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Mike D. Day <ncmike@us.ibm.com>");
@@ -105,9 +106,8 @@
 {
 	char *vm, *val;
 	int ret;
-	extern int xenstored_ready;
 
-	if (!xenstored_ready)
+	if (!is_xenstored_ready())
 		return -EBUSY;
 
 	vm = xenbus_read(XBT_NIL, "vm", "", NULL);
diff -r e68a58170585 drivers/xen/xenbus/xenbus_comms.c
--- a/drivers/xen/xenbus/xenbus_comms.c	Mon Mar 16 17:02:03 2009 +0000
+++ b/drivers/xen/xenbus/xenbus_comms.c	Thu Mar 19 10:46:26 2009 +0000
@@ -50,18 +50,39 @@
 static int xenbus_irq;
 
 extern void xenbus_probe(void *);
-extern int xenstored_ready;
 static DECLARE_WORK(probe_work, xenbus_probe, NULL);
 
 static DECLARE_WAIT_QUEUE_HEAD(xb_waitq);
 
 static irqreturn_t wake_waiting(int irq, void *unused, struct pt_regs *regs)
 {
-	if (unlikely(xenstored_ready == 0)) {
-		xenstored_ready = 1;
-		schedule_work(&probe_work);
+	int old, new;
+
+	old = atomic_read(&xenbus_xsd_state);
+	switch (old) {
+		case XENBUS_XSD_UNCOMMITTED:
+			BUG();
+			return IRQ_HANDLED;
+
+		case XENBUS_XSD_FOREIGN_INIT:
+			new = XENBUS_XSD_FOREIGN_READY;
+			break;
+		
+		case XENBUS_XSD_LOCAL_INIT:
+			new = XENBUS_XSD_LOCAL_READY;
+			break;
+
+		case XENBUS_XSD_FOREIGN_READY:
+		case XENBUS_XSD_LOCAL_READY:
+		default:
+			goto wake;
 	}
 
+	old = atomic_cmpxchg(&xenbus_xsd_state, old, new);
+	if (old != new)
+		schedule_work(&probe_work);
+
+wake:
 	wake_up(&xb_waitq);
 	return IRQ_HANDLED;
 }
diff -r e68a58170585 drivers/xen/xenbus/xenbus_comms.h
--- a/drivers/xen/xenbus/xenbus_comms.h	Mon Mar 16 17:02:03 2009 +0000
+++ b/drivers/xen/xenbus/xenbus_comms.h	Thu Mar 19 10:46:26 2009 +0000
@@ -43,4 +43,20 @@
 extern struct xenstore_domain_interface *xen_store_interface;
 extern int xen_store_evtchn;
 
+/* For xenbus internal use. */
+enum {
+	XENBUS_XSD_UNCOMMITTED = 0,
+	XENBUS_XSD_FOREIGN_INIT,
+	XENBUS_XSD_FOREIGN_READY,
+	XENBUS_XSD_LOCAL_INIT,
+	XENBUS_XSD_LOCAL_READY,
+};
+extern atomic_t xenbus_xsd_state;
+
+static inline int is_xenstored_ready(void)
+{
+	int s = atomic_read(&xenbus_xsd_state);
+	return s == XENBUS_XSD_FOREIGN_READY || s == XENBUS_XSD_LOCAL_READY;
+}
+
 #endif /* _XENBUS_COMMS_H */
diff -r e68a58170585 drivers/xen/xenbus/xenbus_dev.c
--- a/drivers/xen/xenbus/xenbus_dev.c	Mon Mar 16 17:02:03 2009 +0000
+++ b/drivers/xen/xenbus/xenbus_dev.c	Thu Mar 19 10:46:26 2009 +0000
@@ -53,6 +53,8 @@
 #include <xen/platform-compat.h>
 #endif
 
+#include <xen/public/xenbus.h>
+
 struct xenbus_dev_transaction {
 	struct list_head list;
 	struct xenbus_transaction handle;
@@ -95,6 +97,9 @@
 	struct xenbus_dev_data *u = filp->private_data;
 	struct read_buffer *rb;
 	int i, ret;
+
+	if (!is_xenstored_ready())
+		return -ENODEV;
 
 	mutex_lock(&u->reply_mutex);
 	while (list_empty(&u->read_buffers)) {
@@ -205,6 +210,9 @@
 	char *path, *token;
 	struct watch_adapter *watch, *tmp_watch;
 	int err, rc = len;
+
+	if (!is_xenstored_ready())
+		return -ENODEV;
 
 	if ((len + u->len) > sizeof(u->u.buffer)) {
 		rc = -EINVAL;
@@ -391,10 +399,63 @@
 {
 	struct xenbus_dev_data *u = file->private_data;
 
+	if (!is_xenstored_ready())
+		return -ENODEV;
+
 	poll_wait(file, &u->read_waitq, wait);
 	if (!list_empty(&u->read_buffers))
 		return POLLIN | POLLRDNORM;
 	return 0;
+}
+
+static long xenbus_dev_ioctl(struct file *file,
+                             unsigned int cmd, unsigned long data)
+{
+	extern int xenbus_conn(domid_t remote_dom, int *grant_ref,
+	                       evtchn_port_t *local_port);
+	void __user *udata = (void __user *) data;
+	int ret = -ENOTTY;
+	
+	if (!is_initial_xendomain())
+		return -ENODEV;
+
+
+	switch (cmd) {
+	case IOCTL_XENBUS_ALLOC: {
+		xenbus_alloc_t xa;
+		int old;
+
+		old = atomic_cmpxchg(&xenbus_xsd_state,
+		                     XENBUS_XSD_UNCOMMITTED,
+		                     XENBUS_XSD_FOREIGN_INIT);
+		if (old != XENBUS_XSD_UNCOMMITTED)
+			return -EBUSY;
+
+		if (copy_from_user(&xa, udata, sizeof(xa))) {
+			ret = -EFAULT;
+			atomic_set(&xenbus_xsd_state, XENBUS_XSD_UNCOMMITTED);
+			break;
+		}
+
+		ret = xenbus_conn(xa.dom, &xa.grant_ref, &xa.port);
+		if (ret != 0) {
+			atomic_set(&xenbus_xsd_state, XENBUS_XSD_UNCOMMITTED);
+			break;
+		}
+
+		if (copy_to_user(udata, &xa, sizeof(xa))) {
+			ret = -EFAULT;
+			atomic_set(&xenbus_xsd_state, XENBUS_XSD_UNCOMMITTED);
+			break;
+		}
+	}
+	break;
+
+	default:
+		break;
+	}
+
+	return ret;
 }
 
 static const struct file_operations xenbus_dev_file_ops = {
@@ -403,6 +464,7 @@
 	.open = xenbus_dev_open,
 	.release = xenbus_dev_release,
 	.poll = xenbus_dev_poll,
+	.unlocked_ioctl = xenbus_dev_ioctl
 };
 
 int xenbus_dev_init(void)
diff -r e68a58170585 drivers/xen/xenbus/xenbus_probe.c
--- a/drivers/xen/xenbus/xenbus_probe.c	Mon Mar 16 17:02:03 2009 +0000
+++ b/drivers/xen/xenbus/xenbus_probe.c	Thu Mar 19 10:46:26 2009 +0000
@@ -44,6 +44,7 @@
 #include <linux/notifier.h>
 #include <linux/mutex.h>
 #include <linux/module.h>
+#include <xen/gnttab.h>
 
 #include <asm/io.h>
 #include <asm/page.h>
@@ -347,7 +348,12 @@
 
 	DPRINTK("%s", dev->nodename);
 
+/* Commented out since xenstored stubdom is now minios based not linux based
+#define XENSTORE_DOMAIN_SHARES_THIS_KERNEL 
+*/
+#ifndef XENSTORE_DOMAIN_SHARES_THIS_KERNEL 
 	if (is_initial_xendomain())
+#endif
 		return;
 
 	get_device(&dev->dev);
@@ -820,14 +826,13 @@
 EXPORT_SYMBOL_GPL(xenbus_suspend_cancel);
 
 /* A flag to determine if xenstored is 'ready' (i.e. has started) */
-int xenstored_ready = 0;
-
+atomic_t xenbus_xsd_state = ATOMIC_INIT(XENBUS_XSD_UNCOMMITTED);
 
 int register_xenstore_notifier(struct notifier_block *nb)
 {
 	int ret = 0;
 
-	if (xenstored_ready > 0)
+	if (is_xenstored_ready())
 		ret = nb->notifier_call(nb, 0, NULL);
 	else
 		blocking_notifier_chain_register(&xenstore_chain, nb);
@@ -845,7 +850,7 @@
 
 void xenbus_probe(void *unused)
 {
-	BUG_ON((xenstored_ready <= 0));
+	BUG_ON(!is_xenstored_ready());
 
 	/* Enumerate devices in xenstore and watch for changes. */
 	xenbus_probe_devices(&xenbus_frontend);
@@ -865,6 +870,28 @@
 static int xsd_kva_mmap(struct file *file, struct vm_area_struct *vma)
 {
 	size_t size = vma->vm_end - vma->vm_start;
+	int old;
+	int rc;
+
+	old = atomic_cmpxchg(&xenbus_xsd_state,
+	                   XENBUS_XSD_UNCOMMITTED,
+	                   XENBUS_XSD_LOCAL_INIT);
+	switch (old) {
+		case XENBUS_XSD_UNCOMMITTED:
+			rc = xb_init_comms();
+			if (rc != 0)
+				return rc;
+			break;
+
+		case XENBUS_XSD_FOREIGN_INIT:
+		case XENBUS_XSD_FOREIGN_READY:
+			return -EBUSY;
+		
+		case XENBUS_XSD_LOCAL_INIT:
+		case XENBUS_XSD_LOCAL_READY:
+		default:
+			break;
+	}
 
 	if ((size > PAGE_SIZE) || (vma->vm_pgoff != 0))
 		return -EINVAL;
@@ -896,6 +923,62 @@
 	return len;
 }
 #endif
+
+static int xb_free_port(evtchn_port_t port)
+{
+	struct evtchn_close close;
+	close.port = port;
+	return HYPERVISOR_event_channel_op(EVTCHNOP_close, &close);
+}
+
+int xenbus_conn(domid_t remote_dom, unsigned long *grant_ref, evtchn_port_t *local_port)
+{
+	struct evtchn_alloc_unbound alloc_unbound;
+	int rc, rc2;
+
+	BUG_ON(atomic_read(&xenbus_xsd_state) != XENBUS_XSD_FOREIGN_INIT);
+	BUG_ON(!is_initial_xendomain());
+
+#if defined(CONFIG_PROC_FS) && defined(CONFIG_XEN_PRIVILEGED_GUEST)
+	remove_xen_proc_entry("xsd_kva");
+	remove_xen_proc_entry("xsd_port");
+#endif
+
+	rc = xb_free_port(xen_store_evtchn);
+	if (rc != 0)
+		goto fail0;
+
+	alloc_unbound.dom = DOMID_SELF;
+	alloc_unbound.remote_dom = remote_dom;
+	rc = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
+	                                 &alloc_unbound);
+	if (rc != 0)
+		goto fail0;
+	*local_port = xen_store_evtchn = alloc_unbound.port;
+
+	/* keep the old page (xen_store_mfn, xen_store_interface) */
+	rc = gnttab_grant_foreign_access(remote_dom, xen_store_mfn,
+	                                 GTF_permit_access);
+	if (rc < 0)
+		goto fail1;
+	*grant_ref = rc;
+
+	rc = xb_init_comms();
+	if (rc != 0)
+		goto fail1;
+
+	return 0;
+
+fail1:
+	rc2 = xb_free_port(xen_store_evtchn);
+	if (rc2 != 0)
+		printk(KERN_WARNING
+		       "XENBUS: Error freeing xenstore event channel: %d\n",
+		       rc2);
+fail0:
+	xen_store_evtchn = -1;
+	return rc;
+}
 
 static int xenbus_probe_init(void)
 {
@@ -958,7 +1041,7 @@
 #endif
 		xen_store_interface = mfn_to_virt(xen_store_mfn);
 	} else {
-		xenstored_ready = 1;
+		atomic_set(&xenbus_xsd_state, XENBUS_XSD_FOREIGN_READY);
 #ifdef CONFIG_XEN
 		xen_store_evtchn = xen_start_info->store_evtchn;
 		xen_store_mfn = xen_start_info->store_mfn;
@@ -969,8 +1052,11 @@
 		xen_store_interface = ioremap(xen_store_mfn << PAGE_SHIFT,
 					      PAGE_SIZE);
 #endif
+		/* Initialize the shared memory rings to talk to xenstored */
+		err = xb_init_comms();
+		if (err)
+			goto err;
 	}
-
 
 	xenbus_dev_init();
 
diff -r e68a58170585 drivers/xen/xenbus/xenbus_xs.c
--- a/drivers/xen/xenbus/xenbus_xs.c	Mon Mar 16 17:02:03 2009 +0000
+++ b/drivers/xen/xenbus/xenbus_xs.c	Thu Mar 19 10:46:26 2009 +0000
@@ -872,7 +872,6 @@
 
 int xs_init(void)
 {
-	int err;
 	struct task_struct *task;
 
 	INIT_LIST_HEAD(&xs_state.reply_list);
@@ -883,11 +882,6 @@
 	mutex_init(&xs_state.response_mutex);
 	init_rwsem(&xs_state.transaction_mutex);
 	init_rwsem(&xs_state.watch_mutex);
-
-	/* Initialize the shared memory rings to talk to xenstored */
-	err = xb_init_comms();
-	if (err)
-		return err;
 
 	task = kthread_run(xenwatch_thread, NULL, "xenwatch");
 	if (IS_ERR(task))
diff -r e68a58170585 include/xen/public/xenbus.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/include/xen/public/xenbus.h	Thu Mar 19 10:46:26 2009 +0000
@@ -0,0 +1,56 @@
+/******************************************************************************
+ * xenbus.h
+ * 
+ * Interface to /proc/xen/xenbus.
+ * 
+ * Copyright (c) 2008, Diego Ongaro <diego.ongaro@citrix.com>
+ * 
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License version 2
+ * as published by the Free Software Foundation; or, when distributed
+ * separately from the Linux kernel or incorporated into other
+ * software packages, subject to the following license:
+ * 
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this source file (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use, copy, modify,
+ * merge, publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ * 
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#ifndef __LINUX_PUBLIC_XENBUS_H__
+#define __LINUX_PUBLIC_XENBUS_H__
+
+#include <linux/types.h>
+
+#ifndef __user
+#define __user
+#endif
+
+typedef struct xenbus_alloc {
+	domid_t dom;
+	__u32 port;
+	__u32 grant_ref;
+} xenbus_alloc_t;
+
+/*
+ * @cmd: IOCTL_XENBUS_ALLOC
+ * @arg: &xenbus_alloc_t
+ * Return: 0, or -1 for error
+ */
+#define IOCTL_XENBUS_ALLOC					\
+	_IOC(_IOC_NONE, 'X', 0, sizeof(xenbus_alloc_t))
+
+#endif /* __LINUX_PUBLIC_XENBUS_H__ */


[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel