[RFC 1/2] Domain Groups - VMM Support

[RFC 1/2] Domain Groups - VMM Support

[Chris]

[-- Attachment #1: Type: text/plain, Size: 976 bytes --]

diffstat domgrps-vmm.patch

 arch/ia64/xen/xensetup.c                 |    7
 arch/powerpc/powerpc64/hypercall_table.S |    1
 arch/powerpc/setup.c                     |    7
 arch/x86/setup.c                         |    8
 arch/x86/x86_32/entry.S                  |    2
 arch/x86/x86_64/entry.S                  |    3
 common/Makefile                          |    2
 common/domain.c                          |    8
 common/domctl.c                          |   10
 common/domgrp.c                          |  315 +++++++++++++++++++++++++++++++
 common/domgrpctl.c                       |  134 +++++++++++++
 include/public/domctl.h                  |    2
 include/public/domgrpctl.h               |   86 ++++++++
 include/public/xen.h                     |    7
 include/xen/domgrp.h                     |   36 +++
 include/xen/hypercall.h                  |    5
 include/xen/sched.h                      |   26 ++

17 files changed, 657 insertions(+), 2 deletions(-)

[-- Attachment #2: domgrps-vmm.patch --]
[-- Type: text/plain, Size: 28507 bytes --]

diff -urN xen-unstable/xen/arch/ia64/xen/xensetup.c xen-unstable-trp-domgrps-rebase-tip/xen/arch/ia64/xen/xensetup.c
--- xen-unstable/xen/arch/ia64/xen/xensetup.c	2007-10-12 13:09:32.000000000 -0400
+++ xen-unstable-trp-domgrps-rebase-tip/xen/arch/ia64/xen/xensetup.c	2007-11-19 18:42:00.000000000 -0500
@@ -15,6 +15,7 @@
 #include <xen/version.h>
 #include <xen/console.h>
 #include <xen/domain.h>
+#include <xen/domgrp.h>
 #include <xen/serial.h>
 #include <xen/trace.h>
 #include <xen/keyhandler.h>
@@ -550,6 +551,10 @@
 
     expose_p2m_init();
 
+    /* initialize domain groups */
+    if (init_domain_groups())
+        panic("Error creating default groups\n");
+
     /* Create initial domain 0. */
     dom0 = domain_create(0, 0, DOM0_SSIDREF);
     if (dom0 == NULL)
@@ -559,6 +564,8 @@
         panic("Cannot allocate dom0 vcpu 0\n");
 
     dom0->is_privileged = 1;
+    if (add_dom_to_grp(dom0, 0))
+        panic("Error adding dom0 to grp0\n");
 
     /*
      * We're going to setup domain0 using the module(s) that we stashed safely
diff -urN xen-unstable/xen/arch/powerpc/powerpc64/hypercall_table.S xen-unstable-trp-domgrps-rebase-tip/xen/arch/powerpc/powerpc64/hypercall_table.S
--- xen-unstable/xen/arch/powerpc/powerpc64/hypercall_table.S	2007-08-06 17:59:54.000000000 -0400
+++ xen-unstable-trp-domgrps-rebase-tip/xen/arch/powerpc/powerpc64/hypercall_table.S	2007-11-19 18:42:00.000000000 -0500
@@ -41,6 +41,7 @@
         .quad 0 /* do_hvm_op */
         .quad do_sysctl             /* 35 */
         .quad do_domctl
+        .quad do_domgrpctl
         .rept NR_hypercalls-((.-__hypercall_table)/8)
         .quad do_ni_hypercall
         .endr
diff -urN xen-unstable/xen/arch/powerpc/setup.c xen-unstable-trp-domgrps-rebase-tip/xen/arch/powerpc/setup.c
--- xen-unstable/xen/arch/powerpc/setup.c	2007-09-13 14:40:11.000000000 -0400
+++ xen-unstable-trp-domgrps-rebase-tip/xen/arch/powerpc/setup.c	2007-11-19 18:42:00.000000000 -0500
@@ -32,6 +32,7 @@
 #include <xen/trace.h>
 #include <xen/mm.h>
 #include <xen/domain.h>
+#include <xen/domgrp.h>
 #include <xen/gdbstub.h>
 #include <xen/symbols.h>
 #include <xen/keyhandler.h>
@@ -365,6 +366,10 @@
     /* This cannot be called before secondary cpus are marked online.  */
     percpu_free_unused_areas();
 
+    /* initialize domain groups */
+    if (init_domain_groups())
+        panic("Error creating default groups\n");
+
     /* Create initial domain 0. */
     dom0 = domain_create(0, 0, DOM0_SSIDREF);
     if (dom0 == NULL)
@@ -375,6 +380,8 @@
     dom0->vcpu[0]->cpu_affinity = cpumask_of_cpu(0);
 
     dom0->is_privileged = 1;
+    if (add_dom_to_grp(dom0, 0))
+        panic("Error adding dom0 to grp0\n");
 
     /* scrub_heap_pages() requires IRQs enabled, and we're post IRQ setup... */
     local_irq_enable();
diff -urN xen-unstable/xen/arch/x86/setup.c xen-unstable-trp-domgrps-rebase-tip/xen/arch/x86/setup.c
--- xen-unstable/xen/arch/x86/setup.c	2007-12-17 16:45:04.000000000 -0500
+++ xen-unstable-trp-domgrps-rebase-tip/xen/arch/x86/setup.c	2007-12-17 16:44:23.000000000 -0500
@@ -3,6 +3,7 @@
 #include <xen/lib.h>
 #include <xen/sched.h>
 #include <xen/domain.h>
+#include <xen/domgrp.h>
 #include <xen/serial.h>
 #include <xen/softirq.h>
 #include <xen/acpi.h>
@@ -955,6 +956,10 @@
     if ( opt_watchdog ) 
         watchdog_enable();
 
+    /* initialize domain groups */
+    if (init_domain_groups())
+        panic("Error creating default groups\n");
+
     /* Create initial domain 0. */
     dom0 = domain_create(0, 0, DOM0_SSIDREF);
     if ( (dom0 == NULL) || (alloc_vcpu(dom0, 0, 0) == NULL) )
@@ -962,6 +967,9 @@
 
     dom0->is_privileged = 1;
 
+    if (add_dom_to_grp(dom0, 0))
+        panic("Error adding dom0 to grp0\n");
+
     /* Grab the DOM0 command line. */
     cmdline = (char *)(mod[0].string ? __va(mod[0].string) : NULL);
     if ( (cmdline != NULL) || (kextra != NULL) )
diff -urN xen-unstable/xen/arch/x86/x86_32/entry.S xen-unstable-trp-domgrps-rebase-tip/xen/arch/x86/x86_32/entry.S
--- xen-unstable/xen/arch/x86/x86_32/entry.S	2007-11-19 10:38:08.000000000 -0500
+++ xen-unstable-trp-domgrps-rebase-tip/xen/arch/x86/x86_32/entry.S	2007-11-19 18:42:00.000000000 -0500
@@ -682,6 +682,7 @@
         .long do_sysctl             /* 35 */
         .long do_domctl
         .long do_kexec_op
+        .long do_domgrpctl
         .rept NR_hypercalls-((.-hypercall_table)/4)
         .long do_ni_hypercall
         .endr
@@ -725,6 +726,7 @@
         .byte 1 /* do_sysctl            */  /* 35 */
         .byte 1 /* do_domctl            */
         .byte 2 /* do_kexec_op          */
+        .byte 1 /* do_domgrpctl         */
         .rept NR_hypercalls-(.-hypercall_args_table)
         .byte 0 /* do_ni_hypercall      */
         .endr
diff -urN xen-unstable/xen/arch/x86/x86_64/entry.S xen-unstable-trp-domgrps-rebase-tip/xen/arch/x86/x86_64/entry.S
--- xen-unstable/xen/arch/x86/x86_64/entry.S	2007-11-19 10:38:08.000000000 -0500
+++ xen-unstable-trp-domgrps-rebase-tip/xen/arch/x86/x86_64/entry.S	2007-11-19 18:43:06.000000000 -0500
@@ -672,6 +672,7 @@
         .quad do_sysctl             /* 35 */
         .quad do_domctl
         .quad do_kexec_op
+        .quad do_domgrpctl
         .rept NR_hypercalls-((.-hypercall_table)/8)
         .quad do_ni_hypercall
         .endr
@@ -715,7 +716,7 @@
         .byte 1 /* do_sysctl            */  /* 35 */
         .byte 1 /* do_domctl            */
         .byte 2 /* do_kexec             */
-        .byte 1 /* do_xsm_op            */
+        .byte 1 /* do_domgrpctl         */
         .rept NR_hypercalls-(.-hypercall_args_table)
         .byte 0 /* do_ni_hypercall      */
         .endr
diff -urN xen-unstable/xen/common/domain.c xen-unstable-trp-domgrps-rebase-tip/xen/common/domain.c
--- xen-unstable/xen/common/domain.c	2007-11-19 10:38:08.000000000 -0500
+++ xen-unstable-trp-domgrps-rebase-tip/xen/common/domain.c	2007-11-19 18:42:00.000000000 -0500
@@ -11,6 +11,7 @@
 #include <xen/errno.h>
 #include <xen/sched.h>
 #include <xen/domain.h>
+#include <xen/domgrp.h>
 #include <xen/mm.h>
 #include <xen/event.h>
 #include <xen/time.h>
@@ -57,6 +58,7 @@
 
     memset(d, 0, sizeof(*d));
     d->domain_id = domid;
+    d->group_id = INVAL_GROUP_ID;
 
     if ( xsm_alloc_security_domain(d) != 0 )
     {
@@ -241,6 +243,8 @@
         rcu_assign_pointer(*pd, d);
         rcu_assign_pointer(domain_hash[DOMAIN_HASH(domid)], d);
         spin_unlock(&domlist_update_lock);
+
+        add_dom_to_grp(d, NULL_GROUP_ID);
     }
 
     return d;
@@ -388,6 +392,8 @@
 {
     struct vcpu *v;
 
+    del_dom_from_grp(d);
+
     if ( d->domain_id == 0 )
         dom0_shutdown(reason);
 
@@ -522,6 +528,8 @@
     if ( _atomic_read(old) != 0 )
         return;
 
+    del_dom_from_grp(d);
+
     /* Delete from task list and task hashtable. */
     spin_lock(&domlist_update_lock);
     pd = &domain_list;
diff -urN xen-unstable/xen/common/domctl.c xen-unstable-trp-domgrps-rebase-tip/xen/common/domctl.c
--- xen-unstable/xen/common/domctl.c	2007-12-17 16:45:04.000000000 -0500
+++ xen-unstable-trp-domgrps-rebase-tip/xen/common/domctl.c	2007-12-17 16:44:23.000000000 -0500
@@ -12,6 +12,7 @@
 #include <xen/mm.h>
 #include <xen/sched.h>
 #include <xen/domain.h>
+#include <xen/domgrp.h>
 #include <xen/event.h>
 #include <xen/domain_page.h>
 #include <xen/trace.h>
@@ -94,8 +95,10 @@
     u64 cpu_time = 0;
     int flags = XEN_DOMINF_blocked;
     struct vcpu_runstate_info runstate;
-    
+    struct domain_group *grp;
+
     info->domain = d->domain_id;
+    info->group = d->group_id;
     info->nr_online_vcpus = 0;
     
     /* 
@@ -136,6 +139,9 @@
     info->shared_info_frame = mfn_to_gmfn(d, __pa(d->shared_info)>>PAGE_SHIFT);
 
     memcpy(info->handle, d->handle, sizeof(xen_domain_handle_t));
+    grp = find_grp_by_id(info->group);
+    if (grp)
+        memcpy(info->dg_handle, grp->handle, sizeof(xen_domain_group_handle_t));
 }
 
 static unsigned int default_vcpu0_location(void)
@@ -256,6 +262,7 @@
     {
         struct domain *d = rcu_lock_domain_by_id(op->domain);
         ret = -ESRCH;
+
         if ( d != NULL )
         {
             ret = xsm_pausedomain(d);
@@ -426,6 +433,7 @@
     {
         struct domain *d = rcu_lock_domain_by_id(op->domain);
         ret = -ESRCH;
+
         if ( d != NULL )
         {
             ret = xsm_destroydomain(d) ? : domain_kill(d);
diff -urN xen-unstable/xen/common/domgrp.c xen-unstable-trp-domgrps-rebase-tip/xen/common/domgrp.c
--- xen-unstable/xen/common/domgrp.c	1969-12-31 19:00:00.000000000 -0500
+++ xen-unstable-trp-domgrps-rebase-tip/xen/common/domgrp.c	2007-11-26 16:46:46.000000000 -0500
@@ -0,0 +1,315 @@
+/******************************************************************************
+ * domgrp.c
+ *
+ * Generic domain group-handling functions.
+ *
+ * Author: Chris Bookholt (hap10@tycho.ncsc.mil)
+ */
+
+#include <xen/sched.h>
+#include <xen/list.h>
+#include <xen/xmalloc.h>
+#include <public/domgrpctl.h>
+
+DEFINE_SPINLOCK(domgrplist_update_lock);
+DEFINE_RCU_READ_LOCK(domgrplist_read_lock);
+
+struct list_head domgrplist;
+
+#define SERIALIZE_LINKED_LIST(op_name, list_name)		\
+    rcu_read_lock(&grp->op_name##_read_lock);			\
+    memset(&info->op_name, 0, sizeof(domid_t)*MAX_GROUP_SIZE);	\
+    if (!list_empty(&grp->op_name)) {				\
+        i = 0;							\
+        list_for_each_entry(dom, &grp->op_name, list_name) {	\
+            info->op_name[i] = dom->domain_id;			\
+       	    i++;						\
+	}							\
+    } else 							\
+	info->op_name[i] = NULL_GROUP_ID;			\
+    rcu_read_unlock(&grp->op_name##_read_lock);
+
+void get_grp_info(struct domain_group *grp, xen_domgrpctl_getgrpinfo_t * info)
+{
+	struct domain *dom;
+	uint16_t i = 0;
+
+	info->dgid = grp->group_id;
+	info->size = grp->size;
+
+	SERIALIZE_LINKED_LIST(member_list, member);
+
+	memcpy(info->handle, grp->handle, sizeof(xen_domain_group_handle_t));
+}
+
+struct domain_group *alloc_domain_group(void)
+{
+	struct domain_group *grp = NULL;
+
+	if ((grp = xmalloc(struct domain_group)) != NULL)
+		 memset(grp, 0, sizeof(*grp));
+
+	return grp;
+}
+
+struct domain_group *find_grp_by_id(dgid_t dgid)
+{
+	struct domain_group *grp;
+
+	rcu_read_lock(&domgrplist_read_lock);
+	list_for_each_entry(grp, &domgrplist, groups) {
+	    BUG_ON(grp == NULL);
+	    if (grp->group_id == dgid) 
+		goto out;
+	}
+	grp = NULL;
+      out:
+	rcu_read_unlock(&domgrplist_read_lock);
+	return grp;
+}
+
+uint16_t get_new_group_id(void)
+{
+	static DEFINE_SPINLOCK(domgrpcnt_lock);
+	static dgid_t group_counter = 1;
+	dgid_t ret;
+	
+	if (group_counter < NULL_GROUP_ID - 1) {
+		spin_lock(&domgrpcnt_lock);
+	        ret = group_counter++;
+		spin_unlock(&domgrpcnt_lock);
+	} else
+		ret = NULL_GROUP_ID;
+	return ret;
+}
+
+struct domain_group *domain_group_create(dgid_t dgid)
+{
+	struct domain_group *grp = NULL, *tail;
+
+	grp = alloc_domain_group();
+	if (!grp)
+		goto out;
+
+	grp->group_id = dgid;
+	grp->size = 0;
+	if (dgid == 0)
+		memset(grp->handle, 0, sizeof(xen_domain_group_handle_t));
+	else if (dgid == NULL_GROUP_ID)
+		memset(grp->handle, 0xFF, sizeof(xen_domain_group_handle_t));
+
+	/* init group's lists */
+	INIT_LIST_HEAD(&grp->member_list);
+
+	/* init group's locks */
+	spin_lock_init(&grp->grp_big_lock);
+	spin_lock_init(&grp->member_list_update_lock);
+
+	/* add new group to domgrplist *
+	 * TODO: This is a candidate for optimization. Could 
+	 * be optimized by maintaining a ptr to the tail, but 
+	 * list size is small, so search isn't expensive */
+	if (dgid != 0 && dgid != NULL_GROUP_ID) {
+		tail = find_grp_by_id(NULL_GROUP_ID);
+		spin_lock(&domgrplist_update_lock);
+		list_add_tail(&grp->groups, &tail->groups);
+		spin_unlock(&domgrplist_update_lock);
+	} else {
+		spin_lock(&domgrplist_update_lock);
+		list_add_tail(&grp->groups, &domgrplist);
+		spin_unlock(&domgrplist_update_lock);
+	}
+
+      out:
+	return grp;
+}
+
+int dom_in_member_list(domid_t domid, struct domain_group *grp)
+{
+	struct domain *dom;
+	int ret = 0;
+
+	if (grp == NULL) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	rcu_read_lock(&grp->member_list_read_lock);
+	list_for_each_entry(dom, &grp->member_list, member) {
+		if (dom->domain_id == domid) {
+			ret = 1;
+			break;
+		}
+	}
+	rcu_read_unlock(&grp->member_list_read_lock);
+      out:
+	return ret;
+}
+
+#define RM_DOM_FROM_LIST(list_name, entry)	\
+    spin_lock(&grp->list_name##_update_lock);	\
+    if (!list_empty(&grp->list_name)) 		\
+        list_del(&dom->entry);		    	\
+    spin_unlock(&grp->list_name##_update_lock);
+
+void del_dom_from_grp(struct domain *dom)
+{
+	struct domain_group *grp;
+
+	grp = find_grp_by_id(dom->group_id);
+
+	if (grp == NULL)
+		goto out;
+
+	if (dom_in_member_list(dom->domain_id, grp) <= 0)
+		goto out;
+
+	RM_DOM_FROM_LIST(member_list, member);
+	dom->group_id = INVAL_GROUP_ID;
+	grp->size--;
+	
+      out:
+	return;
+}
+
+int add_dom_to_grp(struct domain *dom, dgid_t dgid)
+{
+	struct domain_group *grp = NULL;
+	int ret = 0;
+
+	grp = find_grp_by_id(dgid);
+	if (grp == NULL) {
+		ret = -ESRCH;
+		goto out;
+	} 
+
+	if (grp->size >= MAX_GROUP_SIZE) {
+		ret = -EPERM;
+		goto out;
+	} 
+
+	/* skip it if dom is already a member */
+	if (dom_in_member_list(dom->domain_id, grp))
+		goto out;
+	
+	/* remove dom from old group */
+	if (dom->group_id != INVAL_GROUP_ID)
+		del_dom_from_grp(dom);
+
+	/* add dom to end of new group list */
+	spin_lock(&grp->member_list_update_lock);
+	list_add_tail(&dom->member, &grp->member_list);
+	spin_unlock(&grp->member_list_update_lock);
+
+	dom->group_id = dgid;
+	dom->grp_big_lock = &grp->grp_big_lock;
+
+	grp->size++;
+      out:
+	return ret;
+}
+
+int pause_grp(dgid_t dgid)
+{
+	int ret = 0;
+	struct domain *member;
+	struct domain_group *grp;
+
+	if (dgid == 0) {
+		ret = -EPERM;
+		goto out;
+	}
+
+	grp = find_grp_by_id(dgid);
+
+	if (grp == NULL) {
+		ret = -ESRCH;
+		goto out;
+	}
+
+	spin_lock_recursive(&grp->grp_big_lock);
+	rcu_read_lock(&grp->member_list_read_lock);
+	/* could ignore interupts during this loop to increase atomicity */
+	list_for_each_entry(member, &grp->member_list, member) {
+		if (member != current->domain && member->domain_id != 0)
+			domain_pause_by_systemcontroller(member);
+	}
+	rcu_read_unlock(&grp->member_list_read_lock);
+	spin_unlock_recursive(&grp->grp_big_lock);
+      out:
+	return ret;
+}
+
+int unpause_grp(dgid_t dgid)
+{
+	int ret = 0;
+	struct domain *member;
+	struct domain_group *grp;
+
+	if (dgid == 0) {
+		ret = -EPERM;
+		goto out;
+	}
+
+	grp = find_grp_by_id(dgid);
+
+	if (grp == NULL) {
+		ret = -ESRCH;
+		goto out;
+	}
+
+	spin_lock_recursive(&grp->grp_big_lock);
+	rcu_read_lock(&grp->member_list_read_lock);
+	/* could ignore interupts during this loop to increase atomicity */
+	list_for_each_entry(member, &grp->member_list, member) {
+		if (member != current->domain && member->domain_id != 0)
+			domain_unpause_by_systemcontroller(member);
+	}
+	rcu_read_unlock(&grp->member_list_read_lock);
+	spin_unlock_recursive(&grp->grp_big_lock);
+      out:
+	return ret;
+}
+
+int domain_group_destroy(dgid_t dgid)
+{
+	int ret = 0;
+	struct domain_group *grp;
+
+	grp = find_grp_by_id(dgid);
+	if (grp == NULL) {
+		ret = -ESRCH;
+		goto out;
+	}
+
+	if (grp->size != 0) {
+		ret = -EPERM;
+		printk(KERN_INFO "refusing to destroy non-emtpry group %d\n", grp->group_id);
+		goto out;
+	}
+
+	spin_lock(&domgrplist_update_lock);
+	list_del(&grp->groups);
+	spin_unlock(&domgrplist_update_lock);
+
+	xfree(grp);
+      out:
+	return ret;
+}
+
+int init_domain_groups(void)
+{
+	struct domain_group *grp0, *nullgrp;
+	int ret = 0;
+	INIT_LIST_HEAD(&domgrplist);
+
+	/* order matters for creation of default groups: 
+	 * create default groups in order of ascending dgid so they 
+	 * are added to the group list in the expected order */
+	grp0 = domain_group_create(0);
+	nullgrp = domain_group_create(NULL_GROUP_ID);
+
+	if (!grp0 || !nullgrp)
+	    ret = -ENOMEM;
+	return ret;
+}
diff -urN xen-unstable/xen/common/domgrpctl.c xen-unstable-trp-domgrps-rebase-tip/xen/common/domgrpctl.c
--- xen-unstable/xen/common/domgrpctl.c	1969-12-31 19:00:00.000000000 -0500
+++ xen-unstable-trp-domgrps-rebase-tip/xen/common/domgrpctl.c	2007-11-19 18:42:00.000000000 -0500
@@ -0,0 +1,134 @@
+/******************************************************************************
+ * domgrpctl.c
+ *
+ * Domain group management operations. For use by node control stack.
+ *
+ * Author: Chris Bookholt (hap10@tycho.ncsc.mil)
+ */
+
+#include <xen/sched.h>
+#include <xen/domgrp.h>
+#include <xen/guest_access.h>
+#include <public/domgrpctl.h>
+#include <asm/current.h>
+
+long do_domgrpctl(XEN_GUEST_HANDLE(xen_domgrpctl_t) u_domgrpctl)
+{
+	long ret = 0;
+	struct xen_domgrpctl curop, *op = &curop;
+	static DEFINE_SPINLOCK(domgrpctl_lock);
+	struct domain_group *grp;
+	dgid_t dgid;
+
+	if (!IS_PRIV(current->domain)) {
+		ret = -EPERM;
+		goto out;
+	}
+
+	if (copy_from_guest(op, u_domgrpctl, 1)) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	if (op->interface_version != XEN_DOMGRPCTL_INTERFACE_VERSION) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	spin_lock(&domgrpctl_lock);
+
+	switch (op->cmd) {
+
+	case XEN_DOMGRPCTL_getgrpinfo:
+		{
+			rcu_read_lock(&domgrplist_read_lock);
+			ret = -ESRCH;
+			dgid = op->u.get_grp_info.dgid;
+
+			list_for_each_entry(grp, &domgrplist, groups) {
+				if (grp->group_id >= dgid) {
+					ret = 0;
+					break;
+				}
+			}
+			if (ret)
+				goto getgrpinfo_out;
+
+			get_grp_info(grp, &op->u.get_grp_info);
+
+			if (copy_to_guest(u_domgrpctl, op, 1))
+				ret = -EFAULT;
+
+		      getgrpinfo_out:
+			rcu_read_unlock(&domgrplist_read_lock);
+			break;
+		}
+
+	case XEN_DOMGRPCTL_creategrp:
+		{
+			dgid = get_new_group_id();
+			if (dgid == NULL_GROUP_ID) {
+				ret = -EINVAL;
+				break;
+			}
+
+			grp = domain_group_create(dgid);
+			if (grp == NULL) {
+				ret = -ENOMEM;
+				break;
+			}
+
+			memcpy(grp->handle, op->u.create_grp.handle,
+			       sizeof(xen_domain_group_handle_t));
+
+			op->u.create_grp.dgid = grp->group_id;
+			if (copy_to_guest(u_domgrpctl, op, 1))
+				ret = -EFAULT;
+
+			break;
+		}
+
+	case XEN_DOMGRPCTL_joingrp:
+		{
+			domid_t domid;
+			struct domain *dom;
+
+			domid = op->u.join_grp.domid;
+			dgid = op->u.join_grp.dgid;
+
+			dom = get_domain_by_id(domid);
+			if (dom == NULL)
+				ret = -ESRCH;
+			else
+				ret = add_dom_to_grp(dom, dgid);
+			put_domain(dom);
+
+			break;
+		}
+
+	case XEN_DOMGRPCTL_pausegrp:
+		{
+			ret = pause_grp(op->u.pause_grp.dgid);
+			break;
+		}
+
+	case XEN_DOMGRPCTL_unpausegrp:
+		{
+			ret = unpause_grp(op->u.unpause_grp.dgid);
+			break;
+		}
+
+	case XEN_DOMGRPCTL_destroygrp:
+		{
+			ret = domain_group_destroy(op->u.destroy_grp.dgid);
+			break;
+		}
+
+	default:
+		ret = -EINVAL;
+	}
+
+	spin_unlock(&domgrpctl_lock);
+      out:
+	return ret;
+}
diff -urN xen-unstable/xen/common/Makefile xen-unstable-trp-domgrps-rebase-tip/xen/common/Makefile
--- xen-unstable/xen/common/Makefile	2007-11-19 10:38:08.000000000 -0500
+++ xen-unstable-trp-domgrps-rebase-tip/xen/common/Makefile	2007-11-19 18:42:00.000000000 -0500
@@ -1,6 +1,8 @@
 obj-y += bitmap.o
 obj-y += domctl.o
+obj-y += domgrpctl.o
 obj-y += domain.o
+obj-y += domgrp.o
 obj-y += event_channel.o
 obj-y += grant_table.o
 obj-y += kernel.o
diff -urN xen-unstable/xen/include/public/domctl.h xen-unstable-trp-domgrps-rebase-tip/xen/include/public/domctl.h
--- xen-unstable/xen/include/public/domctl.h	2007-12-17 16:45:05.000000000 -0500
+++ xen-unstable-trp-domgrps-rebase-tip/xen/include/public/domctl.h	2007-12-17 16:44:23.000000000 -0500
@@ -67,6 +67,7 @@
 struct xen_domctl_getdomaininfo {
     /* OUT variables. */
     domid_t  domain;              /* Also echoed in domctl.domain */
+    dgid_t   group;
  /* Domain is scheduled to die. */
 #define _XEN_DOMINF_dying     0
 #define XEN_DOMINF_dying      (1U<<_XEN_DOMINF_dying)
@@ -103,6 +104,7 @@
     uint32_t max_vcpu_id;        /* Maximum VCPUID in use by this domain. */
     uint32_t ssidref;
     xen_domain_handle_t handle;
+    xen_domain_group_handle_t dg_handle;
 };
 typedef struct xen_domctl_getdomaininfo xen_domctl_getdomaininfo_t;
 DEFINE_XEN_GUEST_HANDLE(xen_domctl_getdomaininfo_t);
diff -urN xen-unstable/xen/include/public/domgrpctl.h xen-unstable-trp-domgrps-rebase-tip/xen/include/public/domgrpctl.h
--- xen-unstable/xen/include/public/domgrpctl.h	1969-12-31 19:00:00.000000000 -0500
+++ xen-unstable-trp-domgrps-rebase-tip/xen/include/public/domgrpctl.h	2007-11-19 18:42:00.000000000 -0500
@@ -0,0 +1,86 @@
+/******************************************************************************
+ * domgrpctl.h
+ *
+ * Domain group management operations. For use by node control stack.
+ *
+ * Author: Chris Bookholt (hap10@tycho.ncsc.mil)
+ */
+
+#ifndef __XEN_PUBLIC_DOMGRPCTL_H__
+#define __XEN_PUBLIC_DOMGRPCTL_H__
+
+#if !defined(__XEN__) && !defined(__XEN_TOOLS__)
+#error "domgrpctl operations are intended for use by node control tools only"
+#endif
+
+#include "xen.h"
+
+#define XEN_DOMGRPCTL_INTERFACE_VERSION 0x00000001
+
+#define MAX_GROUP_SIZE			24
+#define NULL_GROUP_ID			(0x7FFFU)
+#define INVAL_GROUP_ID			(0xFFFFU)
+
+#define XEN_DOMGRPCTL_creategrp		1
+struct xen_domgrpctl_creategrp {
+	dgid_t dgid;
+	xen_domain_group_handle_t handle;
+};
+typedef struct xen_domgrpctl_creategrp xen_domgrpctl_creategrp_t;
+DEFINE_XEN_GUEST_HANDLE(xen_domgrpctl_creategrp_t);
+
+#define XEN_DOMGRPCTL_joingrp		2
+struct xen_domgrpctl_joingrp {
+	domid_t domid;
+	dgid_t dgid;
+};
+typedef struct xen_domgrpctl_joingrp xen_domgrpctl_joingrp_t;
+DEFINE_XEN_GUEST_HANDLE(xen_domgrpctl_joingrp_t);
+
+#define XEN_DOMGRPCTL_pausegrp		3
+struct xen_domgrpctl_pausegrp {
+	dgid_t dgid;
+};
+typedef struct xen_domgrpctl_pausegrp xen_domgrpctl_pausegrp_t;
+DEFINE_XEN_GUEST_HANDLE(xen_domgrpctl_pausegrp_t);
+
+#define XEN_DOMGRPCTL_unpausegrp	4
+struct xen_domgrpctl_unpausegrp {
+	dgid_t dgid;
+};
+typedef struct xen_domgrpctl_unpausegrp xen_domgrpctl_unpausegrp_t;
+DEFINE_XEN_GUEST_HANDLE(xen_domgrpctl_unpausegrp_t);
+
+#define XEN_DOMGRPCTL_destroygrp	5
+struct xen_domgrpctl_destroygrp {
+	dgid_t dgid;
+};
+typedef struct xen_domgrpctl_destroygrp xen_domgrpctl_destroygrp_t;
+DEFINE_XEN_GUEST_HANDLE(xen_domgrpctl_destroygrp_t);
+
+#define XEN_DOMGRPCTL_getgrpinfo	6
+struct xen_domgrpctl_getgrpinfo {
+	dgid_t dgid;
+	uint16_t size;
+	domid_t member_list[MAX_GROUP_SIZE];
+	xen_domain_group_handle_t handle;
+};
+typedef struct xen_domgrpctl_getgrpinfo xen_domgrpctl_getgrpinfo_t;
+DEFINE_XEN_GUEST_HANDLE(xen_domgrpctl_getgrpinfo_t);
+
+struct xen_domgrpctl {
+	uint32_t cmd;
+	uint32_t interface_version;
+	union {
+		struct xen_domgrpctl_creategrp create_grp;
+		struct xen_domgrpctl_joingrp join_grp;
+		struct xen_domgrpctl_pausegrp pause_grp;
+		struct xen_domgrpctl_unpausegrp unpause_grp;
+		struct xen_domgrpctl_destroygrp destroy_grp;
+		struct xen_domgrpctl_getgrpinfo get_grp_info;
+	} u;
+};
+typedef struct xen_domgrpctl xen_domgrpctl_t;
+DEFINE_XEN_GUEST_HANDLE(xen_domgrpctl_t);
+
+#endif				/* __XEN_PUBLIC_DOMGRPCTL_H__ */
diff -urN xen-unstable/xen/include/public/xen.h xen-unstable-trp-domgrps-rebase-tip/xen/include/public/xen.h
--- xen-unstable/xen/include/public/xen.h	2007-10-22 15:12:58.000000000 -0400
+++ xen-unstable-trp-domgrps-rebase-tip/xen/include/public/xen.h	2007-11-19 18:42:00.000000000 -0500
@@ -80,6 +80,7 @@
 #define __HYPERVISOR_sysctl               35
 #define __HYPERVISOR_domctl               36
 #define __HYPERVISOR_kexec_op             37
+#define __HYPERVISOR_domgrpctl            38
 
 /* Architecture-specific hypercall definitions. */
 #define __HYPERVISOR_arch_0               48
@@ -297,6 +298,8 @@
 
 typedef uint16_t domid_t;
 
+typedef uint16_t dgid_t;
+
 /* Domain ids >= DOMID_FIRST_RESERVED cannot be used for ordinary domains. */
 #define DOMID_FIRST_RESERVED (0x7FF0U)
 
@@ -520,7 +523,9 @@
     unsigned long nr_pt_frames; /* Number of bootstrap p.t. frames.       */
     unsigned long mfn_list;     /* VIRTUAL address of page-frame list.    */
     unsigned long mod_start;    /* VIRTUAL address of pre-loaded module.  */
+                                /* or VIRTUAL address of module_t array   */
     unsigned long mod_len;      /* Size (bytes) of pre-loaded module.     */
+                                /* or length of module_t array.           */
     int8_t cmd_line[MAX_GUEST_CMDLINE];
 };
 typedef struct start_info start_info_t;
@@ -579,6 +584,8 @@
 
 typedef uint8_t xen_domain_handle_t[16];
 
+typedef uint8_t xen_domain_group_handle_t[16];
+
 /* Turn a plain number into a C unsigned long constant. */
 #define __mk_unsigned_long(x) x ## UL
 #define mk_unsigned_long(x) __mk_unsigned_long(x)
diff -urN xen-unstable/xen/include/xen/domgrp.h xen-unstable-trp-domgrps-rebase-tip/xen/include/xen/domgrp.h
--- xen-unstable/xen/include/xen/domgrp.h	1969-12-31 19:00:00.000000000 -0500
+++ xen-unstable-trp-domgrps-rebase-tip/xen/include/xen/domgrp.h	2007-11-19 18:42:00.000000000 -0500
@@ -0,0 +1,36 @@
+/******************************************************************************
+ * domgrp.h
+ *
+ * Generic domain group-handling functions.
+ *
+ * Author: Chris Bookholt (hap10@tycho.ncsc.mil)
+ */
+
+#ifndef __XEN_DOM_GROUP_H__
+#define __XEN_DOM_GROUP_H__
+
+#include <public/domgrpctl.h>
+
+extern struct list_head domgrplist;
+
+void get_grp_info(struct domain_group *grp, xen_domgrpctl_getgrpinfo_t * info);
+
+struct domain_group *find_grp_by_id(dgid_t dgid);
+
+uint16_t get_new_group_id(void);
+
+struct domain_group *domain_group_create(dgid_t dgid);
+
+int del_dom_from_grp(struct domain *old_dom);
+
+int add_dom_to_grp(struct domain *dom, dgid_t dgid);
+
+int pause_grp(dgid_t dgid);
+
+int unpause_grp(dgid_t dgid);
+
+int domain_group_destroy(dgid_t dgid);
+
+int init_domain_groups(void);
+
+#endif				/* __XEN_DOM_GROUP_H__ */
diff -urN xen-unstable/xen/include/xen/hypercall.h xen-unstable-trp-domgrps-rebase-tip/xen/include/xen/hypercall.h
--- xen-unstable/xen/include/xen/hypercall.h	2007-09-13 14:40:12.000000000 -0400
+++ xen-unstable-trp-domgrps-rebase-tip/xen/include/xen/hypercall.h	2007-11-19 18:42:00.000000000 -0500
@@ -10,6 +10,7 @@
 #include <xen/time.h>
 #include <public/xen.h>
 #include <public/domctl.h>
+#include <public/domgrpctl.h>
 #include <public/sysctl.h>
 #include <public/platform.h>
 #include <public/event_channel.h>
@@ -35,6 +36,10 @@
     XEN_GUEST_HANDLE(xen_domctl_t) u_domctl);
 
 extern long
+do_domgrpctl(
+    XEN_GUEST_HANDLE(xen_domgrpctl_t) u_domgrpctl);
+
+extern long
 do_sysctl(
     XEN_GUEST_HANDLE(xen_sysctl_t) u_sysctl);
 
diff -urN xen-unstable/xen/include/xen/sched.h xen-unstable-trp-domgrps-rebase-tip/xen/include/xen/sched.h
--- xen-unstable/xen/include/xen/sched.h	2007-11-19 10:38:08.000000000 -0500
+++ xen-unstable-trp-domgrps-rebase-tip/xen/include/xen/sched.h	2007-11-19 18:47:23.000000000 -0500
@@ -9,6 +9,7 @@
 #include <xen/shared.h>
 #include <public/xen.h>
 #include <public/domctl.h>
+#include <public/domgrpctl.h>
 #include <public/vcpu.h>
 #include <public/xsm/acm.h>
 #include <xen/time.h>
@@ -19,6 +20,7 @@
 #include <xen/xenoprof.h>
 #include <xen/rcupdate.h>
 #include <xen/irq.h>
+#include <xen/init.h>
 
 #ifdef CONFIG_COMPAT
 #include <compat/vcpu.h>
@@ -27,6 +29,9 @@
 
 extern unsigned long volatile jiffies;
 
+extern spinlock_t domgrplist_update_lock;
+extern rcu_read_lock_t domgrplist_read_lock;
+
 /* A global pointer to the initial domain (DOM0). */
 extern struct domain *dom0;
 
@@ -145,9 +150,13 @@
 {
     domid_t          domain_id;
 
+    dgid_t           group_id;        /* TODO: replace struct group ptr */
+    struct list_head member;
+
     shared_info_t   *shared_info;     /* shared data area */
 
     spinlock_t       big_lock;
+    spinlock_t      *grp_big_lock;
 
     spinlock_t       page_alloc_lock; /* protects all the following fields  */
     struct list_head page_list;       /* linked list, of size tot_pages     */
@@ -233,6 +242,23 @@
     spinlock_t hypercall_deadlock_mutex;
 };
 
+struct domain_group
+{
+    dgid_t                       group_id;
+    uint16_t                     size;
+
+    struct list_head             groups;
+
+    struct list_head             member_list;
+
+    spinlock_t                   grp_big_lock;
+
+    spinlock_t                   member_list_update_lock;
+    rcu_read_lock_t              member_list_read_lock;
+
+    xen_domain_group_handle_t    handle;
+};
+
 struct domain_setup_info
 {
     /* Initialised by caller. */

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel

Re: [RFC 1/2] Domain Groups - VMM Support

[Samuel Thibault]

Hello,

Chris, le Tue 18 Dec 2007 11:02:55 -0500, a écrit :
> @@ -520,7 +523,9 @@
>      unsigned long nr_pt_frames; /* Number of bootstrap p.t. frames.       */
>      unsigned long mfn_list;     /* VIRTUAL address of page-frame list.    */
>      unsigned long mod_start;    /* VIRTUAL address of pre-loaded module.  */
> +                                /* or VIRTUAL address of module_t array   */
>      unsigned long mod_len;      /* Size (bytes) of pre-loaded module.     */
> +                                /* or length of module_t array.           */
>      int8_t cmd_line[MAX_GUEST_CMDLINE];
>  };
>  typedef struct start_info start_info_t;

It looks like this doesn't belong to the patch.  However, I'm interested
in this for the GNU/Mach Xen support, where we would need to have
several modules, with several command lines.  Could you give more
details?

Samuel

Re: [RFC 1/2] Domain Groups - VMM Support

[Chris]

On Dec 18, 2007, at 11:13 AM, Samuel Thibault wrote:
> Chris, le Tue 18 Dec 2007 11:02:55 -0500, a écrit :
>> @@ -520,7 +523,9 @@
>>      unsigned long nr_pt_frames; /* Number of bootstrap p.t.  
>> frames.       */
>>      unsigned long mfn_list;     /* VIRTUAL address of page-frame  
>> list.    */
>>      unsigned long mod_start;    /* VIRTUAL address of pre-loaded  
>> module.  */
>> +                                /* or VIRTUAL address of module_t  
>> array   */
>>      unsigned long mod_len;      /* Size (bytes) of pre-loaded  
>> module.     */
>> +                                /* or length of module_t  
>> array.           */
>>      int8_t cmd_line[MAX_GUEST_CMDLINE];
>>  };
>>  typedef struct start_info start_info_t;
>
> It looks like this doesn't belong to the patch.  However, I'm  
> interested
> in this for the GNU/Mach Xen support, where we would need to have
> several modules, with several command lines.  Could you give more
> details?

Samuel,

You're right, those comments snuck in from another project.  An  
interest of our team is domain decomposition.  Members of the team  
created a domain builder separate from dom0 (conceptually similar to  
Derek Murray's recent project based on Rolf Neugebauer's "initial  
domains" patch) that accepts grub modules as input.  So, your  
suspicion is correct that the comments don't belong in this patch and  
are related to modifications used to accommodate several modules and  
command line arguments.

Thanks for catching my mistake.  I also welcome such an attentive  
critique of the proposed grouping functionality.  :)

Cheers,
Chris

Re: [RESEND] [RFC 1/2] Domain Groups - VMM Support

[Chris]

On Dec 18, 2007, at 11:13 AM, Samuel Thibault wrote:
> Chris, le Tue 18 Dec 2007 11:02:55 -0500, a écrit :
>> @@ -520,7 +523,9 @@
>>      unsigned long nr_pt_frames; /* Number of bootstrap p.t.  
>> frames.       */
>>      unsigned long mfn_list;     /* VIRTUAL address of page-frame  
>> list.    */
>>      unsigned long mod_start;    /* VIRTUAL address of pre-loaded  
>> module.  */
>> +                                /* or VIRTUAL address of module_t  
>> array   */
>>      unsigned long mod_len;      /* Size (bytes) of pre-loaded  
>> module.     */
>> +                                /* or length of module_t  
>> array.           */
>>      int8_t cmd_line[MAX_GUEST_CMDLINE];
>>  };
>>  typedef struct start_info start_info_t;
>
> It looks like this doesn't belong to the patch.  However, I'm  
> interested
> in this for the GNU/Mach Xen support, where we would need to have
> several modules, with several command lines.  Could you give more
> details?

Samuel,

You're right, those comments snuck in from another project.  An  
interest of our team is domain decomposition.  Members of the team  
created a domain builder separate from dom0 (conceptually similar to  
Derek Murray's recent project based on Rolf Neugebauer's "initial  
domains" patch) that accepts grub modules as input.  So, your  
suspicion is correct that the comments don't belong in this patch and  
are related to modifications used to accommodate several modules and  
command line arguments.

Thanks for catching my mistake.  I also welcome such an attentive  
critique of the proposed grouping functionality.  :)

Cheers,
Chris

Re: [RESEND] [RFC 1/2] Domain Groups - VMM Support

[Samuel Thibault]

Hello,

Actually I thought I had already answered this, but I can't find a
trace...

The patch looked sane to me, but maybe it would be useful to see it
merged with the other group patch proposed by Mike D. Day.  I guess it
is mostly a matter of plumbing both together.

Samuel

Re: [RESEND] [RFC 1/2] Domain Groups - VMM Support

[Chris]

On Jan 22, 2008, at 9:40 AM, Samuel Thibault wrote:
> Actually I thought I had already answered this, but I can't find a
> trace...

I think there was some mischief from the xen-devel list server late  
last week.  Please disregard my duplicated response, assuming your  
question was answered.

> The patch looked sane to me, but maybe it would be useful to see it
> merged with the other group patch proposed by Mike D. Day.  I guess it
> is mostly a matter of plumbing both together.

I'm in the process of updating my domgrps patches to facilitate  
scheduling information for compatibility with Mike's schegrps.  I'll  
certainly post the revisions once completed.

Cheers,
Chris

Re: grub-like modules

[Chris]

> How do you recognize that it is a module_t array?  Magic values?

A magic argument ("multimodules") is appended to the VMM boot  
parameters, which signals that the mod_start and mod_len variables  
are to be interpreted in the context of multiple grub modules.

> Would it be possible to see that merged into xen-unstable?

At the moment the changes are not suitable for submission primarily  
because they're applicable to a stale release of xen.  I've asked the  
folks responsible to work something up for general consumption.   
We'll test it internally and then submit it for public discussion.

Cheers,
Chris

Re: Domain Groups - VMM Support

[Mike D. Day]

On 18/12/07 11:02 -0500, Chris wrote:

> --- xen-unstable/xen/arch/x86/x86_32/entry.S	2007-11-19 10:38:08.000000000 -0500
> +++ xen-unstable-trp-domgrps-rebase-tip/xen/arch/x86/x86_32/entry.S	2007-11-19 18:42:00.000000000 -0500
> @@ -682,6 +682,7 @@
>          .long do_sysctl             /* 35 */
>          .long do_domctl
>          .long do_kexec_op
> +        .long do_domgrpctl
>          .rept NR_hypercalls-((.-hypercall_table)/4)
>          .long do_ni_hypercall
>          .endr
> @@ -725,6 +726,7 @@
>          .byte 1 /* do_sysctl            */  /* 35 */
>          .byte 1 /* do_domctl            */
>          .byte 2 /* do_kexec_op          */
> +        .byte 1 /* do_domgrpctl         */
>          .rept NR_hypercalls-(.-hypercall_args_table)


I looked at creating a new hypervisor for scheduling groups, but found
it much simpler to add a new sub-function under the dom_ctl
hypercall. It markedly reduced the size of the patch and I didn't see
any downside. It might make sense to do the same thing here. That way
you wouldn't need to add the new hypercall to every jump table.

> --- xen-unstable/xen/common/domgrp.c	1969-12-31 19:00:00.000000000 -0500
> +++ xen-unstable-trp-domgrps-rebase-tip/xen/common/domgrp.c	2007-11-26 16:46:46.000000000 -0500
> @@ -0,0 +1,315 @@
> +/******************************************************************************
> + * domgrp.c
> + *
> + * Generic domain group-handling functions.
> + *

This is a really good grouping mechanism. To merge with scheduling
groups, it may be nice to have an extensible structure like we do with
schedulers in Xen. The scheduler pattern is a good one to follow, it
separates general scheduling operations from specific
implementations. We could do something like that here. I doubt that
the set of group operations in this patch is the sum total of all
useful group operations on domains so it needs to be simple to add
group operations in the future without breaking any hypercalls and
without disruptive patches.

I think its worth an attempt to merge the scheduling groups with
domain groups. I'll start looking at some of the ideas I mentioned
above, be happy to work with anyone else who wants to have a go.

Mike

-- 
Mike D. Day
IBM LTC
Cell: 919 412-3900
Sametime: ncmike@us.ibm.com AIM: ncmikeday  Yahoo: ultra.runner
PGP key: http://www.ncultra.org/ncmike/pubkey.asc

Re: Re: Domain Groups - VMM Support

[Chris]

On Jan 31, 2008, at 3:04 PM, Mike D. Day wrote:
> I looked at creating a new hypervisor for scheduling groups, but found
> it much simpler to add a new sub-function under the dom_ctl
> hypercall. It markedly reduced the size of the patch and I didn't see
> any downside. It might make sense to do the same thing here. That way
> you wouldn't need to add the new hypercall to every jump table.

I  have no objection to making domgrpctl become a set of sub-ops to  
the existing domain management hypercall.  In fact, with the  
predecessor to domctl (dom0_op) that's how it was implemented.   
During the domctl developmental churn I separated them to make my  
life easier, but now it's just an artifact.

> To merge with scheduling groups, it may be nice to have an  
> extensible structure like we do with schedulers in Xen.

I agree.  There's more to groups than what the two of us have  
released thus far and adding new features should be made as painless  
as possible.  My suggestion is to save this type of overhaul until we  
settle on an agreeable functionality merge.

> I think its worth an attempt to merge the scheduling groups with
> domain groups. I'll start looking at some of the ideas I mentioned
> above, be happy to work with anyone else who wants to have a go.

I'm also having a go at the merge and will release a patch set RFC  
hopefully well before leaving on travel at the end of next week.

With regard to the merge, I've got a question for you.  In schedgrps,  
when a domain joins a group the assumption is that credits are  
thereafter shared among group members (for as long as it's a  
member).  I think there are cases where it makes sense to group  
domains without implicitly sharing credits (the null-group in domgrps  
is such a case).  So, I'd like to make it optional for members to  
delegate their VCPUs to the group.  The question is, do you have an  
opinion whether domains should have to opt-in or opt-out of sharing  
their credits when joining a group?

Cheers,
Chris

[0/3] DomGrp/SchedGrp Merge RFC

[Chris]

On Jan 31, 2008, at 3:04 PM, Mike D. Day wrote:
> I think its worth an attempt to merge the scheduling groups with
> domain groups. I'll start looking at some of the ideas I mentioned
> above, be happy to work with anyone else who wants to have a go.

Hi Mike,

Here's a rough cut of the domgrps/schedgrps merge.  It includes a  
slightly revised version of domgrps with some enhancements for  
performance and compatibility with schedgrps.  I'd appreciate your  
feedback to make sure the merged result accurately represents your  
intent.

DomGrps changes:
- add per-domgrp scheduler-specific data

SchedDom Changes:
- when each domgrp is created, make a csched_dom struct to act as the  
group representative
- when doms join the group, delegate its VCPUs to its representative  
(instead of a master dom)
- removed group management functions, changes to tools, and command  
line interface

As of right now, when a domain joins a group it must relinquish its  
VCPU(s) to the group.  In the near future I'd like to add the following:
- configuration option choose whether a dom will relinquish its VCPUs  
to the group
- modify xm to show whether members have relinquished their VCPUs
- documentation :)

I haven't done any benchmarks, but on the fast path the extra code  
should be nearly identical to your original schedgrps implementation,  
so performance should be the very similar.  Other than the merge  
itself, the practical benefit is that schegrps are now persistent  
across reboots and host migration (although I haven't tested the  
latter).

The attached patches should apply cleanly to xen-unstable.hg cs  
16975.  The intended patch order is: domgrps-vmm.patch,  
schedgrps.patch, domgrps-tools.patch.

Cheers,
Chris

Re: [0/3] DomGrp/SchedGrp Merge RFC

[Keir Fraser]

On 4/2/08 19:14, "Chris" <hap10@tycho.ncsc.mil> wrote:

> The attached patches should apply cleanly to xen-unstable.hg cs
> 16975.  The intended patch order is: domgrps-vmm.patch,
> schedgrps.patch, domgrps-tools.patch.

Hmmm... again the generic domgrp stuff seems like a huge amount of code with
few moving parts that would affect user experience of Xen. The original
schedgrps code by comparison is tiny. I don't mind taking big gobs of code
if there are genuine use cases, but I won't big pieces of 'architecture'
without good reason. I'm inclined to take Mike's patches roughly as he
previously posted them.

 -- Keir

Re: [0/3] DomGrp/SchedGrp Merge RFC

[Chris]

On Feb 4, 2008, at 5:56 PM, Keir Fraser wrote:
> On 4/2/08 19:14, "Chris" <hap10@tycho.ncsc.mil> wrote:
> Hmmm... again the generic domgrp stuff seems like a huge amount of  
> code with
> few moving parts that would affect user experience of Xen. The  
> original
> schedgrps code by comparison is tiny. I don't mind taking big gobs  
> of code
> if there are genuine use cases, but I won't big pieces of  
> 'architecture'
> without good reason. I'm inclined to take Mike's patches roughly as he
> previously posted them.

Use cases for domain groups that exist today:
- domain management (e.g. group migration)
- scheduler integration (Mike's schedgrps)

Use cases coming down the pike:
- XSM integration (e.g. express policy in terms of groups)
- domain relationships (e.g. failure handling)
- resource management (e.g. allocating a resource to a set of domains)
- set_target/IS_PRIV_FOR integration (e.g. to establish privilege  
over a set of domains)

These use cases stem from where we see domain decomposition headed  
and the challenges it creates.

Can some notion of groups can be implemented in each case without an  
underlying VMM-backed, non-hierarchical group architecture?  You  
bet.  However, each implementation is likely to have a separate  
management interface and impose (possibly conflicting) hierarchies  
between domains.  For such independent components to take advantage  
of one another, each would have to translate their respective notions  
of groups, assuming such a translation exists.

On the issue of code size, take Mike's schedgrps for example, which  
was very small as originally posted.  After integration with domgrps,  
it shrank to less than 40% of its original size (259 insertions down  
from 681) and it no longer induced a domain hierarchy.

Could domgrps be smaller?  Yes, and thanks to Mike's recent  
suggestions, I have some concrete ways to make that happen.  If you  
have some specific areas where you think domgrps is unnecessarily  
large, I'm willing to make changes.  Another way to look at the code  
size issue is that the integration with schedgrps shaved off well  
over over 400 LOC, mostly by removing redundant management code from  
schedgrps.  At that rate it would take two, maybe three more group- 
aware projects for the domgrps architecture to break even in terms of  
code size.

But it sounds like the main objection is lack of existing use cases.   
They're coming... slowly.  The best I can say is that I'm working to  
identify and mitigate future challenges before they cause problems.   
Is there critical mass for a generic group architecture yet?  I think  
so, but the case should only get stronger with time.

Cheers,
Chris

Re: [0/3] DomGrp/SchedGrp Merge RFC

[Keir Fraser]

On 5/2/08 22:20, "Chris" <hap10@tycho.ncsc.mil> wrote:

> On the issue of code size, take Mike's schedgrps for example, which
> was very small as originally posted.  After integration with domgrps,
> it shrank to less than 40% of its original size (259 insertions down
> from 681) and it no longer induced a domain hierarchy.

If credit-sharing is made configurable (as you would surely want it to be if
domgrps are to have other uses) then a reasonable number of those lines of
code will reappear, and spread across tools and hypervisor.

> But it sounds like the main objection is lack of existing use cases.
> They're coming... slowly.  The best I can say is that I'm working to
> identify and mitigate future challenges before they cause problems.
> Is there critical mass for a generic group architecture yet?  I think
> so, but the case should only get stronger with time.

I'm driven by concrete use cases. Several of the upcoming uses you mention
need careful consideration of what they are useful for, to determine the
best way to design them into the system. Take resource sharing. Stub domains
sharing scheduler credits with the HVM guest is a rather special case, and
one where a master/slave relationship is not unreasonable (and hence in this
case I think it is arguable whether it is actually a good fit with domgrps
after all). If you really want resource sharing amongst a peer group of
domains, what does that actually mean? Does each domain have a private
allocation of resource plus membership of one or more pools? If so, which
resource set satisfies each resource request by a particular domain? Even if
not, how is resource contention between domains belonging to the resource
pool (e.g., for the static assignment of memory pages to that resource pool)
managed and resolved? The obvious place would be the dom0 control plane --
in which case these resource pools can probably be implemented as a
high-level tools abstraction with no direct hypervisor involvement at all.

My fear is that a nice domgrps infrastructure doesn't itself resolve the
hard devils in the details, for all that it sounds architecturally neat.

 -- Keir

Re: [0/3] DomGrp/SchedGrp Merge RFC

[Samuel Thibault]

Keir Fraser, le Wed 06 Feb 2008 09:20:51 +0000, a écrit :
> On 5/2/08 22:20, "Chris" <hap10@tycho.ncsc.mil> wrote:
> 
> > On the issue of code size, take Mike's schedgrps for example, which
> > was very small as originally posted.  After integration with domgrps,
> > it shrank to less than 40% of its original size (259 insertions down
> > from 681) and it no longer induced a domain hierarchy.
> 
> If credit-sharing is made configurable (as you would surely want it to be if
> domgrps are to have other uses) then a reasonable number of those lines of
> code will reappear, and spread across tools and hypervisor.
> 
> > But it sounds like the main objection is lack of existing use cases.
> > They're coming... slowly.  The best I can say is that I'm working to
> > identify and mitigate future challenges before they cause problems.
> > Is there critical mass for a generic group architecture yet?  I think
> > so, but the case should only get stronger with time.
> 
> I'm driven by concrete use cases. Several of the upcoming uses you mention
> need careful consideration of what they are useful for, to determine the
> best way to design them into the system. Take resource sharing. Stub domains
> sharing scheduler credits with the HVM guest is a rather special case, and
> one where a master/slave relationship is not unreasonable (and hence in this
> case I think it is arguable whether it is actually a good fit with domgrps
> after all).

Actually, in my former research team in Bordeaux, they would like to
write a small domain that computes the scheduling of a bunch of others,
for parallel scientific computing.

Samuel

Re: [0/3] DomGrp/SchedGrp Merge RFC

[Keir Fraser]

On 6/2/08 10:27, "Samuel Thibault" <samuel.thibault@eu.citrix.com> wrote:

>> I'm driven by concrete use cases. Several of the upcoming uses you mention
>> need careful consideration of what they are useful for, to determine the
>> best way to design them into the system. Take resource sharing. Stub domains
>> sharing scheduler credits with the HVM guest is a rather special case, and
>> one where a master/slave relationship is not unreasonable (and hence in this
>> case I think it is arguable whether it is actually a good fit with domgrps
>> after all).
> 
> Actually, in my former research team in Bordeaux, they would like to
> write a small domain that computes the scheduling of a bunch of others,
> for parallel scientific computing.

Well that sounds plausible, but I'm not sure how resource pools or domgrps
would help with that. It sounds like a delegation mechanism (of privilege
and/or resource) would be more appropriate.

 -- Keir

Re: [0/3] DomGrp/SchedGrp Merge RFC

[Samuel Thibault]

Keir Fraser, le Wed 06 Feb 2008 10:37:37 +0000, a écrit :
> On 6/2/08 10:27, "Samuel Thibault" <samuel.thibault@eu.citrix.com> wrote:
> 
> >> I'm driven by concrete use cases. Several of the upcoming uses you mention
> >> need careful consideration of what they are useful for, to determine the
> >> best way to design them into the system. Take resource sharing. Stub domains
> >> sharing scheduler credits with the HVM guest is a rather special case, and
> >> one where a master/slave relationship is not unreasonable (and hence in this
> >> case I think it is arguable whether it is actually a good fit with domgrps
> >> after all).
> > 
> > Actually, in my former research team in Bordeaux, they would like to
> > write a small domain that computes the scheduling of a bunch of others,
> > for parallel scientific computing.
> 
> Well that sounds plausible, but I'm not sure how resource pools or domgrps
> would help with that. It sounds like a delegation mechanism (of privilege
> and/or resource) would be more appropriate.

Sure, delegation will be needed, but there needs to be a way to specify
which domains that scheduler has control on.

Samuel

Re: [0/3] DomGrp/SchedGrp Merge RFC

[Keir Fraser]

On 6/2/08 10:42, "Samuel Thibault" <samuel.thibault@eu.citrix.com> wrote:

>> Well that sounds plausible, but I'm not sure how resource pools or domgrps
>> would help with that. It sounds like a delegation mechanism (of privilege
>> and/or resource) would be more appropriate.
> 
> Sure, delegation will be needed, but there needs to be a way to specify
> which domains that scheduler has control on.

It's not clear to me that supporting any more than a delegation relation
across pairs of domains is needed at the hypervisor level. Nor whether a
richer grouping mechanism in this case would lead to greater run-time
efficiency, fewer lines of code, or clearer code.

 -- Keir

Re: [0/3] DomGrp/SchedGrp Merge RFC

[Chris]

On Feb 6, 2008, at 4:20 AM, Keir Fraser wrote:
> If credit-sharing is made configurable (as you would surely want it  
> to be if
> domgrps are to have other uses) then a reasonable number of those  
> lines of
> code will reappear, and spread across tools and hypervisor.

Without testing, I'd say that making the domgrps/schedgrps merge  
provide logically equivalent functionality to the original schedgrps  
requires the addition of 1 conditional statement in the VMM:

diff -r 82d60475e726 xen/common/domgrp.c
--- a/xen/common/domgrp.c       Wed Feb 06 09:01:54 2008 -0500
+++ b/xen/common/domgrp.c       Wed Feb 06 09:47:38 2008 -0500
@@ -215,7 +215,8 @@ int add_dom_to_grp(struct domain *dom, d
         grp->size++;

         /* add dom to schedgrp */
-       sched_join_group(dom); // FIXME: should be conditional/optional
+       if (dgid == NULL_GROUP_ID)
+               sched_join_group(dom);
        out:
         return ret;
  }

In the domgrps model, all ungrouped domains are in the "null group."   
And in schedgrps model, all grouped domains automatically relinquish  
their VCPUs.  So by preventing domains in the null group from  
relinquishing their VCPUs, the above change make the two  
implementations effectively equivalent: only explicitly grouped  
domains relinquish their VCPUs.

Additional configuration options will be desirable and will take  
code, but in this case domgrps hasn't created any new problems.

> sharing scheduler credits with the HVM guest is a rather special  
> case, and
> one where a master/slave relationship is not unreasonable

I agree.  It's not unreasonable, but it's also not strictly  
necessary.  I would like to avoid any mechanism that induces a domain  
hierarchy without need.

> If you really want resource sharing amongst a peer group of
> domains, what does that actually mean? Does each domain have a private
> allocation of resource plus membership of one or more pools? If so,  
> which
> resource set satisfies each resource request by a particular  
> domain? Even if
> not, how is resource contention between domains belonging to the  
> resource
> pool (e.g., for the static assignment of memory pages to that  
> resource pool)
> managed and resolved? The obvious place would be the dom0 control  
> plane --
> in which case these resource pools can probably be implemented as a
> high-level tools abstraction with no direct hypervisor involvement  
> at all.

It might be that, in the case of resource management, the VMM doesn't  
have to be involved, but the tool chain would benefit from a common  
configuration system and API to converse about groups of domains.

> My fear is that a nice domgrps infrastructure doesn't itself  
> resolve the
> hard devils in the details, for all that it sounds architecturally  
> neat.

You're right, it definitely doesn't make the devils go away.  My goal  
is more modest.  I'm advocating for a common framework where tools  
and the VMM can use group information in a way that's relationship  
agnostic.

-Chris