[RFH] xen domain0 failover stuff

[Don Zickus <dzickus@redhat.com>]

[-- Attachment #1: Type: text/plain, Size: 1543 bytes --]

Hello,

I am ignorantly trying to find a way to start a second domain0 when the
first domain0 crashes.  My original intent was to find a way to collect a
core file much in the same way the domainU core files are collected on a
linux box.  

I managed to hack in some hooks to get a second domain0 loaded and started
upon a crash.  However, I am stuck without console output, ie printk() is
not working.  Oddly, early_printk() works fine and I can sort of track the
progress of the second domain0 as it boots.  So my first question is how
does the serial console (not vga) magic work for a domain0 kernel?  Do I
have to set some internal domain bits to have the console output
redirected to the serial port as opposed to userspace and xend (ala
domainU kernels)?  

I attached my small dirty hack below, if it helps. The jist of it is I
added a new parameter, "failover", to be read in from a domainU config
file (ie /etc/xen/guest).  The second domain0 is treated just like a
domainU kernel in regards to loading and installing the userspace disk.
The only thing different is "xm create -c guest" will load a domain0
kernel instead of a domainU kernel.  Combined with the "failover" flag
above, this triggers the hypervisor to reserve this domain as a backup for
later and _not_ unpause the domain once its cpu context is loaded.  Upon
the first domain0 crash, the hypervisor looks up the backup domain0,
mindlessly sets some interesting bits, and unpauses it.  

Pretty simple.  

Any help or tips or pointers would be great.

Cheers,
Don

 

[-- Attachment #2: xend.patch --]
[-- Type: text/plain, Size: 2098 bytes --]

--- ./python/xen/xm/create.py.dz	2006-03-24 12:37:19.000000000 -0500
+++ ./python/xen/xm/create.py	2006-03-27 16:43:57.000000000 -0500
@@ -125,6 +125,10 @@ gopts.var('bootloader', val='FILE',
           fn=set_value, default=None,
           use="Path to bootloader.")
 
+gopts.var('failover', val='FAILOVER',
+          fn=set_int, default=0,
+          use="Enable failover domain.")
+
 gopts.var('bootentry', val='NAME',
           fn=set_value, default=None,
           use="Entry to boot via boot loader")
@@ -587,7 +591,7 @@ def make_config(vals):
                 config.append([n, v])
 
     map(add_conf, ['name', 'memory', 'ssidref', 'maxmem', 'restart',
-                   'on_poweroff', 'on_reboot', 'on_crash', 'vcpus'])
+                   'on_poweroff', 'on_reboot', 'on_crash', 'vcpus', 'failover'])
     
     if vals.uuid is not None:
         config.append(['uuid', vals.uuid])
--- ./python/xen/xend/XendDomainInfo.py.dz	2006-03-20 15:09:54.000000000 -0500
+++ ./python/xen/xend/XendDomainInfo.py	2006-03-24 14:11:55.000000000 -0500
@@ -126,6 +126,7 @@ ROUNDTRIPPING_CONFIG_ENTRIES = [
     ('memory',     int),
     ('maxmem',     int),
     ('bootloader', str),
+    ('failover',   int),
     ]
 
 ROUNDTRIPPING_CONFIG_ENTRIES += VM_CONFIG_PARAMS
@@ -558,6 +559,7 @@ class XendDomainInfo:
             defaultInfo('memory',       lambda: 0)
             defaultInfo('maxmem',       lambda: 0)
             defaultInfo('bootloader',   lambda: None)
+	    defaultInfo('failover',     lambda: 0)
             defaultInfo('backend',      lambda: [])
             defaultInfo('device',       lambda: [])
             defaultInfo('image',        lambda: None)
@@ -1141,8 +1143,12 @@ class XendDomainInfo:
                   self.domid,
                   self.info['ssidref'])
 
+	dom = 0
+        if self.info['failover']:
+	    dom = 0x7FF3
+	    
         self.domid = xc.domain_create(
-            dom = 0, ssidref = self.info['ssidref'],
+            dom, ssidref = self.info['ssidref'],
             handle = uuid.fromString(self.info['uuid']))
 
         if self.domid < 0:

[-- Attachment #3: xen.patch --]
[-- Type: text/plain, Size: 4961 bytes --]

--- ./xen/common/dom0_ops.c.dz	2006-03-23 17:00:59.000000000 -0500
+++ ./xen/common/dom0_ops.c	2006-03-23 17:07:32.000000000 -0500
@@ -178,7 +178,13 @@ long do_dom0_op(struct dom0_op *u_dom0_o
             return -EINVAL;
 
         dom = op->u.createdomain.domain;
-        if ( (dom > 0) && (dom < DOMID_FIRST_RESERVED) )
+	if ( dom == DOMID_FAILOVER )
+	{
+	    ret = -EINVAL;
+	    if ( find_domain_by_id(dom) != NULL )
+	    	break;
+	}
+        else if ( (dom > 0) && (dom < DOMID_FIRST_RESERVED) )
         {
             ret = -EINVAL;
             if ( !is_free_domid(dom) )
--- ./xen/arch/x86/domain_build.c.dz	2006-03-23 18:20:41.000000000 -0500
+++ ./xen/arch/x86/domain_build.c	2006-03-23 18:28:30.000000000 -0500
@@ -867,6 +867,60 @@ int elf_sanity_check(Elf_Ehdr *ehdr)
     return 1;
 }
 
+#include <acm/acm_hooks.h>
+
+void arch_domain_failover_setup(void)
+{
+    int rc, i;
+    unsigned long mfn;
+    
+    set_bit(_DOMF_privileged, &dom0->domain_flags);
+    /* post-create hooks sets security label */
+    acm_post_domain0_create(dom0->domain_id);
+
+    /* random useful?? bits copied from construct_dom0 */    
+    rc = 0;
+
+    /* DOM0 is permitted full I/O capabilities. */
+    rc |= ioports_permit_access(dom0, 0, 0xFFFF);
+    rc |= iomem_permit_access(dom0, 0UL, ~0UL);
+    rc |= irqs_permit_access(dom0, 0, NR_PIRQS-1);
+
+    /*
+     * Modify I/O port access permissions.
+     */
+    /* Master Interrupt Controller (PIC). */
+    rc |= ioports_deny_access(dom0, 0x20, 0x21);
+    /* Slave Interrupt Controller (PIC). */
+    rc |= ioports_deny_access(dom0, 0xA0, 0xA1);
+    /* Interval Timer (PIT). */
+    rc |= ioports_deny_access(dom0, 0x40, 0x43);
+    /* PIT Channel 2 / PC Speaker Control. */
+    rc |= ioports_deny_access(dom0, 0x61, 0x61);
+    /* Command-line I/O ranges. */
+    process_dom0_ioports_disable();
+
+    /*
+     * Modify I/O memory access permissions.
+     */
+    /* Local APIC. */
+    if ( mp_lapic_addr != 0 )
+    {
+        mfn = paddr_to_pfn(mp_lapic_addr);
+        rc |= iomem_deny_access(dom0, mfn, mfn);
+    }
+    /* I/O APICs. */
+    for ( i = 0; i < nr_ioapics; i++ )
+    {
+        mfn = paddr_to_pfn(mp_ioapics[i].mpc_apicaddr);
+        if ( smp_found_config )
+            rc |= iomem_deny_access(dom0, mfn, mfn);
+    }
+
+    BUG_ON(rc != 0);
+
+}
+
 /*
  * Local variables:
  * mode: C
--- ./xen/include/xen/sched.h.dz	2006-03-23 18:23:35.000000000 -0500
+++ ./xen/include/xen/sched.h	2006-03-23 18:24:51.000000000 -0500
@@ -232,6 +232,7 @@ extern void domain_destroy(struct domain
 extern void domain_kill(struct domain *d);
 extern void domain_shutdown(struct domain *d, u8 reason);
 extern void domain_pause_for_debugger(void);
+extern void arch_domain_failover_setup(void);
 
 /*
  * Mark specified domain as crashed. This function always returns, even if the
--- ./xen/include/public/xen.h.dz	2006-03-23 17:08:03.000000000 -0500
+++ ./xen/include/public/xen.h	2006-03-23 17:08:41.000000000 -0500
@@ -250,6 +250,7 @@ typedef uint16_t domid_t;
  * the caller is privileged.
  */
 #define DOMID_XEN  (0x7FF2U)
+#define DOMID_FAILOVER (0x7FF3U)
 
 /*
  * Send an array of these to HYPERVISOR_mmu_update().
--- ./xen/common/domain.c.dz	2006-03-23 18:17:53.000000000 -0500
+++ ./xen/common/domain.c	2006-03-27 16:35:26.000000000 -0500
@@ -120,6 +120,20 @@ struct domain *find_domain_by_id(domid_t
     return d;
 }
 
+void domain_failover_setup(struct domain *d, struct domain *d_fo)
+{
+    /* swap dom0 and the failover domain */
+    dom0->domain_id = DOMID_FAILOVER;
+    d_fo->domain_id = 0;
+    dom0 = d_fo;
+
+    arch_domain_failover_setup();
+    
+    /* cross our fingers.. */
+    domain_unpause(dom0);
+    printk("Domain 0 failover has been released!!\n");
+}
+
 void domain_kill(struct domain *d)
 {
     struct vcpu *v;
@@ -202,10 +216,11 @@ static __init int domain_shutdown_finali
 }
 __initcall(domain_shutdown_finaliser_init);
 
-
+#include <xen/sched-if.h>
 void domain_shutdown(struct domain *d, u8 reason)
 {
     struct vcpu *v;
+    struct domain *d_fo = NULL;
 
     if ( d->domain_id == 0 )
     {
@@ -219,6 +234,27 @@ void domain_shutdown(struct domain *d, u
             printk("Domain 0 halted: halting machine.\n");
             machine_halt();
         }
+	if ( reason == SHUTDOWN_crash )
+	{
+	    printk("Domain 0 is crashing..v=%d, c=%d\n", current->vcpu_id,
+	    		smp_processor_id());
+			
+	    /*dirty hack to get around delivering a crash event for each cpu*/
+	    if (current->vcpu_id != 1)
+	        return;
+	    d_fo = find_domain_by_id(DOMID_FAILOVER);
+	    if ( d_fo != NULL )
+	    {
+		printk("Found a failover Domain 0..\n");
+        	domain_failover_setup(d, d_fo);
+		return;
+	    }
+	    else
+	    {
+                printk("Domain 0 shutdown: rebooting machine.\n");
+                machine_restart(0);
+	    }
+	}	    
         else
         {
             printk("Domain 0 shutdown: rebooting machine.\n");

[-- Attachment #4: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel