From mboxrd@z Thu Jan 1 00:00:00 1970 From: Don Zickus Subject: [RFH] xen domain0 failover stuff Date: Mon, 27 Mar 2006 17:08:42 -0500 Message-ID: <20060327220842.GG9941@redhat.com> Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="VbJkn9YxBvnuCH5J" Return-path: Content-Disposition: inline List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Sender: xen-devel-bounces@lists.xensource.com Errors-To: xen-devel-bounces@lists.xensource.com To: xen-devel@lists.xensource.com List-Id: xen-devel@lists.xenproject.org --VbJkn9YxBvnuCH5J Content-Type: text/plain; charset=us-ascii Content-Disposition: inline Hello, I am ignorantly trying to find a way to start a second domain0 when the first domain0 crashes. My original intent was to find a way to collect a core file much in the same way the domainU core files are collected on a linux box. I managed to hack in some hooks to get a second domain0 loaded and started upon a crash. However, I am stuck without console output, ie printk() is not working. Oddly, early_printk() works fine and I can sort of track the progress of the second domain0 as it boots. So my first question is how does the serial console (not vga) magic work for a domain0 kernel? Do I have to set some internal domain bits to have the console output redirected to the serial port as opposed to userspace and xend (ala domainU kernels)? I attached my small dirty hack below, if it helps. The jist of it is I added a new parameter, "failover", to be read in from a domainU config file (ie /etc/xen/guest). The second domain0 is treated just like a domainU kernel in regards to loading and installing the userspace disk. The only thing different is "xm create -c guest" will load a domain0 kernel instead of a domainU kernel. Combined with the "failover" flag above, this triggers the hypervisor to reserve this domain as a backup for later and _not_ unpause the domain once its cpu context is loaded. Upon the first domain0 crash, the hypervisor looks up the backup domain0, mindlessly sets some interesting bits, and unpauses it. Pretty simple. Any help or tips or pointers would be great. Cheers, Don --VbJkn9YxBvnuCH5J Content-Type: text/plain; charset=us-ascii Content-Disposition: attachment; filename="xend.patch" --- ./python/xen/xm/create.py.dz 2006-03-24 12:37:19.000000000 -0500 +++ ./python/xen/xm/create.py 2006-03-27 16:43:57.000000000 -0500 @@ -125,6 +125,10 @@ gopts.var('bootloader', val='FILE', fn=set_value, default=None, use="Path to bootloader.") +gopts.var('failover', val='FAILOVER', + fn=set_int, default=0, + use="Enable failover domain.") + gopts.var('bootentry', val='NAME', fn=set_value, default=None, use="Entry to boot via boot loader") @@ -587,7 +591,7 @@ def make_config(vals): config.append([n, v]) map(add_conf, ['name', 'memory', 'ssidref', 'maxmem', 'restart', - 'on_poweroff', 'on_reboot', 'on_crash', 'vcpus']) + 'on_poweroff', 'on_reboot', 'on_crash', 'vcpus', 'failover']) if vals.uuid is not None: config.append(['uuid', vals.uuid]) --- ./python/xen/xend/XendDomainInfo.py.dz 2006-03-20 15:09:54.000000000 -0500 +++ ./python/xen/xend/XendDomainInfo.py 2006-03-24 14:11:55.000000000 -0500 @@ -126,6 +126,7 @@ ROUNDTRIPPING_CONFIG_ENTRIES = [ ('memory', int), ('maxmem', int), ('bootloader', str), + ('failover', int), ] ROUNDTRIPPING_CONFIG_ENTRIES += VM_CONFIG_PARAMS @@ -558,6 +559,7 @@ class XendDomainInfo: defaultInfo('memory', lambda: 0) defaultInfo('maxmem', lambda: 0) defaultInfo('bootloader', lambda: None) + defaultInfo('failover', lambda: 0) defaultInfo('backend', lambda: []) defaultInfo('device', lambda: []) defaultInfo('image', lambda: None) @@ -1141,8 +1143,12 @@ class XendDomainInfo: self.domid, self.info['ssidref']) + dom = 0 + if self.info['failover']: + dom = 0x7FF3 + self.domid = xc.domain_create( - dom = 0, ssidref = self.info['ssidref'], + dom, ssidref = self.info['ssidref'], handle = uuid.fromString(self.info['uuid'])) if self.domid < 0: --VbJkn9YxBvnuCH5J Content-Type: text/plain; charset=us-ascii Content-Disposition: attachment; filename="xen.patch" --- ./xen/common/dom0_ops.c.dz 2006-03-23 17:00:59.000000000 -0500 +++ ./xen/common/dom0_ops.c 2006-03-23 17:07:32.000000000 -0500 @@ -178,7 +178,13 @@ long do_dom0_op(struct dom0_op *u_dom0_o return -EINVAL; dom = op->u.createdomain.domain; - if ( (dom > 0) && (dom < DOMID_FIRST_RESERVED) ) + if ( dom == DOMID_FAILOVER ) + { + ret = -EINVAL; + if ( find_domain_by_id(dom) != NULL ) + break; + } + else if ( (dom > 0) && (dom < DOMID_FIRST_RESERVED) ) { ret = -EINVAL; if ( !is_free_domid(dom) ) --- ./xen/arch/x86/domain_build.c.dz 2006-03-23 18:20:41.000000000 -0500 +++ ./xen/arch/x86/domain_build.c 2006-03-23 18:28:30.000000000 -0500 @@ -867,6 +867,60 @@ int elf_sanity_check(Elf_Ehdr *ehdr) return 1; } +#include + +void arch_domain_failover_setup(void) +{ + int rc, i; + unsigned long mfn; + + set_bit(_DOMF_privileged, &dom0->domain_flags); + /* post-create hooks sets security label */ + acm_post_domain0_create(dom0->domain_id); + + /* random useful?? bits copied from construct_dom0 */ + rc = 0; + + /* DOM0 is permitted full I/O capabilities. */ + rc |= ioports_permit_access(dom0, 0, 0xFFFF); + rc |= iomem_permit_access(dom0, 0UL, ~0UL); + rc |= irqs_permit_access(dom0, 0, NR_PIRQS-1); + + /* + * Modify I/O port access permissions. + */ + /* Master Interrupt Controller (PIC). */ + rc |= ioports_deny_access(dom0, 0x20, 0x21); + /* Slave Interrupt Controller (PIC). */ + rc |= ioports_deny_access(dom0, 0xA0, 0xA1); + /* Interval Timer (PIT). */ + rc |= ioports_deny_access(dom0, 0x40, 0x43); + /* PIT Channel 2 / PC Speaker Control. */ + rc |= ioports_deny_access(dom0, 0x61, 0x61); + /* Command-line I/O ranges. */ + process_dom0_ioports_disable(); + + /* + * Modify I/O memory access permissions. + */ + /* Local APIC. */ + if ( mp_lapic_addr != 0 ) + { + mfn = paddr_to_pfn(mp_lapic_addr); + rc |= iomem_deny_access(dom0, mfn, mfn); + } + /* I/O APICs. */ + for ( i = 0; i < nr_ioapics; i++ ) + { + mfn = paddr_to_pfn(mp_ioapics[i].mpc_apicaddr); + if ( smp_found_config ) + rc |= iomem_deny_access(dom0, mfn, mfn); + } + + BUG_ON(rc != 0); + +} + /* * Local variables: * mode: C --- ./xen/include/xen/sched.h.dz 2006-03-23 18:23:35.000000000 -0500 +++ ./xen/include/xen/sched.h 2006-03-23 18:24:51.000000000 -0500 @@ -232,6 +232,7 @@ extern void domain_destroy(struct domain extern void domain_kill(struct domain *d); extern void domain_shutdown(struct domain *d, u8 reason); extern void domain_pause_for_debugger(void); +extern void arch_domain_failover_setup(void); /* * Mark specified domain as crashed. This function always returns, even if the --- ./xen/include/public/xen.h.dz 2006-03-23 17:08:03.000000000 -0500 +++ ./xen/include/public/xen.h 2006-03-23 17:08:41.000000000 -0500 @@ -250,6 +250,7 @@ typedef uint16_t domid_t; * the caller is privileged. */ #define DOMID_XEN (0x7FF2U) +#define DOMID_FAILOVER (0x7FF3U) /* * Send an array of these to HYPERVISOR_mmu_update(). --- ./xen/common/domain.c.dz 2006-03-23 18:17:53.000000000 -0500 +++ ./xen/common/domain.c 2006-03-27 16:35:26.000000000 -0500 @@ -120,6 +120,20 @@ struct domain *find_domain_by_id(domid_t return d; } +void domain_failover_setup(struct domain *d, struct domain *d_fo) +{ + /* swap dom0 and the failover domain */ + dom0->domain_id = DOMID_FAILOVER; + d_fo->domain_id = 0; + dom0 = d_fo; + + arch_domain_failover_setup(); + + /* cross our fingers.. */ + domain_unpause(dom0); + printk("Domain 0 failover has been released!!\n"); +} + void domain_kill(struct domain *d) { struct vcpu *v; @@ -202,10 +216,11 @@ static __init int domain_shutdown_finali } __initcall(domain_shutdown_finaliser_init); - +#include void domain_shutdown(struct domain *d, u8 reason) { struct vcpu *v; + struct domain *d_fo = NULL; if ( d->domain_id == 0 ) { @@ -219,6 +234,27 @@ void domain_shutdown(struct domain *d, u printk("Domain 0 halted: halting machine.\n"); machine_halt(); } + if ( reason == SHUTDOWN_crash ) + { + printk("Domain 0 is crashing..v=%d, c=%d\n", current->vcpu_id, + smp_processor_id()); + + /*dirty hack to get around delivering a crash event for each cpu*/ + if (current->vcpu_id != 1) + return; + d_fo = find_domain_by_id(DOMID_FAILOVER); + if ( d_fo != NULL ) + { + printk("Found a failover Domain 0..\n"); + domain_failover_setup(d, d_fo); + return; + } + else + { + printk("Domain 0 shutdown: rebooting machine.\n"); + machine_restart(0); + } + } else { printk("Domain 0 shutdown: rebooting machine.\n"); --VbJkn9YxBvnuCH5J Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Content-Disposition: inline _______________________________________________ Xen-devel mailing list Xen-devel@lists.xensource.com http://lists.xensource.com/xen-devel --VbJkn9YxBvnuCH5J--