[PATCH RFC 2/9] x86/traps: Make panic and reboot paths safe during early boot

[Andrew Cooper <andrew.cooper3@citrix.com>]

Reverse two conditions in show_registers().  For an early crash, it is not
safe to dereference current for its HVM status before knowing that it is a
guest vcpu.

Introduce SYS_STATE_smp_boot to help distinguish whether APs need to be
considered during boot.  This involves tweaking quite a few system_state
checks so their semantics remain identical.

Make use of SYS_STATE_smp_boot to help machine_{halt,restart}() know if/when
it is safe to enable interrupts and access the local apic to send IPIs.
Before system_state == SYS_STATE_smp_boot, we can be certain that only the BSP
is running.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
CC: Keir Fraser <keir@xen.org>
CC: Jan Beulich <JBeulich@suse.com>
CC: Tim Deegan <tim@xen.org>
---
 xen/arch/x86/mm.c           |   10 +++++-----
 xen/arch/x86/setup.c        |    2 ++
 xen/arch/x86/shutdown.c     |   38 +++++++++++++++++++++++---------------
 xen/arch/x86/x86_64/traps.c |    2 +-
 xen/common/symbols.c        |    2 +-
 xen/include/xen/kernel.h    |    1 +
 6 files changed, 33 insertions(+), 22 deletions(-)

diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c
index 1a8a5e0..bd67cde 100644
--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -5246,7 +5246,7 @@ static l3_pgentry_t *virt_to_xen_l3e(unsigned long v)
     pl4e = &idle_pg_table[l4_table_offset(v)];
     if ( !(l4e_get_flags(*pl4e) & _PAGE_PRESENT) )
     {
-        bool_t locking = system_state > SYS_STATE_boot;
+        bool_t locking = system_state >= SYS_STATE_active;
         l3_pgentry_t *pl3e = alloc_xen_pagetable();
 
         if ( !pl3e )
@@ -5278,7 +5278,7 @@ static l2_pgentry_t *virt_to_xen_l2e(unsigned long v)
 
     if ( !(l3e_get_flags(*pl3e) & _PAGE_PRESENT) )
     {
-        bool_t locking = system_state > SYS_STATE_boot;
+        bool_t locking = system_state >= SYS_STATE_active;
         l2_pgentry_t *pl2e = alloc_xen_pagetable();
 
         if ( !pl2e )
@@ -5311,7 +5311,7 @@ l1_pgentry_t *virt_to_xen_l1e(unsigned long v)
 
     if ( !(l2e_get_flags(*pl2e) & _PAGE_PRESENT) )
     {
-        bool_t locking = system_state > SYS_STATE_boot;
+        bool_t locking = system_state >= SYS_STATE_active;
         l1_pgentry_t *pl1e = alloc_xen_pagetable();
 
         if ( !pl1e )
@@ -5353,7 +5353,7 @@ int map_pages_to_xen(
     unsigned long nr_mfns,
     unsigned int flags)
 {
-    bool_t locking = system_state > SYS_STATE_boot;
+    bool_t locking = system_state >= SYS_STATE_active;
     l2_pgentry_t *pl2e, ol2e;
     l1_pgentry_t *pl1e, ol1e;
     unsigned int  i;
@@ -5670,7 +5670,7 @@ int map_pages_to_xen(
 
 void destroy_xen_mappings(unsigned long s, unsigned long e)
 {
-    bool_t locking = system_state > SYS_STATE_boot;
+    bool_t locking = system_state >= SYS_STATE_active;
     l2_pgentry_t *pl2e;
     l1_pgentry_t *pl1e;
     unsigned int  i;
diff --git a/xen/arch/x86/setup.c b/xen/arch/x86/setup.c
index d8598a3..a864b9f 100644
--- a/xen/arch/x86/setup.c
+++ b/xen/arch/x86/setup.c
@@ -1319,6 +1319,8 @@ void __init noreturn __start_xen(unsigned long mbi_p)
 
     console_init_postirq();
 
+    system_state = SYS_STATE_smp_boot;
+
     do_presmp_initcalls();
 
     for_each_present_cpu ( i )
diff --git a/xen/arch/x86/shutdown.c b/xen/arch/x86/shutdown.c
index 827515d..44bcd7f 100644
--- a/xen/arch/x86/shutdown.c
+++ b/xen/arch/x86/shutdown.c
@@ -96,8 +96,13 @@ void machine_halt(void)
 {
     watchdog_disable();
     console_start_sync();
-    local_irq_enable();
-    smp_call_function(__machine_halt, NULL, 0);
+
+    if ( system_state >= SYS_STATE_smp_boot )
+    {
+        local_irq_enable();
+        smp_call_function(__machine_halt, NULL, 0);
+    }
+
     __machine_halt(NULL);
 }
 
@@ -466,18 +471,6 @@ void machine_restart(unsigned int delay_millisecs)
     console_start_sync();
     spin_debug_disable();
 
-    local_irq_enable();
-
-    /* Ensure we are the boot CPU. */
-    if ( get_apic_id() != boot_cpu_physical_apicid )
-    {
-        /* Send IPI to the boot CPU (logical cpu 0). */
-        on_selected_cpus(cpumask_of(0), __machine_restart,
-                         &delay_millisecs, 0);
-        for ( ; ; )
-            halt();
-    }
-
     /*
      * We may be called from an interrupt context, and various functions we
      * may need to call (alloc_domheap_pages, map_domain_page, ...) assert that
@@ -485,7 +478,22 @@ void machine_restart(unsigned int delay_millisecs)
      */
     local_irq_count(0) = 0;
 
-    smp_send_stop();
+    if ( system_state >= SYS_STATE_smp_boot )
+    {
+        local_irq_enable();
+
+        /* Ensure we are the boot CPU. */
+        if ( get_apic_id() != boot_cpu_physical_apicid )
+        {
+            /* Send IPI to the boot CPU (logical cpu 0). */
+            on_selected_cpus(cpumask_of(0), __machine_restart,
+                             &delay_millisecs, 0);
+            for ( ; ; )
+                halt();
+        }
+
+        smp_send_stop();
+    }
 
     mdelay(delay_millisecs);
 
diff --git a/xen/arch/x86/x86_64/traps.c b/xen/arch/x86/x86_64/traps.c
index cdaf1e6..e1c7b3b 100644
--- a/xen/arch/x86/x86_64/traps.c
+++ b/xen/arch/x86/x86_64/traps.c
@@ -86,7 +86,7 @@ void show_registers(struct cpu_user_regs *regs)
     enum context context;
     struct vcpu *v = current;
 
-    if ( has_hvm_container_vcpu(v) && guest_mode(regs) )
+    if ( guest_mode(regs) && has_hvm_container_vcpu(v) )
     {
         struct segment_register sreg;
         context = CTXT_hvm_guest;
diff --git a/xen/common/symbols.c b/xen/common/symbols.c
index 45941e1..bc2fde6 100644
--- a/xen/common/symbols.c
+++ b/xen/common/symbols.c
@@ -96,7 +96,7 @@ static unsigned int get_symbol_offset(unsigned long pos)
 bool_t is_active_kernel_text(unsigned long addr)
 {
     return (is_kernel_text(addr) ||
-            (system_state == SYS_STATE_boot && is_kernel_inittext(addr)));
+            (system_state < SYS_STATE_active && is_kernel_inittext(addr)));
 }
 
 const char *symbols_lookup(unsigned long addr,
diff --git a/xen/include/xen/kernel.h b/xen/include/xen/kernel.h
index 54e88dd..2c6d448 100644
--- a/xen/include/xen/kernel.h
+++ b/xen/include/xen/kernel.h
@@ -92,6 +92,7 @@ extern char _sinittext[], _einittext[];
 extern enum system_state {
     SYS_STATE_early_boot,
     SYS_STATE_boot,
+    SYS_STATE_smp_boot,
     SYS_STATE_active,
     SYS_STATE_suspend,
     SYS_STATE_resume
-- 
1.7.10.4