Re: domU and dom0 hung with Xen console interrupt binding showing in-flight=1, (---M)

[Keir Fraser <keir.fraser@eu.citrix.com>]

[-- Attachment #1: Type: text/plain, Size: 582 bytes --]

On 19/08/2010 14:42, "Bruce Edge" <bruce.edge@gmail.com> wrote:

> Is there any more information that I can provide that would be helpful in
> diagnosing the direct cause and the appropriate fix?
> Possibly adding instrumentation or trace code to detect the trigger
> conditions?
> This is very repeatable on our target systems after a few hours of load.

You can try the attached Xen patch which should clear out the in-flight
interrupt after a short timeout. It might kick things back into action, and
is probably a fix we should have in the tree anyhow.

 -- Keir


[-- Attachment #2: 00-irq-eoi-timer --]
[-- Type: application/octet-stream, Size: 3312 bytes --]

diff -r 7b8b976f534e xen/arch/x86/irq.c
--- a/xen/arch/x86/irq.c	Tue Aug 17 19:27:20 2010 +0100
+++ b/xen/arch/x86/irq.c	Thu Aug 19 16:45:47 2010 +0100
@@ -784,13 +784,55 @@
     desc->handler->enable(irq);
 }
 
+static void set_eoi_ready(void *data);
+
 static void irq_guest_eoi_timer_fn(void *data)
 {
     struct irq_desc *desc = data;
+    unsigned int irq = desc - irq_desc;
+    irq_guest_action_t *action;
+    cpumask_t cpu_eoi_map;
     unsigned long flags;
 
     spin_lock_irqsave(&desc->lock, flags);
-    _irq_guest_eoi(desc);
+    
+    if ( !(desc->status & IRQ_GUEST) )
+        goto out;
+
+    action = (irq_guest_action_t *)desc->action;
+
+    if ( action->ack_type != ACKTYPE_NONE )
+    {
+        unsigned int i;
+        for ( i = 0; i < action->nr_guests; i++ )
+        {
+            struct domain *d = action->guest[i];
+            unsigned int pirq = domain_irq_to_pirq(d, irq);
+            if ( test_and_clear_bit(pirq, d->pirq_mask) )
+                action->in_flight--;
+        }
+    }
+
+    if ( action->in_flight != 0 )
+        goto out;
+
+    switch ( action->ack_type )
+    {
+    case ACKTYPE_UNMASK:
+        desc->handler->end(irq);
+        break;
+    case ACKTYPE_EOI:
+        cpu_eoi_map = action->cpu_eoi_map;
+        spin_unlock_irq(&desc->lock);
+        on_selected_cpus(&cpu_eoi_map, set_eoi_ready, desc, 0);
+        spin_lock_irq(&desc->lock);
+        break;
+    case ACKTYPE_NONE:
+        _irq_guest_eoi(desc);
+        break;
+    }
+
+ out:
     spin_unlock_irqrestore(&desc->lock, flags);
 }
 
@@ -847,9 +889,11 @@
         }
     }
 
-    if ( already_pending == action->nr_guests )
+    stop_timer(&irq_guest_eoi_timer[irq]);
+
+    if ( (action->ack_type == ACKTYPE_NONE) &&
+         (already_pending == action->nr_guests) )
     {
-        stop_timer(&irq_guest_eoi_timer[irq]);
         desc->handler->disable(irq);
         desc->status |= IRQ_GUEST_EOI_PENDING;
         for ( i = 0; i < already_pending; ++i )
@@ -865,10 +909,11 @@
              * - skip the timer setup below.
              */
         }
-        init_timer(&irq_guest_eoi_timer[irq],
-                   irq_guest_eoi_timer_fn, desc, smp_processor_id());
-        set_timer(&irq_guest_eoi_timer[irq], NOW() + MILLISECS(1));
     }
+
+    init_timer(&irq_guest_eoi_timer[irq],
+               irq_guest_eoi_timer_fn, desc, smp_processor_id());
+    set_timer(&irq_guest_eoi_timer[irq], NOW() + MILLISECS(1));
 }
 
 /*
@@ -1306,9 +1351,7 @@
     BUG_ON(!cpus_empty(action->cpu_eoi_map));
 
     desc->action = NULL;
-    desc->status &= ~IRQ_GUEST;
-    desc->status &= ~IRQ_INPROGRESS;
-    kill_timer(&irq_guest_eoi_timer[irq]);
+    desc->status &= ~(IRQ_GUEST|IRQ_GUEST_EOI_PENDING|IRQ_INPROGRESS);
     desc->handler->shutdown(irq);
 
     /* Caller frees the old guest descriptor block. */
@@ -1342,7 +1385,10 @@
     spin_unlock_irq(&desc->lock);
 
     if ( oldaction != NULL )
+    {
+        kill_timer(&irq_guest_eoi_timer[desc-irq_desc]);
         xfree(oldaction);
+    }
 }
 
 static int pirq_guest_force_unbind(struct domain *d, int irq)
@@ -1380,7 +1426,10 @@
     spin_unlock_irq(&desc->lock);
 
     if ( oldaction != NULL )
+    {
+        kill_timer(&irq_guest_eoi_timer[desc-irq_desc]);
         xfree(oldaction);
+    }
 
     return bound;
 }

[-- Attachment #3: Type: text/plain, Size: 138 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xensource.com
http://lists.xensource.com/xen-devel