Re: cpuidle and un-eoid interrupts at the local apic

["Thimo E." <abc@digithi.de>]

[-- Attachment #1: Type: text/plain, Size: 447 bytes --]

Hello,

Andrew sent it somewhere yesterday into another branch of this thread, 
attached you'll find that patch that corresponds to my debugging output.

Best regards
   Thimo

Am 13.08.2013 03:43, schrieb Zhang, Yang Z:
> Andrew Cooper wrote on 2013-08-12:
>> I already have.  That would be "Marked {foo} ready" debugging in the
>> PEOI stack section.
> I didn't find your debug patch that add PEOI stack tracing. Could you resend it? thanks.
>


[-- Attachment #2: ca-107844-debug.patch --]
[-- Type: text/plain, Size: 5875 bytes --]

# HG changeset patch
# Parent bbd6b6d05c06f6331974467467cf567d60915b3d

diff -r bbd6b6d05c06 xen/arch/x86/io_apic.c
--- a/xen/arch/x86/io_apic.c
+++ b/xen/arch/x86/io_apic.c
@@ -1176,7 +1176,7 @@ static inline void UNEXPECTED_IO_APIC(vo
 {
 }
 
-static void /*__init*/ __print_IO_APIC(void)
+void /*__init*/ __print_IO_APIC(void)
 {
     int apic, i;
     union IO_APIC_reg_00 reg_00;
diff -r bbd6b6d05c06 xen/arch/x86/irq.c
--- a/xen/arch/x86/irq.c
+++ b/xen/arch/x86/irq.c
@@ -1003,6 +1003,46 @@ static void irq_guest_eoi_timer_fn(void 
     spin_unlock_irqrestore(&desc->lock, flags);
 }
 
+struct peoi_record {
+    enum { PEOI_PUSH,
+           PEOI_SETREADY,
+           PEOI_FLUSH,
+           PEOI_POP } action;
+    unsigned sp, irq, vector;
+};
+
+static void print_peoi_record(const struct peoi_record *r)
+{
+    switch ( r->action )
+    {
+    case PEOI_PUSH:
+        printk("  Pushed {sp %d, irq %d, vec 0x%02x}\n",
+               r->sp, r->irq, r->vector);
+        break;
+    case PEOI_SETREADY:
+        printk("  Marked {sp %d, irq %d, vec 0x%02x} ready\n",
+               r->sp, r->irq, r->vector);
+        break;
+    case PEOI_FLUSH:
+        printk("  Fushed %d -> 0 \n", r->sp);
+        break;
+    case PEOI_POP:
+        printk("  Poped entry {sp %d, irq %d, vec 0x%02x}\n",
+               r->sp, r->irq, r->vector);
+        break;
+    default:
+        printk("  Unknown: {%d, %d, %d, 0x%02x}\n",
+               r->action, r->sp, r->irq, r->vector);
+        break;
+    }
+}
+
+#define NR_PEOI_RECORDS 32
+static DEFINE_PER_CPU(struct peoi_record, _peoi_dbg[NR_PEOI_RECORDS]) = {{0}};
+static DEFINE_PER_CPU(unsigned int, _peoi_dbg_idx) = 0;
+
+static void dump_irqs(unsigned char key);
+void __print_IO_APIC(void);
 static void __do_IRQ_guest(int irq)
 {
     struct irq_desc         *desc = irq_to_desc(irq);
@@ -1024,13 +1064,53 @@ static void __do_IRQ_guest(int irq)
     if ( action->ack_type == ACKTYPE_EOI )
     {
         sp = pending_eoi_sp(peoi);
-        ASSERT((sp == 0) || (peoi[sp-1].vector < vector));
+        if ( unlikely( !((sp == 0) || (peoi[sp-1].vector < vector)) ))
+        {
+            int p;
+            unsigned i, idx;
+            printk("**Pending EOI error\n");
+            printk("  irq %d, vector 0x%x\n", irq, vector);
+
+            for ( p = sp-1; p >= 0; --p )
+            {
+                printk("  s[%d] irq %d, vec 0x%x, ready %u, "
+                       "ISR %08"PRIx32", TMR %08"PRIx32", IRR %08"PRIx32"\n",
+                       p, peoi[p].irq, peoi[p].vector, peoi[p].ready,
+                       apic_isr_read(peoi[p].vector),
+                       apic_tmr_read(peoi[p].vector),
+                       apic_irr_read(peoi[p].vector) );
+            }
+
+            printk("All LAPIC state:\n");
+            printk("[vector] %8s %8s %8s\n", "ISR", "TMR", "IRR");
+            for ( i = 0; i < APIC_ISR_NR; ++i )
+                printk("[%02x:%02x] %08"PRIx32" %08"PRIx32" %08"PRIx32"\n",
+                       (i * 32)+31, i*32,
+                       apic_read(APIC_ISR + i*0x10),
+                       apic_read(APIC_TMR + i*0x10),
+                       apic_read(APIC_IRR + i*0x10) );
+
+            printk("Peoi stack trace records:\n");
+            idx = this_cpu(_peoi_dbg_idx);
+            for ( i = 1; i <= NR_PEOI_RECORDS; ++i )
+                print_peoi_record(&this_cpu(_peoi_dbg)[(idx - i) &
+                                                       (NR_PEOI_RECORDS-1)] );
+
+            spin_unlock(&desc->lock);
+            dump_irqs('i');
+            __print_IO_APIC();
+
+            panic("CA-107844");
+        }
         ASSERT(sp < (NR_DYNAMIC_VECTORS-1));
         peoi[sp].irq = irq;
         peoi[sp].vector = vector;
         peoi[sp].ready = 0;
         pending_eoi_sp(peoi) = sp+1;
         cpu_set(smp_processor_id(), action->cpu_eoi_map);
+
+        this_cpu(_peoi_dbg)[(this_cpu(_peoi_dbg_idx)++) & (NR_PEOI_RECORDS-1)]
+            = (struct peoi_record){PEOI_PUSH, sp, irq, peoi[sp].vector};
     }
 
     for ( i = 0; i < action->nr_guests; i++ )
@@ -1130,6 +1210,9 @@ static void flush_ready_eoi(void)
         spin_lock(&desc->lock);
         desc->handler->end(irq, peoi[sp].vector);
         spin_unlock(&desc->lock);
+
+        this_cpu(_peoi_dbg)[(this_cpu(_peoi_dbg_idx)++) & (NR_PEOI_RECORDS-1)]
+            = (struct peoi_record){PEOI_POP, sp+1, irq, peoi[sp].vector};
     }
 
     pending_eoi_sp(peoi) = sp+1;
@@ -1155,6 +1238,9 @@ static void __set_eoi_ready(struct irq_d
     } while ( peoi[--sp].irq != irq );
     ASSERT(!peoi[sp].ready);
     peoi[sp].ready = 1;
+
+    this_cpu(_peoi_dbg)[(this_cpu(_peoi_dbg_idx)++) & (NR_PEOI_RECORDS-1)]
+        = (struct peoi_record){PEOI_SETREADY, sp, irq, desc->chip_data->vector};
 }
 
 /* Mark specified IRQ as ready-for-EOI (if it really is) and attempt to EOI. */
@@ -1976,6 +2062,8 @@ void fixup_irqs(void)
 
     /* Flush the interrupt EOI stack. */
     peoi = this_cpu(pending_eoi);
+    this_cpu(_peoi_dbg)[(this_cpu(_peoi_dbg_idx)++) & (NR_PEOI_RECORDS-1)]
+        = (struct peoi_record){PEOI_FLUSH, pending_eoi_sp(peoi)};
     for ( sp = 0; sp < pending_eoi_sp(peoi); sp++ )
         peoi[sp].ready = 1;
     flush_ready_eoi();
diff -r bbd6b6d05c06 xen/include/asm-x86/apic.h
--- a/xen/include/asm-x86/apic.h
+++ b/xen/include/asm-x86/apic.h
@@ -152,6 +152,18 @@ static __inline bool_t apic_isr_read(u8 
             (vector & 0x1f)) & 1;
 }
 
+static __inline bool_t apic_tmr_read(u8 vector)
+{
+    return (apic_read(APIC_TMR + ((vector & ~0x1f) >> 1)) >>
+            (vector & 0x1f)) & 1;
+}
+
+static __inline bool_t apic_irr_read(u8 vector)
+{
+    return (apic_read(APIC_IRR + ((vector & ~0x1f) >> 1)) >>
+            (vector & 0x1f)) & 1;
+}
+
 static __inline u32 get_apic_id(void) /* Get the physical APIC id */
 {
     u32 id = apic_read(APIC_ID);

[-- Attachment #3: Type: text/plain, Size: 126 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel