From: "Thimo E." <abc@digithi.de>
To: "Zhang, Yang Z" <yang.z.zhang@intel.com>
Cc: Keir Fraser <keir@xen.org>, Jan Beulich <JBeulich@suse.com>,
Andrew Cooper <andrew.cooper3@citrix.com>,
"Dong, Eddie" <eddie.dong@intel.com>,
Xen-develList <xen-devel@lists.xen.org>,
"Nakajima, Jun" <jun.nakajima@intel.com>,
"Zhang, Xiantao" <xiantao.zhang@intel.com>
Subject: Re: cpuidle and un-eoid interrupts at the local apic
Date: Tue, 13 Aug 2013 08:39:27 +0200 [thread overview]
Message-ID: <5209D49F.7020808@digithi.de> (raw)
In-Reply-To: <A9667DDFB95DB7438FA9D7D576C3D87E0A8E1E39@SHSMSX104.ccr.corp.intel.com>
[-- Attachment #1: Type: text/plain, Size: 447 bytes --]
Hello,
Andrew sent it somewhere yesterday into another branch of this thread,
attached you'll find that patch that corresponds to my debugging output.
Best regards
Thimo
Am 13.08.2013 03:43, schrieb Zhang, Yang Z:
> Andrew Cooper wrote on 2013-08-12:
>> I already have. That would be "Marked {foo} ready" debugging in the
>> PEOI stack section.
> I didn't find your debug patch that add PEOI stack tracing. Could you resend it? thanks.
>
[-- Attachment #2: ca-107844-debug.patch --]
[-- Type: text/plain, Size: 5875 bytes --]
# HG changeset patch
# Parent bbd6b6d05c06f6331974467467cf567d60915b3d
diff -r bbd6b6d05c06 xen/arch/x86/io_apic.c
--- a/xen/arch/x86/io_apic.c
+++ b/xen/arch/x86/io_apic.c
@@ -1176,7 +1176,7 @@ static inline void UNEXPECTED_IO_APIC(vo
{
}
-static void /*__init*/ __print_IO_APIC(void)
+void /*__init*/ __print_IO_APIC(void)
{
int apic, i;
union IO_APIC_reg_00 reg_00;
diff -r bbd6b6d05c06 xen/arch/x86/irq.c
--- a/xen/arch/x86/irq.c
+++ b/xen/arch/x86/irq.c
@@ -1003,6 +1003,46 @@ static void irq_guest_eoi_timer_fn(void
spin_unlock_irqrestore(&desc->lock, flags);
}
+struct peoi_record {
+ enum { PEOI_PUSH,
+ PEOI_SETREADY,
+ PEOI_FLUSH,
+ PEOI_POP } action;
+ unsigned sp, irq, vector;
+};
+
+static void print_peoi_record(const struct peoi_record *r)
+{
+ switch ( r->action )
+ {
+ case PEOI_PUSH:
+ printk(" Pushed {sp %d, irq %d, vec 0x%02x}\n",
+ r->sp, r->irq, r->vector);
+ break;
+ case PEOI_SETREADY:
+ printk(" Marked {sp %d, irq %d, vec 0x%02x} ready\n",
+ r->sp, r->irq, r->vector);
+ break;
+ case PEOI_FLUSH:
+ printk(" Fushed %d -> 0 \n", r->sp);
+ break;
+ case PEOI_POP:
+ printk(" Poped entry {sp %d, irq %d, vec 0x%02x}\n",
+ r->sp, r->irq, r->vector);
+ break;
+ default:
+ printk(" Unknown: {%d, %d, %d, 0x%02x}\n",
+ r->action, r->sp, r->irq, r->vector);
+ break;
+ }
+}
+
+#define NR_PEOI_RECORDS 32
+static DEFINE_PER_CPU(struct peoi_record, _peoi_dbg[NR_PEOI_RECORDS]) = {{0}};
+static DEFINE_PER_CPU(unsigned int, _peoi_dbg_idx) = 0;
+
+static void dump_irqs(unsigned char key);
+void __print_IO_APIC(void);
static void __do_IRQ_guest(int irq)
{
struct irq_desc *desc = irq_to_desc(irq);
@@ -1024,13 +1064,53 @@ static void __do_IRQ_guest(int irq)
if ( action->ack_type == ACKTYPE_EOI )
{
sp = pending_eoi_sp(peoi);
- ASSERT((sp == 0) || (peoi[sp-1].vector < vector));
+ if ( unlikely( !((sp == 0) || (peoi[sp-1].vector < vector)) ))
+ {
+ int p;
+ unsigned i, idx;
+ printk("**Pending EOI error\n");
+ printk(" irq %d, vector 0x%x\n", irq, vector);
+
+ for ( p = sp-1; p >= 0; --p )
+ {
+ printk(" s[%d] irq %d, vec 0x%x, ready %u, "
+ "ISR %08"PRIx32", TMR %08"PRIx32", IRR %08"PRIx32"\n",
+ p, peoi[p].irq, peoi[p].vector, peoi[p].ready,
+ apic_isr_read(peoi[p].vector),
+ apic_tmr_read(peoi[p].vector),
+ apic_irr_read(peoi[p].vector) );
+ }
+
+ printk("All LAPIC state:\n");
+ printk("[vector] %8s %8s %8s\n", "ISR", "TMR", "IRR");
+ for ( i = 0; i < APIC_ISR_NR; ++i )
+ printk("[%02x:%02x] %08"PRIx32" %08"PRIx32" %08"PRIx32"\n",
+ (i * 32)+31, i*32,
+ apic_read(APIC_ISR + i*0x10),
+ apic_read(APIC_TMR + i*0x10),
+ apic_read(APIC_IRR + i*0x10) );
+
+ printk("Peoi stack trace records:\n");
+ idx = this_cpu(_peoi_dbg_idx);
+ for ( i = 1; i <= NR_PEOI_RECORDS; ++i )
+ print_peoi_record(&this_cpu(_peoi_dbg)[(idx - i) &
+ (NR_PEOI_RECORDS-1)] );
+
+ spin_unlock(&desc->lock);
+ dump_irqs('i');
+ __print_IO_APIC();
+
+ panic("CA-107844");
+ }
ASSERT(sp < (NR_DYNAMIC_VECTORS-1));
peoi[sp].irq = irq;
peoi[sp].vector = vector;
peoi[sp].ready = 0;
pending_eoi_sp(peoi) = sp+1;
cpu_set(smp_processor_id(), action->cpu_eoi_map);
+
+ this_cpu(_peoi_dbg)[(this_cpu(_peoi_dbg_idx)++) & (NR_PEOI_RECORDS-1)]
+ = (struct peoi_record){PEOI_PUSH, sp, irq, peoi[sp].vector};
}
for ( i = 0; i < action->nr_guests; i++ )
@@ -1130,6 +1210,9 @@ static void flush_ready_eoi(void)
spin_lock(&desc->lock);
desc->handler->end(irq, peoi[sp].vector);
spin_unlock(&desc->lock);
+
+ this_cpu(_peoi_dbg)[(this_cpu(_peoi_dbg_idx)++) & (NR_PEOI_RECORDS-1)]
+ = (struct peoi_record){PEOI_POP, sp+1, irq, peoi[sp].vector};
}
pending_eoi_sp(peoi) = sp+1;
@@ -1155,6 +1238,9 @@ static void __set_eoi_ready(struct irq_d
} while ( peoi[--sp].irq != irq );
ASSERT(!peoi[sp].ready);
peoi[sp].ready = 1;
+
+ this_cpu(_peoi_dbg)[(this_cpu(_peoi_dbg_idx)++) & (NR_PEOI_RECORDS-1)]
+ = (struct peoi_record){PEOI_SETREADY, sp, irq, desc->chip_data->vector};
}
/* Mark specified IRQ as ready-for-EOI (if it really is) and attempt to EOI. */
@@ -1976,6 +2062,8 @@ void fixup_irqs(void)
/* Flush the interrupt EOI stack. */
peoi = this_cpu(pending_eoi);
+ this_cpu(_peoi_dbg)[(this_cpu(_peoi_dbg_idx)++) & (NR_PEOI_RECORDS-1)]
+ = (struct peoi_record){PEOI_FLUSH, pending_eoi_sp(peoi)};
for ( sp = 0; sp < pending_eoi_sp(peoi); sp++ )
peoi[sp].ready = 1;
flush_ready_eoi();
diff -r bbd6b6d05c06 xen/include/asm-x86/apic.h
--- a/xen/include/asm-x86/apic.h
+++ b/xen/include/asm-x86/apic.h
@@ -152,6 +152,18 @@ static __inline bool_t apic_isr_read(u8
(vector & 0x1f)) & 1;
}
+static __inline bool_t apic_tmr_read(u8 vector)
+{
+ return (apic_read(APIC_TMR + ((vector & ~0x1f) >> 1)) >>
+ (vector & 0x1f)) & 1;
+}
+
+static __inline bool_t apic_irr_read(u8 vector)
+{
+ return (apic_read(APIC_IRR + ((vector & ~0x1f) >> 1)) >>
+ (vector & 0x1f)) & 1;
+}
+
static __inline u32 get_apic_id(void) /* Get the physical APIC id */
{
u32 id = apic_read(APIC_ID);
[-- Attachment #3: Type: text/plain, Size: 126 bytes --]
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel
next prev parent reply other threads:[~2013-08-13 6:39 UTC|newest]
Thread overview: 63+ messages / expand[flat|nested] mbox.gz Atom feed top
2013-05-31 20:32 cpuidle and un-eoid interrupts at the local apic Andrew Cooper
2013-06-03 14:30 ` Jan Beulich
2013-07-31 8:30 ` Thimo E.
2013-07-31 9:47 ` Andrew Cooper
2013-08-02 22:50 ` Thimo E.
2013-08-02 23:32 ` Andrew Cooper
2013-08-05 12:45 ` Jan Beulich
2013-08-05 14:51 ` Andrew Cooper
2013-08-09 21:27 ` Thimo E.
2013-08-09 21:40 ` Andrew Cooper
2013-08-09 21:44 ` Andrew Cooper
2013-08-11 17:46 ` Thimo E.
2013-08-12 6:02 ` Zhang, Yang Z
2013-08-12 8:49 ` Zhang, Yang Z
2013-08-12 8:57 ` Jan Beulich
2013-08-12 11:52 ` Thimo E
2013-08-12 12:04 ` Andrew Cooper
2013-08-19 15:14 ` Thimo E.
2013-08-20 5:43 ` Thimo Eichstädt
2013-08-20 8:40 ` Jan Beulich
2013-08-20 8:50 ` Zhang, Yang Z
2013-08-23 7:22 ` Thimo Eichstädt
2013-08-23 7:30 ` Zhang, Yang Z
2013-08-27 1:03 ` Zhang, Yang Z
2013-09-04 18:32 ` Thimo E.
2013-09-04 18:55 ` Andrew Cooper
2013-09-04 19:56 ` Thimo E.
2013-09-04 20:54 ` Andrew Cooper
2013-09-05 1:45 ` Zhang, Yang Z
2013-09-05 7:20 ` Thimo E.
2013-09-05 1:15 ` Zhang, Yang Z
2013-09-17 2:09 ` Zhang, Yang Z
2013-09-17 7:39 ` Thimo E.
2013-09-17 7:43 ` Zhang, Yang Z
2013-09-17 21:04 ` Thimo E.
2013-09-18 1:18 ` Zhang, Xiantao
2013-09-18 17:24 ` Thimo E.
2013-09-18 12:06 ` Andrew Cooper
2013-08-12 13:54 ` Thimo E
2013-08-12 14:06 ` Andrew Cooper
2013-08-13 1:43 ` Zhang, Yang Z
2013-08-13 6:39 ` Thimo E. [this message]
2013-08-13 11:39 ` Wu, Feng
2013-08-13 12:46 ` Andrew Cooper
2013-08-12 9:10 ` Andrew Cooper
2013-08-12 5:50 ` Zhang, Yang Z
2013-08-12 8:20 ` Jan Beulich
2013-08-12 9:28 ` Andrew Cooper
2013-08-12 10:05 ` Jan Beulich
2013-08-12 10:27 ` Andrew Cooper
2013-08-14 2:53 ` Zhang, Yang Z
2013-08-14 7:51 ` Thimo E.
2013-08-14 9:52 ` Andrew Cooper
2013-09-07 13:27 ` Thimo E.
2013-09-07 17:02 ` Andrew Cooper
2013-09-07 23:37 ` Thimo E.
2013-09-08 9:53 ` Andrew Cooper
2013-09-08 10:24 ` Thimo E.
2013-09-09 13:16 ` Andrew Cooper
2013-09-09 14:48 ` Thimo Eichstädt
2013-09-09 15:12 ` Andrew Cooper
2013-09-09 7:59 ` Jan Beulich
2013-09-09 12:53 ` Andrew Cooper
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=5209D49F.7020808@digithi.de \
--to=abc@digithi.de \
--cc=JBeulich@suse.com \
--cc=andrew.cooper3@citrix.com \
--cc=eddie.dong@intel.com \
--cc=jun.nakajima@intel.com \
--cc=keir@xen.org \
--cc=xen-devel@lists.xen.org \
--cc=xiantao.zhang@intel.com \
--cc=yang.z.zhang@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).