[PATCH] create /proc/all-interrupts

[PATCH] create /proc/all-interrupts

[Joe Korty]

Create /proc/all-interrupts for some architectures.

Create a version of /proc/interrupts that displays _every_
IRQ vector, not just those that someone thought might be
interesting, and add an entry in the commentary column
for those vectors which lacked such a comment.

Rationale: /proc/interrupts is not truly useful unless it
displays every IRQ vector, not just those somebody thought
would be interesting.  For example, since /proc/interrupts
does not display the rescheduling interrupt, the occurance
of rescheduling interrupt floods ends up affecting
latencies, yet without an entry in /proc/interrupts, it
is difficult to discern why latencies are being affected.

Rather than modify /proc/interrupts, this patch creates
a new version of /proc/interrupts, on the off-chance
that adding new lines to /proc/interrupts, and appending
new fields to the end of old lines, might break some
longstanding script.  However, these kinds of changes
traditionally do not affect scripts, so it might be
reasonable to fold /proc/all-interrupts back into
/proc/interrupts.

Signed-off-by: Joe Korty <joe.korty@ccur.com>

Index: 2.6.22.1-rt8/arch/i386/kernel/apic.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/i386/kernel/apic.c	2007-07-26 11:57:13.000000000 -0400
+++ 2.6.22.1-rt8/arch/i386/kernel/apic.c	2007-07-26 11:57:14.000000000 -0400
@@ -1268,6 +1268,8 @@
 {
 	unsigned long v;
 
+	atomic_inc(&__get_cpu_var(irq_spur_counts));
+
 	irq_enter();
 	/*
 	 * Check if this really is a spurious interrupt and ACK it
@@ -1297,7 +1299,7 @@
 	apic_write(APIC_ESR, 0);
 	v1 = apic_read(APIC_ESR);
 	ack_APIC_irq();
-	atomic_inc(&irq_err_count);
+	atomic_inc(&__get_cpu_var(irq_err_counts));
 
 	/* Here is what the APIC error bits mean:
 	   0: Send CS error
Index: 2.6.22.1-rt8/arch/i386/kernel/cpu/mcheck/p4.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/i386/kernel/cpu/mcheck/p4.c	2007-07-26 11:57:13.000000000 -0400
+++ 2.6.22.1-rt8/arch/i386/kernel/cpu/mcheck/p4.c	2007-07-26 11:57:14.000000000 -0400
@@ -60,6 +60,7 @@
 fastcall void smp_thermal_interrupt(struct pt_regs *regs)
 {
 	irq_enter();
+	atomic_inc(&__get_cpu_var(irq_thermal_counts));
 	vendor_thermal_interrupt(regs);
 	irq_exit();
 }
Index: 2.6.22.1-rt8/arch/i386/kernel/i8259.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/i386/kernel/i8259.c	2007-07-26 11:57:13.000000000 -0400
+++ 2.6.22.1-rt8/arch/i386/kernel/i8259.c	2007-07-26 11:57:14.000000000 -0400
@@ -209,7 +209,7 @@
 			printk(KERN_DEBUG "spurious 8259A interrupt: IRQ%d.\n", irq);
 			spurious_irq_mask |= irqmask;
 		}
-		atomic_inc(&irq_err_count);
+		atomic_inc(&__get_cpu_var(irq_err_counts));
 		/*
 		 * Theoretically we do not have to handle this IRQ,
 		 * but in Linux this does not cause problems and is
Index: 2.6.22.1-rt8/arch/i386/kernel/io_apic.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/i386/kernel/io_apic.c	2007-07-26 11:57:13.000000000 -0400
+++ 2.6.22.1-rt8/arch/i386/kernel/io_apic.c	2007-07-26 11:57:14.000000000 -0400
@@ -51,7 +51,6 @@
 #include "io_ports.h"
 
 int (*ioapic_renumber_irq)(int ioapic, int irq);
-atomic_t irq_mis_count;
 
 /* Where if anywhere is the i8259 connect in external int mode */
 static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
@@ -2031,7 +2030,7 @@
 	ack_APIC_irq();
 
 	if (!(v & (1 << (i & 0x1f)))) {
-		atomic_inc(&irq_mis_count);
+		atomic_inc(&__get_cpu_var(irq_mis_counts));
 		spin_lock(&ioapic_lock);
 		/* mask = 1, trigger = 0 */
 		__modify_IO_APIC_irq(irq, 0x00010000, 0x00008000);
Index: 2.6.22.1-rt8/arch/i386/kernel/irq.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/i386/kernel/irq.c	2007-07-26 11:57:13.000000000 -0400
+++ 2.6.22.1-rt8/arch/i386/kernel/irq.c	2007-07-26 13:13:22.000000000 -0400
@@ -12,6 +12,8 @@
 
 #include <linux/module.h>
 #include <linux/seq_file.h>
+#include <linux/fs.h>
+#include <linux/proc_fs.h>
 #include <linux/interrupt.h>
 #include <linux/kernel_stat.h>
 #include <linux/notifier.h>
@@ -252,15 +254,22 @@
  * Interrupt statistics:
  */
 
-atomic_t irq_err_count;
+DEFINE_PER_CPU(atomic_t, irq_resched_counts);
+DEFINE_PER_CPU(atomic_t, irq_call_counts);
+DEFINE_PER_CPU(atomic_t, irq_spur_counts);
+DEFINE_PER_CPU(atomic_t, irq_tlb_counts);
+DEFINE_PER_CPU(atomic_t, irq_thermal_counts);
+DEFINE_PER_CPU(atomic_t, irq_err_counts);
+DEFINE_PER_CPU(atomic_t, irq_mis_counts);
 
 /*
- * /proc/interrupts printing:
+ * /proc/interrupts and /proc/all-interrupts printing.  Done this
+ * way to preserve the original /proc/interrupts layout.
  */
 
-int show_interrupts(struct seq_file *p, void *v)
+int show_selected_interrupts(struct seq_file *p, int i, int full)
 {
-	int i = *(loff_t *) v, j;
+	int j;
 	struct irqaction * action;
 	unsigned long flags;
 
@@ -297,22 +306,162 @@
 		seq_printf(p, "NMI: ");
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ", nmi_count(j));
+		if (full)
+			seq_printf(p, "  Non-maskable interrupts");
 		seq_putc(p, '\n');
+
 #ifdef CONFIG_X86_LOCAL_APIC
 		seq_printf(p, "LOC: ");
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ",
 				per_cpu(irq_stat,j).apic_timer_irqs);
+		if (full)
+			seq_printf(p, "  Local interrupts");
 		seq_putc(p, '\n');
 #endif
-		seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count));
-#if defined(CONFIG_X86_IO_APIC)
-		seq_printf(p, "MIS: %10u\n", atomic_read(&irq_mis_count));
+#ifdef CONFIG_SMP
+#ifdef RESCHEDULE_VECTOR
+		if (full) {
+			seq_printf(p, "RES: ");
+			for_each_online_cpu(j)
+				seq_printf(p, "%10u ",
+					atomic_read(&per_cpu(irq_resched_counts, j)));
+			seq_printf(p, "  Rescheduling interrupts\n");
+		}
+#endif
+#endif
+#ifdef CONFIG_SMP
+#ifdef CALL_FUNCTION_VECTOR
+		if (full) {
+			seq_printf(p, "CAL: ");
+			for_each_online_cpu(j)
+				seq_printf(p, "%10u ",
+					atomic_read(&per_cpu(irq_call_counts, j)));
+			seq_printf(p, "  function call interrupts\n");
+		}
+#endif
+#endif
+#ifdef CONFIG_SMP
+#ifdef INVALIDATE_TLB_VECTOR
+		if (full) {
+			seq_printf(p, "TLB: ");
+			for_each_online_cpu(j)
+				seq_printf(p, "%10u ",
+					atomic_read(&per_cpu(irq_tlb_counts, j)));
+			seq_printf(p, "  TLB shootdowns\n");
+		}
+#endif
+#endif
+#ifdef THERMAL_APIC_VECTOR
+		if (full) {
+			seq_printf(p, "TRM: ");
+			for_each_online_cpu(j)
+				seq_printf(p, "%10u ",
+					atomic_read(&per_cpu(irq_thermal_counts, j)));
+			seq_printf(p, "  Thermal event interrupts\n");
+		}
+#endif
+#ifdef SPURIOUS_APIC_VECTOR
+		if (full) {
+			seq_printf(p, "SPU: ");
+			for_each_online_cpu(j)
+				seq_printf(p, "%10u ",
+					atomic_read(&per_cpu(irq_spur_counts, j)));
+			seq_printf(p, "  Spurious interrupts\n");
+		}
+#endif
+#ifdef ERROR_APIC_VECTOR
+		seq_printf(p, "ERR: ");
+		if (full) {
+			for_each_online_cpu(j)
+				seq_printf(p, "%10u ",
+					atomic_read(&per_cpu(irq_err_counts, j)));
+			seq_printf(p, "  Error interrupts\n");
+		} else {
+			unsigned count = 0;
+			for_each_online_cpu(j)
+				count += atomic_read(&per_cpu(irq_err_counts, j));
+			seq_printf(p, "%10u\n", count);
+		}
+#endif
+#if defined(CONFIG_X86_IO_APIC) && defined(APIC_MISMATCH_DEBUG)
+		seq_printf(p, "MIS: ");
+		if (full) {
+			for_each_online_cpu(j)
+				seq_printf(p, "%10u ",
+					atomic_read(&per_cpu(irq_mis_counts, j)));
+			seq_printf(p, "  APIC mismatch fixups\n");
+		} else {
+			unsigned count = 0;
+			for_each_online_cpu(j)
+				count += atomic_read(&per_cpu(irq_mis_counts, j)));
+			seq_printf(p, "%10u\n");
+		}
 #endif
 	}
 	return 0;
 }
 
+int show_interrupts(struct seq_file *p, void *v)
+{
+	int i  = *(loff_t *)v;
+	return show_selected_interrupts(p, i, 0);
+}
+
+int show_all_interrupts(struct seq_file *p, void *v)
+{
+	int i  = *(loff_t *)v;
+	return show_selected_interrupts(p, i, 1);
+}
+
+static void *show_irq_start(struct seq_file *f, loff_t *pos)
+{
+	return (*pos <= NR_IRQS) ? pos : NULL;
+}
+
+static void *show_irq_next(struct seq_file *f, void *v, loff_t *pos)
+{
+	(*pos)++;
+	if (*pos > NR_IRQS)
+		return NULL;
+	return pos;
+}
+
+static void show_irq_stop(struct seq_file *f, void *v)
+{
+	/* Nothing to do */
+}
+
+static struct seq_operations show_all_interrupts_ops = {
+	.start = show_irq_start,
+	.next  = show_irq_next,
+	.stop  = show_irq_stop,
+	.show  = show_all_interrupts
+};
+
+static int show_all_interrupts_open(struct inode *inode, struct file *filp)
+{
+	return seq_open(filp, &show_all_interrupts_ops);
+}
+
+static const struct file_operations show_all_interrupts_proc_ops = {
+	.open		= show_all_interrupts_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= seq_release,
+};
+
+static int __init show_all_interrupts_register(void)
+{
+	struct proc_dir_entry *entry;
+
+	entry = create_proc_entry("all-interrupts", 0, NULL);
+	if (entry)
+		entry->proc_fops = &show_all_interrupts_proc_ops;
+	return 0;
+}
+core_initcall(show_all_interrupts_register);
+
 #ifdef CONFIG_HOTPLUG_CPU
 #include <mach_apic.h>
 
Index: 2.6.22.1-rt8/arch/i386/kernel/smp.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/i386/kernel/smp.c	2007-07-26 11:57:13.000000000 -0400
+++ 2.6.22.1-rt8/arch/i386/kernel/smp.c	2007-07-26 11:57:14.000000000 -0400
@@ -315,6 +315,7 @@
 	unsigned long cpu;
 
 	cpu = get_cpu();
+	atomic_inc(&__get_cpu_var(irq_tlb_counts));
 
 	if (!cpu_isset(cpu, flush_cpumask))
 		goto out;
@@ -658,6 +659,7 @@
 {
 	trace_special(regs->eip, 0, 0);
 	ack_APIC_irq();
+	atomic_inc(&__get_cpu_var(irq_resched_counts));
 	set_tsk_need_resched(current);
 }
 
@@ -668,6 +670,7 @@
 	int wait = call_data->wait;
 
 	ack_APIC_irq();
+	atomic_inc(&__get_cpu_var(irq_call_counts));
 	/*
 	 * Notify initiating CPU that I've grabbed the data and am
 	 * about to execute the function
Index: 2.6.22.1-rt8/arch/x86_64/kernel/apic.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/x86_64/kernel/apic.c	2007-07-26 11:57:13.000000000 -0400
+++ 2.6.22.1-rt8/arch/x86_64/kernel/apic.c	2007-07-26 11:57:14.000000000 -0400
@@ -1099,6 +1099,9 @@
 asmlinkage void smp_spurious_interrupt(void)
 {
 	unsigned int v;
+
+	atomic_inc(&__get_cpu_var(irq_spur_counts));
+
 	exit_idle();
 	irq_enter();
 	/*
@@ -1127,7 +1130,7 @@
 	apic_write(APIC_ESR, 0);
 	v1 = apic_read(APIC_ESR);
 	ack_APIC_irq();
-	atomic_inc(&irq_err_count);
+	atomic_inc(&__get_cpu_var(irq_err_counts));
 
 	/* Here is what the APIC error bits mean:
 	   0: Send CS error
Index: 2.6.22.1-rt8/arch/x86_64/kernel/i8259.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/x86_64/kernel/i8259.c	2007-07-26 11:57:13.000000000 -0400
+++ 2.6.22.1-rt8/arch/x86_64/kernel/i8259.c	2007-07-26 11:57:14.000000000 -0400
@@ -273,7 +273,7 @@
 			printk(KERN_DEBUG "spurious 8259A interrupt: IRQ%d.\n", irq);
 			spurious_irq_mask |= irqmask;
 		}
-		atomic_inc(&irq_err_count);
+		atomic_inc(&__get_cpu_var(irq_err_counts));
 		/*
 		 * Theoretically we do not have to handle this IRQ,
 		 * but in Linux this does not cause problems and is
Index: 2.6.22.1-rt8/arch/x86_64/kernel/irq.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/x86_64/kernel/irq.c	2007-07-26 11:57:13.000000000 -0400
+++ 2.6.22.1-rt8/arch/x86_64/kernel/irq.c	2007-07-26 12:29:50.000000000 -0400
@@ -13,6 +13,8 @@
 #include <linux/kernel_stat.h>
 #include <linux/interrupt.h>
 #include <linux/seq_file.h>
+#include <linux/fs.h>
+#include <linux/proc_fs.h>
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <asm/uaccess.h>
@@ -20,7 +22,18 @@
 #include <asm/idle.h>
 #include <asm/smp.h>
 
-atomic_t irq_err_count;
+DEFINE_PER_CPU(atomic_t, irq_resched_counts);
+DEFINE_PER_CPU(atomic_t, irq_call_counts);
+DEFINE_PER_CPU(atomic_t, irq_spur_counts);
+DEFINE_PER_CPU(atomic_t, irq_tlb_counts);
+DEFINE_PER_CPU(atomic_t, irq_threshold_counts);
+DEFINE_PER_CPU(atomic_t, irq_thermal_counts);
+DEFINE_PER_CPU(atomic_t, irq_err_counts);
+#ifdef CONFIG_X86_IO_APIC
+#ifdef APIC_MISMATCH_DEBUG
+DEFINE_PER_CPU(atomic_t, irq_mis_counts);
+#endif
+#endif
 
 #ifdef CONFIG_DEBUG_STACKOVERFLOW
 /*
@@ -47,12 +60,13 @@
 #endif
 
 /*
- * Generic, controller-independent functions:
+ * /proc/interrupts and /proc/all-interrupts printing.  Done this
+ * way to preserve the original /proc/interrupts layout.
  */
 
-int show_interrupts(struct seq_file *p, void *v)
+int show_selected_interrupts(struct seq_file *p, int i, int full)
 {
-	int i = *(loff_t *) v, j;
+	int j;
 	struct irqaction * action;
 	unsigned long flags;
 
@@ -88,16 +102,163 @@
 		seq_printf(p, "NMI: ");
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ", cpu_pda(j)->__nmi_count);
+		if (full)
+			seq_printf(p, "  Non-maskable interrupts");
 		seq_putc(p, '\n');
+
 		seq_printf(p, "LOC: ");
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ", cpu_pda(j)->apic_timer_irqs);
+		if (full)
+			seq_printf(p, "  Local interrupts");
 		seq_putc(p, '\n');
-		seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count));
+
+#ifdef CONFIG_SMP
+#ifdef RESCHEDULE_VECTOR
+		if (full) {
+			seq_printf(p, "RES: ");
+			for_each_online_cpu(j)
+				seq_printf(p, "%10u ", atomic_read(&per_cpu(irq_resched_counts, j)));
+			seq_printf(p, "  Rescheduling interrupts\n");
+		}
+#endif
+#endif
+#ifdef CONFIG_SMP
+#ifdef CALL_FUNCTION_VECTOR
+		if (full) {
+			seq_printf(p, "CAL: ");
+			for_each_online_cpu(j)
+				seq_printf(p, "%10u ", atomic_read(&per_cpu(irq_call_counts, j)));
+			seq_printf(p, "  function call interrupts\n");
+		}
+#endif
+#endif
+#ifdef CONFIG_SMP
+#ifdef INVALIDATE_TLB_VECTOR_START
+		if (full) {
+			seq_printf(p, "TLB: ");
+			for_each_online_cpu(j)
+				seq_printf(p, "%10u ", atomic_read(&per_cpu(irq_tlb_counts, j)));
+			seq_printf(p, "  TLB shootdowns\n");
+		}
+#endif
+#endif
+#ifdef THERMAL_APIC_VECTOR
+		if (full) {
+			seq_printf(p, "TRM: ");
+			for_each_online_cpu(j)
+				seq_printf(p, "%10u ", atomic_read(&per_cpu(irq_thermal_counts, j)));
+			seq_printf(p, "  Thermal event interrupts\n");
+		}
+#endif
+#ifdef THRESHOLD_APIC_VECTOR
+		if (full) {
+			seq_printf(p, "THR: ");
+			for_each_online_cpu(j)
+				seq_printf(p, "%10u ", atomic_read(&per_cpu(irq_threshold_counts, j)));
+			seq_printf(p, "  Threshold APIC interrupts\n");
+		}
+#endif
+#ifdef SPURIOUS_APIC_VECTOR
+		if (full) {
+			seq_printf(p, "SPU: ");
+			for_each_online_cpu(j)
+				seq_printf(p, "%10u ", atomic_read(&per_cpu(irq_spur_counts, j)));
+			seq_printf(p, "  Spurious interrupts\n");
+		}
+#endif
+#ifdef ERROR_APIC_VECTOR
+		seq_printf(p, "ERR: ");
+		if (full) {
+			for_each_online_cpu(j)
+				seq_printf(p, "%10u ", atomic_read(&per_cpu(irq_err_counts, j)));
+			seq_printf(p, "  Error interrupts\n");
+		} else {
+			unsigned count = 0;
+			for_each_online_cpu(j)
+				count += atomic_read(&per_cpu(irq_err_counts, j));
+			seq_printf(p, "%10u\n", count);
+		}
+#endif
+#ifdef CONFIG_X86_IO_APIC
+#ifdef APIC_MISMATCH_DEBUG
+		seq_printf(p, "MIS: ");
+		if (full) {
+			for_each_online_cpu(j)
+				seq_printf(p, "%10u ", atomic_read(&per_cpu(irq_mis_counts, j)));
+			seq_printf(p, "  APIC mismatch fixups\n");
+		} else {
+			unsigned count = 0;
+			for_each_online_cpu(j)
+				count += atomic_read(&per_cpu(irq_mis_counts, j));
+			seq_printf(p, "%10u\n", count);
+		}
+#endif
+#endif
 	}
 	return 0;
 }
 
+int show_interrupts(struct seq_file *p, void *v)
+{
+	int i  = *(loff_t *)v;
+	return show_selected_interrupts(p, i, 0);
+}
+
+static int show_all_interrupts(struct seq_file *p, void *v)
+{
+	int i  = *(loff_t *)v;
+	return show_selected_interrupts(p, i, 1);
+}
+
+static void *show_irq_start(struct seq_file *f, loff_t *pos)
+{
+	return (*pos <= NR_IRQS) ? pos : NULL;
+}
+
+static void *show_irq_next(struct seq_file *f, void *v, loff_t *pos)
+{
+	(*pos)++;
+	if (*pos > NR_IRQS)
+		return NULL;
+	return pos;
+}
+
+static void show_irq_stop(struct seq_file *f, void *v)
+{
+	/* Nothing to do */
+}
+
+static struct seq_operations show_all_interrupts_ops = {
+	.start = show_irq_start,
+	.next  = show_irq_next,
+	.stop  = show_irq_stop,
+	.show  = show_all_interrupts
+};
+
+static int show_all_interrupts_open(struct inode *inode, struct file *filp)
+{
+	return seq_open(filp, &show_all_interrupts_ops);
+}
+
+static const struct file_operations show_all_interrupts_proc_ops = {
+	.open		= show_all_interrupts_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= seq_release,
+};
+
+static int __init show_all_interrupts_register(void)
+{
+	struct proc_dir_entry *entry;
+
+	entry = create_proc_entry("all-interrupts", 0, NULL);
+	if (entry)
+		entry->proc_fops = &show_all_interrupts_proc_ops;
+	return 0;
+}
+core_initcall(show_all_interrupts_register);
+
 /*
  * do_IRQ handles all normal device IRQ's (the special
  * SMP cross-CPU interrupts have their own specific
Index: 2.6.22.1-rt8/arch/x86_64/kernel/mce_amd.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/x86_64/kernel/mce_amd.c	2007-07-26 11:57:13.000000000 -0400
+++ 2.6.22.1-rt8/arch/x86_64/kernel/mce_amd.c	2007-07-26 11:57:14.000000000 -0400
@@ -186,6 +186,8 @@
 	exit_idle();
 	irq_enter();
 
+	atomic_inc(&__get_cpu_var(irq_threshold_counts));
+
 	memset(&m, 0, sizeof(m));
 	rdtscll(m.tsc);
 	m.cpu = smp_processor_id();
Index: 2.6.22.1-rt8/arch/x86_64/kernel/mce_intel.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/x86_64/kernel/mce_intel.c	2007-07-26 11:57:13.000000000 -0400
+++ 2.6.22.1-rt8/arch/x86_64/kernel/mce_intel.c	2007-07-26 11:57:14.000000000 -0400
@@ -21,6 +21,7 @@
 
 	exit_idle();
 	irq_enter();
+	atomic_inc(&__get_cpu_var(irq_thermal_counts));
 
 	rdmsrl(MSR_IA32_THERM_STATUS, msr_val);
 	if (therm_throt_process(msr_val & 1))
Index: 2.6.22.1-rt8/include/asm-i386/apic.h
===================================================================
--- 2.6.22.1-rt8.orig/include/asm-i386/apic.h	2007-07-26 11:57:13.000000000 -0400
+++ 2.6.22.1-rt8/include/asm-i386/apic.h	2007-07-26 11:57:14.000000000 -0400
@@ -7,6 +7,7 @@
 #include <asm/apicdef.h>
 #include <asm/processor.h>
 #include <asm/system.h>
+#include <asm/atomic.h>
 
 #define Dprintk(x...)
 
@@ -118,6 +119,9 @@
 
 extern void smp_send_nmi_allbutself(void);
 
+DECLARE_PER_CPU(atomic_t, irq_spur_counts);
+DECLARE_PER_CPU(atomic_t, irq_thermal_counts);
+
 #else /* !CONFIG_X86_LOCAL_APIC */
 static inline void lapic_shutdown(void) { }
 
Index: 2.6.22.1-rt8/include/asm-i386/hw_irq.h
===================================================================
--- 2.6.22.1-rt8.orig/include/asm-i386/hw_irq.h	2007-07-26 11:57:13.000000000 -0400
+++ 2.6.22.1-rt8/include/asm-i386/hw_irq.h	2007-07-26 11:57:14.000000000 -0400
@@ -13,6 +13,7 @@
  */
 
 #include <linux/profile.h>
+#include <linux/percpu.h>
 #include <asm/atomic.h>
 #include <asm/irq.h>
 #include <asm/sections.h>
@@ -58,8 +59,11 @@
 
 extern unsigned long io_apic_irqs;
 
-extern atomic_t irq_err_count;
-extern atomic_t irq_mis_count;
+DECLARE_PER_CPU(atomic_t, irq_resched_counts);
+DECLARE_PER_CPU(atomic_t, irq_call_counts);
+DECLARE_PER_CPU(atomic_t, irq_tlb_counts);
+DECLARE_PER_CPU(atomic_t, irq_err_counts);
+DECLARE_PER_CPU(atomic_t, irq_mis_counts);
 
 #define IO_APIC_IRQ(x) (((x) >= 16) || ((1<<(x)) & io_apic_irqs))
 
Index: 2.6.22.1-rt8/include/asm-x86_64/apic.h
===================================================================
--- 2.6.22.1-rt8.orig/include/asm-x86_64/apic.h	2007-07-26 11:57:13.000000000 -0400
+++ 2.6.22.1-rt8/include/asm-x86_64/apic.h	2007-07-26 11:57:14.000000000 -0400
@@ -6,6 +6,7 @@
 #include <asm/fixmap.h>
 #include <asm/apicdef.h>
 #include <asm/system.h>
+#include <asm/atomic.h>
 
 #define Dprintk(x...)
 
@@ -101,4 +102,7 @@
 extern unsigned boot_cpu_id;
 extern int local_apic_timer_c2_ok;
 
+DECLARE_PER_CPU(atomic_t, irq_spur_counts);
+DECLARE_PER_CPU(atomic_t, irq_thermal_counts);
+
 #endif /* __ASM_APIC_H */
Index: 2.6.22.1-rt8/include/asm-x86_64/hw_irq.h
===================================================================
--- 2.6.22.1-rt8.orig/include/asm-x86_64/hw_irq.h	2007-07-26 11:57:13.000000000 -0400
+++ 2.6.22.1-rt8/include/asm-x86_64/hw_irq.h	2007-07-26 11:57:14.000000000 -0400
@@ -123,8 +123,12 @@
 
 extern unsigned long io_apic_irqs;
 
-extern atomic_t irq_err_count;
-extern atomic_t irq_mis_count;
+DECLARE_PER_CPU(atomic_t, irq_resched_counts);
+DECLARE_PER_CPU(atomic_t, irq_call_counts);
+DECLARE_PER_CPU(atomic_t, irq_tlb_counts);
+DECLARE_PER_CPU(atomic_t, irq_threshold_counts);
+DECLARE_PER_CPU(atomic_t, irq_err_counts);
+DECLARE_PER_CPU(atomic_t, irq_mis_counts);
 
 #define IO_APIC_IRQ(x) (((x) >= 16) || ((1<<(x)) & io_apic_irqs))
 
Index: 2.6.22.1-rt8/arch/x86_64/kernel/smp.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/x86_64/kernel/smp.c	2007-07-26 11:57:13.000000000 -0400
+++ 2.6.22.1-rt8/arch/x86_64/kernel/smp.c	2007-07-26 11:57:14.000000000 -0400
@@ -133,6 +133,8 @@
 	int sender;
 	union smp_flush_state *f;
 
+	atomic_inc(&__get_cpu_var(irq_tlb_counts));
+
 	cpu = smp_processor_id();
 	/*
 	 * orig_rax contains the negated interrupt vector.
@@ -508,6 +510,7 @@
 asmlinkage void smp_reschedule_interrupt(void)
 {
 	ack_APIC_irq();
+	atomic_inc(&__get_cpu_var(irq_resched_counts));
 }
 
 asmlinkage void smp_call_function_interrupt(void)
@@ -517,6 +520,7 @@
 	int wait = call_data->wait;
 
 	ack_APIC_irq();
+	atomic_inc(&__get_cpu_var(irq_call_counts));
 	/*
 	 * Notify initiating CPU that I've grabbed the data and am
 	 * about to execute the function
Index: 2.6.22.1-rt8/Documentation/filesystems/proc.txt
===================================================================
--- 2.6.22.1-rt8.orig/Documentation/filesystems/proc.txt	2007-07-08 19:32:17.000000000 -0400
+++ 2.6.22.1-rt8/Documentation/filesystems/proc.txt	2007-07-26 13:02:17.000000000 -0400
@@ -295,7 +295,15 @@
 the IO-APIC automatically retry the transmission, so it should not be a big
 problem, but you should read the SMP-FAQ.
 
-In this context it could be interesting to note the new irq directory in 2.4.
+In 2.6.* a new file, /proc/all-interrupts appears on some architectures.
+This adds to the /proc/interrupts output all the system interrupt vectors
+that /proc/interrupts missed, and it fills in the commentary field for each
+system vector that lacked it.  This added functionality was made a seperate
+file, rather than merged into /proc/interrupts, as /proc/interrupts has
+not changed in a long time, thus raising the specter that _any_ change
+might break ancient, longstanding scripts and programs.
+
+An interesting addition to 2.4 is the new irq directory.
 It could be used to set IRQ to CPU affinity, this means that you can "hook" an
 IRQ to only one CPU, or to exclude a CPU of handling IRQs. The contents of the
 irq subdir is one subdir for each IRQ, and one file; prof_cpu_mask

Re: [PATCH] create /proc/all-interrupts

[H. Peter Anvin]

Joe Korty wrote:
> Create /proc/all-interrupts for some architectures.
> 
> Create a version of /proc/interrupts that displays _every_
> IRQ vector, not just those that someone thought might be
> interesting, and add an entry in the commentary column
> for those vectors which lacked such a comment.
> 
> Rationale: /proc/interrupts is not truly useful unless it
> displays every IRQ vector, not just those somebody thought
> would be interesting.  For example, since /proc/interrupts
> does not display the rescheduling interrupt, the occurance
> of rescheduling interrupt floods ends up affecting
> latencies, yet without an entry in /proc/interrupts, it
> is difficult to discern why latencies are being affected.
> 
> Rather than modify /proc/interrupts, this patch creates
> a new version of /proc/interrupts, on the off-chance
> that adding new lines to /proc/interrupts, and appending
> new fields to the end of old lines, might break some
> longstanding script.  However, these kinds of changes
> traditionally do not affect scripts, so it might be
> reasonable to fold /proc/all-interrupts back into
> /proc/interrupts.

I think that would be the right thing to do.  We have added things to
/proc/interrupts in the past.

	-hpa

Re: [PATCH] create /proc/all-interrupts

[Sven-Thorsten Dietrich]

[-- Attachment #1: Type: text/plain, Size: 1434 bytes --]

On Thu, 2007-07-26 at 11:56 -0700, H. Peter Anvin wrote: 
> Joe Korty wrote:
> > Create /proc/all-interrupts for some architectures.
> > 
> > Create a version of /proc/interrupts that displays _every_
> > IRQ vector, not just those that someone thought might be
> > interesting, and add an entry in the commentary column
> > for those vectors which lacked such a comment.
> > 
> > Rationale: /proc/interrupts is not truly useful unless it
> > displays every IRQ vector, not just those somebody thought
> > would be interesting.  For example, since /proc/interrupts
> > does not display the rescheduling interrupt, the occurance
> > of rescheduling interrupt floods ends up affecting
> > latencies, yet without an entry in /proc/interrupts, it
> > is difficult to discern why latencies are being affected.
> > 
> > Rather than modify /proc/interrupts, this patch creates
> > a new version of /proc/interrupts, on the off-chance
> > that adding new lines to /proc/interrupts, and appending
> > new fields to the end of old lines, might break some
> > longstanding script.  However, these kinds of changes
> > traditionally do not affect scripts, so it might be
> > reasonable to fold /proc/all-interrupts back into
> > /proc/interrupts.
> 
> I think that would be the right thing to do.  We have added things to
> /proc/interrupts in the past.
> 
Hi Andrew,

Would it make sense to drop this patch into -mm for feedback?

Thanks,

Sven

[-- Attachment #2: Attached message - [PATCH 2/3] proc-interrupts-missing-vectors --]
[-- Type: message/rfc822, Size: 16626 bytes --]

From: Joe Korty <joe.korty@ccur.com>
To: mingo@elte.hu
Cc: tglx@linutronix.de, sdietrich@novell.com, linux-rt-users@vger.kernel.org
Subject: [PATCH 2/3] proc-interrupts-missing-vectors
Date: Wed, 25 Jul 2007 16:58:16 -0400
Message-ID: <20070725205816.GA9032@tsunami.ccur.com>

Add missing IRQs to /proc/interrupts.

/proc/interrupts is not truly useful unless it displays
every IRQ vector, not just those somebody thought
would be interesting.  For example, since the default
/proc/interrupts setup did not display the rescheduling
interrupt, the occurance of rescheduling interrupt floods
ends up affecting latencies, yet without an entry in
/proc/interrupts, it is difficult to discern why latencies
are being affected.

Also does a silly little expansion of the 'ERR:' display
to be per cpu.  This makes its layout identical to all
the other display lines in /proc/interrupts.

Signed-off-by: Joe Korty <joe.korty@ccur.com>

Index: 2.6.22.1-rt8/arch/i386/kernel/apic.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/i386/kernel/apic.c	2007-07-25 16:07:23.000000000 -0400
+++ 2.6.22.1-rt8/arch/i386/kernel/apic.c	2007-07-25 16:07:43.000000000 -0400
@@ -1268,6 +1268,8 @@
 {
 	unsigned long v;
 
+	atomic_inc(&__get_cpu_var(irq_spur_counts));
+
 	irq_enter();
 	/*
 	 * Check if this really is a spurious interrupt and ACK it
@@ -1297,7 +1299,7 @@
 	apic_write(APIC_ESR, 0);
 	v1 = apic_read(APIC_ESR);
 	ack_APIC_irq();
-	atomic_inc(&irq_err_count);
+	atomic_inc(&__get_cpu_var(irq_err_counts));
 
 	/* Here is what the APIC error bits mean:
 	   0: Send CS error
Index: 2.6.22.1-rt8/arch/i386/kernel/cpu/mcheck/p4.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/i386/kernel/cpu/mcheck/p4.c	2007-07-25 16:07:23.000000000 -0400
+++ 2.6.22.1-rt8/arch/i386/kernel/cpu/mcheck/p4.c	2007-07-25 16:07:43.000000000 -0400
@@ -60,6 +60,7 @@
 fastcall void smp_thermal_interrupt(struct pt_regs *regs)
 {
 	irq_enter();
+	atomic_inc(&__get_cpu_var(irq_thermal_counts));
 	vendor_thermal_interrupt(regs);
 	irq_exit();
 }
Index: 2.6.22.1-rt8/arch/i386/kernel/i8259.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/i386/kernel/i8259.c	2007-07-25 16:07:23.000000000 -0400
+++ 2.6.22.1-rt8/arch/i386/kernel/i8259.c	2007-07-25 16:07:43.000000000 -0400
@@ -209,7 +209,7 @@
 			printk(KERN_DEBUG "spurious 8259A interrupt: IRQ%d.\n", irq);
 			spurious_irq_mask |= irqmask;
 		}
-		atomic_inc(&irq_err_count);
+		atomic_inc(&__get_cpu_var(irq_err_counts));
 		/*
 		 * Theoretically we do not have to handle this IRQ,
 		 * but in Linux this does not cause problems and is
Index: 2.6.22.1-rt8/arch/i386/kernel/io_apic.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/i386/kernel/io_apic.c	2007-07-25 16:07:23.000000000 -0400
+++ 2.6.22.1-rt8/arch/i386/kernel/io_apic.c	2007-07-25 16:07:43.000000000 -0400
@@ -51,7 +51,6 @@
 #include "io_ports.h"
 
 int (*ioapic_renumber_irq)(int ioapic, int irq);
-atomic_t irq_mis_count;
 
 /* Where if anywhere is the i8259 connect in external int mode */
 static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
@@ -2031,7 +2030,7 @@
 	ack_APIC_irq();
 
 	if (!(v & (1 << (i & 0x1f)))) {
-		atomic_inc(&irq_mis_count);
+		atomic_inc(&__get_cpu_var(irq_mis_counts));
 		spin_lock(&ioapic_lock);
 		/* mask = 1, trigger = 0 */
 		__modify_IO_APIC_irq(irq, 0x00010000, 0x00008000);
Index: 2.6.22.1-rt8/arch/i386/kernel/irq.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/i386/kernel/irq.c	2007-07-25 16:07:23.000000000 -0400
+++ 2.6.22.1-rt8/arch/i386/kernel/irq.c	2007-07-25 16:23:12.000000000 -0400
@@ -252,7 +252,13 @@
  * Interrupt statistics:
  */
 
-atomic_t irq_err_count;
+DEFINE_PER_CPU(atomic_t, irq_resched_counts);
+DEFINE_PER_CPU(atomic_t, irq_call_counts);
+DEFINE_PER_CPU(atomic_t, irq_spur_counts);
+DEFINE_PER_CPU(atomic_t, irq_tlb_counts);
+DEFINE_PER_CPU(atomic_t, irq_thermal_counts);
+DEFINE_PER_CPU(atomic_t, irq_err_counts);
+DEFINE_PER_CPU(atomic_t, irq_mis_counts);
 
 /*
  * /proc/interrupts printing:
@@ -305,9 +311,60 @@
 				per_cpu(irq_stat,j).apic_timer_irqs);
 		seq_putc(p, '\n');
 #endif
-		seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count));
-#if defined(CONFIG_X86_IO_APIC)
-		seq_printf(p, "MIS: %10u\n", atomic_read(&irq_mis_count));
+#ifdef CONFIG_SMP
+#ifdef RESCHEDULE_VECTOR
+		seq_printf(p, "RES: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				atomic_read(&per_cpu(irq_resched_counts, j)));
+		seq_putc(p, '\n');
+#endif
+#endif
+#ifdef CONFIG_SMP
+#ifdef CALL_FUNCTION_VECTOR
+		seq_printf(p, "CAL: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				atomic_read(&per_cpu(irq_call_counts, j)));
+		seq_putc(p, '\n');
+#endif
+#endif
+#ifdef CONFIG_SMP
+#ifdef INVALIDATE_TLB_VECTOR
+		seq_printf(p, "TLB: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				atomic_read(&per_cpu(irq_tlb_counts, j)));
+		seq_putc(p, '\n');
+#endif
+#endif
+#ifdef THERMAL_APIC_VECTOR
+		seq_printf(p, "TRM: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				atomic_read(&per_cpu(irq_thermal_counts, j)));
+		seq_putc(p, '\n');
+#endif
+#ifdef SPURIOUS_APIC_VECTOR
+		seq_printf(p, "SPU: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				atomic_read(&per_cpu(irq_spur_counts, j)));
+		seq_putc(p, '\n');
+#endif
+#ifdef ERROR_APIC_VECTOR
+		seq_printf(p, "ERR: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				atomic_read(&per_cpu(irq_err_counts, j)));
+		seq_putc(p, '\n');
+#endif
+#if defined(CONFIG_X86_IO_APIC) && defined(APIC_MISMATCH_DEBUG)
+		seq_printf(p, "MIS: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				atomic_read(&per_cpu(irq_mis_counts, j)));
+		seq_putc(p, '\n');
 #endif
 	}
 	return 0;
Index: 2.6.22.1-rt8/arch/i386/kernel/smp.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/i386/kernel/smp.c	2007-07-25 16:07:23.000000000 -0400
+++ 2.6.22.1-rt8/arch/i386/kernel/smp.c	2007-07-25 16:07:43.000000000 -0400
@@ -315,6 +315,7 @@
 	unsigned long cpu;
 
 	cpu = get_cpu();
+	atomic_inc(&__get_cpu_var(irq_tlb_counts));
 
 	if (!cpu_isset(cpu, flush_cpumask))
 		goto out;
@@ -658,6 +659,7 @@
 {
 	trace_special(regs->eip, 0, 0);
 	ack_APIC_irq();
+	atomic_inc(&__get_cpu_var(irq_resched_counts));
 	set_tsk_need_resched(current);
 }
 
@@ -668,6 +670,7 @@
 	int wait = call_data->wait;
 
 	ack_APIC_irq();
+	atomic_inc(&__get_cpu_var(irq_call_counts));
 	/*
 	 * Notify initiating CPU that I've grabbed the data and am
 	 * about to execute the function
Index: 2.6.22.1-rt8/arch/x86_64/kernel/apic.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/x86_64/kernel/apic.c	2007-07-25 16:07:23.000000000 -0400
+++ 2.6.22.1-rt8/arch/x86_64/kernel/apic.c	2007-07-25 16:07:43.000000000 -0400
@@ -1099,6 +1099,9 @@
 asmlinkage void smp_spurious_interrupt(void)
 {
 	unsigned int v;
+
+	atomic_inc(&__get_cpu_var(irq_spur_counts));
+
 	exit_idle();
 	irq_enter();
 	/*
@@ -1127,7 +1130,7 @@
 	apic_write(APIC_ESR, 0);
 	v1 = apic_read(APIC_ESR);
 	ack_APIC_irq();
-	atomic_inc(&irq_err_count);
+	atomic_inc(&__get_cpu_var(irq_err_counts));
 
 	/* Here is what the APIC error bits mean:
 	   0: Send CS error
Index: 2.6.22.1-rt8/arch/x86_64/kernel/i8259.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/x86_64/kernel/i8259.c	2007-07-25 16:07:23.000000000 -0400
+++ 2.6.22.1-rt8/arch/x86_64/kernel/i8259.c	2007-07-25 16:07:43.000000000 -0400
@@ -273,7 +273,7 @@
 			printk(KERN_DEBUG "spurious 8259A interrupt: IRQ%d.\n", irq);
 			spurious_irq_mask |= irqmask;
 		}
-		atomic_inc(&irq_err_count);
+		atomic_inc(&__get_cpu_var(irq_err_counts));
 		/*
 		 * Theoretically we do not have to handle this IRQ,
 		 * but in Linux this does not cause problems and is
Index: 2.6.22.1-rt8/arch/x86_64/kernel/irq.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/x86_64/kernel/irq.c	2007-07-25 16:07:23.000000000 -0400
+++ 2.6.22.1-rt8/arch/x86_64/kernel/irq.c	2007-07-25 16:23:12.000000000 -0400
@@ -20,7 +20,18 @@
 #include <asm/idle.h>
 #include <asm/smp.h>
 
-atomic_t irq_err_count;
+DEFINE_PER_CPU(atomic_t, irq_resched_counts);
+DEFINE_PER_CPU(atomic_t, irq_call_counts);
+DEFINE_PER_CPU(atomic_t, irq_spur_counts);
+DEFINE_PER_CPU(atomic_t, irq_tlb_counts);
+DEFINE_PER_CPU(atomic_t, irq_threshold_counts);
+DEFINE_PER_CPU(atomic_t, irq_thermal_counts);
+DEFINE_PER_CPU(atomic_t, irq_err_counts);
+#ifdef CONFIG_X86_IO_APIC
+#ifdef APIC_MISMATCH_DEBUG
+DEFINE_PER_CPU(atomic_t, irq_mis_counts);
+#endif
+#endif
 
 #ifdef CONFIG_DEBUG_STACKOVERFLOW
 /*
@@ -93,7 +104,62 @@
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ", cpu_pda(j)->apic_timer_irqs);
 		seq_putc(p, '\n');
-		seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count));
+#ifdef CONFIG_SMP
+#ifdef RESCHEDULE_VECTOR
+		seq_printf(p, "RES: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", atomic_read(&per_cpu(irq_resched_counts, j)));
+		seq_putc(p, '\n');
+#endif
+#endif
+#ifdef CONFIG_SMP
+#ifdef CALL_FUNCTION_VECTOR
+		seq_printf(p, "CAL: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", atomic_read(&per_cpu(irq_call_counts, j)));
+		seq_putc(p, '\n');
+#endif
+#endif
+#ifdef CONFIG_SMP
+#ifdef INVALIDATE_TLB_VECTOR_START
+		seq_printf(p, "TLB: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", atomic_read(&per_cpu(irq_tlb_counts, j)));
+		seq_putc(p, '\n');
+#endif
+#endif
+#ifdef THERMAL_APIC_VECTOR
+		seq_printf(p, "TRM: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", atomic_read(&per_cpu(irq_thermal_counts, j)));
+		seq_putc(p, '\n');
+#endif
+#ifdef THRESHOLD_APIC_VECTOR
+		seq_printf(p, "THR: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", atomic_read(&per_cpu(irq_threshold_counts, j)));
+		seq_putc(p, '\n');
+#endif
+#ifdef SPURIOUS_APIC_VECTOR
+		seq_printf(p, "SPU: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", atomic_read(&per_cpu(irq_spur_counts, j)));
+		seq_putc(p, '\n');
+#endif
+#ifdef ERROR_APIC_VECTOR
+		seq_printf(p, "ERR: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", atomic_read(&per_cpu(irq_err_counts, j)));
+		seq_putc(p, '\n');
+#endif
+#ifdef CONFIG_X86_IO_APIC
+#ifdef APIC_MISMATCH_DEBUG
+		seq_printf(p, "MIS: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", atomic_read(&per_cpu(irq_mis_counts, j)));
+		seq_putc(p, '\n');
+#endif
+#endif
 	}
 	return 0;
 }
Index: 2.6.22.1-rt8/arch/x86_64/kernel/mce_amd.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/x86_64/kernel/mce_amd.c	2007-07-25 16:07:23.000000000 -0400
+++ 2.6.22.1-rt8/arch/x86_64/kernel/mce_amd.c	2007-07-25 16:07:43.000000000 -0400
@@ -186,6 +186,8 @@
 	exit_idle();
 	irq_enter();
 
+	atomic_inc(&__get_cpu_var(irq_threshold_counts));
+
 	memset(&m, 0, sizeof(m));
 	rdtscll(m.tsc);
 	m.cpu = smp_processor_id();
Index: 2.6.22.1-rt8/arch/x86_64/kernel/mce_intel.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/x86_64/kernel/mce_intel.c	2007-07-25 16:07:23.000000000 -0400
+++ 2.6.22.1-rt8/arch/x86_64/kernel/mce_intel.c	2007-07-25 16:07:43.000000000 -0400
@@ -21,6 +21,7 @@
 
 	exit_idle();
 	irq_enter();
+	atomic_inc(&__get_cpu_var(irq_thermal_counts));
 
 	rdmsrl(MSR_IA32_THERM_STATUS, msr_val);
 	if (therm_throt_process(msr_val & 1))
Index: 2.6.22.1-rt8/include/asm-i386/apic.h
===================================================================
--- 2.6.22.1-rt8.orig/include/asm-i386/apic.h	2007-07-25 16:07:23.000000000 -0400
+++ 2.6.22.1-rt8/include/asm-i386/apic.h	2007-07-25 16:07:43.000000000 -0400
@@ -7,6 +7,7 @@
 #include <asm/apicdef.h>
 #include <asm/processor.h>
 #include <asm/system.h>
+#include <asm/atomic.h>
 
 #define Dprintk(x...)
 
@@ -118,6 +119,9 @@
 
 extern void smp_send_nmi_allbutself(void);
 
+DECLARE_PER_CPU(atomic_t, irq_spur_counts);
+DECLARE_PER_CPU(atomic_t, irq_thermal_counts);
+
 #else /* !CONFIG_X86_LOCAL_APIC */
 static inline void lapic_shutdown(void) { }
 
Index: 2.6.22.1-rt8/include/asm-i386/hw_irq.h
===================================================================
--- 2.6.22.1-rt8.orig/include/asm-i386/hw_irq.h	2007-07-25 16:07:23.000000000 -0400
+++ 2.6.22.1-rt8/include/asm-i386/hw_irq.h	2007-07-25 16:07:43.000000000 -0400
@@ -13,6 +13,7 @@
  */
 
 #include <linux/profile.h>
+#include <linux/percpu.h>
 #include <asm/atomic.h>
 #include <asm/irq.h>
 #include <asm/sections.h>
@@ -58,8 +59,11 @@
 
 extern unsigned long io_apic_irqs;
 
-extern atomic_t irq_err_count;
-extern atomic_t irq_mis_count;
+DECLARE_PER_CPU(atomic_t, irq_resched_counts);
+DECLARE_PER_CPU(atomic_t, irq_call_counts);
+DECLARE_PER_CPU(atomic_t, irq_tlb_counts);
+DECLARE_PER_CPU(atomic_t, irq_err_counts);
+DECLARE_PER_CPU(atomic_t, irq_mis_counts);
 
 #define IO_APIC_IRQ(x) (((x) >= 16) || ((1<<(x)) & io_apic_irqs))
 
Index: 2.6.22.1-rt8/include/asm-x86_64/apic.h
===================================================================
--- 2.6.22.1-rt8.orig/include/asm-x86_64/apic.h	2007-07-25 16:07:23.000000000 -0400
+++ 2.6.22.1-rt8/include/asm-x86_64/apic.h	2007-07-25 16:07:43.000000000 -0400
@@ -6,6 +6,7 @@
 #include <asm/fixmap.h>
 #include <asm/apicdef.h>
 #include <asm/system.h>
+#include <asm/atomic.h>
 
 #define Dprintk(x...)
 
@@ -101,4 +102,7 @@
 extern unsigned boot_cpu_id;
 extern int local_apic_timer_c2_ok;
 
+DECLARE_PER_CPU(atomic_t, irq_spur_counts);
+DECLARE_PER_CPU(atomic_t, irq_thermal_counts);
+
 #endif /* __ASM_APIC_H */
Index: 2.6.22.1-rt8/include/asm-x86_64/hw_irq.h
===================================================================
--- 2.6.22.1-rt8.orig/include/asm-x86_64/hw_irq.h	2007-07-25 16:07:23.000000000 -0400
+++ 2.6.22.1-rt8/include/asm-x86_64/hw_irq.h	2007-07-25 16:07:43.000000000 -0400
@@ -123,8 +123,12 @@
 
 extern unsigned long io_apic_irqs;
 
-extern atomic_t irq_err_count;
-extern atomic_t irq_mis_count;
+DECLARE_PER_CPU(atomic_t, irq_resched_counts);
+DECLARE_PER_CPU(atomic_t, irq_call_counts);
+DECLARE_PER_CPU(atomic_t, irq_tlb_counts);
+DECLARE_PER_CPU(atomic_t, irq_threshold_counts);
+DECLARE_PER_CPU(atomic_t, irq_err_counts);
+DECLARE_PER_CPU(atomic_t, irq_mis_counts);
 
 #define IO_APIC_IRQ(x) (((x) >= 16) || ((1<<(x)) & io_apic_irqs))
 
Index: 2.6.22.1-rt8/arch/x86_64/kernel/smp.c
===================================================================
--- 2.6.22.1-rt8.orig/arch/x86_64/kernel/smp.c	2007-07-25 16:07:23.000000000 -0400
+++ 2.6.22.1-rt8/arch/x86_64/kernel/smp.c	2007-07-25 16:07:43.000000000 -0400
@@ -133,6 +133,8 @@
 	int sender;
 	union smp_flush_state *f;
 
+	atomic_inc(&__get_cpu_var(irq_tlb_counts));
+
 	cpu = smp_processor_id();
 	/*
 	 * orig_rax contains the negated interrupt vector.
@@ -508,6 +510,7 @@
 asmlinkage void smp_reschedule_interrupt(void)
 {
 	ack_APIC_irq();
+	atomic_inc(&__get_cpu_var(irq_resched_counts));
 }
 
 asmlinkage void smp_call_function_interrupt(void)
@@ -517,6 +520,7 @@
 	int wait = call_data->wait;
 
 	ack_APIC_irq();
+	atomic_inc(&__get_cpu_var(irq_call_counts));
 	/*
 	 * Notify initiating CPU that I've grabbed the data and am
 	 * about to execute the function

Re: [PATCH] create /proc/all-interrupts

[Andrew Morton]

On Mon, 30 Jul 2007 10:33:17 -0700
Sven-Thorsten Dietrich <sdietrich@novell.com> wrote:

> On Thu, 2007-07-26 at 11:56 -0700, H. Peter Anvin wrote: 
> > Joe Korty wrote:
> > > Create /proc/all-interrupts for some architectures.
> > > 
> > > Create a version of /proc/interrupts that displays _every_
> > > IRQ vector, not just those that someone thought might be
> > > interesting, and add an entry in the commentary column
> > > for those vectors which lacked such a comment.
> > > 
> > > Rationale: /proc/interrupts is not truly useful unless it
> > > displays every IRQ vector, not just those somebody thought
> > > would be interesting.  For example, since /proc/interrupts
> > > does not display the rescheduling interrupt, the occurance
> > > of rescheduling interrupt floods ends up affecting
> > > latencies, yet without an entry in /proc/interrupts, it
> > > is difficult to discern why latencies are being affected.
> > > 
> > > Rather than modify /proc/interrupts, this patch creates
> > > a new version of /proc/interrupts, on the off-chance
> > > that adding new lines to /proc/interrupts, and appending
> > > new fields to the end of old lines, might break some
> > > longstanding script.  However, these kinds of changes
> > > traditionally do not affect scripts, so it might be
> > > reasonable to fold /proc/all-interrupts back into
> > > /proc/interrupts.
> > 
> > I think that would be the right thing to do.  We have added things to
> > /proc/interrupts in the past.
> > 
> Hi Andrew,
> 
> Would it make sense to drop this patch into -mm for feedback?
> 

It's a lot of code for something which might be useful to someone sometime.

It's a bit of a crappy changelog too.  I'd at least like to see a list of
all the new fields.

It should be OK to add new lines to /proc/interrupts?  That file varies a
lot between machines adn between architectures - as long as the new lines
have similar layout it is unlikely that anything will break.



+	atomic_inc(&__get_cpu_var(irq_thermal_counts));

The patch does atomic ops on cpu-local variables.  This isn't needed, and
is expensive.

If the field is only ever modified from hard interrupt context then you can
make the field unsigned long and use plain old `foo++'.

If the field is modified from both hard-IRQ and from non-IRQ then use a
local_t and local_inc.

Or even, given that this is just a statistic and grrat precision is not
needed, use unsigned long and f++ even if that _is_ racy.  Because the
consequences of a race will just be a single lost count, which we dont'
care about enough to add the additional overhead of an atomic op.


But I'm a bit dubious about the whole thing, really.  As I said, it's a lot
of code.  More justfication is needed for its addition, I'd suggest.

Re: [PATCH] create /proc/all-interrupts

[Joe Korty]

On Mon, Jul 30, 2007 at 12:32:06PM -0700, Andrew Morton wrote:
> On Mon, 30 Jul 2007 10:33:17 -0700
> Sven-Thorsten Dietrich <sdietrich@novell.com> wrote:
> > On Thu, 2007-07-26 at 11:56 -0700, H. Peter Anvin wrote: 
> > > Joe Korty wrote:
> > > > Create /proc/all-interrupts for some architectures.

> > Would it make sense to drop this patch into -mm for feedback?
> > 
> 
> It's a lot of code for something which might be useful to someone sometime.
> 
> It's a bit of a crappy changelog too.  I'd at least like to see a list of
> all the new fields.
> 
> It should be OK to add new lines to /proc/interrupts?  That file varies a
> lot between machines adn between architectures - as long as the new lines
> have similar layout it is unlikely that anything will break.
> 
> +	atomic_inc(&__get_cpu_var(irq_thermal_counts));
> 
> The patch does atomic ops on cpu-local variables.  This isn't needed, and
> is expensive.
> 
> If the field is only ever modified from hard interrupt context then you can
> make the field unsigned long and use plain old `foo++'.
> 
> If the field is modified from both hard-IRQ and from non-IRQ then use a
> local_t and local_inc.
> 
> Or even, given that this is just a statistic and grrat precision is not
> needed, use unsigned long and f++ even if that _is_ racy.  Because the
> consequences of a race will just be a single lost count, which we dont'
> care about enough to add the additional overhead of an atomic op.

Hi Andrew,
Thanks for the comments.  I'll, at least, make the changes you suggested.

(the /proc/interrupts version has the benefit of being smaller too).

Joe

[PATCH] expand /proc/interrupts to include missing vectors, v2

[Joe Korty]

Add missing IRQs and IRQ descriptions to /proc/interrupts.

/proc/interrupts is most useful when it displays every
IRQ vector in use by the system, not just those somebody
thought would be interesting.

This patch inserts the following vector displays to the
i386 and x86_64 platforms, as appropriate:
	
	rescheduling interrupts
	TLB flush interrupts
	function call interrupts
	thermal event interrupts
	threshold interrupts
	spurious interrupts

A threshold interrupt occurs when ECC memory correction
is occuring at too high a frequency.  Thresholds are used
by the ECC hardware as occasional ECC failures are part
of normal operation, but long sequences of ECC failures
usually indicate a memory chip that is about to fail.

Thermal event interrupts occur when a temperature threshold
has been exceeded for some CPU chip.  IIRC, a thermal
interrupt is also generated when the temperature drops
back to a normal level.

A spurious interrupt is an interrupt that was raised then
lowered by the device before it could be fully processed
by the APIC.  Hence the apic sees the interrupt but does
not know what device it came from.  For this case the APIC
hardware will assume a vector of 0xff.

Rescheduling, call, and TLB flush interrupts are sent from
one CPU to another per the needs of the OS.  Typically,
their statistics would be used to discover if an
interrupt flood of the given type has been occuring.

Signed-off-by: Joe Korty <joe.korty@ccur.com>

Index: 2.6.23-rc1-git7/arch/i386/kernel/apic.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/kernel/apic.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/kernel/apic.c	2007-07-30 19:08:07.000000000 -0400
@@ -1280,6 +1280,7 @@
 	printk(KERN_INFO "spurious APIC interrupt on CPU#%d, "
 	       "should never happen.\n", smp_processor_id());
 	irq_exit();
+	__get_cpu_var(irq_stat).irq_spur_counts++;
 }
 
 /*
Index: 2.6.23-rc1-git7/arch/i386/kernel/cpu/mcheck/p4.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/kernel/cpu/mcheck/p4.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/kernel/cpu/mcheck/p4.c	2007-07-30 19:08:07.000000000 -0400
@@ -62,6 +62,7 @@
 	irq_enter();
 	vendor_thermal_interrupt(regs);
 	irq_exit();
+	__get_cpu_var(irq_stat).irq_thermal_counts++;
 }
 
 /* P4/Xeon Thermal regulation detect and init */
Index: 2.6.23-rc1-git7/arch/i386/kernel/irq.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/kernel/irq.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/kernel/irq.c	2007-07-30 19:08:07.000000000 -0400
@@ -284,14 +284,45 @@
 		seq_printf(p, "NMI: ");
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ", nmi_count(j));
-		seq_putc(p, '\n');
+		seq_printf(p, "  Non-maskable interrupts\n");
 #ifdef CONFIG_X86_LOCAL_APIC
 		seq_printf(p, "LOC: ");
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ",
 				per_cpu(irq_stat,j).apic_timer_irqs);
-		seq_putc(p, '\n');
+		seq_printf(p, "  Local interrupts\n");
 #endif
+#ifdef CONFIG_SMP
+		seq_printf(p, "RES: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				per_cpu(irq_stat,j).irq_resched_counts);
+		seq_printf(p, "  Rescheduling interrupts\n");
+#endif
+#ifdef CONFIG_SMP
+		seq_printf(p, "CAL: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				per_cpu(irq_stat,j).irq_call_counts);
+		seq_printf(p, "  function call interrupts\n");
+#endif
+#ifdef CONFIG_SMP
+		seq_printf(p, "TLB: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				per_cpu(irq_stat,j).irq_tlb_counts);
+		seq_printf(p, "  TLB shootdowns\n");
+#endif
+		seq_printf(p, "TRM: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				per_cpu(irq_stat,j).irq_thermal_counts);
+		seq_printf(p, "  Thermal event interrupts\n");
+		seq_printf(p, "SPU: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				per_cpu(irq_stat,j).irq_spur_counts);
+		seq_printf(p, "  Spurious interrupts\n");
 		seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count));
 #if defined(CONFIG_X86_IO_APIC)
 		seq_printf(p, "MIS: %10u\n", atomic_read(&irq_mis_count));
Index: 2.6.23-rc1-git7/arch/i386/kernel/smp.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/kernel/smp.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/kernel/smp.c	2007-07-30 19:08:07.000000000 -0400
@@ -342,6 +342,7 @@
 	smp_mb__after_clear_bit();
 out:
 	put_cpu_no_resched();
+	__get_cpu_var(irq_stat).irq_tlb_counts++;
 }
 
 void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
@@ -640,6 +641,7 @@
 fastcall void smp_reschedule_interrupt(struct pt_regs *regs)
 {
 	ack_APIC_irq();
+	__get_cpu_var(irq_stat).irq_resched_counts++;
 }
 
 fastcall void smp_call_function_interrupt(struct pt_regs *regs)
@@ -666,6 +668,7 @@
 		mb();
 		atomic_inc(&call_data->finished);
 	}
+	__get_cpu_var(irq_stat).irq_call_counts++;
 }
 
 static int convert_apicid_to_cpu(int apic_id)
Index: 2.6.23-rc1-git7/arch/x86_64/kernel/apic.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/x86_64/kernel/apic.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/x86_64/kernel/apic.c	2007-07-30 19:08:07.000000000 -0400
@@ -1118,6 +1118,7 @@
 asmlinkage void smp_spurious_interrupt(void)
 {
 	unsigned int v;
+
 	exit_idle();
 	irq_enter();
 	/*
@@ -1130,6 +1131,7 @@
 		ack_APIC_irq();
 
 	irq_exit();
+	add_pda(irq_spur_counts, 1);
 }
 
 /*
Index: 2.6.23-rc1-git7/arch/x86_64/kernel/irq.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/x86_64/kernel/irq.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/x86_64/kernel/irq.c	2007-07-30 19:10:22.000000000 -0400
@@ -88,11 +88,41 @@
 		seq_printf(p, "NMI: ");
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ", cpu_pda(j)->__nmi_count);
-		seq_putc(p, '\n');
+		seq_printf(p, "  Non-maskable interrupts\n");
 		seq_printf(p, "LOC: ");
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ", cpu_pda(j)->apic_timer_irqs);
-		seq_putc(p, '\n');
+		seq_printf(p, "  Local interrupts\n");
+#ifdef CONFIG_SMP
+		seq_printf(p, "RES: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_resched_counts);
+		seq_printf(p, "  Rescheduling interrupts\n");
+#endif
+#ifdef CONFIG_SMP
+		seq_printf(p, "CAL: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_call_counts);
+		seq_printf(p, "  function call interrupts\n");
+#endif
+#ifdef CONFIG_SMP
+		seq_printf(p, "TLB: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_tlb_counts);
+		seq_printf(p, "  TLB shootdowns\n");
+#endif
+		seq_printf(p, "TRM: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_thermal_counts);
+		seq_printf(p, "  Thermal event interrupts\n");
+		seq_printf(p, "THR: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_threshold_counts);
+		seq_printf(p, "  Threshold APIC interrupts\n");
+		seq_printf(p, "SPU: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_spur_counts);
+		seq_printf(p, "  Spurious interrupts\n");
 		seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count));
 	}
 	return 0;
Index: 2.6.23-rc1-git7/arch/x86_64/kernel/mce_amd.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/x86_64/kernel/mce_amd.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/x86_64/kernel/mce_amd.c	2007-07-30 19:08:07.000000000 -0400
@@ -238,6 +238,7 @@
 	}
 out:
 	irq_exit();
+	add_pda(irq_threshold_counts, 1);
 }
 
 /*
Index: 2.6.23-rc1-git7/arch/x86_64/kernel/mce_intel.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/x86_64/kernel/mce_intel.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/x86_64/kernel/mce_intel.c	2007-07-30 19:08:07.000000000 -0400
@@ -27,6 +27,7 @@
 		mce_log_therm_throt_event(smp_processor_id(), msr_val);
 
 	irq_exit();
+	add_pda(irq_thermal_counts, 1);
 }
 
 static void __cpuinit intel_init_thermal(struct cpuinfo_x86 *c)
Index: 2.6.23-rc1-git7/arch/x86_64/kernel/smp.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/x86_64/kernel/smp.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/x86_64/kernel/smp.c	2007-07-30 19:08:07.000000000 -0400
@@ -163,6 +163,7 @@
 out:
 	ack_APIC_irq();
 	cpu_clear(cpu, f->flush_cpumask);
+	add_pda(irq_tlb_counts, 1);
 }
 
 static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
@@ -493,6 +494,7 @@
 asmlinkage void smp_reschedule_interrupt(void)
 {
 	ack_APIC_irq();
+	add_pda(irq_resched_counts, 1);
 }
 
 asmlinkage void smp_call_function_interrupt(void)
@@ -519,5 +521,6 @@
 		mb();
 		atomic_inc(&call_data->finished);
 	}
+	add_pda(irq_call_counts, 1);
 }
 
Index: 2.6.23-rc1-git7/include/asm-i386/hardirq.h
===================================================================
--- 2.6.23-rc1-git7.orig/include/asm-i386/hardirq.h	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/include/asm-i386/hardirq.h	2007-07-30 19:08:07.000000000 -0400
@@ -9,6 +9,11 @@
 	unsigned long idle_timestamp;
 	unsigned int __nmi_count;	/* arch dependent */
 	unsigned int apic_timer_irqs;	/* arch dependent */
+	unsigned int irq_resched_counts;
+	unsigned int irq_call_counts;
+	unsigned int irq_tlb_counts;
+	unsigned int irq_thermal_counts;
+	unsigned int irq_spur_counts;
 } ____cacheline_aligned irq_cpustat_t;
 
 DECLARE_PER_CPU(irq_cpustat_t, irq_stat);
Index: 2.6.23-rc1-git7/include/asm-x86_64/pda.h
===================================================================
--- 2.6.23-rc1-git7.orig/include/asm-x86_64/pda.h	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/include/asm-x86_64/pda.h	2007-07-30 19:08:07.000000000 -0400
@@ -29,6 +29,12 @@
 	short isidle;
 	struct mm_struct *active_mm;
 	unsigned apic_timer_irqs;
+	unsigned irq_resched_counts;
+	unsigned irq_call_counts;
+	unsigned irq_tlb_counts;
+	unsigned irq_thermal_counts;
+	unsigned irq_threshold_counts;
+	unsigned irq_spur_counts;
 } ____cacheline_aligned_in_smp;
 
 extern struct x8664_pda *_cpu_pda[];
Index: 2.6.23-rc1-git7/arch/i386/mach-voyager/voyager_smp.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/mach-voyager/voyager_smp.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/mach-voyager/voyager_smp.c	2007-07-30 19:08:07.000000000 -0400
@@ -1042,6 +1042,7 @@
 		mb();
 		clear_bit(cpu, &call_data->finished);
 	}
+	__get_cpu_var(irq_stat).irq_call_counts++;
 }
 
 static int
Index: 2.6.23-rc1-git7/arch/i386/xen/smp.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/xen/smp.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/xen/smp.c	2007-07-30 19:08:07.000000000 -0400
@@ -352,6 +352,7 @@
 		mb();		/* commit everything before setting finished */
 		atomic_inc(&call_data->finished);
 	}
+	__get_cpu_var(irq_stat).irq_call_counts++;
 
 	return IRQ_HANDLED;
 }

Re: [PATCH] expand /proc/interrupts to include missing vectors, v2

[Gabriel C]

Joe Korty wrote:
> +#ifdef CONFIG_SMP
> +		seq_printf(p, "RES: ");
> +		for_each_online_cpu(j)
> +			seq_printf(p, "%10u ",
> +				per_cpu(irq_stat,j).irq_resched_counts);
> +		seq_printf(p, "  Rescheduling interrupts\n");
> +#endif
> +#ifdef CONFIG_SMP
> +		seq_printf(p, "CAL: ");
> +		for_each_online_cpu(j)
> +			seq_printf(p, "%10u ",
> +				per_cpu(irq_stat,j).irq_call_counts);
> +		seq_printf(p, "  function call interrupts\n");
> +#endif
> +#ifdef CONFIG_SMP
> +		seq_printf(p, "TLB: ");
> +		for_each_online_cpu(j)
> +			seq_printf(p, "%10u ",
> +				per_cpu(irq_stat,j).irq_tlb_counts);
> +		seq_printf(p, "  TLB shootdowns\n");
> +#endif


How about :

#ifdef CONFIG_SMP
		seq_printf(p, "RES: ");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ",
				per_cpu(irq_stat,j).irq_resched_counts);
		seq_printf(p, "  Rescheduling interrupts\n");

		seq_printf(p, "CAL: ");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ",
				per_cpu(irq_stat,j).irq_call_counts);
		seq_printf(p, "  function call interrupts\n");

		seq_printf(p, "TLB: ");
		for_each_online_cpu(j)
			seq_printf(p, "%10u ",
				per_cpu(irq_stat,j).irq_tlb_counts);
		seq_printf(p, "  TLB shootdowns\n");
#endif


should to the same.


Regards,

Gabriel C

Re: [PATCH] expand /proc/interrupts to include missing vectors, v2

[Andrew Morton]

On Mon, 30 Jul 2007 20:17:18 -0400 Joe Korty <joe.korty@ccur.com> wrote:

> +		seq_printf(p, "RES: ");
> +		seq_printf(p, "CAL: ");
> +		seq_printf(p, "TLB: ");
> +		seq_printf(p, "TRM: ");

Documentation/filesystems/proc.txt presently describes NMI:, LOC: and ERR:.
Are you thinking what I'm thinking?  ;)

Re: [PATCH] expand /proc/interrupts to include missing vectors, v3

[Joe Korty]

[ changes from v2:
    added documentation
    merged some #ifdef CONFIG_SMP's
]

Add missing IRQs and IRQ descriptions to /proc/interrupts.

/proc/interrupts is most useful when it displays every
IRQ vector in use by the system, not just those somebody
thought would be interesting.

This patch inserts the following vector displays to the
i386 and x86_64 platforms, as appropriate:
	
	rescheduling interrupts
	TLB flush interrupts
	function call interrupts
	thermal event interrupts
	threshold interrupts
	spurious interrupts

A threshold interrupt occurs when ECC memory correction
is occuring at too high a frequency.  Thresholds are used
by the ECC hardware as occasional ECC failures are part
of normal operation, but long sequences of ECC failures
usually indicate a memory chip that is about to fail.

Thermal event interrupts occur when a temperature threshold
has been exceeded for some CPU chip.  IIRC, thermal interrupts
can also be generated when the temperature drops back to
the normal range.

A spurious interrupt is an interrupt that was raised then
lowered by the device before it could be fully processed
by the APIC.  Hence the apic sees the interrupt but does
not know what device it came from.  For this case the APIC
hardware will assume a vector of 0xff.

Rescheduling, call, and TLB flush interrupts are sent from
one CPU to another per the needs of the OS.  Typically,
their statistics would be used to discover interrupt
flooding.

Signed-off-by: Joe Korty <joe.korty@ccur.com>

Index: 2.6.23-rc1-git7/arch/i386/kernel/apic.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/kernel/apic.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/kernel/apic.c	2007-07-30 19:08:07.000000000 -0400
@@ -1280,6 +1280,7 @@
 	printk(KERN_INFO "spurious APIC interrupt on CPU#%d, "
 	       "should never happen.\n", smp_processor_id());
 	irq_exit();
+	__get_cpu_var(irq_stat).irq_spur_counts++;
 }
 
 /*
Index: 2.6.23-rc1-git7/arch/i386/kernel/cpu/mcheck/p4.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/kernel/cpu/mcheck/p4.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/kernel/cpu/mcheck/p4.c	2007-07-30 19:08:07.000000000 -0400
@@ -62,6 +62,7 @@
 	irq_enter();
 	vendor_thermal_interrupt(regs);
 	irq_exit();
+	__get_cpu_var(irq_stat).irq_thermal_counts++;
 }
 
 /* P4/Xeon Thermal regulation detect and init */
Index: 2.6.23-rc1-git7/arch/i386/kernel/irq.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/kernel/irq.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/kernel/irq.c	2007-07-31 09:40:58.000000000 -0400
@@ -284,14 +284,41 @@
 		seq_printf(p, "NMI: ");
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ", nmi_count(j));
-		seq_putc(p, '\n');
+		seq_printf(p, "  Non-maskable interrupts\n");
 #ifdef CONFIG_X86_LOCAL_APIC
 		seq_printf(p, "LOC: ");
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ",
 				per_cpu(irq_stat,j).apic_timer_irqs);
-		seq_putc(p, '\n');
+		seq_printf(p, "  Local interrupts\n");
 #endif
+#ifdef CONFIG_SMP
+		seq_printf(p, "RES: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				per_cpu(irq_stat,j).irq_resched_counts);
+		seq_printf(p, "  Rescheduling interrupts\n");
+		seq_printf(p, "CAL: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				per_cpu(irq_stat,j).irq_call_counts);
+		seq_printf(p, "  function call interrupts\n");
+		seq_printf(p, "TLB: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				per_cpu(irq_stat,j).irq_tlb_counts);
+		seq_printf(p, "  TLB shootdowns\n");
+#endif
+		seq_printf(p, "TRM: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				per_cpu(irq_stat,j).irq_thermal_counts);
+		seq_printf(p, "  Thermal event interrupts\n");
+		seq_printf(p, "SPU: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				per_cpu(irq_stat,j).irq_spur_counts);
+		seq_printf(p, "  Spurious interrupts\n");
 		seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count));
 #if defined(CONFIG_X86_IO_APIC)
 		seq_printf(p, "MIS: %10u\n", atomic_read(&irq_mis_count));
Index: 2.6.23-rc1-git7/arch/i386/kernel/smp.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/kernel/smp.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/kernel/smp.c	2007-07-30 19:08:07.000000000 -0400
@@ -342,6 +342,7 @@
 	smp_mb__after_clear_bit();
 out:
 	put_cpu_no_resched();
+	__get_cpu_var(irq_stat).irq_tlb_counts++;
 }
 
 void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
@@ -640,6 +641,7 @@
 fastcall void smp_reschedule_interrupt(struct pt_regs *regs)
 {
 	ack_APIC_irq();
+	__get_cpu_var(irq_stat).irq_resched_counts++;
 }
 
 fastcall void smp_call_function_interrupt(struct pt_regs *regs)
@@ -666,6 +668,7 @@
 		mb();
 		atomic_inc(&call_data->finished);
 	}
+	__get_cpu_var(irq_stat).irq_call_counts++;
 }
 
 static int convert_apicid_to_cpu(int apic_id)
Index: 2.6.23-rc1-git7/arch/x86_64/kernel/apic.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/x86_64/kernel/apic.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/x86_64/kernel/apic.c	2007-07-30 19:08:07.000000000 -0400
@@ -1118,6 +1118,7 @@
 asmlinkage void smp_spurious_interrupt(void)
 {
 	unsigned int v;
+
 	exit_idle();
 	irq_enter();
 	/*
@@ -1130,6 +1131,7 @@
 		ack_APIC_irq();
 
 	irq_exit();
+	add_pda(irq_spur_counts, 1);
 }
 
 /*
Index: 2.6.23-rc1-git7/arch/x86_64/kernel/irq.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/x86_64/kernel/irq.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/x86_64/kernel/irq.c	2007-07-31 09:41:14.000000000 -0400
@@ -88,11 +88,37 @@
 		seq_printf(p, "NMI: ");
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ", cpu_pda(j)->__nmi_count);
-		seq_putc(p, '\n');
+		seq_printf(p, "  Non-maskable interrupts\n");
 		seq_printf(p, "LOC: ");
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ", cpu_pda(j)->apic_timer_irqs);
-		seq_putc(p, '\n');
+		seq_printf(p, "  Local interrupts\n");
+#ifdef CONFIG_SMP
+		seq_printf(p, "RES: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_resched_counts);
+		seq_printf(p, "  Rescheduling interrupts\n");
+		seq_printf(p, "CAL: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_call_counts);
+		seq_printf(p, "  function call interrupts\n");
+		seq_printf(p, "TLB: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_tlb_counts);
+		seq_printf(p, "  TLB shootdowns\n");
+#endif
+		seq_printf(p, "TRM: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_thermal_counts);
+		seq_printf(p, "  Thermal event interrupts\n");
+		seq_printf(p, "THR: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_threshold_counts);
+		seq_printf(p, "  Threshold APIC interrupts\n");
+		seq_printf(p, "SPU: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_spur_counts);
+		seq_printf(p, "  Spurious interrupts\n");
 		seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count));
 	}
 	return 0;
Index: 2.6.23-rc1-git7/arch/x86_64/kernel/mce_amd.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/x86_64/kernel/mce_amd.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/x86_64/kernel/mce_amd.c	2007-07-30 19:08:07.000000000 -0400
@@ -238,6 +238,7 @@
 	}
 out:
 	irq_exit();
+	add_pda(irq_threshold_counts, 1);
 }
 
 /*
Index: 2.6.23-rc1-git7/arch/x86_64/kernel/mce_intel.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/x86_64/kernel/mce_intel.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/x86_64/kernel/mce_intel.c	2007-07-30 19:08:07.000000000 -0400
@@ -27,6 +27,7 @@
 		mce_log_therm_throt_event(smp_processor_id(), msr_val);
 
 	irq_exit();
+	add_pda(irq_thermal_counts, 1);
 }
 
 static void __cpuinit intel_init_thermal(struct cpuinfo_x86 *c)
Index: 2.6.23-rc1-git7/arch/x86_64/kernel/smp.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/x86_64/kernel/smp.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/x86_64/kernel/smp.c	2007-07-30 19:08:07.000000000 -0400
@@ -163,6 +163,7 @@
 out:
 	ack_APIC_irq();
 	cpu_clear(cpu, f->flush_cpumask);
+	add_pda(irq_tlb_counts, 1);
 }
 
 static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
@@ -493,6 +494,7 @@
 asmlinkage void smp_reschedule_interrupt(void)
 {
 	ack_APIC_irq();
+	add_pda(irq_resched_counts, 1);
 }
 
 asmlinkage void smp_call_function_interrupt(void)
@@ -519,5 +521,6 @@
 		mb();
 		atomic_inc(&call_data->finished);
 	}
+	add_pda(irq_call_counts, 1);
 }
 
Index: 2.6.23-rc1-git7/include/asm-i386/hardirq.h
===================================================================
--- 2.6.23-rc1-git7.orig/include/asm-i386/hardirq.h	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/include/asm-i386/hardirq.h	2007-07-30 19:08:07.000000000 -0400
@@ -9,6 +9,11 @@
 	unsigned long idle_timestamp;
 	unsigned int __nmi_count;	/* arch dependent */
 	unsigned int apic_timer_irqs;	/* arch dependent */
+	unsigned int irq_resched_counts;
+	unsigned int irq_call_counts;
+	unsigned int irq_tlb_counts;
+	unsigned int irq_thermal_counts;
+	unsigned int irq_spur_counts;
 } ____cacheline_aligned irq_cpustat_t;
 
 DECLARE_PER_CPU(irq_cpustat_t, irq_stat);
Index: 2.6.23-rc1-git7/include/asm-x86_64/pda.h
===================================================================
--- 2.6.23-rc1-git7.orig/include/asm-x86_64/pda.h	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/include/asm-x86_64/pda.h	2007-07-30 19:08:07.000000000 -0400
@@ -29,6 +29,12 @@
 	short isidle;
 	struct mm_struct *active_mm;
 	unsigned apic_timer_irqs;
+	unsigned irq_resched_counts;
+	unsigned irq_call_counts;
+	unsigned irq_tlb_counts;
+	unsigned irq_thermal_counts;
+	unsigned irq_threshold_counts;
+	unsigned irq_spur_counts;
 } ____cacheline_aligned_in_smp;
 
 extern struct x8664_pda *_cpu_pda[];
Index: 2.6.23-rc1-git7/arch/i386/mach-voyager/voyager_smp.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/mach-voyager/voyager_smp.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/mach-voyager/voyager_smp.c	2007-07-30 19:08:07.000000000 -0400
@@ -1042,6 +1042,7 @@
 		mb();
 		clear_bit(cpu, &call_data->finished);
 	}
+	__get_cpu_var(irq_stat).irq_call_counts++;
 }
 
 static int
Index: 2.6.23-rc1-git7/arch/i386/xen/smp.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/xen/smp.c	2007-07-30 19:08:05.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/xen/smp.c	2007-07-30 19:08:07.000000000 -0400
@@ -352,6 +352,7 @@
 		mb();		/* commit everything before setting finished */
 		atomic_inc(&call_data->finished);
 	}
+	__get_cpu_var(irq_stat).irq_call_counts++;
 
 	return IRQ_HANDLED;
 }
Index: 2.6.23-rc1-git7/Documentation/filesystems/proc.txt
===================================================================
--- 2.6.23-rc1-git7.orig/Documentation/filesystems/proc.txt	2007-07-30 17:32:48.000000000 -0400
+++ 2.6.23-rc1-git7/Documentation/filesystems/proc.txt	2007-07-31 10:12:48.000000000 -0400
@@ -347,7 +347,37 @@
 the IO-APIC automatically retry the transmission, so it should not be a big
 problem, but you should read the SMP-FAQ.
 
-In this context it could be interesting to note the new irq directory in 2.4.
+In 2.6.2* /proc/interrupts was expanded again.  This time the goal was for
+/proc/interrupts to display every IRQ vector in use by the system, not
+just those considered 'most important'.  The new vectors are:
+
+  THR -- a threshold interrupt occurs when ECC memory correction is
+  occuring at too high a frequency.  Threshold interrupt machinery is
+  often put into the ECC logic, as occasional ECC memory failures are
+  part of normal memory operation, but sequences of ECC failures over
+  some short interval usually indicate a memory chip that is about to fail.
+
+  TRM -- a thermal event interrupt occurs when a temperature threshold
+  has been exceeded for some CPU chip.  This interrupt may also be generated
+  when the temperature drops back to normal.
+
+  SPU -- a spurious interrupt is some interrupt that was raised then lowered
+  by some IO device before it could be fully processed by the APIC.  Hence
+  the APIC sees the interrupt but does not know what device it came from.
+  For this case the APIC will generate the interrupt with a IRQ vector
+  of 0xff.
+
+  RES, CAL, TLB -- rescheduling, call and tlb flush interrupts are
+  sent from one CPU to another per the needs of the OS.  Typically,
+  their statistics are used by kernel developers and interested users to
+  determine the occurance of interrupt floods of the given type.
+
+The above IRQ vectors are displayed only when relevent.  For example,
+the threshold vector does not exist on x86_64 platforms.  Others are
+suppressed when the system is a uniprocessor.  As of this writing, only
+i386 and x86_64 platforms support the new IRQ vector displays.
+
+Of some interest is the introduction of the /proc/irq directory to 2.4.
 It could be used to set IRQ to CPU affinity, this means that you can "hook" an
 IRQ to only one CPU, or to exclude a CPU of handling IRQs. The contents of the
 irq subdir is one subdir for each IRQ, and one file; prof_cpu_mask

Re: [PATCH] expand /proc/interrupts to include missing vectors, v3

[Andi Kleen]

Joe Korty <joe.korty@ccur.com> writes:
> 	spurious interrupts
> 
> A threshold interrupt occurs when ECC memory correction
> is occuring at too high a frequency. 

It's configurable and the default is off. Also 
it's only on AMD hardware.

Your description is a little misleading.

> Thresholds are used
> by the ECC hardware as occasional ECC failures are part
> of normal operation,

Not really.


> ===================================================================
> --- 2.6.23-rc1-git7.orig/arch/i386/kernel/apic.c	2007-07-30 19:08:05.000000000 -0400
> +++ 2.6.23-rc1-git7/arch/i386/kernel/apic.c	2007-07-30 19:08:07.000000000 -0400
> @@ -1280,6 +1280,7 @@
>  	printk(KERN_INFO "spurious APIC interrupt on CPU#%d, "
>  	       "should never happen.\n", smp_processor_id());
>  	irq_exit();
> +	__get_cpu_var(irq_stat).irq_spur_counts++;

Wouldn't it be safer on preemptible kernels to have that inside
the irq_exit? 

>  }
>  
>  /*
> Index: 2.6.23-rc1-git7/arch/i386/kernel/cpu/mcheck/p4.c
> ===================================================================
> --- 2.6.23-rc1-git7.orig/arch/i386/kernel/cpu/mcheck/p4.c	2007-07-30 19:08:05.000000000 -0400
> +++ 2.6.23-rc1-git7/arch/i386/kernel/cpu/mcheck/p4.c	2007-07-30 19:08:07.000000000 -0400
> @@ -62,6 +62,7 @@
>  	irq_enter();
>  	vendor_thermal_interrupt(regs);
>  	irq_exit();
> +	__get_cpu_var(irq_stat).irq_thermal_counts++;
>  }
>  
>  /* P4/Xeon Thermal regulation detect and init */
> Index: 2.6.23-rc1-git7/arch/i386/kernel/irq.c
> ===================================================================
> --- 2.6.23-rc1-git7.orig/arch/i386/kernel/irq.c	2007-07-30 19:08:05.000000000 -0400
> +++ 2.6.23-rc1-git7/arch/i386/kernel/irq.c	2007-07-31 09:40:58.000000000 -0400
> @@ -284,14 +284,41 @@
>  		seq_printf(p, "NMI: ");
>  		for_each_online_cpu(j)
>  			seq_printf(p, "%10u ", nmi_count(j));
> -		seq_putc(p, '\n');
> +		seq_printf(p, "  Non-maskable interrupts\n");
>  #ifdef CONFIG_X86_LOCAL_APIC
>  		seq_printf(p, "LOC: ");
>  		for_each_online_cpu(j)
>  			seq_printf(p, "%10u ",
>  				per_cpu(irq_stat,j).apic_timer_irqs);
> -		seq_putc(p, '\n');
> +		seq_printf(p, "  Local interrupts\n");
>  #endif
> +#ifdef CONFIG_SMP
> +		seq_printf(p, "RES: ");

I think it would be better to use 5-6 char identifiers even
when it whacks the columns a bit; otherwise
nobody will know what it means. e.g. SCHED here. 

Also there you should update proc(5) and send a patch
to the manpage maintainer.

> Index: 2.6.23-rc1-git7/arch/x86_64/kernel/apic.c
> ===================================================================
> --- 2.6.23-rc1-git7.orig/arch/x86_64/kernel/apic.c	2007-07-30 19:08:05.000000000 -0400
> +++ 2.6.23-rc1-git7/arch/x86_64/kernel/apic.c	2007-07-30 19:08:07.000000000 -0400
> @@ -1118,6 +1118,7 @@
>  asmlinkage void smp_spurious_interrupt(void)
>  {
>  	unsigned int v;
> +

Don't add white space.


-Andi

Re: [PATCH] expand /proc/interrupts to include missing vectors, v3

[Joe Korty]

On Tue, Jul 31, 2007 at 07:02:01PM +0200, Andi Kleen wrote:

Hi Andi,
Thanks for the review.  I implemented many of your suggestions and for
the rest, here mention why not, in case you want to respond further.

Regards,
Joe

> Joe Korty <joe.korty@ccur.com> writes:
> > A threshold interrupt occurs when ECC memory correction
> > is occuring at too high a frequency. 
> 
> It's configurable and the default is off. Also 
> it's only on AMD hardware.

v4 now has a comment to the Documentation section noting
this.


> > Thresholds are used
> > by the ECC hardware as occasional ECC failures are part
> > of normal operation,

Occasional ECC _corrections_ are normal (due to stray alpha particles)
but ECC _failures_ are not.  Document corrected.

> >  	irq_exit();
> > +	__get_cpu_var(irq_stat).irq_spur_counts++;
> 
> Wouldn't it be safer on preemptible kernels to have that inside
> the irq_exit? 

Although irq_exit() releases the preemption block, it doesn't seem to
release the APIC interrupt block, at least for i386.  And as an interrupt
block also blocks preemption and process migration, it seems that it would
be safe to do the increments after the irq_exit().  But I've moved them
all inside in v4, just in case I am wrong, or this changes in the future
(eg, PREEMPT_RT).

> > +		seq_printf(p, "RES: ");
> 
> I think it would be better to use 5-6 char identifiers
> even when it whacks the columns a bit; otherwise nobody
> will know what it means. e.g. SCHED here.

v3 addresses this.  The normally empty 'description' column at the end of
each line now holds a description of each vector.  The three-character
line-prefix names are there only to make the new lines match the syntax
and format of the other lines in /proc/interrupts.

> Also there you should update proc(5) and send a patch
> to the manpage maintainer.

Will do.

Thanks,
Joe

PS: also fixed up the whitespace.

[PATCH] expand /proc/interrupts to include missing vectors, v4

[Joe Korty]

[ v3-v4 changelog:
    s/irq_spur_counts/irq_spurious_counts/g (Andrew Morton)
    tweaked documentation (Andi Kleen)
    moved increments before irq_exit as appropriate (Andi Kleen)
    whitespace cleanup (Andi Kleen)
]

Add missing IRQs and IRQ descriptions to /proc/interrupts,
version 4.

/proc/interrupts is most useful when it displays every
IRQ vector in use by the system, not just those somebody
thought would be interesting.

This patch inserts the following vector displays to the
i386 and x86_64 platforms.
	
	rescheduling interrupts
	TLB flush interrupts
	function call interrupts
	thermal event interrupts
	threshold interrupts
	spurious interrupts

A threshold interrupt occurs when ECC memory correction
is occuring at too high a frequency.  Thresholds are used
by the ECC hardware as occasional ECC corrections are part
of normal operation (alpha particles), but long sequences
of ECC corrections usually indicate a memory chip that
is about to fail.  Note that not every system has ECC
threshold logic, and those that do, can require it to
be specifically enabled.

Thermal event interrupts occur when a temperature threshold
has been exceeded for some CPU chip.  I am not sure,
but I think a thermal interrupt is also generated when
the temperature drops back to a normal level.

A spurious interrupt is an interrupt that was raised then
lowered by the device before it could be fully processed
by the APIC.  Hence the apic sees the interrupt but does
not know what device it came from.  For this case the APIC
hardware will assume a vector of 0xff.

Rescheduling, call, and TLB flush interrupts are sent from
one CPU to another per the needs of the OS.  Typically,
their statistics would be used to discover interrupt
flooding.

Signed-off-by: Joe Korty <joe.korty@ccur.com>

Index: 2.6.23-rc1-git7/arch/i386/kernel/apic.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/kernel/apic.c	2007-07-31 16:31:09.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/kernel/apic.c	2007-07-31 16:45:30.000000000 -0400
@@ -1279,6 +1279,7 @@
 	/* see sw-dev-man vol 3, chapter 7.4.13.5 */
 	printk(KERN_INFO "spurious APIC interrupt on CPU#%d, "
 	       "should never happen.\n", smp_processor_id());
+	__get_cpu_var(irq_stat).irq_spurious_counts++;
 	irq_exit();
 }
 
Index: 2.6.23-rc1-git7/arch/i386/kernel/cpu/mcheck/p4.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/kernel/cpu/mcheck/p4.c	2007-07-31 16:31:09.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/kernel/cpu/mcheck/p4.c	2007-07-31 16:45:30.000000000 -0400
@@ -61,6 +61,7 @@
 {
 	irq_enter();
 	vendor_thermal_interrupt(regs);
+	__get_cpu_var(irq_stat).irq_thermal_counts++;
 	irq_exit();
 }
 
Index: 2.6.23-rc1-git7/arch/i386/kernel/irq.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/kernel/irq.c	2007-07-31 16:31:09.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/kernel/irq.c	2007-07-31 16:45:30.000000000 -0400
@@ -284,14 +284,41 @@
 		seq_printf(p, "NMI: ");
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ", nmi_count(j));
-		seq_putc(p, '\n');
+		seq_printf(p, "  Non-maskable interrupts\n");
 #ifdef CONFIG_X86_LOCAL_APIC
 		seq_printf(p, "LOC: ");
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ",
 				per_cpu(irq_stat,j).apic_timer_irqs);
-		seq_putc(p, '\n');
+		seq_printf(p, "  Local interrupts\n");
 #endif
+#ifdef CONFIG_SMP
+		seq_printf(p, "RES: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				per_cpu(irq_stat,j).irq_resched_counts);
+		seq_printf(p, "  Rescheduling interrupts\n");
+		seq_printf(p, "CAL: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				per_cpu(irq_stat,j).irq_call_counts);
+		seq_printf(p, "  function call interrupts\n");
+		seq_printf(p, "TLB: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				per_cpu(irq_stat,j).irq_tlb_counts);
+		seq_printf(p, "  TLB shootdowns\n");
+#endif
+		seq_printf(p, "TRM: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				per_cpu(irq_stat,j).irq_thermal_counts);
+		seq_printf(p, "  Thermal event interrupts\n");
+		seq_printf(p, "SPU: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ",
+				per_cpu(irq_stat,j).irq_spurious_counts);
+		seq_printf(p, "  Spurious interrupts\n");
 		seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count));
 #if defined(CONFIG_X86_IO_APIC)
 		seq_printf(p, "MIS: %10u\n", atomic_read(&irq_mis_count));
Index: 2.6.23-rc1-git7/arch/i386/kernel/smp.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/kernel/smp.c	2007-07-31 16:31:09.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/kernel/smp.c	2007-07-31 16:45:30.000000000 -0400
@@ -342,6 +342,7 @@
 	smp_mb__after_clear_bit();
 out:
 	put_cpu_no_resched();
+	__get_cpu_var(irq_stat).irq_tlb_counts++;
 }
 
 void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
@@ -640,6 +641,7 @@
 fastcall void smp_reschedule_interrupt(struct pt_regs *regs)
 {
 	ack_APIC_irq();
+	__get_cpu_var(irq_stat).irq_resched_counts++;
 }
 
 fastcall void smp_call_function_interrupt(struct pt_regs *regs)
@@ -660,6 +662,7 @@
 	 */
 	irq_enter();
 	(*func)(info);
+	__get_cpu_var(irq_stat).irq_call_counts++;
 	irq_exit();
 
 	if (wait) {
Index: 2.6.23-rc1-git7/arch/x86_64/kernel/apic.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/x86_64/kernel/apic.c	2007-07-31 16:31:09.000000000 -0400
+++ 2.6.23-rc1-git7/arch/x86_64/kernel/apic.c	2007-07-31 16:45:30.000000000 -0400
@@ -1129,6 +1129,7 @@
 	if (v & (1 << (SPURIOUS_APIC_VECTOR & 0x1f)))
 		ack_APIC_irq();
 
+	add_pda(irq_spurious_counts, 1);
 	irq_exit();
 }
 
Index: 2.6.23-rc1-git7/arch/x86_64/kernel/irq.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/x86_64/kernel/irq.c	2007-07-31 16:31:09.000000000 -0400
+++ 2.6.23-rc1-git7/arch/x86_64/kernel/irq.c	2007-07-31 16:45:30.000000000 -0400
@@ -88,11 +88,37 @@
 		seq_printf(p, "NMI: ");
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ", cpu_pda(j)->__nmi_count);
-		seq_putc(p, '\n');
+		seq_printf(p, "  Non-maskable interrupts\n");
 		seq_printf(p, "LOC: ");
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ", cpu_pda(j)->apic_timer_irqs);
-		seq_putc(p, '\n');
+		seq_printf(p, "  Local interrupts\n");
+#ifdef CONFIG_SMP
+		seq_printf(p, "RES: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_resched_counts);
+		seq_printf(p, "  Rescheduling interrupts\n");
+		seq_printf(p, "CAL: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_call_counts);
+		seq_printf(p, "  function call interrupts\n");
+		seq_printf(p, "TLB: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_tlb_counts);
+		seq_printf(p, "  TLB shootdowns\n");
+#endif
+		seq_printf(p, "TRM: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_thermal_counts);
+		seq_printf(p, "  Thermal event interrupts\n");
+		seq_printf(p, "THR: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_threshold_counts);
+		seq_printf(p, "  Threshold APIC interrupts\n");
+		seq_printf(p, "SPU: ");
+		for_each_online_cpu(j)
+			seq_printf(p, "%10u ", cpu_pda(j)->irq_spurious_counts);
+		seq_printf(p, "  Spurious interrupts\n");
 		seq_printf(p, "ERR: %10u\n", atomic_read(&irq_err_count));
 	}
 	return 0;
Index: 2.6.23-rc1-git7/arch/x86_64/kernel/mce_amd.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/x86_64/kernel/mce_amd.c	2007-07-31 16:31:09.000000000 -0400
+++ 2.6.23-rc1-git7/arch/x86_64/kernel/mce_amd.c	2007-07-31 16:45:30.000000000 -0400
@@ -237,6 +237,7 @@
 		}
 	}
 out:
+	add_pda(irq_threshold_counts, 1);
 	irq_exit();
 }
 
Index: 2.6.23-rc1-git7/arch/x86_64/kernel/mce_intel.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/x86_64/kernel/mce_intel.c	2007-07-31 16:31:09.000000000 -0400
+++ 2.6.23-rc1-git7/arch/x86_64/kernel/mce_intel.c	2007-07-31 16:45:30.000000000 -0400
@@ -26,6 +26,7 @@
 	if (therm_throt_process(msr_val & 1))
 		mce_log_therm_throt_event(smp_processor_id(), msr_val);
 
+	add_pda(irq_thermal_counts, 1);
 	irq_exit();
 }
 
Index: 2.6.23-rc1-git7/arch/x86_64/kernel/smp.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/x86_64/kernel/smp.c	2007-07-31 16:31:09.000000000 -0400
+++ 2.6.23-rc1-git7/arch/x86_64/kernel/smp.c	2007-07-31 16:45:30.000000000 -0400
@@ -163,6 +163,7 @@
 out:
 	ack_APIC_irq();
 	cpu_clear(cpu, f->flush_cpumask);
+	add_pda(irq_tlb_counts, 1);
 }
 
 static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
@@ -493,6 +494,7 @@
 asmlinkage void smp_reschedule_interrupt(void)
 {
 	ack_APIC_irq();
+	add_pda(irq_resched_counts, 1);
 }
 
 asmlinkage void smp_call_function_interrupt(void)
@@ -514,6 +516,7 @@
 	exit_idle();
 	irq_enter();
 	(*func)(info);
+	add_pda(irq_call_counts, 1);
 	irq_exit();
 	if (wait) {
 		mb();
Index: 2.6.23-rc1-git7/include/asm-i386/hardirq.h
===================================================================
--- 2.6.23-rc1-git7.orig/include/asm-i386/hardirq.h	2007-07-31 16:31:09.000000000 -0400
+++ 2.6.23-rc1-git7/include/asm-i386/hardirq.h	2007-07-31 16:45:30.000000000 -0400
@@ -9,6 +9,11 @@
 	unsigned long idle_timestamp;
 	unsigned int __nmi_count;	/* arch dependent */
 	unsigned int apic_timer_irqs;	/* arch dependent */
+	unsigned int irq_resched_counts;
+	unsigned int irq_call_counts;
+	unsigned int irq_tlb_counts;
+	unsigned int irq_thermal_counts;
+	unsigned int irq_spurious_counts;
 } ____cacheline_aligned irq_cpustat_t;
 
 DECLARE_PER_CPU(irq_cpustat_t, irq_stat);
Index: 2.6.23-rc1-git7/include/asm-x86_64/pda.h
===================================================================
--- 2.6.23-rc1-git7.orig/include/asm-x86_64/pda.h	2007-07-31 16:31:09.000000000 -0400
+++ 2.6.23-rc1-git7/include/asm-x86_64/pda.h	2007-07-31 16:45:30.000000000 -0400
@@ -29,6 +29,12 @@
 	short isidle;
 	struct mm_struct *active_mm;
 	unsigned apic_timer_irqs;
+	unsigned irq_resched_counts;
+	unsigned irq_call_counts;
+	unsigned irq_tlb_counts;
+	unsigned irq_thermal_counts;
+	unsigned irq_threshold_counts;
+	unsigned irq_spurious_counts;
 } ____cacheline_aligned_in_smp;
 
 extern struct x8664_pda *_cpu_pda[];
Index: 2.6.23-rc1-git7/arch/i386/mach-voyager/voyager_smp.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/mach-voyager/voyager_smp.c	2007-07-31 16:31:09.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/mach-voyager/voyager_smp.c	2007-07-31 16:45:30.000000000 -0400
@@ -1037,6 +1037,7 @@
 	 */
 	irq_enter();
 	(*func)(info);
+	__get_cpu_var(irq_stat).irq_call_counts++;
 	irq_exit();
 	if (wait) {
 		mb();
Index: 2.6.23-rc1-git7/arch/i386/xen/smp.c
===================================================================
--- 2.6.23-rc1-git7.orig/arch/i386/xen/smp.c	2007-07-31 16:31:09.000000000 -0400
+++ 2.6.23-rc1-git7/arch/i386/xen/smp.c	2007-07-31 16:45:30.000000000 -0400
@@ -346,6 +346,7 @@
 	 */
 	irq_enter();
 	(*func)(info);
+	__get_cpu_var(irq_stat).irq_call_counts++;
 	irq_exit();
 
 	if (wait) {
Index: 2.6.23-rc1-git7/Documentation/filesystems/proc.txt
===================================================================
--- 2.6.23-rc1-git7.orig/Documentation/filesystems/proc.txt	2007-07-31 16:31:09.000000000 -0400
+++ 2.6.23-rc1-git7/Documentation/filesystems/proc.txt	2007-07-31 16:45:30.000000000 -0400
@@ -347,7 +347,40 @@
 the IO-APIC automatically retry the transmission, so it should not be a big
 problem, but you should read the SMP-FAQ.
 
-In this context it could be interesting to note the new irq directory in 2.4.
+In 2.6.2* /proc/interrupts was expanded again.  This time the goal was for
+/proc/interrupts to display every IRQ vector in use by the system, not
+just those considered 'most important'.  The new vectors are:
+
+  THR -- a threshold interrupt occurs when ECC memory correction is occuring
+  at too high a frequency.  Threshold interrupt machinery is often put
+  into the ECC logic, as occasional ECC memory corrections are part of
+  normal operation (due to random alpha particles), but sequences of
+  ECC corrections or outright failures over some short interval usually
+  indicate a memory chip that is about to fail.  Note that not every
+  platform has ECC threshold logic, and those that do generally require
+  it to be explicitly turned on.
+
+  TRM -- a thermal event interrupt occurs when a temperature threshold
+  has been exceeded for some CPU chip.  This interrupt may also be generated
+  when the temperature drops back to normal.
+
+  SPU -- a spurious interrupt is some interrupt that was raised then lowered
+  by some IO device before it could be fully processed by the APIC.  Hence
+  the APIC sees the interrupt but does not know what device it came from.
+  For this case the APIC will generate the interrupt with a IRQ vector
+  of 0xff.
+
+  RES, CAL, TLB -- rescheduling, call and tlb flush interrupts are
+  sent from one CPU to another per the needs of the OS.  Typically,
+  their statistics are used by kernel developers and interested users to
+  determine the occurance of interrupt floods of the given type.
+
+The above IRQ vectors are displayed only when relevent.  For example,
+the threshold vector does not exist on x86_64 platforms.  Others are
+suppressed when the system is a uniprocessor.  As of this writing, only
+i386 and x86_64 platforms support the new IRQ vector displays.
+
+Of some interest is the introduction of the /proc/irq directory to 2.4.
 It could be used to set IRQ to CPU affinity, this means that you can "hook" an
 IRQ to only one CPU, or to exclude a CPU of handling IRQs. The contents of the
 irq subdir is one subdir for each IRQ, and one file; prof_cpu_mask

Re: [PATCH] expand /proc/interrupts to include missing vectors, v2

[Lennart Sorensen]

On Mon, Jul 30, 2007 at 08:17:18PM -0400, Joe Korty wrote:
> Add missing IRQs and IRQ descriptions to /proc/interrupts.
> 
> /proc/interrupts is most useful when it displays every
> IRQ vector in use by the system, not just those somebody
> thought would be interesting.
> 
> This patch inserts the following vector displays to the
> i386 and x86_64 platforms, as appropriate:
> 	
> 	rescheduling interrupts
> 	TLB flush interrupts
> 	function call interrupts
> 	thermal event interrupts
> 	threshold interrupts
> 	spurious interrupts
> 
> A threshold interrupt occurs when ECC memory correction
> is occuring at too high a frequency.  Thresholds are used
> by the ECC hardware as occasional ECC failures are part
> of normal operation, but long sequences of ECC failures
> usually indicate a memory chip that is about to fail.
> 
> Thermal event interrupts occur when a temperature threshold
> has been exceeded for some CPU chip.  IIRC, a thermal
> interrupt is also generated when the temperature drops
> back to a normal level.
> 
> A spurious interrupt is an interrupt that was raised then
> lowered by the device before it could be fully processed
> by the APIC.  Hence the apic sees the interrupt but does
> not know what device it came from.  For this case the APIC
> hardware will assume a vector of 0xff.
> 
> Rescheduling, call, and TLB flush interrupts are sent from
> one CPU to another per the needs of the OS.  Typically,
> their statistics would be used to discover if an
> interrupt flood of the given type has been occuring.

Since you are playing with /proc/interrupts, is there any chance you
could make it make sense too?

Right now I have the annoying issue that /proc/interrupts only shows
interrupts that it sees a handler for, and ignores the rest.  For some
reason irq 3 never shows up in /proc/interrupts on my system, even
though I have serial ports on both IRQ 3 and 4.  Only irq 4 shows
'serial' as the handler, and is the only one shown.  irq 3 is getting
lots of interrupts, and the serial driver is handling them, it just
doesn't show up in /proc/interrupts.  procinfo does show irq 3 although
with no handler listed.  I think any interrupt with a handler AND any
interrupt with a non zero count ought to be shown.  I only happened to
try procinfo by chance and discover that while I thought IRQ 3 wasn't
working, the only issue was that /proc/interrupts simply wasn't showing
all the relevant information.

Of course I would also love to know why irq 3 isn't showing serial as
the handler, but I have no idea why that is.  This is all using Debian's
2.6.18 kernel.

--
Len Sorensen