[PATCH] x86, UV: add uv_setup_irq() and uv_teardown_irq() functions

[PATCH] x86, UV: add uv_setup_irq() and uv_teardown_irq() functions

[Dean Nelson]

Provide a means for UV interrupt MMRs to be setup with the message to be sent
when an MSI is raised.

Signed-off-by: Dean Nelson <dcn@sgi.com>

---

This functionality is needed by drivers/misc/sgi-xp. And a patch will be
submitted shortly.

 arch/x86/kernel/Makefile    |    2 
 arch/x86/kernel/io_apic.c   |   95 ++++++++++++++++++++++++++++++++++++++++
 arch/x86/kernel/uv_irq.c    |   50 +++++++++++++++++++++
 include/asm-x86/uv/uv_irq.h |   34 ++++++++++++++
 kernel/irq/chip.c           |    1 
 5 files changed, 181 insertions(+), 1 deletion(-)

Index: linux/arch/x86/kernel/io_apic.c
===================================================================
--- linux.orig/arch/x86/kernel/io_apic.c	2008-09-30 09:07:42.000000000 -0500
+++ linux/arch/x86/kernel/io_apic.c	2008-09-30 12:54:12.000000000 -0500
@@ -58,6 +58,8 @@
 #include <asm/setup.h>
 #include <asm/irq_remapping.h>
 #include <asm/hpet.h>
+#include <asm/uv/uv_hub.h>
+#include <asm/uv/uv_irq.h>
 
 #include <mach_ipi.h>
 #include <mach_apic.h>
@@ -3694,6 +3696,99 @@ int arch_setup_ht_irq(unsigned int irq, 
 }
 #endif /* CONFIG_HT_IRQ */
 
+#ifdef CONFIG_X86_64
+static void noop(unsigned int irq)
+{
+}
+
+static unsigned int noop_ret(unsigned int irq)
+{
+	return 0;
+}
+
+static void ack_apic(unsigned int irq)
+{
+	ack_APIC_irq();
+}
+
+static struct irq_chip uv_irq_chip = {
+	.name		= "UV_MSI",
+	.startup	= noop_ret,
+	.shutdown	= noop,
+	.enable		= noop,
+	.disable	= noop,
+	.ack		= noop,
+	.mask		= noop,
+	.unmask		= noop,
+	.eoi		= ack_apic,
+	.end		= noop,
+};
+
+/*
+ * Re-target the irq to the specified CPU and enable the specified MMR located
+ * on the specified blade to allow the sending of MSIs to the specified CPU.
+ */
+int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
+		       unsigned long mmr_offset)
+{
+	const cpumask_t *eligible_cpu = get_cpu_mask(cpu);
+	struct irq_cfg *cfg;
+	int mmr_pnode;
+	unsigned long mmr_value;
+	struct uv_IO_APIC_route_entry *entry;
+	unsigned long flags;
+	int err;
+
+	err = assign_irq_vector(irq, *eligible_cpu);
+	if (err != 0)
+		return err;
+
+	spin_lock_irqsave(&vector_lock, flags);
+	set_irq_chip_and_handler_name(irq, &uv_irq_chip, handle_percpu_irq,
+				      irq_name);
+	spin_unlock_irqrestore(&vector_lock, flags);
+
+	cfg = irq_cfg(irq);
+
+	mmr_value = 0;
+	entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
+	BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
+
+	entry->vector = cfg->vector;
+	entry->delivery_mode = INT_DELIVERY_MODE;
+	entry->dest_mode = INT_DEST_MODE;
+	entry->polarity = 0;
+	entry->trigger = 0;
+	entry->mask = 0;
+	entry->dest = cpu_mask_to_apicid(*eligible_cpu);
+
+	mmr_pnode = uv_blade_to_pnode(mmr_blade);
+	uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
+
+	return irq;
+}
+
+/*
+ * Disable the specified MMR located on the specified blade so that MSIs are
+ * longer allowed to be sent.
+ */
+void arch_disable_uv_irq(int mmr_blade, unsigned long mmr_offset)
+{
+	unsigned long mmr_value;
+	struct uv_IO_APIC_route_entry *entry;
+	int mmr_pnode;
+
+	mmr_value = 0;
+	entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
+	BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
+
+	entry->mask = 1;
+
+	mmr_pnode = uv_blade_to_pnode(mmr_blade);
+	uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
+}
+#endif /* CONFIG_X86_64 */
+
 int __init io_apic_get_redir_entries (int ioapic)
 {
 	union IO_APIC_reg_01	reg_01;
Index: linux/kernel/irq/chip.c
===================================================================
--- linux.orig/kernel/irq/chip.c	2008-09-30 09:07:42.000000000 -0500
+++ linux/kernel/irq/chip.c	2008-09-30 09:07:44.000000000 -0500
@@ -79,6 +79,7 @@ void dynamic_irq_cleanup(unsigned int ir
 	desc->chip_data = NULL;
 	desc->handle_irq = handle_bad_irq;
 	desc->chip = &no_irq_chip;
+	desc->name = "none";
 	spin_unlock_irqrestore(&desc->lock, flags);
 }
 
Index: linux/arch/x86/kernel/uv_irq.c
===================================================================
--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux/arch/x86/kernel/uv_irq.c	2008-10-01 06:35:31.000000000 -0500
@@ -0,0 +1,50 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * SGI UV IRQ functions
+ *
+ * Copyright (C) 2008 Silicon Graphics, Inc. All rights reserved.
+ */
+
+#include <linux/module.h>
+#include <linux/irq.h>
+#include <asm/uv/uv_irq.h>
+
+/*
+ * Set up a mapping of an available irq and vector, and enable the specified
+ * MMR that defines the MSI that is to be sent to the specified CPU when an
+ * interrupt is raised.
+ */
+int uv_setup_irq(char *irq_name, int cpu, int mmr_blade,
+		 unsigned long mmr_offset)
+{
+	int irq;
+	int ret;
+
+	irq = create_irq();
+	if (irq <= 0)
+		return -EBUSY;
+
+	ret = arch_enable_uv_irq(irq_name, irq, cpu, mmr_blade, mmr_offset);
+	if (ret != irq)
+		destroy_irq(irq);
+
+	return ret;
+}
+EXPORT_SYMBOL(uv_setup_irq);
+
+/*
+ * Tear down a mapping of an irq and vector, and disable the specified MMR that
+ * defined the MSI that was to be sent to the specified CPU when an interrupt
+ * was raised.
+ *
+ * Set mmr_blade and mmr_offset to what was passed in on uv_setup_irq().
+ */
+void uv_teardown_irq(unsigned int irq, int mmr_blade, unsigned long mmr_offset)
+{
+	arch_disable_uv_irq(mmr_blade, mmr_offset);
+	destroy_irq(irq);
+}
+EXPORT_SYMBOL(uv_teardown_irq);
Index: linux/include/asm-x86/uv/uv_irq.h
===================================================================
--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux/include/asm-x86/uv/uv_irq.h	2008-09-30 09:07:44.000000000 -0500
@@ -0,0 +1,34 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * SGI UV IRQ definitions
+ *
+ * Copyright (C) 2008 Silicon Graphics, Inc. All rights reserved.
+ */
+
+#ifndef ASM_X86__UV__UV_IRQ_H
+#define ASM_X86__UV__UV_IRQ_H
+
+/* If a generic version of this structure gets defined, eliminate this one. */
+struct uv_IO_APIC_route_entry {
+	__u64	vector		:  8,
+		delivery_mode	:  3,
+		dest_mode	:  1,
+		delivery_status	:  1,
+		polarity	:  1,
+		__reserved_1	:  1,
+		trigger		:  1,
+		mask		:  1,
+		__reserved_2	: 15,
+		dest		: 32;
+};
+
+extern int arch_enable_uv_irq(char *, unsigned int, int, int, unsigned long);
+extern void arch_disable_uv_irq(int, unsigned long);
+
+extern int uv_setup_irq(char *, int, int, unsigned long);
+extern void uv_teardown_irq(unsigned int, int, unsigned long);
+
+#endif /* ASM_X86__UV__UV_IRQ_H */
Index: linux/arch/x86/kernel/Makefile
===================================================================
--- linux.orig/arch/x86/kernel/Makefile	2008-09-30 09:07:42.000000000 -0500
+++ linux/arch/x86/kernel/Makefile	2008-09-30 09:07:44.000000000 -0500
@@ -107,7 +107,7 @@ obj-$(CONFIG_OLPC)		+= olpc.o
 # 64 bit specific files
 ifeq ($(CONFIG_X86_64),y)
         obj-y				+= genapic_64.o genapic_flat_64.o genx2apic_uv_x.o tlb_uv.o
-	obj-y				+= bios_uv.o uv_sysfs.o
+	obj-y				+= bios_uv.o uv_sysfs.o uv_irq.o
         obj-y				+= genx2apic_cluster.o
         obj-y				+= genx2apic_phys.o
         obj-$(CONFIG_X86_PM_TIMER)	+= pmtimer_64.o

Re: [PATCH] x86, UV: add uv_setup_irq() and uv_teardown_irq() functions

[Ingo Molnar]

* Dean Nelson <dcn@sgi.com> wrote:

> Provide a means for UV interrupt MMRs to be setup with the message to 
> be sent when an MSI is raised.
> 
> Signed-off-by: Dean Nelson <dcn@sgi.com>
> 
> ---
> 
> This functionality is needed by drivers/misc/sgi-xp. And a patch will be
> submitted shortly.
> 
>  arch/x86/kernel/Makefile    |    2 
>  arch/x86/kernel/io_apic.c   |   95 ++++++++++++++++++++++++++++++++++++++++
>  arch/x86/kernel/uv_irq.c    |   50 +++++++++++++++++++++
>  include/asm-x86/uv/uv_irq.h |   34 ++++++++++++++
>  kernel/irq/chip.c           |    1 
>  5 files changed, 181 insertions(+), 1 deletion(-)
> 
> Index: linux/arch/x86/kernel/io_apic.c
> ===================================================================
> --- linux.orig/arch/x86/kernel/io_apic.c	2008-09-30 09:07:42.000000000 -0500
> +++ linux/arch/x86/kernel/io_apic.c	2008-09-30 12:54:12.000000000 -0500
> @@ -58,6 +58,8 @@
>  #include <asm/setup.h>
>  #include <asm/irq_remapping.h>
>  #include <asm/hpet.h>
> +#include <asm/uv/uv_hub.h>
> +#include <asm/uv/uv_irq.h>
>  
>  #include <mach_ipi.h>
>  #include <mach_apic.h>
> @@ -3694,6 +3696,99 @@ int arch_setup_ht_irq(unsigned int irq, 
>  }
>  #endif /* CONFIG_HT_IRQ */
>  
> +#ifdef CONFIG_X86_64
> +static void noop(unsigned int irq)
> +{
> +}
> +
> +static unsigned int noop_ret(unsigned int irq)
> +{
> +	return 0;
> +}
> +
> +static void ack_apic(unsigned int irq)
> +{
> +	ack_APIC_irq();
> +}
> +
> +static struct irq_chip uv_irq_chip = {
> +	.name		= "UV_MSI",
> +	.startup	= noop_ret,
> +	.shutdown	= noop,
> +	.enable		= noop,
> +	.disable	= noop,
> +	.ack		= noop,
> +	.mask		= noop,
> +	.unmask		= noop,
> +	.eoi		= ack_apic,
> +	.end		= noop,
> +};

hm, why isnt this in uv_irq.c?

> Index: linux/kernel/irq/chip.c
> ===================================================================
> --- linux.orig/kernel/irq/chip.c	2008-09-30 09:07:42.000000000 -0500
> +++ linux/kernel/irq/chip.c	2008-09-30 09:07:44.000000000 -0500
> @@ -79,6 +79,7 @@ void dynamic_irq_cleanup(unsigned int ir
>  	desc->chip_data = NULL;
>  	desc->handle_irq = handle_bad_irq;
>  	desc->chip = &no_irq_chip;
> +	desc->name = "none";
>  	spin_unlock_irqrestore(&desc->lock, flags);

unrelated change that belongs into a separate patch.

patch looks quite clean otherwise.

	Ingo

[PATCH] x86, UV: add uv_setup_irq() and uv_teardown_irq() functions v.2

[Dean Nelson]

Provide a means for UV interrupt MMRs to be setup with the message to be sent
when an MSI is raised.

Signed-off-by: Dean Nelson <dcn@sgi.com>

---

On Thu, Oct 02, 2008 at 10:42:14AM +0200, Ingo Molnar wrote:
>  
> * Dean Nelson <dcn@sgi.com> wrote:
> 
> > +static struct irq_chip uv_irq_chip = {
> > +   .name           = "UV_MSI",
> > +   .startup        = noop_ret,
> > +   .shutdown       = noop,
> > +   .enable         = noop,
> > +   .disable        = noop,
> > +   .ack            = noop,
> > +   .mask           = noop,
> > +   .unmask         = noop,
> > +   .eoi            = ack_apic,
> > +   .end            = noop,
> > +};
>
> hm, why isnt this in uv_irq.c?

Good question. Sometimes the obvious gets overlooked in the midst of
having rewritten a patch a dozen different ways. It's now in uv_irq.c.

> > Index: linux/kernel/irq/chip.c
> > ===================================================================
> > --- linux.orig/kernel/irq/chip.c    2008-09-30 09:07:42.000000000 -0500
> > +++ linux/kernel/irq/chip.c 2008-09-30 09:07:44.000000000 -0500
> > @@ -79,6 +79,7 @@ void dynamic_irq_cleanup(unsigned int ir
> >     desc->chip_data = NULL;
> >     desc->handle_irq = handle_bad_irq;
> >     desc->chip = &no_irq_chip;
> > +   desc->name = "none";
> >     spin_unlock_irqrestore(&desc->lock, flags);
>
> unrelated change that belongs into a separate patch.

Agreed. The change is needed by drivers/misc/sgi-xp once it has been
modified to call uv_setup_irq() and uv_teardown_irq(). I'll move this
change into that patch (or should it be in a patch all by itself?).

The patch to drivers/misc/sgi-xp to call the functions introduced by
this patch, will be submitted shortly.

Thanks,
Dean

 arch/x86/kernel/Makefile    |    2 -
 arch/x86/kernel/io_apic.c   |   68 +++++++++++++++++++++++++++++++++++
 arch/x86/kernel/uv_irq.c    |   77 ++++++++++++++++++++++++++++++++++++++++
 include/asm-x86/uv/uv_irq.h |   36 ++++++++++++++++++
 4 files changed, 182 insertions(+), 1 deletion(-)

Index: linux/arch/x86/kernel/io_apic.c
===================================================================
--- linux.orig/arch/x86/kernel/io_apic.c	2008-10-02 06:12:54.000000000 -0500
+++ linux/arch/x86/kernel/io_apic.c	2008-10-02 06:15:48.000000000 -0500
@@ -58,6 +58,8 @@
 #include <asm/setup.h>
 #include <asm/irq_remapping.h>
 #include <asm/hpet.h>
+#include <asm/uv/uv_hub.h>
+#include <asm/uv/uv_irq.h>
 
 #include <mach_ipi.h>
 #include <mach_apic.h>
@@ -3694,6 +3696,72 @@ int arch_setup_ht_irq(unsigned int irq, 
 }
 #endif /* CONFIG_HT_IRQ */
 
+#ifdef CONFIG_X86_64
+/*
+ * Re-target the irq to the specified CPU and enable the specified MMR located
+ * on the specified blade to allow the sending of MSIs to the specified CPU.
+ */
+int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
+		       unsigned long mmr_offset)
+{
+	const cpumask_t *eligible_cpu = get_cpu_mask(cpu);
+	struct irq_cfg *cfg;
+	int mmr_pnode;
+	unsigned long mmr_value;
+	struct uv_IO_APIC_route_entry *entry;
+	unsigned long flags;
+	int err;
+
+	err = assign_irq_vector(irq, *eligible_cpu);
+	if (err != 0)
+		return err;
+
+	spin_lock_irqsave(&vector_lock, flags);
+	set_irq_chip_and_handler_name(irq, &uv_irq_chip, handle_percpu_irq,
+				      irq_name);
+	spin_unlock_irqrestore(&vector_lock, flags);
+
+	cfg = irq_cfg(irq);
+
+	mmr_value = 0;
+	entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
+	BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
+
+	entry->vector = cfg->vector;
+	entry->delivery_mode = INT_DELIVERY_MODE;
+	entry->dest_mode = INT_DEST_MODE;
+	entry->polarity = 0;
+	entry->trigger = 0;
+	entry->mask = 0;
+	entry->dest = cpu_mask_to_apicid(*eligible_cpu);
+
+	mmr_pnode = uv_blade_to_pnode(mmr_blade);
+	uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
+
+	return irq;
+}
+
+/*
+ * Disable the specified MMR located on the specified blade so that MSIs are
+ * longer allowed to be sent.
+ */
+void arch_disable_uv_irq(int mmr_blade, unsigned long mmr_offset)
+{
+	unsigned long mmr_value;
+	struct uv_IO_APIC_route_entry *entry;
+	int mmr_pnode;
+
+	mmr_value = 0;
+	entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
+	BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
+
+	entry->mask = 1;
+
+	mmr_pnode = uv_blade_to_pnode(mmr_blade);
+	uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
+}
+#endif /* CONFIG_X86_64 */
+
 int __init io_apic_get_redir_entries (int ioapic)
 {
 	union IO_APIC_reg_01	reg_01;
Index: linux/arch/x86/kernel/uv_irq.c
===================================================================
--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux/arch/x86/kernel/uv_irq.c	2008-10-02 06:22:14.000000000 -0500
@@ -0,0 +1,77 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * SGI UV IRQ functions
+ *
+ * Copyright (C) 2008 Silicon Graphics, Inc. All rights reserved.
+ */
+
+#include <linux/module.h>
+#include <linux/irq.h>
+#include <asm/uv/uv_irq.h>
+
+static void uv_noop(unsigned int irq)
+{
+}
+
+static unsigned int uv_noop_ret(unsigned int irq)
+{
+	return 0;
+}
+
+static void uv_ack_apic(unsigned int irq)
+{
+	ack_APIC_irq();
+}
+
+struct irq_chip uv_irq_chip = {
+	.name		= "UV_MSI",
+	.startup	= uv_noop_ret,
+	.shutdown	= uv_noop,
+	.enable		= uv_noop,
+	.disable	= uv_noop,
+	.ack		= uv_noop,
+	.mask		= uv_noop,
+	.unmask		= uv_noop,
+	.eoi		= uv_ack_apic,
+	.end		= uv_noop,
+};
+
+/*
+ * Set up a mapping of an available irq and vector, and enable the specified
+ * MMR that defines the MSI that is to be sent to the specified CPU when an
+ * interrupt is raised.
+ */
+int uv_setup_irq(char *irq_name, int cpu, int mmr_blade,
+		 unsigned long mmr_offset)
+{
+	int irq;
+	int ret;
+
+	irq = create_irq();
+	if (irq <= 0)
+		return -EBUSY;
+
+	ret = arch_enable_uv_irq(irq_name, irq, cpu, mmr_blade, mmr_offset);
+	if (ret != irq)
+		destroy_irq(irq);
+
+	return ret;
+}
+EXPORT_SYMBOL(uv_setup_irq);
+
+/*
+ * Tear down a mapping of an irq and vector, and disable the specified MMR that
+ * defined the MSI that was to be sent to the specified CPU when an interrupt
+ * was raised.
+ *
+ * Set mmr_blade and mmr_offset to what was passed in on uv_setup_irq().
+ */
+void uv_teardown_irq(unsigned int irq, int mmr_blade, unsigned long mmr_offset)
+{
+	arch_disable_uv_irq(mmr_blade, mmr_offset);
+	destroy_irq(irq);
+}
+EXPORT_SYMBOL(uv_teardown_irq);
Index: linux/include/asm-x86/uv/uv_irq.h
===================================================================
--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux/include/asm-x86/uv/uv_irq.h	2008-10-02 06:15:53.000000000 -0500
@@ -0,0 +1,36 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * SGI UV IRQ definitions
+ *
+ * Copyright (C) 2008 Silicon Graphics, Inc. All rights reserved.
+ */
+
+#ifndef ASM_X86__UV__UV_IRQ_H
+#define ASM_X86__UV__UV_IRQ_H
+
+/* If a generic version of this structure gets defined, eliminate this one. */
+struct uv_IO_APIC_route_entry {
+	__u64	vector		:  8,
+		delivery_mode	:  3,
+		dest_mode	:  1,
+		delivery_status	:  1,
+		polarity	:  1,
+		__reserved_1	:  1,
+		trigger		:  1,
+		mask		:  1,
+		__reserved_2	: 15,
+		dest		: 32;
+};
+
+extern struct irq_chip uv_irq_chip;
+
+extern int arch_enable_uv_irq(char *, unsigned int, int, int, unsigned long);
+extern void arch_disable_uv_irq(int, unsigned long);
+
+extern int uv_setup_irq(char *, int, int, unsigned long);
+extern void uv_teardown_irq(unsigned int, int, unsigned long);
+
+#endif /* ASM_X86__UV__UV_IRQ_H */
Index: linux/arch/x86/kernel/Makefile
===================================================================
--- linux.orig/arch/x86/kernel/Makefile	2008-10-02 06:12:54.000000000 -0500
+++ linux/arch/x86/kernel/Makefile	2008-10-02 06:13:01.000000000 -0500
@@ -107,7 +107,7 @@ obj-$(CONFIG_OLPC)		+= olpc.o
 # 64 bit specific files
 ifeq ($(CONFIG_X86_64),y)
         obj-y				+= genapic_64.o genapic_flat_64.o genx2apic_uv_x.o tlb_uv.o
-	obj-y				+= bios_uv.o uv_sysfs.o
+	obj-y				+= bios_uv.o uv_sysfs.o uv_irq.o
         obj-y				+= genx2apic_cluster.o
         obj-y				+= genx2apic_phys.o
         obj-$(CONFIG_X86_PM_TIMER)	+= pmtimer_64.o

Re: [PATCH] x86, UV: add uv_setup_irq() and uv_teardown_irq() functions

[Yinghai Lu]

On Wed, Oct 1, 2008 at 4:44 AM, Dean Nelson <dcn@sgi.com> wrote:
> Provide a means for UV interrupt MMRs to be setup with the message to be sent
> when an MSI is raised.
>
> Signed-off-by: Dean Nelson <dcn@sgi.com>
>
> ---
>
> This functionality is needed by drivers/misc/sgi-xp. And a patch will be
> submitted shortly.
>
>  arch/x86/kernel/Makefile    |    2
>  arch/x86/kernel/io_apic.c   |   95 ++++++++++++++++++++++++++++++++++++++++
>  arch/x86/kernel/uv_irq.c    |   50 +++++++++++++++++++++
>  include/asm-x86/uv/uv_irq.h |   34 ++++++++++++++
>  kernel/irq/chip.c           |    1
>  5 files changed, 181 insertions(+), 1 deletion(-)
>
> Index: linux/arch/x86/kernel/io_apic.c
> ===================================================================
> --- linux.orig/arch/x86/kernel/io_apic.c        2008-09-30 09:07:42.000000000 -0500
> +++ linux/arch/x86/kernel/io_apic.c     2008-09-30 12:54:12.000000000 -0500
> @@ -58,6 +58,8 @@
>  #include <asm/setup.h>
>  #include <asm/irq_remapping.h>
>  #include <asm/hpet.h>
> +#include <asm/uv/uv_hub.h>
> +#include <asm/uv/uv_irq.h>
>
>  #include <mach_ipi.h>
>  #include <mach_apic.h>
> @@ -3694,6 +3696,99 @@ int arch_setup_ht_irq(unsigned int irq,
>  }
>  #endif /* CONFIG_HT_IRQ */
>
> +#ifdef CONFIG_X86_64
> +static void noop(unsigned int irq)
> +{
> +}
> +
> +static unsigned int noop_ret(unsigned int irq)
> +{
> +       return 0;
> +}
> +
> +static void ack_apic(unsigned int irq)
> +{
> +       ack_APIC_irq();
> +}
> +
> +static struct irq_chip uv_irq_chip = {
> +       .name           = "UV_MSI",

anything to do with MSI?

YH

Re: [PATCH] x86, UV: add uv_setup_irq() and uv_teardown_irq() functions

[H. Peter Anvin]

Yinghai Lu wrote:
>> +
>> +static struct irq_chip uv_irq_chip = {
>> +       .name           = "UV_MSI",
> 
> anything to do with MSI?
> 

Not really.  It probably should be called "UV-CORE" or something like that.

	-hpa

[PATCH] x86, UV: add uv_setup_irq() and uv_teardown_irq() functions v.3

[Dean Nelson]

Provide a means for UV interrupt MMRs to be setup with the message to be sent
when an MSI is raised.

Signed-off-by: Dean Nelson <dcn@sgi.com>

---

On Thu, Oct 02, 2008 at 10:42:14AM +0200, Ingo Molnar wrote:
>  
> * Dean Nelson <dcn@sgi.com> wrote:
> 
> > +static struct irq_chip uv_irq_chip = {
> > +   .name           = "UV_MSI",
> > +   .startup        = noop_ret,
> > +   .shutdown       = noop,
> > +   .enable         = noop,
> > +   .disable        = noop,
> > +   .ack            = noop,
> > +   .mask           = noop,
> > +   .unmask         = noop,
> > +   .eoi            = ack_apic,
> > +   .end            = noop,
> > +};
>
> hm, why isnt this in uv_irq.c?

Good question. Sometimes the obvious gets overlooked in the midst of
having rewritten a patch a dozen different ways. It's now in uv_irq.c.

> > Index: linux/kernel/irq/chip.c
> > ===================================================================
> > --- linux.orig/kernel/irq/chip.c    2008-09-30 09:07:42.000000000 -0500
> > +++ linux/kernel/irq/chip.c 2008-09-30 09:07:44.000000000 -0500
> > @@ -79,6 +79,7 @@ void dynamic_irq_cleanup(unsigned int ir
> >     desc->chip_data = NULL;
> >     desc->handle_irq = handle_bad_irq;
> >     desc->chip = &no_irq_chip;
> > +   desc->name = "none";
> >     spin_unlock_irqrestore(&desc->lock, flags);
>
> unrelated change that belongs into a separate patch.

Agreed. The change is needed by drivers/misc/sgi-xp once it has been
modified to call uv_setup_irq() and uv_teardown_irq(). I'll move this
change into that patch (or should it be in a patch all by itself?).

On Thu, Oct 02, 2008 at 09:44:03AM -0700, H. Peter Anvin wrote:
> Yinghai Lu wrote:
> >>+
> >>+static struct irq_chip uv_irq_chip = {
> >>+       .name           = "UV_MSI",
> >
> >anything to do with MSI?
> >
> 
> Not really.  It probably should be called "UV-CORE" or something like that.

Agreed and changed.

The patch to drivers/misc/sgi-xp to call the functions introduced by
this patch, will be submitted shortly.

Thanks,
Dean

 arch/x86/kernel/Makefile    |    2 -
 arch/x86/kernel/io_apic.c   |   68 +++++++++++++++++++++++++++++++++++
 arch/x86/kernel/uv_irq.c    |   77 ++++++++++++++++++++++++++++++++++++++++
 include/asm-x86/uv/uv_irq.h |   36 ++++++++++++++++++
 4 files changed, 182 insertions(+), 1 deletion(-)

Index: linux/arch/x86/kernel/io_apic.c
===================================================================
--- linux.orig/arch/x86/kernel/io_apic.c	2008-10-02 06:12:54.000000000 -0500
+++ linux/arch/x86/kernel/io_apic.c	2008-10-02 06:15:48.000000000 -0500
@@ -58,6 +58,8 @@
 #include <asm/setup.h>
 #include <asm/irq_remapping.h>
 #include <asm/hpet.h>
+#include <asm/uv/uv_hub.h>
+#include <asm/uv/uv_irq.h>
 
 #include <mach_ipi.h>
 #include <mach_apic.h>
@@ -3694,6 +3696,72 @@ int arch_setup_ht_irq(unsigned int irq, 
 }
 #endif /* CONFIG_HT_IRQ */
 
+#ifdef CONFIG_X86_64
+/*
+ * Re-target the irq to the specified CPU and enable the specified MMR located
+ * on the specified blade to allow the sending of MSIs to the specified CPU.
+ */
+int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
+		       unsigned long mmr_offset)
+{
+	const cpumask_t *eligible_cpu = get_cpu_mask(cpu);
+	struct irq_cfg *cfg;
+	int mmr_pnode;
+	unsigned long mmr_value;
+	struct uv_IO_APIC_route_entry *entry;
+	unsigned long flags;
+	int err;
+
+	err = assign_irq_vector(irq, *eligible_cpu);
+	if (err != 0)
+		return err;
+
+	spin_lock_irqsave(&vector_lock, flags);
+	set_irq_chip_and_handler_name(irq, &uv_irq_chip, handle_percpu_irq,
+				      irq_name);
+	spin_unlock_irqrestore(&vector_lock, flags);
+
+	cfg = irq_cfg(irq);
+
+	mmr_value = 0;
+	entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
+	BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
+
+	entry->vector = cfg->vector;
+	entry->delivery_mode = INT_DELIVERY_MODE;
+	entry->dest_mode = INT_DEST_MODE;
+	entry->polarity = 0;
+	entry->trigger = 0;
+	entry->mask = 0;
+	entry->dest = cpu_mask_to_apicid(*eligible_cpu);
+
+	mmr_pnode = uv_blade_to_pnode(mmr_blade);
+	uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
+
+	return irq;
+}
+
+/*
+ * Disable the specified MMR located on the specified blade so that MSIs are
+ * longer allowed to be sent.
+ */
+void arch_disable_uv_irq(int mmr_blade, unsigned long mmr_offset)
+{
+	unsigned long mmr_value;
+	struct uv_IO_APIC_route_entry *entry;
+	int mmr_pnode;
+
+	mmr_value = 0;
+	entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
+	BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
+
+	entry->mask = 1;
+
+	mmr_pnode = uv_blade_to_pnode(mmr_blade);
+	uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
+}
+#endif /* CONFIG_X86_64 */
+
 int __init io_apic_get_redir_entries (int ioapic)
 {
 	union IO_APIC_reg_01	reg_01;
Index: linux/arch/x86/kernel/uv_irq.c
===================================================================
--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux/arch/x86/kernel/uv_irq.c	2008-10-02 12:08:04.000000000 -0500
@@ -0,0 +1,77 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * SGI UV IRQ functions
+ *
+ * Copyright (C) 2008 Silicon Graphics, Inc. All rights reserved.
+ */
+
+#include <linux/module.h>
+#include <linux/irq.h>
+#include <asm/uv/uv_irq.h>
+
+static void uv_noop(unsigned int irq)
+{
+}
+
+static unsigned int uv_noop_ret(unsigned int irq)
+{
+	return 0;
+}
+
+static void uv_ack_apic(unsigned int irq)
+{
+	ack_APIC_irq();
+}
+
+struct irq_chip uv_irq_chip = {
+	.name		= "UV-CORE",
+	.startup	= uv_noop_ret,
+	.shutdown	= uv_noop,
+	.enable		= uv_noop,
+	.disable	= uv_noop,
+	.ack		= uv_noop,
+	.mask		= uv_noop,
+	.unmask		= uv_noop,
+	.eoi		= uv_ack_apic,
+	.end		= uv_noop,
+};
+
+/*
+ * Set up a mapping of an available irq and vector, and enable the specified
+ * MMR that defines the MSI that is to be sent to the specified CPU when an
+ * interrupt is raised.
+ */
+int uv_setup_irq(char *irq_name, int cpu, int mmr_blade,
+		 unsigned long mmr_offset)
+{
+	int irq;
+	int ret;
+
+	irq = create_irq();
+	if (irq <= 0)
+		return -EBUSY;
+
+	ret = arch_enable_uv_irq(irq_name, irq, cpu, mmr_blade, mmr_offset);
+	if (ret != irq)
+		destroy_irq(irq);
+
+	return ret;
+}
+EXPORT_SYMBOL(uv_setup_irq);
+
+/*
+ * Tear down a mapping of an irq and vector, and disable the specified MMR that
+ * defined the MSI that was to be sent to the specified CPU when an interrupt
+ * was raised.
+ *
+ * Set mmr_blade and mmr_offset to what was passed in on uv_setup_irq().
+ */
+void uv_teardown_irq(unsigned int irq, int mmr_blade, unsigned long mmr_offset)
+{
+	arch_disable_uv_irq(mmr_blade, mmr_offset);
+	destroy_irq(irq);
+}
+EXPORT_SYMBOL(uv_teardown_irq);
Index: linux/include/asm-x86/uv/uv_irq.h
===================================================================
--- /dev/null	1970-01-01 00:00:00.000000000 +0000
+++ linux/include/asm-x86/uv/uv_irq.h	2008-10-02 06:15:53.000000000 -0500
@@ -0,0 +1,36 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License.  See the file "COPYING" in the main directory of this archive
+ * for more details.
+ *
+ * SGI UV IRQ definitions
+ *
+ * Copyright (C) 2008 Silicon Graphics, Inc. All rights reserved.
+ */
+
+#ifndef ASM_X86__UV__UV_IRQ_H
+#define ASM_X86__UV__UV_IRQ_H
+
+/* If a generic version of this structure gets defined, eliminate this one. */
+struct uv_IO_APIC_route_entry {
+	__u64	vector		:  8,
+		delivery_mode	:  3,
+		dest_mode	:  1,
+		delivery_status	:  1,
+		polarity	:  1,
+		__reserved_1	:  1,
+		trigger		:  1,
+		mask		:  1,
+		__reserved_2	: 15,
+		dest		: 32;
+};
+
+extern struct irq_chip uv_irq_chip;
+
+extern int arch_enable_uv_irq(char *, unsigned int, int, int, unsigned long);
+extern void arch_disable_uv_irq(int, unsigned long);
+
+extern int uv_setup_irq(char *, int, int, unsigned long);
+extern void uv_teardown_irq(unsigned int, int, unsigned long);
+
+#endif /* ASM_X86__UV__UV_IRQ_H */
Index: linux/arch/x86/kernel/Makefile
===================================================================
--- linux.orig/arch/x86/kernel/Makefile	2008-10-02 06:12:54.000000000 -0500
+++ linux/arch/x86/kernel/Makefile	2008-10-02 06:13:01.000000000 -0500
@@ -107,7 +107,7 @@ obj-$(CONFIG_OLPC)		+= olpc.o
 # 64 bit specific files
 ifeq ($(CONFIG_X86_64),y)
         obj-y				+= genapic_64.o genapic_flat_64.o genx2apic_uv_x.o tlb_uv.o
-	obj-y				+= bios_uv.o uv_sysfs.o
+	obj-y				+= bios_uv.o uv_sysfs.o uv_irq.o
         obj-y				+= genx2apic_cluster.o
         obj-y				+= genx2apic_phys.o
         obj-$(CONFIG_X86_PM_TIMER)	+= pmtimer_64.o

Re: [PATCH] x86, UV: add uv_setup_irq() and uv_teardown_irq() functions v.3

[Ingo Molnar]

* Dean Nelson <dcn@sgi.com> wrote:

> Provide a means for UV interrupt MMRs to be setup with the message to be sent
> when an MSI is raised.
> 
> Signed-off-by: Dean Nelson <dcn@sgi.com>

v3 looks good - applied to tip/x86/uv, thanks Dean!

changed one small detail:

> +EXPORT_SYMBOL(uv_setup_irq);

changed that to a _GPL export. This is rather lowlevel functionality.

	Ingo

x86, UV: add uv_setup_irq() and uv_teardown_irq() functions, v3, fix

[Ingo Molnar]

* Ingo Molnar <mingo@elte.hu> wrote:

> 
> * Dean Nelson <dcn@sgi.com> wrote:
> 
> > Provide a means for UV interrupt MMRs to be setup with the message to be sent
> > when an MSI is raised.
> > 
> > Signed-off-by: Dean Nelson <dcn@sgi.com>
> 
> v3 looks good - applied to tip/x86/uv, thanks Dean!

-tip testing found a build error - fixlet below.

	Ingo

------------>
>From 1b2127d363a68a388e7c57610bb5ceeb743c89d2 Mon Sep 17 00:00:00 2001
From: Ingo Molnar <mingo@elte.hu>
Date: Fri, 3 Oct 2008 11:38:37 +0200
Subject: [PATCH] x86, UV: add uv_setup_irq() and uv_teardown_irq() functions, v3, fix

fix:

 arch/x86/kernel/uv_irq.c: In function 'uv_ack_apic':
 arch/x86/kernel/uv_irq.c:26: error: implicit declaration of function 'ack_APIC_irq'

Signed-off-by: Ingo Molnar <mingo@elte.hu>
---
 arch/x86/kernel/uv_irq.c |    2 ++
 1 files changed, 2 insertions(+), 0 deletions(-)

diff --git a/arch/x86/kernel/uv_irq.c b/arch/x86/kernel/uv_irq.c
index 6bd26c9..aeef529 100644
--- a/arch/x86/kernel/uv_irq.c
+++ b/arch/x86/kernel/uv_irq.c
@@ -10,6 +10,8 @@
 
 #include <linux/module.h>
 #include <linux/irq.h>
+
+#include <asm/apic.h>
 #include <asm/uv/uv_irq.h>
 
 static void uv_noop(unsigned int irq)

sgi-gru drivers need of up to 8192 irqs

[Dean Nelson]

Hi Ingo,

You may remember a patch that I submitted not so long ago that added
uv_setup_irq() and uv_teardown_irq() functions for UV on x86.

It gave us (SGI) a way to 'allocate' an irq/vector pair and write an MMR
with a CPU's APCID and the vector number. (Two functions were added to
/arch/x86/kernel/io_apic.c to accomplish this -- arch_enable_uv_irq() and
arch_disable_uv_irq().)

This solution was what was suggested by the community after much discussion.

I'm now trying to modify the GRU driver to call uv_setup_irq(). One of the
issues that is arising is that we need to be able to allocate lots of irqs.
For the GRU alone we need up to 32 irqs per blade (based on the number of CPUs)
and there can be up to 256 blades per SSI. (A total of 8192 irqs per SSI.)

Currently I'm finding that NR_IRQS=4352. (It is constrained to the lesser of
NR_CPUS and MAX_IO_APICS.) But the system really runs off of nr_irqs which
gets set down by probe_nr_irqs() to 96 in one configuration I've been running.
This function bases its answer on io_apic_get_redir_entries().

For the GRU there isn't an io_apic component, its needs are based on NR_CPUS.
So there isn't something readily available to cause probe_nr_irqs() to
generate a larger number.

Anyway, I'm looking for any suggestions as to what we should do. (Which may
include abandoning the using of irqs for the GRU driver and just go with two
hardwired system vectors?

Also, what has happened to the sparse irq patch? It doesn't seem to exist
anywhere. Has it been abandoned? I thought it was the means by which we would
be able to have huge numbers of irqs.

Thanks,
Dean

Re: sgi-gru drivers need of up to 8192 irqs

[Ingo Molnar]

(Cc:-ed Yinghai and Mike Travis.)

* Dean Nelson <dcn@sgi.com> wrote:

> Hi Ingo,
> 
> You may remember a patch that I submitted not so long ago that added 
> uv_setup_irq() and uv_teardown_irq() functions for UV on x86.
> 
> It gave us (SGI) a way to 'allocate' an irq/vector pair and write an 
> MMR with a CPU's APCID and the vector number. (Two functions were 
> added to /arch/x86/kernel/io_apic.c to accomplish this -- 
> arch_enable_uv_irq() and arch_disable_uv_irq().)
> 
> This solution was what was suggested by the community after much 
> discussion.
> 
> I'm now trying to modify the GRU driver to call uv_setup_irq(). One 
> of the issues that is arising is that we need to be able to allocate 
> lots of irqs. For the GRU alone we need up to 32 irqs per blade 
> (based on the number of CPUs) and there can be up to 256 blades per 
> SSI. (A total of 8192 irqs per SSI.)
> 
> Currently I'm finding that NR_IRQS=4352. (It is constrained to the 
> lesser of NR_CPUS and MAX_IO_APICS.) But the system really runs off 
> of nr_irqs which gets set down by probe_nr_irqs() to 96 in one 
> configuration I've been running. This function bases its answer on 
> io_apic_get_redir_entries().
> 
> For the GRU there isn't an io_apic component, its needs are based on 
> NR_CPUS. So there isn't something readily available to cause 
> probe_nr_irqs() to generate a larger number.
> 
> Anyway, I'm looking for any suggestions as to what we should do. 
> (Which may include abandoning the using of irqs for the GRU driver 
> and just go with two hardwired system vectors?
> 
> Also, what has happened to the sparse irq patch? It doesn't seem to 
> exist anywhere. Has it been abandoned? I thought it was the means by 
> which we would be able to have huge numbers of irqs.

it's being worked on, with the first half of the changes upstream 
already, and the latest version of the second half of sparseirq (patch 
v11) was posted by Yinghai to lkml today, under this thread:

Subject: [PATCH] sparse_irq aka dyn_irq v11

	Ingo