* [patch 1/5] x84_64, apic: Use probe routines to simplify apic selection
2011-05-13 18:04 [patch 0/5] x2apic cluster optimization patches Suresh Siddha
@ 2011-05-13 18:04 ` Suresh Siddha
2011-05-16 8:08 ` Ingo Molnar
2011-05-13 18:04 ` [patch 2/5] x86, x2apic: Remove duplicate code for IPI mask routines Suresh Siddha
` (3 subsequent siblings)
4 siblings, 1 reply; 14+ messages in thread
From: Suresh Siddha @ 2011-05-13 18:04 UTC (permalink / raw)
To: mingo, tglx, hpa, steiner, yinghai; +Cc: linux-kernel, gorcunov, Suresh Siddha
[-- Attachment #1: x64_apic_probe.patch --]
[-- Type: text/plain, Size: 4459 bytes --]
From: Suresh Siddha <suresh.b.siddha@intel.com>
Subject: x84_64, apic: Use probe routines to simplify apic selection
Use the unused probe routine in the apic driver to finalize the apic model
selection. This cleans up the default_setup_apic_routing() and this probe
routine in future can also be used for doing any apic model specific
initialization.
Acked-by: Cyrill Gorcunov <gorcunov@openvz.org>
Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
---
arch/x86/kernel/apic/apic_flat_64.c | 10 +++++++++-
arch/x86/kernel/apic/probe_64.c | 22 +++++++---------------
arch/x86/kernel/apic/x2apic_cluster.c | 7 ++++++-
arch/x86/kernel/apic/x2apic_phys.c | 10 +++++++++-
arch/x86/kernel/apic/x2apic_uv_x.c | 7 ++++++-
5 files changed, 37 insertions(+), 19 deletions(-)
Index: linux-2.6-tip/arch/x86/kernel/apic/apic_flat_64.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/apic_flat_64.c
+++ linux-2.6-tip/arch/x86/kernel/apic/apic_flat_64.c
@@ -312,10 +312,18 @@ physflat_cpu_mask_to_apicid_and(const st
return per_cpu(x86_cpu_to_apicid, cpu);
}
+static int physflat_probe(void)
+{
+ if (apic == &apic_physflat || num_possible_cpus() > 8)
+ return 1;
+
+ return 0;
+}
+
struct apic apic_physflat = {
.name = "physical flat",
- .probe = NULL,
+ .probe = physflat_probe,
.acpi_madt_oem_check = physflat_acpi_madt_oem_check,
.apic_id_registered = flat_apic_id_registered,
Index: linux-2.6-tip/arch/x86/kernel/apic/probe_64.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/probe_64.c
+++ linux-2.6-tip/arch/x86/kernel/apic/probe_64.c
@@ -54,26 +54,18 @@ static int apicid_phys_pkg_id(int initia
*/
void __init default_setup_apic_routing(void)
{
+ int i;
enable_IR_x2apic();
-#ifdef CONFIG_X86_X2APIC
- if (x2apic_mode
-#ifdef CONFIG_X86_UV
- && apic != &apic_x2apic_uv_x
-#endif
- ) {
- if (x2apic_phys)
- apic = &apic_x2apic_phys;
- else
- apic = &apic_x2apic_cluster;
+ for (i = 0; apic_probe[i]; ++i) {
+ if (apic_probe[i]->probe()) {
+ apic = apic_probe[i];
+ break;
+ }
}
-#endif
-
- if (apic == &apic_flat && num_possible_cpus() > 8)
- apic = &apic_physflat;
- printk(KERN_INFO "Setting APIC routing to %s\n", apic->name);
+ printk(KERN_INFO "APIC routing finalized to %s.\n", apic->name);
if (is_vsmp_box()) {
/* need to update phys_pkg_id */
Index: linux-2.6-tip/arch/x86/kernel/apic/x2apic_cluster.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/x2apic_cluster.c
+++ linux-2.6-tip/arch/x86/kernel/apic/x2apic_cluster.c
@@ -184,10 +184,15 @@ static void init_x2apic_ldr(void)
per_cpu(x86_cpu_to_logical_apicid, cpu) = apic_read(APIC_LDR);
}
+static int x2apic_cluster_probe(void)
+{
+ return x2apic_mode;
+}
+
struct apic apic_x2apic_cluster = {
.name = "cluster x2apic",
- .probe = NULL,
+ .probe = x2apic_cluster_probe,
.acpi_madt_oem_check = x2apic_acpi_madt_oem_check,
.apic_id_registered = x2apic_apic_id_registered,
Index: linux-2.6-tip/arch/x86/kernel/apic/x2apic_phys.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/x2apic_phys.c
+++ linux-2.6-tip/arch/x86/kernel/apic/x2apic_phys.c
@@ -173,10 +173,18 @@ static void init_x2apic_ldr(void)
{
}
+static int x2apic_phys_probe(void)
+{
+ if (x2apic_mode && x2apic_phys)
+ return 1;
+
+ return apic == &apic_x2apic_phys;
+}
+
struct apic apic_x2apic_phys = {
.name = "physical x2apic",
- .probe = NULL,
+ .probe = x2apic_phys_probe,
.acpi_madt_oem_check = x2apic_acpi_madt_oem_check,
.apic_id_registered = x2apic_apic_id_registered,
Index: linux-2.6-tip/arch/x86/kernel/apic/x2apic_uv_x.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/x2apic_uv_x.c
+++ linux-2.6-tip/arch/x86/kernel/apic/x2apic_uv_x.c
@@ -326,10 +326,15 @@ static void uv_send_IPI_self(int vector)
apic_write(APIC_SELF_IPI, vector);
}
+static int uv_probe(void)
+{
+ return apic == &apic_x2apic_uv_x;
+}
+
struct apic __refdata apic_x2apic_uv_x = {
.name = "UV large system",
- .probe = NULL,
+ .probe = uv_probe,
.acpi_madt_oem_check = uv_acpi_madt_oem_check,
.apic_id_registered = uv_apic_id_registered,
^ permalink raw reply [flat|nested] 14+ messages in thread* Re: [patch 1/5] x84_64, apic: Use probe routines to simplify apic selection
2011-05-13 18:04 ` [patch 1/5] x84_64, apic: Use probe routines to simplify apic selection Suresh Siddha
@ 2011-05-16 8:08 ` Ingo Molnar
2011-05-16 8:45 ` Cyrill Gorcunov
2011-05-16 23:51 ` Suresh Siddha
0 siblings, 2 replies; 14+ messages in thread
From: Ingo Molnar @ 2011-05-16 8:08 UTC (permalink / raw)
To: Suresh Siddha; +Cc: tglx, hpa, steiner, yinghai, linux-kernel, gorcunov
* Suresh Siddha <suresh.b.siddha@intel.com> wrote:
> From: Suresh Siddha <suresh.b.siddha@intel.com>
> Subject: x84_64, apic: Use probe routines to simplify apic selection
>
> Use the unused probe routine in the apic driver to finalize the apic model
> selection. This cleans up the default_setup_apic_routing() and this probe
> routine in future can also be used for doing any apic model specific
> initialization.
>
> Acked-by: Cyrill Gorcunov <gorcunov@openvz.org>
> Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
> ---
> arch/x86/kernel/apic/apic_flat_64.c | 10 +++++++++-
> arch/x86/kernel/apic/probe_64.c | 22 +++++++---------------
> arch/x86/kernel/apic/x2apic_cluster.c | 7 ++++++-
> arch/x86/kernel/apic/x2apic_phys.c | 10 +++++++++-
> arch/x86/kernel/apic/x2apic_uv_x.c | 7 ++++++-
> 5 files changed, 37 insertions(+), 19 deletions(-)
Ok, looks like a step forward in the right direction.
Wouldnt it be more self-contained if the probe function returned an apic driver
pointer?
We could add an __apicdriver section trick to 'know' about all APIC probing
functions - and we'd call them one by one and use the first one that returns a
non-NULL result.
Then we'd have the generic fallback APIC drivers as well - not marked
__apicdriver but used by the probe function directly.
Or we could have them as __apicdriver as well, but then the .o link ordering
matters to probing order.
Thanks,
Ingo
^ permalink raw reply [flat|nested] 14+ messages in thread
* Re: [patch 1/5] x84_64, apic: Use probe routines to simplify apic selection
2011-05-16 8:08 ` Ingo Molnar
@ 2011-05-16 8:45 ` Cyrill Gorcunov
2011-05-16 9:03 ` Ingo Molnar
2011-05-16 23:51 ` Suresh Siddha
1 sibling, 1 reply; 14+ messages in thread
From: Cyrill Gorcunov @ 2011-05-16 8:45 UTC (permalink / raw)
To: Ingo Molnar; +Cc: Suresh Siddha, tglx, hpa, steiner, yinghai, linux-kernel
On Mon, May 16, 2011 at 12:08 PM, Ingo Molnar <mingo@elte.hu> wrote:
>
> * Suresh Siddha <suresh.b.siddha@intel.com> wrote:
>
>> From: Suresh Siddha <suresh.b.siddha@intel.com>
>> Subject: x84_64, apic: Use probe routines to simplify apic selection
>>
>> Use the unused probe routine in the apic driver to finalize the apic model
>> selection. This cleans up the default_setup_apic_routing() and this probe
>> routine in future can also be used for doing any apic model specific
>> initialization.
>>
>> Acked-by: Cyrill Gorcunov <gorcunov@openvz.org>
>> Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
>> ---
>> arch/x86/kernel/apic/apic_flat_64.c | 10 +++++++++-
>> arch/x86/kernel/apic/probe_64.c | 22 +++++++---------------
>> arch/x86/kernel/apic/x2apic_cluster.c | 7 ++++++-
>> arch/x86/kernel/apic/x2apic_phys.c | 10 +++++++++-
>> arch/x86/kernel/apic/x2apic_uv_x.c | 7 ++++++-
>> 5 files changed, 37 insertions(+), 19 deletions(-)
>
> Ok, looks like a step forward in the right direction.
>
> Wouldnt it be more self-contained if the probe function returned an apic driver
> pointer?
>
> We could add an __apicdriver section trick to 'know' about all APIC probing
> functions - and we'd call them one by one and use the first one that returns a
> non-NULL result.
>
> Then we'd have the generic fallback APIC drivers as well - not marked
> __apicdriver but used by the probe function directly.
>
> Or we could have them as __apicdriver as well, but then the .o link ordering
> matters to probing order.
>
> Thanks,
>
> Ingo
>
Yes, this should be a cleaner step but it requires some more code churn that
is why I think Suresh' approach is pretty well done! Ingo, probably we could
do so on *top* of this changes as only time permits? The problem is that I
personally can't guarantee I continue working on the task this week, if Suresh
is able to -- this would be great, if not -- probably even such
minimal disturbing
patch worth merging, it definitley makes code better I think ;)
^ permalink raw reply [flat|nested] 14+ messages in thread
* Re: [patch 1/5] x84_64, apic: Use probe routines to simplify apic selection
2011-05-16 8:45 ` Cyrill Gorcunov
@ 2011-05-16 9:03 ` Ingo Molnar
2011-05-16 9:48 ` Cyrill Gorcunov
0 siblings, 1 reply; 14+ messages in thread
From: Ingo Molnar @ 2011-05-16 9:03 UTC (permalink / raw)
To: Cyrill Gorcunov; +Cc: Suresh Siddha, tglx, hpa, steiner, yinghai, linux-kernel
* Cyrill Gorcunov <gorcunov@openvz.org> wrote:
> Yes, this should be a cleaner step but it requires some more code churn
> that is why I think Suresh' approach is pretty well done! Ingo, probably we
> could do so on *top* of this changes as only time permits? The problem is
> that I personally can't guarantee I continue working on the task this week,
> if Suresh is able to -- this would be great, if not -- probably even such
> minimal disturbing patch worth merging, it definitley makes code better I
> think ;)
We can do it on top, but within the series please. The APIC mess is not
improving so we only want to merge features that are done cleanly.
Thanks,
Ingo
^ permalink raw reply [flat|nested] 14+ messages in thread
* Re: [patch 1/5] x84_64, apic: Use probe routines to simplify apic selection
2011-05-16 9:03 ` Ingo Molnar
@ 2011-05-16 9:48 ` Cyrill Gorcunov
0 siblings, 0 replies; 14+ messages in thread
From: Cyrill Gorcunov @ 2011-05-16 9:48 UTC (permalink / raw)
To: Ingo Molnar
Cc: Cyrill Gorcunov, Suresh Siddha, tglx@linutronix.de, hpa@zytor.com,
steiner@sgi.com, yinghai@kernel.org, linux-kernel@vger.kernel.org
ok, thanks Ingo.
On Monday, May 16, 2011, Ingo Molnar <mingo@elte.hu> wrote:
>
> * Cyrill Gorcunov <gorcunov@openvz.org> wrote:
>
>> Yes, this should be a cleaner step but it requires some more code churn
>> that is why I think Suresh' approach is pretty well done! Ingo, probably we
>> could do so on *top* of this changes as only time permits? The problem is
>> that I personally can't guarantee I continue working on the task this week,
>> if Suresh is able to -- this would be great, if not -- probably even such
>> minimal disturbing patch worth merging, it definitley makes code better I
>> think ;)
>
> We can do it on top, but within the series please. The APIC mess is not
> improving so we only want to merge features that are done cleanly.
>
> Thanks,
>
> Ingo
>
^ permalink raw reply [flat|nested] 14+ messages in thread
* Re: [patch 1/5] x84_64, apic: Use probe routines to simplify apic selection
2011-05-16 8:08 ` Ingo Molnar
2011-05-16 8:45 ` Cyrill Gorcunov
@ 2011-05-16 23:51 ` Suresh Siddha
2011-05-17 12:28 ` Ingo Molnar
2011-05-19 23:54 ` Suresh Siddha
1 sibling, 2 replies; 14+ messages in thread
From: Suresh Siddha @ 2011-05-16 23:51 UTC (permalink / raw)
To: Ingo Molnar
Cc: tglx@linutronix.de, hpa@zytor.com, steiner@sgi.com,
yinghai@kernel.org, linux-kernel@vger.kernel.org,
gorcunov@openvz.org
On Mon, 2011-05-16 at 01:08 -0700, Ingo Molnar wrote:
> Ok, looks like a step forward in the right direction.
>
> Wouldnt it be more self-contained if the probe function returned an apic driver
> pointer?
>
> We could add an __apicdriver section trick to 'know' about all APIC probing
> functions - and we'd call them one by one and use the first one that returns a
> non-NULL result.
>
> Then we'd have the generic fallback APIC drivers as well - not marked
> __apicdriver but used by the probe function directly.
>
> Or we could have them as __apicdriver as well, but then the .o link ordering
> matters to probing order.
>
So how does something like the appended look? There are multiple apic
driver routines and apic name string that is being looked at by the
generic code. So I exported the struct apic * using the section trick.
Untested for now. If you are ok, then I will post with a better
changelog.
Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
---
arch/x86/include/asm/apic.h | 11 +++--
arch/x86/kernel/apic/Makefile | 17 +++++---
arch/x86/kernel/apic/apic_flat_64.c | 12 +++++-
arch/x86/kernel/apic/bigsmp_32.c | 2 +
arch/x86/kernel/apic/es7000_32.c | 7 ++-
arch/x86/kernel/apic/numaq_32.c | 7 ++-
arch/x86/kernel/apic/probe_32.c | 65 +++++++++++-----------------------
arch/x86/kernel/apic/probe_64.c | 43 ++++++----------------
arch/x86/kernel/apic/summit_32.c | 6 ++-
arch/x86/kernel/apic/x2apic_cluster.c | 4 +-
arch/x86/kernel/apic/x2apic_phys.c | 6 ++-
arch/x86/kernel/apic/x2apic_uv_x.c | 4 +-
arch/x86/kernel/vmlinux.lds.S | 6 +++
13 files changed, 94 insertions(+), 96 deletions(-)
Index: linux-2.6-tip/arch/x86/include/asm/apic.h
===================================================================
--- linux-2.6-tip.orig/arch/x86/include/asm/apic.h
+++ linux-2.6-tip/arch/x86/include/asm/apic.h
@@ -380,6 +380,12 @@ struct apic {
*/
extern struct apic *apic;
+#define apicdriver(sym) \
+ static struct apic *__apicdriver_##sym __used \
+ __attribute__ ((__section__(".apicdrivers"))) = &sym
+
+extern struct apic *__apicdrivers[], *__apicdrivers_end[];
+
/*
* APIC functionality to boot other CPUs - only used on SMP:
*/
@@ -458,15 +464,10 @@ static inline unsigned default_get_apic_
#define DEFAULT_TRAMPOLINE_PHYS_HIGH 0x469
#ifdef CONFIG_X86_64
-extern struct apic apic_flat;
-extern struct apic apic_physflat;
-extern struct apic apic_x2apic_cluster;
-extern struct apic apic_x2apic_phys;
extern int default_acpi_madt_oem_check(char *, char *);
extern void apic_send_IPI_self(int vector);
-extern struct apic apic_x2apic_uv_x;
DECLARE_PER_CPU(int, x2apic_extra_bits);
extern int default_cpu_present_to_apicid(int mps_cpu);
Index: linux-2.6-tip/arch/x86/kernel/apic/Makefile
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/Makefile
+++ linux-2.6-tip/arch/x86/kernel/apic/Makefile
@@ -2,20 +2,25 @@
# Makefile for local APIC drivers and for the IO-APIC code
#
-obj-$(CONFIG_X86_LOCAL_APIC) += apic.o apic_noop.o probe_$(BITS).o ipi.o
+obj-$(CONFIG_X86_LOCAL_APIC) += apic.o apic_noop.o ipi.o
obj-y += hw_nmi.o
obj-$(CONFIG_X86_IO_APIC) += io_apic.o
obj-$(CONFIG_SMP) += ipi.o
ifeq ($(CONFIG_X86_64),y)
-obj-y += apic_flat_64.o
-obj-$(CONFIG_X86_X2APIC) += x2apic_cluster.o
-obj-$(CONFIG_X86_X2APIC) += x2apic_phys.o
+# APIC probe will depend on the listing order here
obj-$(CONFIG_X86_UV) += x2apic_uv_x.o
+obj-$(CONFIG_X86_X2APIC) += x2apic_phys.o
+obj-$(CONFIG_X86_X2APIC) += x2apic_cluster.o
+obj-y += apic_flat_64.o
endif
-obj-$(CONFIG_X86_BIGSMP) += bigsmp_32.o
+# APIC probe will depend on the listing order here
obj-$(CONFIG_X86_NUMAQ) += numaq_32.o
-obj-$(CONFIG_X86_ES7000) += es7000_32.o
obj-$(CONFIG_X86_SUMMIT) += summit_32.o
+obj-$(CONFIG_X86_BIGSMP) += bigsmp_32.o
+obj-$(CONFIG_X86_ES7000) += es7000_32.o
+
+# For 32bit, probe_32 need to be listed last
+obj-$(CONFIG_X86_LOCAL_APIC) += probe_$(BITS).o
Index: linux-2.6-tip/arch/x86/kernel/apic/apic_flat_64.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/apic_flat_64.c
+++ linux-2.6-tip/arch/x86/kernel/apic/apic_flat_64.c
@@ -24,6 +24,8 @@
#include <acpi/acpi_bus.h>
#endif
+static struct apic apic_physflat;
+
static int flat_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
{
return 1;
@@ -164,7 +166,7 @@ static int flat_phys_pkg_id(int initial_
return initial_apic_id >> index_msb;
}
-struct apic apic_flat = {
+static struct apic apic_flat = {
.name = "flat",
.probe = NULL,
.acpi_madt_oem_check = flat_acpi_madt_oem_check,
@@ -320,7 +322,7 @@ static int physflat_probe(void)
return 0;
}
-struct apic apic_physflat = {
+static struct apic apic_physflat = {
.name = "physical flat",
.probe = physflat_probe,
@@ -377,3 +379,9 @@ struct apic apic_physflat = {
.wait_icr_idle = native_apic_wait_icr_idle,
.safe_wait_icr_idle = native_safe_apic_wait_icr_idle,
};
+
+/*
+ * We need to check for Physflat first, so don't change this order!
+ */
+apicdriver(apic_physflat);
+apicdriver(apic_flat);
Index: linux-2.6-tip/arch/x86/kernel/apic/probe_64.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/probe_64.c
+++ linux-2.6-tip/arch/x86/kernel/apic/probe_64.c
@@ -23,27 +23,9 @@
#include <asm/ipi.h>
#include <asm/setup.h>
-extern struct apic apic_flat;
-extern struct apic apic_physflat;
-extern struct apic apic_x2xpic_uv_x;
-extern struct apic apic_x2apic_phys;
-extern struct apic apic_x2apic_cluster;
-
-struct apic __read_mostly *apic = &apic_flat;
+struct apic __read_mostly *apic;
EXPORT_SYMBOL_GPL(apic);
-static struct apic *apic_probe[] __initdata = {
-#ifdef CONFIG_X86_UV
- &apic_x2apic_uv_x,
-#endif
-#ifdef CONFIG_X86_X2APIC
- &apic_x2apic_phys,
- &apic_x2apic_cluster,
-#endif
- &apic_physflat,
- NULL,
-};
-
static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
{
return hard_smp_processor_id() >> index_msb;
@@ -54,19 +36,19 @@ static int apicid_phys_pkg_id(int initia
*/
void __init default_setup_apic_routing(void)
{
- int i;
+ struct apic **list;
enable_IR_x2apic();
- for (i = 0; apic_probe[i]; ++i) {
- if (apic_probe[i]->probe()) {
- apic = apic_probe[i];
+ for (list = __apicdrivers; list < __apicdrivers_end; list++) {
+ if ((*list)->probe()) {
+ apic = *list;
+ printk(KERN_INFO "APIC routing finalized to %s.\n",
+ apic->name);
break;
}
}
- printk(KERN_INFO "APIC routing finalized to %s.\n", apic->name);
-
if (is_vsmp_box()) {
/* need to update phys_pkg_id */
apic->phys_pkg_id = apicid_phys_pkg_id;
@@ -82,13 +64,14 @@ void apic_send_IPI_self(int vector)
int __init default_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
{
- int i;
+ struct apic **list;
- for (i = 0; apic_probe[i]; ++i) {
- if (apic_probe[i]->acpi_madt_oem_check(oem_id, oem_table_id)) {
- apic = apic_probe[i];
+ for (list = __apicdrivers; list < __apicdrivers_end; list++) {
+ if ((*list)->acpi_madt_oem_check(oem_id, oem_table_id)) {
+ apic = *list;
printk(KERN_INFO "Setting APIC routing to %s.\n",
- apic->name);
+ apic->name);
+
return 1;
}
}
Index: linux-2.6-tip/arch/x86/kernel/apic/x2apic_cluster.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/x2apic_cluster.c
+++ linux-2.6-tip/arch/x86/kernel/apic/x2apic_cluster.c
@@ -208,7 +208,7 @@ static int x2apic_cluster_probe(void)
return 0;
}
-struct apic apic_x2apic_cluster = {
+static struct apic apic_x2apic_cluster = {
.name = "cluster x2apic",
.probe = x2apic_cluster_probe,
@@ -264,3 +264,5 @@ struct apic apic_x2apic_cluster = {
.wait_icr_idle = native_x2apic_wait_icr_idle,
.safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle,
};
+
+apicdriver(apic_x2apic_cluster);
Index: linux-2.6-tip/arch/x86/kernel/apic/x2apic_phys.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/x2apic_phys.c
+++ linux-2.6-tip/arch/x86/kernel/apic/x2apic_phys.c
@@ -11,6 +11,8 @@
int x2apic_phys;
+static struct apic apic_x2apic_phys;
+
static int set_x2apic_phys_mode(char *arg)
{
x2apic_phys = 1;
@@ -112,7 +114,7 @@ static int x2apic_phys_probe(void)
return apic == &apic_x2apic_phys;
}
-struct apic apic_x2apic_phys = {
+static struct apic apic_x2apic_phys = {
.name = "physical x2apic",
.probe = x2apic_phys_probe,
@@ -168,3 +170,5 @@ struct apic apic_x2apic_phys = {
.wait_icr_idle = native_x2apic_wait_icr_idle,
.safe_wait_icr_idle = native_safe_x2apic_wait_icr_idle,
};
+
+apicdriver(apic_x2apic_phys);
Index: linux-2.6-tip/arch/x86/kernel/apic/x2apic_uv_x.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/x2apic_uv_x.c
+++ linux-2.6-tip/arch/x86/kernel/apic/x2apic_uv_x.c
@@ -331,7 +331,7 @@ static int uv_probe(void)
return apic == &apic_x2apic_uv_x;
}
-struct apic __refdata apic_x2apic_uv_x = {
+static struct apic apic_x2apic_uv_x = {
.name = "UV large system",
.probe = uv_probe,
@@ -864,3 +864,5 @@ void __init uv_system_init(void)
if (is_kdump_kernel())
reboot_type = BOOT_ACPI;
}
+
+apicdriver(apic_x2apic_uv_x);
Index: linux-2.6-tip/arch/x86/kernel/vmlinux.lds.S
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/vmlinux.lds.S
+++ linux-2.6-tip/arch/x86/kernel/vmlinux.lds.S
@@ -305,6 +305,12 @@ SECTIONS
__iommu_table_end = .;
}
+ .apicdrivers : AT(ADDR(.apicdrivers) - LOAD_OFFSET) {
+ __apicdrivers = .;
+ *(.apicdrivers);
+ __apicdrivers_end = .;
+ }
+
. = ALIGN(8);
/*
* .exit.text is discard at runtime, not link time, to deal with
Index: linux-2.6-tip/arch/x86/kernel/apic/bigsmp_32.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/bigsmp_32.c
+++ linux-2.6-tip/arch/x86/kernel/apic/bigsmp_32.c
@@ -254,3 +254,5 @@ struct apic apic_bigsmp = {
.x86_32_early_logical_apicid = bigsmp_early_logical_apicid,
};
+
+apicdriver(apic_bigsmp);
Index: linux-2.6-tip/arch/x86/kernel/apic/es7000_32.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/es7000_32.c
+++ linux-2.6-tip/arch/x86/kernel/apic/es7000_32.c
@@ -620,7 +620,7 @@ static int es7000_mps_oem_check_cluster(
}
/* We've been warned by a false positive warning.Use __refdata to keep calm. */
-struct apic __refdata apic_es7000_cluster = {
+static struct apic __refdata apic_es7000_cluster = {
.name = "es7000",
.probe = probe_es7000,
@@ -685,7 +685,7 @@ struct apic __refdata apic_es7000_cluste
.x86_32_early_logical_apicid = es7000_early_logical_apicid,
};
-struct apic __refdata apic_es7000 = {
+static struct apic __refdata apic_es7000 = {
.name = "es7000",
.probe = probe_es7000,
@@ -747,3 +747,6 @@ struct apic __refdata apic_es7000 = {
.x86_32_early_logical_apicid = es7000_early_logical_apicid,
};
+
+apicdriver(apic_es7000);
+apicdriver(apic_es7000_cluster);
Index: linux-2.6-tip/arch/x86/kernel/apic/numaq_32.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/numaq_32.c
+++ linux-2.6-tip/arch/x86/kernel/apic/numaq_32.c
@@ -73,7 +73,6 @@ int mp_bus_id_to_node[MAX_MP_BUSSES]
int mp_bus_id_to_local[MAX_MP_BUSSES];
int quad_local_to_mp_bus_id[NR_CPUS/4][4];
-
static inline void numaq_register_node(int node, struct sys_cfg_data *scd)
{
struct eachquadmem *eq = scd->eq + node;
@@ -472,8 +471,8 @@ static void numaq_setup_portio_remap(voi
(u_long) xquad_portio, (u_long) num_quads*XQUAD_PORTIO_QUAD);
}
-/* Use __refdata to keep false positive warning calm. */
-struct apic __refdata apic_numaq = {
+/* Use __refdata to keep false positive warning calm. */
+static struct apic __refdata apic_numaq = {
.name = "NUMAQ",
.probe = probe_numaq,
@@ -537,3 +536,5 @@ struct apic __refdata apic_numaq = {
.x86_32_early_logical_apicid = noop_x86_32_early_logical_apicid,
.x86_32_numa_cpu_node = numaq_numa_cpu_node,
};
+
+apicdriver(apic_numaq);
Index: linux-2.6-tip/arch/x86/kernel/apic/probe_32.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/probe_32.c
+++ linux-2.6-tip/arch/x86/kernel/apic/probe_32.c
@@ -174,44 +174,20 @@ struct apic apic_default = {
.x86_32_early_logical_apicid = default_x86_32_early_logical_apicid,
};
-extern struct apic apic_numaq;
-extern struct apic apic_summit;
-extern struct apic apic_bigsmp;
-extern struct apic apic_es7000;
-extern struct apic apic_es7000_cluster;
-
struct apic *apic = &apic_default;
EXPORT_SYMBOL_GPL(apic);
-static struct apic *apic_probe[] __initdata = {
-#ifdef CONFIG_X86_NUMAQ
- &apic_numaq,
-#endif
-#ifdef CONFIG_X86_SUMMIT
- &apic_summit,
-#endif
-#ifdef CONFIG_X86_BIGSMP
- &apic_bigsmp,
-#endif
-#ifdef CONFIG_X86_ES7000
- &apic_es7000,
- &apic_es7000_cluster,
-#endif
- &apic_default, /* must be last */
- NULL,
-};
-
static int cmdline_apic __initdata;
static int __init parse_apic(char *arg)
{
- int i;
+ struct apic **list;
if (!arg)
return -EINVAL;
- for (i = 0; apic_probe[i]; i++) {
- if (!strcmp(apic_probe[i]->name, arg)) {
- apic = apic_probe[i];
+ for (list = __apicdrivers; list < __apicdrivers_end; list++) {
+ if (!strcmp((*list)->name, arg)) {
+ apic = *list;
cmdline_apic = 1;
return 0;
}
@@ -222,6 +198,8 @@ static int __init parse_apic(char *arg)
}
early_param("apic", parse_apic);
+extern struct apic apic_bigsmp;
+
void __init generic_bigsmp_probe(void)
{
#ifdef CONFIG_X86_BIGSMP
@@ -245,15 +223,16 @@ void __init generic_bigsmp_probe(void)
void __init generic_apic_probe(void)
{
if (!cmdline_apic) {
- int i;
- for (i = 0; apic_probe[i]; i++) {
- if (apic_probe[i]->probe()) {
- apic = apic_probe[i];
+ struct apic **list;
+
+ for (list = __apicdrivers; list < __apicdrivers_end; list++) {
+ if ((*list)->probe()) {
+ apic = *list;
break;
}
}
/* Not visible without early console */
- if (!apic_probe[i])
+ if (list == __apicdrivers_end)
panic("Didn't find an APIC driver");
}
printk(KERN_INFO "Using APIC driver %s\n", apic->name);
@@ -264,16 +243,16 @@ void __init generic_apic_probe(void)
int __init
generic_mps_oem_check(struct mpc_table *mpc, char *oem, char *productid)
{
- int i;
+ struct apic **list;
- for (i = 0; apic_probe[i]; ++i) {
- if (!apic_probe[i]->mps_oem_check)
+ for (list = __apicdrivers; list < __apicdrivers_end; list++) {
+ if (!((*list)->mps_oem_check))
continue;
- if (!apic_probe[i]->mps_oem_check(mpc, oem, productid))
+ if (!(*list)->mps_oem_check(mpc, oem, productid))
continue;
if (!cmdline_apic) {
- apic = apic_probe[i];
+ apic = *list;
printk(KERN_INFO "Switched to APIC driver `%s'.\n",
apic->name);
}
@@ -284,16 +263,16 @@ generic_mps_oem_check(struct mpc_table *
int __init default_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
{
- int i;
+ struct apic **list;
- for (i = 0; apic_probe[i]; ++i) {
- if (!apic_probe[i]->acpi_madt_oem_check)
+ for (list = __apicdrivers; list < __apicdrivers_end; list++) {
+ if (!(*list)->acpi_madt_oem_check)
continue;
- if (!apic_probe[i]->acpi_madt_oem_check(oem_id, oem_table_id))
+ if (!(*list)->acpi_madt_oem_check(oem_id, oem_table_id))
continue;
if (!cmdline_apic) {
- apic = apic_probe[i];
+ apic = *list;
printk(KERN_INFO "Switched to APIC driver `%s'.\n",
apic->name);
}
Index: linux-2.6-tip/arch/x86/kernel/apic/summit_32.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/summit_32.c
+++ linux-2.6-tip/arch/x86/kernel/apic/summit_32.c
@@ -93,7 +93,7 @@ static int summit_mps_oem_check(struct m
}
/* Hook from generic ACPI tables.c */
-static int summit_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
+static struct apic *summit_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
{
if (!strncmp(oem_id, "IBM", 3) &&
(!strncmp(oem_table_id, "SERVIGIL", 8)
@@ -491,7 +491,7 @@ void setup_summit(void)
}
#endif
-struct apic apic_summit = {
+static struct apic apic_summit = {
.name = "summit",
.probe = probe_summit,
@@ -552,3 +552,5 @@ struct apic apic_summit = {
.x86_32_early_logical_apicid = summit_early_logical_apicid,
};
+
+apicdriver(apic_summit);
^ permalink raw reply [flat|nested] 14+ messages in thread* Re: [patch 1/5] x84_64, apic: Use probe routines to simplify apic selection
2011-05-16 23:51 ` Suresh Siddha
@ 2011-05-17 12:28 ` Ingo Molnar
2011-05-17 13:02 ` Thomas Gleixner
2011-05-19 23:54 ` Suresh Siddha
1 sibling, 1 reply; 14+ messages in thread
From: Ingo Molnar @ 2011-05-17 12:28 UTC (permalink / raw)
To: Suresh Siddha
Cc: tglx@linutronix.de, hpa@zytor.com, steiner@sgi.com,
yinghai@kernel.org, linux-kernel@vger.kernel.org,
gorcunov@openvz.org
* Suresh Siddha <suresh.b.siddha@intel.com> wrote:
> On Mon, 2011-05-16 at 01:08 -0700, Ingo Molnar wrote:
> > Ok, looks like a step forward in the right direction.
> >
> > Wouldnt it be more self-contained if the probe function returned an apic driver
> > pointer?
> >
> > We could add an __apicdriver section trick to 'know' about all APIC probing
> > functions - and we'd call them one by one and use the first one that returns a
> > non-NULL result.
> >
> > Then we'd have the generic fallback APIC drivers as well - not marked
> > __apicdriver but used by the probe function directly.
> >
> > Or we could have them as __apicdriver as well, but then the .o link ordering
> > matters to probing order.
> >
>
> So how does something like the appended look? There are multiple apic
> driver routines and apic name string that is being looked at by the
> generic code. So I exported the struct apic * using the section trick.
>
> Untested for now. If you are ok, then I will post with a better
> changelog.
Yeah, this looks very nice to me.
Thanks,
Ingo
^ permalink raw reply [flat|nested] 14+ messages in thread
* Re: [patch 1/5] x84_64, apic: Use probe routines to simplify apic selection
2011-05-17 12:28 ` Ingo Molnar
@ 2011-05-17 13:02 ` Thomas Gleixner
0 siblings, 0 replies; 14+ messages in thread
From: Thomas Gleixner @ 2011-05-17 13:02 UTC (permalink / raw)
To: Ingo Molnar
Cc: Suresh Siddha, hpa@zytor.com, steiner@sgi.com, yinghai@kernel.org,
linux-kernel@vger.kernel.org, gorcunov@openvz.org
On Tue, 17 May 2011, Ingo Molnar wrote:
>
> * Suresh Siddha <suresh.b.siddha@intel.com> wrote:
>
> > On Mon, 2011-05-16 at 01:08 -0700, Ingo Molnar wrote:
> > > Ok, looks like a step forward in the right direction.
> > >
> > > Wouldnt it be more self-contained if the probe function returned an apic driver
> > > pointer?
> > >
> > > We could add an __apicdriver section trick to 'know' about all APIC probing
> > > functions - and we'd call them one by one and use the first one that returns a
> > > non-NULL result.
> > >
> > > Then we'd have the generic fallback APIC drivers as well - not marked
> > > __apicdriver but used by the probe function directly.
> > >
> > > Or we could have them as __apicdriver as well, but then the .o link ordering
> > > matters to probing order.
> > >
> >
> > So how does something like the appended look? There are multiple apic
> > driver routines and apic name string that is being looked at by the
> > generic code. So I exported the struct apic * using the section trick.
> >
> > Untested for now. If you are ok, then I will post with a better
> > changelog.
>
> Yeah, this looks very nice to me.
Ditto.
^ permalink raw reply [flat|nested] 14+ messages in thread
* Re: [patch 1/5] x84_64, apic: Use probe routines to simplify apic selection
2011-05-16 23:51 ` Suresh Siddha
2011-05-17 12:28 ` Ingo Molnar
@ 2011-05-19 23:54 ` Suresh Siddha
1 sibling, 0 replies; 14+ messages in thread
From: Suresh Siddha @ 2011-05-19 23:54 UTC (permalink / raw)
To: Ingo Molnar
Cc: tglx@linutronix.de, hpa@zytor.com, steiner@sgi.com,
yinghai@kernel.org, linux-kernel@vger.kernel.org,
gorcunov@openvz.org
Ingo, Thomas:
On Mon, 2011-05-16 at 16:51 -0700, Suresh Siddha wrote:
> +
> +/*
> + * We need to check for Physflat first, so don't change this order!
> + */
> +apicdriver(apic_physflat);
> +apicdriver(apic_flat);
For the couple of files (one in 32-bit arch and another in 64-bit arch)
that were listing two apic drivers (which were sharing most of the code)
in the same file, the above ordering didn't do the trick to enforce the
order.
So I ended up defining one more macro for this case and it looks like
this below.
/*
* APIC drivers are probed based on how they are listed in the .apicdrivers
* section. So the order is important and enforced by the ordering
* of different apic driver files in the Makefile.
*
* For the files having two apic drivers, we use apic_drivers()
* to enforce the order with in them.
*/
#define apic_driver(sym) \
static struct apic *__apicdrivers_##sym __used \
__aligned(sizeof(struct apic *)) \
__section(.apicdrivers) = { &sym }
#define apic_drivers(sym1, sym2) \
static struct apic *__apicdrivers_##sym1##sym2[2] __used \
__aligned(sizeof(struct apic *)) \
__section(.apicdrivers) = { &sym1, &sym2 }
Posted the refreshed patchset.
thanks,
suresh
^ permalink raw reply [flat|nested] 14+ messages in thread
* [patch 2/5] x86, x2apic: Remove duplicate code for IPI mask routines
2011-05-13 18:04 [patch 0/5] x2apic cluster optimization patches Suresh Siddha
2011-05-13 18:04 ` [patch 1/5] x84_64, apic: Use probe routines to simplify apic selection Suresh Siddha
@ 2011-05-13 18:04 ` Suresh Siddha
2011-05-13 18:04 ` [patch 3/5] x86, x2apic: Track the x2apic cluster sibling map Suresh Siddha
` (2 subsequent siblings)
4 siblings, 0 replies; 14+ messages in thread
From: Suresh Siddha @ 2011-05-13 18:04 UTC (permalink / raw)
To: mingo, tglx, hpa, steiner, yinghai; +Cc: linux-kernel, gorcunov, Suresh Siddha
[-- Attachment #1: consolidate_x2apic_ipi_mask_routines.patch --]
[-- Type: text/plain, Size: 5069 bytes --]
From: Suresh Siddha <suresh.b.siddha@intel.com>
Subject: x86, x2apic: Remove duplicate code for IPI mask routines
Acked-by: Cyrill Gorcunov <gorcunov@openvz.org>
Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
---
arch/x86/kernel/apic/x2apic_cluster.c | 48 +++++++++++-----------------------
arch/x86/kernel/apic/x2apic_phys.c | 45 ++++++++++---------------------
2 files changed, 31 insertions(+), 62 deletions(-)
Index: linux-2.6-tip/arch/x86/kernel/apic/x2apic_cluster.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/x2apic_cluster.c
+++ linux-2.6-tip/arch/x86/kernel/apic/x2apic_cluster.c
@@ -54,64 +54,48 @@ static void
* at once. We have 16 cpu's in a cluster. This will minimize IPI register
* writes.
*/
-static void x2apic_send_IPI_mask(const struct cpumask *mask, int vector)
+static void
+__x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest)
{
unsigned long query_cpu;
+ unsigned long this_cpu;
unsigned long flags;
x2apic_wrmsr_fence();
local_irq_save(flags);
+
+ this_cpu = smp_processor_id();
for_each_cpu(query_cpu, mask) {
+ if (apic_dest == APIC_DEST_ALLBUT && query_cpu == this_cpu)
+ continue;
__x2apic_send_IPI_dest(
per_cpu(x86_cpu_to_logical_apicid, query_cpu),
vector, apic->dest_logical);
}
+
local_irq_restore(flags);
}
+static void x2apic_send_IPI_mask(const struct cpumask *mask, int vector)
+{
+ __x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLINC);
+}
+
static void
x2apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector)
{
- unsigned long this_cpu = smp_processor_id();
- unsigned long query_cpu;
- unsigned long flags;
-
- x2apic_wrmsr_fence();
-
- local_irq_save(flags);
- for_each_cpu(query_cpu, mask) {
- if (query_cpu == this_cpu)
- continue;
- __x2apic_send_IPI_dest(
- per_cpu(x86_cpu_to_logical_apicid, query_cpu),
- vector, apic->dest_logical);
- }
- local_irq_restore(flags);
+ __x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLBUT);
}
static void x2apic_send_IPI_allbutself(int vector)
{
- unsigned long this_cpu = smp_processor_id();
- unsigned long query_cpu;
- unsigned long flags;
-
- x2apic_wrmsr_fence();
-
- local_irq_save(flags);
- for_each_online_cpu(query_cpu) {
- if (query_cpu == this_cpu)
- continue;
- __x2apic_send_IPI_dest(
- per_cpu(x86_cpu_to_logical_apicid, query_cpu),
- vector, apic->dest_logical);
- }
- local_irq_restore(flags);
+ __x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLBUT);
}
static void x2apic_send_IPI_all(int vector)
{
- x2apic_send_IPI_mask(cpu_online_mask, vector);
+ __x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLINC);
}
static int x2apic_apic_id_registered(void)
Index: linux-2.6-tip/arch/x86/kernel/apic/x2apic_phys.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/x2apic_phys.c
+++ linux-2.6-tip/arch/x86/kernel/apic/x2apic_phys.c
@@ -55,61 +55,46 @@ static void __x2apic_send_IPI_dest(unsig
native_x2apic_icr_write(cfg, apicid);
}
-static void x2apic_send_IPI_mask(const struct cpumask *mask, int vector)
+static void
+__x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest)
{
unsigned long query_cpu;
+ unsigned long this_cpu;
unsigned long flags;
x2apic_wrmsr_fence();
local_irq_save(flags);
+
+ this_cpu = smp_processor_id();
for_each_cpu(query_cpu, mask) {
+ if (apic_dest == APIC_DEST_ALLBUT && this_cpu == query_cpu)
+ continue;
__x2apic_send_IPI_dest(per_cpu(x86_cpu_to_apicid, query_cpu),
vector, APIC_DEST_PHYSICAL);
}
local_irq_restore(flags);
}
+static void x2apic_send_IPI_mask(const struct cpumask *mask, int vector)
+{
+ __x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLINC);
+}
+
static void
x2apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector)
{
- unsigned long this_cpu = smp_processor_id();
- unsigned long query_cpu;
- unsigned long flags;
-
- x2apic_wrmsr_fence();
-
- local_irq_save(flags);
- for_each_cpu(query_cpu, mask) {
- if (query_cpu != this_cpu)
- __x2apic_send_IPI_dest(
- per_cpu(x86_cpu_to_apicid, query_cpu),
- vector, APIC_DEST_PHYSICAL);
- }
- local_irq_restore(flags);
+ __x2apic_send_IPI_mask(mask, vector, APIC_DEST_ALLBUT);
}
static void x2apic_send_IPI_allbutself(int vector)
{
- unsigned long this_cpu = smp_processor_id();
- unsigned long query_cpu;
- unsigned long flags;
-
- x2apic_wrmsr_fence();
-
- local_irq_save(flags);
- for_each_online_cpu(query_cpu) {
- if (query_cpu == this_cpu)
- continue;
- __x2apic_send_IPI_dest(per_cpu(x86_cpu_to_apicid, query_cpu),
- vector, APIC_DEST_PHYSICAL);
- }
- local_irq_restore(flags);
+ __x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLBUT);
}
static void x2apic_send_IPI_all(int vector)
{
- x2apic_send_IPI_mask(cpu_online_mask, vector);
+ __x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLINC);
}
static int x2apic_apic_id_registered(void)
^ permalink raw reply [flat|nested] 14+ messages in thread* [patch 3/5] x86, x2apic: Track the x2apic cluster sibling map
2011-05-13 18:04 [patch 0/5] x2apic cluster optimization patches Suresh Siddha
2011-05-13 18:04 ` [patch 1/5] x84_64, apic: Use probe routines to simplify apic selection Suresh Siddha
2011-05-13 18:04 ` [patch 2/5] x86, x2apic: Remove duplicate code for IPI mask routines Suresh Siddha
@ 2011-05-13 18:04 ` Suresh Siddha
2011-05-13 18:04 ` [patch 4/5] x86, x2apic: Minimize IPI register writes using cluster groups Suresh Siddha
2011-05-13 18:04 ` [patch 5/5] x86, x2apic: Move the common bits to x2apic.h Suresh Siddha
4 siblings, 0 replies; 14+ messages in thread
From: Suresh Siddha @ 2011-05-13 18:04 UTC (permalink / raw)
To: mingo, tglx, hpa, steiner, yinghai; +Cc: linux-kernel, gorcunov, Suresh Siddha
[-- Attachment #1: track_x2apic_cluster_info.patch --]
[-- Type: text/plain, Size: 3396 bytes --]
From: Cyrill Gorcunov <gorcunov@openvz.org>
Subject: x86, x2apic: Track the x2apic cluster sibling map
In the case of x2apic cluster mode, we can group IPI register writes based on
the cluster group instead of individual per-cpu destination messages.
For this purpose, track the cpu's that belong to the same x2apic cluster.
Signed-off-by: Cyrill Gorcunov <gorcunov@openvz.org>
Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
---
arch/x86/kernel/apic/x2apic_cluster.c | 72 +++++++++++++++++++++++++++++++++-
1 file changed, 70 insertions(+), 2 deletions(-)
Index: linux-2.6-tip/arch/x86/kernel/apic/x2apic_cluster.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/x2apic_cluster.c
+++ linux-2.6-tip/arch/x86/kernel/apic/x2apic_cluster.c
@@ -11,6 +11,7 @@
#include <asm/ipi.h>
static DEFINE_PER_CPU(u32, x86_cpu_to_logical_apicid);
+static DEFINE_PER_CPU(cpumask_var_t, cpus_in_cluster);
static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
{
@@ -48,6 +49,11 @@ static void
native_x2apic_icr_write(cfg, apicid);
}
+static inline u32 x2apic_cluster(int cpu)
+{
+ return per_cpu(x86_cpu_to_logical_apicid, cpu) >> 16;
+}
+
/*
* for now, we send the IPI's one by one in the cpumask.
* TBD: Based on the cpu mask, we can send the IPI's to the cluster group
@@ -163,14 +169,76 @@ static void x2apic_send_IPI_self(int vec
static void init_x2apic_ldr(void)
{
+ unsigned int this_cpu = smp_processor_id();
+ unsigned int cpu;
+
+ per_cpu(x86_cpu_to_logical_apicid, this_cpu) = apic_read(APIC_LDR);
+
+ __cpu_set(this_cpu, per_cpu(cpus_in_cluster, this_cpu));
+ for_each_online_cpu(cpu) {
+ if (x2apic_cluster(this_cpu) != x2apic_cluster(cpu))
+ continue;
+ __cpu_set(this_cpu, per_cpu(cpus_in_cluster, cpu));
+ __cpu_set(cpu, per_cpu(cpus_in_cluster, this_cpu));
+ }
+}
+
+ /*
+ * At CPU state changes, update the x2apic cluster sibling info.
+ */
+static int __cpuinit
+update_clusterinfo(struct notifier_block *nfb, unsigned long action, void *hcpu)
+{
+ unsigned int this_cpu = (unsigned long)hcpu;
+ unsigned int cpu;
+ int err = 0;
+
+ switch (action) {
+ case CPU_UP_PREPARE:
+ if (!zalloc_cpumask_var(&per_cpu(cpus_in_cluster, this_cpu),
+ GFP_KERNEL)) {
+ err = -ENOMEM;
+ }
+ break;
+ case CPU_UP_CANCELED:
+ case CPU_UP_CANCELED_FROZEN:
+ case CPU_DEAD:
+ for_each_online_cpu(cpu) {
+ if (x2apic_cluster(this_cpu) != x2apic_cluster(cpu))
+ continue;
+ __cpu_clear(this_cpu, per_cpu(cpus_in_cluster, cpu));
+ __cpu_clear(cpu, per_cpu(cpus_in_cluster, this_cpu));
+ }
+ free_cpumask_var(per_cpu(cpus_in_cluster, this_cpu));
+ break;
+ }
+
+ return notifier_from_errno(err);
+}
+
+static struct notifier_block __refdata x2apic_cpu_notifier = {
+ .notifier_call = update_clusterinfo,
+};
+
+static int x2apic_init_cpu_notifier(void)
+{
int cpu = smp_processor_id();
- per_cpu(x86_cpu_to_logical_apicid, cpu) = apic_read(APIC_LDR);
+ zalloc_cpumask_var(&per_cpu(cpus_in_cluster, cpu), GFP_KERNEL);
+
+ BUG_ON(!per_cpu(cpus_in_cluster, cpu));
+
+ __cpu_set(cpu, per_cpu(cpus_in_cluster, cpu));
+ register_hotcpu_notifier(&x2apic_cpu_notifier);
+ return 1;
}
static int x2apic_cluster_probe(void)
{
- return x2apic_mode;
+ if (x2apic_mode)
+ return x2apic_init_cpu_notifier();
+ else
+ return 0;
}
struct apic apic_x2apic_cluster = {
^ permalink raw reply [flat|nested] 14+ messages in thread* [patch 4/5] x86, x2apic: Minimize IPI register writes using cluster groups
2011-05-13 18:04 [patch 0/5] x2apic cluster optimization patches Suresh Siddha
` (2 preceding siblings ...)
2011-05-13 18:04 ` [patch 3/5] x86, x2apic: Track the x2apic cluster sibling map Suresh Siddha
@ 2011-05-13 18:04 ` Suresh Siddha
2011-05-13 18:04 ` [patch 5/5] x86, x2apic: Move the common bits to x2apic.h Suresh Siddha
4 siblings, 0 replies; 14+ messages in thread
From: Suresh Siddha @ 2011-05-13 18:04 UTC (permalink / raw)
To: mingo, tglx, hpa, steiner, yinghai; +Cc: linux-kernel, gorcunov, Suresh Siddha
[-- Attachment #1: minimize_IPI_register_writes_using_cluster_groups.patch --]
[-- Type: text/plain, Size: 4483 bytes --]
From: Cyrill Gorcunov <gorcunov@openvz.org>
Subjct: x86, x2apic: Minimize IPI register writes using cluster groups
In the case of x2apic cluster mode we can group IPI register writes based
on the cluster group instead of individual per-cpu destination messages.
This reduces the apic register writes and reduces the amount of IPI messages
(in the best case we can reduce it by a factor of 16).
With this change, the cost of flush_tlb_others(), with the flush tlb IPI being
sent from a cpu in the socket-1 to all the logical cpus in socket-2 (on a
Westmere-EX system that has 20 logical cpus in a socket) is 3x times better now
(compared to the former 'send one-by-one' algorithm).
Signed-off-by: Cyrill Gorcunov <gorcunov@openvz.org>
Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
---
arch/x86/kernel/apic/x2apic_cluster.c | 58 +++++++++++++++++++++++++---------
1 file changed, 44 insertions(+), 14 deletions(-)
Index: linux-2.6-tip/arch/x86/kernel/apic/x2apic_cluster.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/x2apic_cluster.c
+++ linux-2.6-tip/arch/x86/kernel/apic/x2apic_cluster.c
@@ -5,6 +5,7 @@
#include <linux/ctype.h>
#include <linux/init.h>
#include <linux/dmar.h>
+#include <linux/cpu.h>
#include <asm/smp.h>
#include <asm/apic.h>
@@ -12,6 +13,7 @@
static DEFINE_PER_CPU(u32, x86_cpu_to_logical_apicid);
static DEFINE_PER_CPU(cpumask_var_t, cpus_in_cluster);
+static DEFINE_PER_CPU(cpumask_var_t, ipi_mask);
static int x2apic_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
{
@@ -54,30 +56,52 @@ static inline u32 x2apic_cluster(int cpu
return per_cpu(x86_cpu_to_logical_apicid, cpu) >> 16;
}
-/*
- * for now, we send the IPI's one by one in the cpumask.
- * TBD: Based on the cpu mask, we can send the IPI's to the cluster group
- * at once. We have 16 cpu's in a cluster. This will minimize IPI register
- * writes.
- */
static void
__x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest)
{
- unsigned long query_cpu;
- unsigned long this_cpu;
+ struct cpumask *cpus_in_cluster_ptr;
+ struct cpumask *ipi_mask_ptr;
+ unsigned int cpu, this_cpu;
unsigned long flags;
+ u32 dest;
x2apic_wrmsr_fence();
local_irq_save(flags);
this_cpu = smp_processor_id();
- for_each_cpu(query_cpu, mask) {
- if (apic_dest == APIC_DEST_ALLBUT && query_cpu == this_cpu)
+
+ /*
+ * We are to modify mask, so we need an own copy
+ * and be sure it's manipulated with irq off.
+ */
+ ipi_mask_ptr = __raw_get_cpu_var(ipi_mask);
+ cpumask_copy(ipi_mask_ptr, mask);
+
+ /*
+ * The idea is to send one IPI per cluster.
+ */
+ for_each_cpu(cpu, ipi_mask_ptr) {
+ unsigned long i;
+
+ cpus_in_cluster_ptr = per_cpu(cpus_in_cluster, cpu);
+ dest = 0;
+
+ /* Collect cpus in cluster. */
+ for_each_cpu_and(i, ipi_mask_ptr, cpus_in_cluster_ptr) {
+ if (apic_dest == APIC_DEST_ALLINC || i != this_cpu)
+ dest |= per_cpu(x86_cpu_to_logical_apicid, i);
+ }
+
+ if (!dest)
continue;
- __x2apic_send_IPI_dest(
- per_cpu(x86_cpu_to_logical_apicid, query_cpu),
- vector, apic->dest_logical);
+
+ __x2apic_send_IPI_dest(dest, vector, apic->dest_logical);
+ /*
+ * Cluster sibling cpus should be discared now so
+ * we would not send IPI them second time.
+ */
+ cpumask_andnot(ipi_mask_ptr, ipi_mask_ptr, cpus_in_cluster_ptr);
}
local_irq_restore(flags);
@@ -198,6 +222,10 @@ update_clusterinfo(struct notifier_block
if (!zalloc_cpumask_var(&per_cpu(cpus_in_cluster, this_cpu),
GFP_KERNEL)) {
err = -ENOMEM;
+ } else if (!zalloc_cpumask_var(&per_cpu(ipi_mask, this_cpu),
+ GFP_KERNEL)) {
+ free_cpumask_var(per_cpu(cpus_in_cluster, this_cpu));
+ err = -ENOMEM;
}
break;
case CPU_UP_CANCELED:
@@ -210,6 +238,7 @@ update_clusterinfo(struct notifier_block
__cpu_clear(cpu, per_cpu(cpus_in_cluster, this_cpu));
}
free_cpumask_var(per_cpu(cpus_in_cluster, this_cpu));
+ free_cpumask_var(per_cpu(ipi_mask, this_cpu));
break;
}
@@ -225,8 +254,9 @@ static int x2apic_init_cpu_notifier(void
int cpu = smp_processor_id();
zalloc_cpumask_var(&per_cpu(cpus_in_cluster, cpu), GFP_KERNEL);
+ zalloc_cpumask_var(&per_cpu(ipi_mask, cpu), GFP_KERNEL);
- BUG_ON(!per_cpu(cpus_in_cluster, cpu));
+ BUG_ON(!per_cpu(cpus_in_cluster, cpu) || !per_cpu(ipi_mask, cpu));
__cpu_set(cpu, per_cpu(cpus_in_cluster, cpu));
register_hotcpu_notifier(&x2apic_cpu_notifier);
^ permalink raw reply [flat|nested] 14+ messages in thread* [patch 5/5] x86, x2apic: Move the common bits to x2apic.h
2011-05-13 18:04 [patch 0/5] x2apic cluster optimization patches Suresh Siddha
` (3 preceding siblings ...)
2011-05-13 18:04 ` [patch 4/5] x86, x2apic: Minimize IPI register writes using cluster groups Suresh Siddha
@ 2011-05-13 18:04 ` Suresh Siddha
4 siblings, 0 replies; 14+ messages in thread
From: Suresh Siddha @ 2011-05-13 18:04 UTC (permalink / raw)
To: mingo, tglx, hpa, steiner, yinghai; +Cc: linux-kernel, gorcunov, Suresh Siddha
[-- Attachment #1: move_the_common_bits_of_physical_and_cluster_modes_to_x2apic.patch --]
[-- Type: text/plain, Size: 7066 bytes --]
From: Cyrill Gorcunov <gorcunov@openvz.org>
Subjct: x86, x2apic: Move the common bits to x2apic.h
To eliminate code duplication.
Signed-off-by: Cyrill Gorcunov <gorcunov@openvz.org>
Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
---
arch/x86/include/asm/x2apic.h | 62 +++++++++++++++++++++++++++++
arch/x86/kernel/apic/x2apic_cluster.c | 71 +---------------------------------
arch/x86/kernel/apic/x2apic_phys.c | 60 +---------------------------
3 files changed, 69 insertions(+), 124 deletions(-)
Index: linux-2.6-tip/arch/x86/include/asm/x2apic.h
===================================================================
--- /dev/null
+++ linux-2.6-tip/arch/x86/include/asm/x2apic.h
@@ -0,0 +1,62 @@
+/*
+ * Common bits for X2APIC cluster/physical modes.
+ */
+
+#ifndef _ASM_X86_X2APIC_H
+#define _ASM_X86_X2APIC_H
+
+#include <asm/apic.h>
+#include <asm/ipi.h>
+#include <linux/cpumask.h>
+
+/*
+ * Need to use more than cpu 0, because we need more vectors
+ * when MSI-X are used.
+ */
+static const struct cpumask *x2apic_target_cpus(void)
+{
+ return cpu_online_mask;
+}
+
+static int x2apic_apic_id_registered(void)
+{
+ return 1;
+}
+
+/*
+ * For now each logical cpu is in its own vector allocation domain.
+ */
+static void x2apic_vector_allocation_domain(int cpu, struct cpumask *retmask)
+{
+ cpumask_clear(retmask);
+ cpumask_set_cpu(cpu, retmask);
+}
+
+static void
+__x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest)
+{
+ unsigned long cfg = __prepare_ICR(0, vector, dest);
+ native_x2apic_icr_write(cfg, apicid);
+}
+
+static unsigned int x2apic_get_apic_id(unsigned long id)
+{
+ return id;
+}
+
+static unsigned long x2apic_set_apic_id(unsigned int id)
+{
+ return id;
+}
+
+static int x2apic_phys_pkg_id(int initial_apicid, int index_msb)
+{
+ return initial_apicid >> index_msb;
+}
+
+static void x2apic_send_IPI_self(int vector)
+{
+ apic_write(APIC_SELF_IPI, vector);
+}
+
+#endif /* _ASM_X86_X2APIC_H */
Index: linux-2.6-tip/arch/x86/kernel/apic/x2apic_cluster.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/x2apic_cluster.c
+++ linux-2.6-tip/arch/x86/kernel/apic/x2apic_cluster.c
@@ -8,8 +8,7 @@
#include <linux/cpu.h>
#include <asm/smp.h>
-#include <asm/apic.h>
-#include <asm/ipi.h>
+#include <asm/x2apic.h>
static DEFINE_PER_CPU(u32, x86_cpu_to_logical_apicid);
static DEFINE_PER_CPU(cpumask_var_t, cpus_in_cluster);
@@ -20,37 +19,6 @@ static int x2apic_acpi_madt_oem_check(ch
return x2apic_enabled();
}
-/*
- * need to use more than cpu 0, because we need more vectors when
- * MSI-X are used.
- */
-static const struct cpumask *x2apic_target_cpus(void)
-{
- return cpu_online_mask;
-}
-
-/*
- * for now each logical cpu is in its own vector allocation domain.
- */
-static void x2apic_vector_allocation_domain(int cpu, struct cpumask *retmask)
-{
- cpumask_clear(retmask);
- cpumask_set_cpu(cpu, retmask);
-}
-
-static void
- __x2apic_send_IPI_dest(unsigned int apicid, int vector, unsigned int dest)
-{
- unsigned long cfg;
-
- cfg = __prepare_ICR(0, vector, dest);
-
- /*
- * send the IPI.
- */
- native_x2apic_icr_write(cfg, apicid);
-}
-
static inline u32 x2apic_cluster(int cpu)
{
return per_cpu(x86_cpu_to_logical_apicid, cpu) >> 16;
@@ -128,11 +96,6 @@ static void x2apic_send_IPI_all(int vect
__x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLINC);
}
-static int x2apic_apic_id_registered(void)
-{
- return 1;
-}
-
static unsigned int x2apic_cpu_mask_to_apicid(const struct cpumask *cpumask)
{
/*
@@ -165,32 +128,6 @@ x2apic_cpu_mask_to_apicid_and(const stru
return per_cpu(x86_cpu_to_logical_apicid, cpu);
}
-static unsigned int x2apic_cluster_phys_get_apic_id(unsigned long x)
-{
- unsigned int id;
-
- id = x;
- return id;
-}
-
-static unsigned long set_apic_id(unsigned int id)
-{
- unsigned long x;
-
- x = id;
- return x;
-}
-
-static int x2apic_cluster_phys_pkg_id(int initial_apicid, int index_msb)
-{
- return initial_apicid >> index_msb;
-}
-
-static void x2apic_send_IPI_self(int vector)
-{
- apic_write(APIC_SELF_IPI, vector);
-}
-
static void init_x2apic_ldr(void)
{
unsigned int this_cpu = smp_processor_id();
@@ -298,11 +235,11 @@ struct apic apic_x2apic_cluster = {
.setup_portio_remap = NULL,
.check_phys_apicid_present = default_check_phys_apicid_present,
.enable_apic_mode = NULL,
- .phys_pkg_id = x2apic_cluster_phys_pkg_id,
+ .phys_pkg_id = x2apic_phys_pkg_id,
.mps_oem_check = NULL,
- .get_apic_id = x2apic_cluster_phys_get_apic_id,
- .set_apic_id = set_apic_id,
+ .get_apic_id = x2apic_get_apic_id,
+ .set_apic_id = x2apic_set_apic_id,
.apic_id_mask = 0xFFFFFFFFu,
.cpu_mask_to_apicid = x2apic_cpu_mask_to_apicid,
Index: linux-2.6-tip/arch/x86/kernel/apic/x2apic_phys.c
===================================================================
--- linux-2.6-tip.orig/arch/x86/kernel/apic/x2apic_phys.c
+++ linux-2.6-tip/arch/x86/kernel/apic/x2apic_phys.c
@@ -7,8 +7,7 @@
#include <linux/dmar.h>
#include <asm/smp.h>
-#include <asm/apic.h>
-#include <asm/ipi.h>
+#include <asm/x2apic.h>
int x2apic_phys;
@@ -27,34 +26,6 @@ static int x2apic_acpi_madt_oem_check(ch
return 0;
}
-/*
- * need to use more than cpu 0, because we need more vectors when
- * MSI-X are used.
- */
-static const struct cpumask *x2apic_target_cpus(void)
-{
- return cpu_online_mask;
-}
-
-static void x2apic_vector_allocation_domain(int cpu, struct cpumask *retmask)
-{
- cpumask_clear(retmask);
- cpumask_set_cpu(cpu, retmask);
-}
-
-static void __x2apic_send_IPI_dest(unsigned int apicid, int vector,
- unsigned int dest)
-{
- unsigned long cfg;
-
- cfg = __prepare_ICR(0, vector, dest);
-
- /*
- * send the IPI.
- */
- native_x2apic_icr_write(cfg, apicid);
-}
-
static void
__x2apic_send_IPI_mask(const struct cpumask *mask, int vector, int apic_dest)
{
@@ -97,11 +68,6 @@ static void x2apic_send_IPI_all(int vect
__x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLINC);
}
-static int x2apic_apic_id_registered(void)
-{
- return 1;
-}
-
static unsigned int x2apic_cpu_mask_to_apicid(const struct cpumask *cpumask)
{
/*
@@ -134,26 +100,6 @@ x2apic_cpu_mask_to_apicid_and(const stru
return per_cpu(x86_cpu_to_apicid, cpu);
}
-static unsigned int x2apic_phys_get_apic_id(unsigned long x)
-{
- return x;
-}
-
-static unsigned long set_apic_id(unsigned int id)
-{
- return id;
-}
-
-static int x2apic_phys_pkg_id(int initial_apicid, int index_msb)
-{
- return initial_apicid >> index_msb;
-}
-
-static void x2apic_send_IPI_self(int vector)
-{
- apic_write(APIC_SELF_IPI, vector);
-}
-
static void init_x2apic_ldr(void)
{
}
@@ -196,8 +142,8 @@ struct apic apic_x2apic_phys = {
.phys_pkg_id = x2apic_phys_pkg_id,
.mps_oem_check = NULL,
- .get_apic_id = x2apic_phys_get_apic_id,
- .set_apic_id = set_apic_id,
+ .get_apic_id = x2apic_get_apic_id,
+ .set_apic_id = x2apic_set_apic_id,
.apic_id_mask = 0xFFFFFFFFu,
.cpu_mask_to_apicid = x2apic_cpu_mask_to_apicid,
^ permalink raw reply [flat|nested] 14+ messages in thread