[PATCH 0/3] x86/cpumask: fixups and additions

[PATCH 0/3] x86/cpumask: fixups and additions

[Mike Travis]

The following patches are included:

  * x86: fix cpu_mask_to_apicid_and to include cpu_online_mask.
	Fixes potential problem when destination apicid of an
	offline cpu is returned.

  * x86: use possible_cpus=NUM to extend the number of possible cpus.
	This conforms with the existing s390 argument.

  * cpumask: add sysfs displays for configured and disabled cpu maps
	Display both the configured kernel_max cpus (NR_CPUS-1) as well
	as the cpus in the system that are disabled because they exceed
	the NR_CPUS limit.

Based on tip/cpus4096 + .../rusty/linux-2.6-for-ingo.git/master +
	 x86-only-patches sent 12/15.

Signed-off-by: Mike Travis <travis@sgi.com>

-- 

[PATCH 1/3] x86: fix cpu_mask_to_apicid_and to include cpu_online_mask

[Mike Travis]

[-- Attachment #1: x86:fix-cpu_mask_to_apicid_and.patch --]
[-- Type: text/plain, Size: 8700 bytes --]

Impact: fix potential problem.

In determining the destination apicid, there are usually three cpumasks
that are considered: the incoming cpumask arg, cfg->domain and the
cpu_online_mask.  Since we are just introducing the cpu_mask_to_apicid_and
function, make sure it includes the cpu_online_mask in it's evaluation.
[Added with this patch.]

There are two io_apic.c functions that did not previously use the
cpu_online_mask:  setup_IO_APIC_irq and msi_compose_msg.  Both of these
simply used cpu_mask_to_apicid(cfg->domain & TARGET_CPUS), and all but
one arch (NUMAQ[*]) returns only online cpus in the TARGET_CPUS mask,
so the behavior is identical for all cases.

[*-This sounds like an error in NUMAQ?  Or perhaps it cannot offline cpus?]

Note that alloc_cpumask_var is only used for the 32-bit cases where
it's highly likely that the cpumask set size will be small and therefore
CPUMASK_OFFSTACK=n.  But if that's not the case, failing the allocate
will cause the same return value as the default.

Based on tip/cpus4096 + .../rusty/linux-2.6-for-ingo.git/master +
	 x86-only-patches sent 12/15.

Signed-off-by: Mike Travis <travis@sgi.com>
---
 arch/x86/include/asm/bigsmp/apic.h            |    4 +-
 arch/x86/include/asm/es7000/apic.h            |   40 +++++++++++---------------
 arch/x86/include/asm/mach-default/mach_apic.h |    3 +
 arch/x86/include/asm/summit/apic.h            |   30 +++++++++++--------
 arch/x86/kernel/genapic_flat_64.c             |    4 +-
 arch/x86/kernel/genx2apic_cluster.c           |    4 +-
 arch/x86/kernel/genx2apic_phys.c              |    4 +-
 arch/x86/kernel/genx2apic_uv_x.c              |    4 +-
 8 files changed, 52 insertions(+), 41 deletions(-)

--- linux-2.6-for-ingo.orig/arch/x86/include/asm/bigsmp/apic.h
+++ linux-2.6-for-ingo/arch/x86/include/asm/bigsmp/apic.h
@@ -138,7 +138,9 @@ static inline unsigned int cpu_mask_to_a
 	 * We're using fixed IRQ delivery, can only return one phys APIC ID.
 	 * May as well be the first.
 	 */
-	cpu = cpumask_any_and(cpumask, andmask);
+	for_each_cpu_and(cpu, cpumask, andmask)
+		if (cpumask_test_cpu(cpu, cpu_online_mask))
+			break;
 	if (cpu < nr_cpu_ids)
 		return cpu_to_logical_apicid(cpu);
 
--- linux-2.6-for-ingo.orig/arch/x86/include/asm/es7000/apic.h
+++ linux-2.6-for-ingo/arch/x86/include/asm/es7000/apic.h
@@ -214,51 +214,47 @@ static inline unsigned int cpu_mask_to_a
 	return apicid;
 }
 
-static inline unsigned int cpu_mask_to_apicid_and(const struct cpumask *cpumask,
+
+static inline unsigned int cpu_mask_to_apicid_and(const struct cpumask *inmask,
 						  const struct cpumask *andmask)
 {
 	int num_bits_set;
-	int num_bits_set2;
 	int cpus_found = 0;
 	int cpu;
-	int apicid = 0;
+	int apicid = cpu_to_logical_apicid(0);
+	cpumask_var_t cpumask;
+
+	if (!alloc_cpumask_var(&cpumask, GFP_ATOMIC))
+		return apicid;
+
+	cpumask_and(cpumask, inmask, andmask);
+	cpumask_and(cpumask, cpumask, cpu_online_mask);
 
 	num_bits_set = cpumask_weight(cpumask);
-	num_bits_set2 = cpumask_weight(andmask);
-	num_bits_set = min(num_bits_set, num_bits_set2);
 	/* Return id to all */
-	if (num_bits_set >= nr_cpu_ids)
-#if defined CONFIG_ES7000_CLUSTERED_APIC
-		return 0xFF;
-#else
-		return cpu_to_logical_apicid(0);
-#endif
+	if (num_bits_set == NR_CPUS)
+		goto exit;
 	/*
 	 * The cpus in the mask must all be on the apic cluster.  If are not
 	 * on the same apicid cluster return default value of TARGET_CPUS.
 	 */
-	cpu = cpumask_first_and(cpumask, andmask);
+	cpu = cpumask_first(cpumask);
 	apicid = cpu_to_logical_apicid(cpu);
-
 	while (cpus_found < num_bits_set) {
-		if (cpumask_test_cpu(cpu, cpumask) &&
-		    cpumask_test_cpu(cpu, andmask)) {
+		if (cpumask_test_cpu(cpu, cpumask)) {
 			int new_apicid = cpu_to_logical_apicid(cpu);
 			if (apicid_cluster(apicid) !=
-					apicid_cluster(new_apicid)) {
-				printk(KERN_WARNING
-					"%s: Not a valid mask!\n", __func__);
-#if defined CONFIG_ES7000_CLUSTERED_APIC
-				return 0xFF;
-#else
+					apicid_cluster(new_apicid)){
+				printk ("%s: Not a valid mask!\n", __func__);
 				return cpu_to_logical_apicid(0);
-#endif
 			}
 			apicid = new_apicid;
 			cpus_found++;
 		}
 		cpu++;
 	}
+exit:
+	free_cpumask_var(cpumask);
 	return apicid;
 }
 
--- linux-2.6-for-ingo.orig/arch/x86/include/asm/mach-default/mach_apic.h
+++ linux-2.6-for-ingo/arch/x86/include/asm/mach-default/mach_apic.h
@@ -72,8 +72,9 @@ static inline unsigned int cpu_mask_to_a
 {
 	unsigned long mask1 = cpumask_bits(cpumask)[0];
 	unsigned long mask2 = cpumask_bits(andmask)[0];
+	unsigned long mask3 = cpumask_bits(cpu_online_mask)[0];
 
-	return (unsigned int)(mask1 & mask2);
+	return (unsigned int)(mask1 & mask2 & mask3);
 }
 
 static inline u32 phys_pkg_id(u32 cpuid_apic, int index_msb)
--- linux-2.6-for-ingo.orig/arch/x86/include/asm/summit/apic.h
+++ linux-2.6-for-ingo/arch/x86/include/asm/summit/apic.h
@@ -170,35 +170,37 @@ static inline unsigned int cpu_mask_to_a
 	return apicid;
 }
 
-static inline unsigned int cpu_mask_to_apicid_and(const struct cpumask *cpumask,
+static inline unsigned int cpu_mask_to_apicid_and(const struct cpumask *inmask,
 						  const struct cpumask *andmask)
 {
 	int num_bits_set;
-	int num_bits_set2;
 	int cpus_found = 0;
 	int cpu;
-	int apicid = 0;
+	int apicid = 0xFF;
+	cpumask_var_t cpumask;
+
+	if (!alloc_cpumask_var(&cpumask, GFP_ATOMIC))
+		return (int) 0xFF;
+
+	cpumask_and(cpumask, inmask, andmask);
+	cpumask_and(cpumask, cpumask, cpu_online_mask);
 
 	num_bits_set = cpumask_weight(cpumask);
-	num_bits_set2 = cpumask_weight(andmask);
-	num_bits_set = min(num_bits_set, num_bits_set2);
 	/* Return id to all */
-	if (num_bits_set >= nr_cpu_ids)
-		return 0xFF;
+	if (num_bits_set == nr_cpu_ids)
+		goto exit;
 	/*
 	 * The cpus in the mask must all be on the apic cluster.  If are not
 	 * on the same apicid cluster return default value of TARGET_CPUS.
 	 */
-	cpu = cpumask_first_and(cpumask, andmask);
+	cpu = cpumask_first(cpumask);
 	apicid = cpu_to_logical_apicid(cpu);
 	while (cpus_found < num_bits_set) {
-		if (cpumask_test_cpu(cpu, cpumask)
-		    && cpumask_test_cpu(cpu, andmask)) {
+		if (cpumask_test_cpu(cpu, cpumask)) {
 			int new_apicid = cpu_to_logical_apicid(cpu);
 			if (apicid_cluster(apicid) !=
-					apicid_cluster(new_apicid)) {
-				printk(KERN_WARNING
-					"%s: Not a valid mask!\n", __func__);
+					apicid_cluster(new_apicid)){
+				printk ("%s: Not a valid mask!\n", __func__);
 				return 0xFF;
 			}
 			apicid = apicid | new_apicid;
@@ -206,6 +208,8 @@ static inline unsigned int cpu_mask_to_a
 		}
 		cpu++;
 	}
+exit:
+	free_cpumask_var(cpumask);
 	return apicid;
 }
 
--- linux-2.6-for-ingo.orig/arch/x86/kernel/genapic_flat_64.c
+++ linux-2.6-for-ingo/arch/x86/kernel/genapic_flat_64.c
@@ -276,7 +276,9 @@ physflat_cpu_mask_to_apicid_and(const st
 	 * We're using fixed IRQ delivery, can only return one phys APIC ID.
 	 * May as well be the first.
 	 */
-	cpu = cpumask_any_and(cpumask, andmask);
+	for_each_cpu_and(cpu, cpumask, andmask)
+		if (cpumask_test_cpu(cpu, cpu_online_mask))
+			break;
 	if (cpu < nr_cpu_ids)
 		return per_cpu(x86_cpu_to_apicid, cpu);
 	return BAD_APICID;
--- linux-2.6-for-ingo.orig/arch/x86/kernel/genx2apic_cluster.c
+++ linux-2.6-for-ingo/arch/x86/kernel/genx2apic_cluster.c
@@ -133,7 +133,9 @@ static unsigned int x2apic_cpu_mask_to_a
 	 * We're using fixed IRQ delivery, can only return one phys APIC ID.
 	 * May as well be the first.
 	 */
-	cpu = cpumask_any_and(cpumask, andmask);
+	for_each_cpu_and(cpu, cpumask, andmask)
+		if (cpumask_test_cpu(cpu, cpu_online_mask))
+			break;
 	if (cpu < nr_cpu_ids)
 		return per_cpu(x86_cpu_to_apicid, cpu);
 	return BAD_APICID;
--- linux-2.6-for-ingo.orig/arch/x86/kernel/genx2apic_phys.c
+++ linux-2.6-for-ingo/arch/x86/kernel/genx2apic_phys.c
@@ -132,7 +132,9 @@ static unsigned int x2apic_cpu_mask_to_a
 	 * We're using fixed IRQ delivery, can only return one phys APIC ID.
 	 * May as well be the first.
 	 */
-	cpu = cpumask_any_and(cpumask, andmask);
+	for_each_cpu_and(cpu, cpumask, andmask)
+		if (cpumask_test_cpu(cpu, cpu_online_mask))
+			break;
 	if (cpu < nr_cpu_ids)
 		return per_cpu(x86_cpu_to_apicid, cpu);
 	return BAD_APICID;
--- linux-2.6-for-ingo.orig/arch/x86/kernel/genx2apic_uv_x.c
+++ linux-2.6-for-ingo/arch/x86/kernel/genx2apic_uv_x.c
@@ -188,7 +188,9 @@ static unsigned int uv_cpu_mask_to_apici
 	 * We're using fixed IRQ delivery, can only return one phys APIC ID.
 	 * May as well be the first.
 	 */
-	cpu = cpumask_any_and(cpumask, andmask);
+	for_each_cpu_and(cpu, cpumask, andmask)
+		if (cpumask_test_cpu(cpu, cpu_online_mask))
+			break;
 	if (cpu < nr_cpu_ids)
 		return per_cpu(x86_cpu_to_apicid, cpu);
 	return BAD_APICID;

-- 

[PATCH 2/3] x86: use possible_cpus=NUM to extend the possible cpus allowed

[Mike Travis]

[-- Attachment #1: x86:add-setup_possible_cpus.patch --]
[-- Type: text/plain, Size: 4875 bytes --]

Impact: allow adding additional cpus.

Use possible_cpus=NUM kernel parameter to extend the number of possible
cpus.

The ability to HOTPLUG ON cpus that are "possible" but not "present" is
dealt with in a later patch.

Based on tip/cpus4096 + .../rusty/linux-2.6-for-ingo.git/master +
	 x86-only-patches sent 12/15.

Signed-off-by: Mike Travis <travis@sgi.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Ashok Raj <ashok.raj@intel.com>
Cc: Rusty Russell <rusty@rustycorp.com.au>
---
 Documentation/cpu-hotplug.txt |   17 +++++++++--------
 arch/x86/kernel/apic.c        |   16 ++++++++++------
 arch/x86/kernel/smpboot.c     |   25 +++++++++++++++++++++----
 3 files changed, 40 insertions(+), 18 deletions(-)

--- linux-2.6-for-ingo.orig/Documentation/cpu-hotplug.txt
+++ linux-2.6-for-ingo/Documentation/cpu-hotplug.txt
@@ -50,16 +50,17 @@ additional_cpus=n (*)	Use this to limit 
   			cpu_possible_map = cpu_present_map + additional_cpus
 
 (*) Option valid only for following architectures
-- x86_64, ia64
+- ia64
 
-ia64 and x86_64 use the number of disabled local apics in ACPI tables MADT
-to determine the number of potentially hot-pluggable cpus. The implementation
-should only rely on this to count the # of cpus, but *MUST* not rely on the
-apicid values in those tables for disabled apics. In the event BIOS doesn't
-mark such hot-pluggable cpus as disabled entries, one could use this
-parameter "additional_cpus=x" to represent those cpus in the cpu_possible_map.
+ia64 uses the number of disabled local apics in ACPI tables MADT to
+determine the number of potentially hot-pluggable cpus. The implementation
+should only rely on this to count the # of cpus, but *MUST* not rely
+on the apicid values in those tables for disabled apics. In the event
+BIOS doesn't mark such hot-pluggable cpus as disabled entries, one could
+use this parameter "additional_cpus=x" to represent those cpus in the
+cpu_possible_map.
 
-possible_cpus=n		[s390 only] use this to set hotpluggable cpus.
+possible_cpus=n		[s390,x86_64] use this to set hotpluggable cpus.
 			This option sets possible_cpus bits in
 			cpu_possible_map. Thus keeping the numbers of bits set
 			constant even if the machine gets rebooted.
--- linux-2.6-for-ingo.orig/arch/x86/kernel/apic.c
+++ linux-2.6-for-ingo/arch/x86/kernel/apic.c
@@ -1832,7 +1832,6 @@ void disconnect_bsp_APIC(int virt_wire_s
 void __cpuinit generic_processor_info(int apicid, int version)
 {
 	int cpu;
-	cpumask_t tmp_map;
 
 	/*
 	 * Validate version
@@ -1845,15 +1844,20 @@ void __cpuinit generic_processor_info(in
 	}
 	apic_version[apicid] = version;
 
-	if (num_processors >= NR_CPUS) {
-		printk(KERN_WARNING "WARNING: NR_CPUS limit of %i reached."
-			"  Processor ignored.\n", NR_CPUS);
+	if (num_processors >= nr_cpu_ids) {
+		int max = nr_cpu_ids;
+		int thiscpu = max + disabled_cpus;
+
+		printk(KERN_WARNING
+			"ACPI: NR_CPUS/possible_cpus limit of %i reached."
+			"  Processor %d/0x%x ignored.\n", max, thiscpu, apicid);
+
+		disabled_cpus++;
 		return;
 	}
 
 	num_processors++;
-	cpus_complement(tmp_map, cpu_present_map);
-	cpu = first_cpu(tmp_map);
+	cpu = cpumask_next_zero(-1, cpu_present_mask);
 
 	physid_set(apicid, phys_cpu_present_map);
 	if (apicid == boot_cpu_physical_apicid) {
--- linux-2.6-for-ingo.orig/arch/x86/kernel/smpboot.c
+++ linux-2.6-for-ingo/arch/x86/kernel/smpboot.c
@@ -1254,6 +1254,15 @@ void __init native_smp_cpus_done(unsigne
 	check_nmi_watchdog();
 }
 
+static int __initdata setup_possible_cpus = -1;
+static int __init _setup_possible_cpus(char *str)
+{
+	get_option(&str, &setup_possible_cpus);
+	return 0;
+}
+early_param("possible_cpus", _setup_possible_cpus);
+
+
 /*
  * cpu_possible_map should be static, it cannot change as cpu's
  * are onlined, or offlined. The reason is per-cpu data-structures
@@ -1266,7 +1275,7 @@ void __init native_smp_cpus_done(unsigne
  *
  * Three ways to find out the number of additional hotplug CPUs:
  * - If the BIOS specified disabled CPUs in ACPI/mptables use that.
- * - The user can overwrite it with additional_cpus=NUM
+ * - The user can overwrite it with possible_cpus=NUM
  * - Otherwise don't reserve additional CPUs.
  * We do this because additional CPUs waste a lot of memory.
  * -AK
@@ -1279,9 +1288,17 @@ __init void prefill_possible_map(void)
 	if (!num_processors)
 		num_processors = 1;
 
-	possible = num_processors + disabled_cpus;
-	if (possible > NR_CPUS)
-		possible = NR_CPUS;
+	if (setup_possible_cpus == -1)
+		possible = num_processors + disabled_cpus;
+	else
+		possible = setup_possible_cpus;
+
+	if (possible > CONFIG_NR_CPUS) {
+		printk(KERN_WARNING
+			"%d Processors exceeds NR_CPUS limit of %d\n",
+			possible, CONFIG_NR_CPUS);
+		possible = CONFIG_NR_CPUS;
+	}
 
 	printk(KERN_INFO "SMP: Allowing %d CPUs, %d hotplug CPUs\n",
 		possible, max_t(int, possible - num_processors, 0));

-- 

[PATCH 3/3] cpumask: add sysfs displays for configured and disabled cpu maps

[Mike Travis]

[-- Attachment #1: cpumask:add-sysfs-files.patch --]
[-- Type: text/plain, Size: 4026 bytes --]

Impact: add new functionality.

Add sysfs files "kernel_max" and "offline" to display the max CPU index
allowed (NR_CPUS-1), and the map of cpus that are offline. 

Cpus can be offlined via HOTPLUG, disabled by the BIOS ACPI tables, or
if they exceed the number of cpus allowed by the NR_CPUS config option,
or the "maxcpus=NUM" kernel start parameter.

The "possible_cpus=NUM" parameter can also extend the number of possible
cpus allowed, in which case the cpus not present at startup will be
in the offline state.  (These cpus can be HOTPLUGGED ON after system
startup [pending a follow-on patch to provide the capability via the
/sys/devices/sys/cpu/cpuN/online mechanism to bring them online.])

By design, the "offlined cpus > possible cpus" display will always
use the following formats:

  * all possible cpus online:   "x$"    or "x-y$" 
  * some possible cpus offline: ".*,x$" or ".*,x-y$"

where:
  x == number of possible cpus (nr_cpu_ids); and
  y == number of cpus >= NR_CPUS or maxcpus (if y > x).

One use of this feature is for distros to select (or configure) the
appropriate kernel to install for the resident system.

Notes:
  * cpus offlined <= possible cpus will be printed for all architectures.
  * cpus offlined >  possible cpus will only be printed for arches that
  	set 'total_cpus' [X86 only in this patch].

Based on tip/cpus4096 + .../rusty/linux-2.6-for-ingo.git/master +
	 x86-only-patches sent 12/15.

Signed-off-by: Mike Travis <travis@sgi.com>
Acked-by: Rusty Russell <rusty@rustcorp.com.au>
---
 arch/x86/kernel/smpboot.c |    2 ++
 drivers/base/cpu.c        |   45 +++++++++++++++++++++++++++++++++++++++++++++
 include/linux/smp.h       |    3 +++
 3 files changed, 50 insertions(+)

--- linux-2.6-for-ingo.orig/arch/x86/kernel/smpboot.c
+++ linux-2.6-for-ingo/arch/x86/kernel/smpboot.c
@@ -1293,6 +1293,8 @@ __init void prefill_possible_map(void)
 	else
 		possible = setup_possible_cpus;
 
+	total_cpus = max_t(int, possible, num_processors + disabled_cpus);
+
 	if (possible > CONFIG_NR_CPUS) {
 		printk(KERN_WARNING
 			"%d Processors exceeds NR_CPUS limit of %d\n",
--- linux-2.6-for-ingo.orig/drivers/base/cpu.c
+++ linux-2.6-for-ingo/drivers/base/cpu.c
@@ -128,10 +128,55 @@ print_cpus_func(online);
 print_cpus_func(possible);
 print_cpus_func(present);
 
+/*
+ * Print values for NR_CPUS and offlined cpus
+ */
+static ssize_t print_cpus_kernel_max(struct sysdev_class *class, char *buf)
+{
+	int n = snprintf(buf, PAGE_SIZE-2, "%d\n", CONFIG_NR_CPUS - 1);
+	return n;
+}
+static SYSDEV_CLASS_ATTR(kernel_max, 0444, print_cpus_kernel_max, NULL);
+
+/* arch-optional setting to enable display of offline cpus >= nr_cpu_ids */
+unsigned int total_cpus;
+
+static ssize_t print_cpus_offline(struct sysdev_class *class, char *buf)
+{
+	int n = 0, len = PAGE_SIZE-2;
+	cpumask_var_t offline;
+
+	/* display offline cpus < nr_cpu_ids */
+	if (!alloc_cpumask_var(&offline, GFP_KERNEL))
+		goto next;
+	cpumask_complement(offline, cpu_online_mask);
+	n = cpulist_scnprintf(buf, len, offline);
+	free_cpumask_var(offline);
+
+next:
+	/* display offline cpus >= nr_cpu_ids */
+	if (total_cpus && nr_cpu_ids < total_cpus) {
+		if (n && n < len)
+			buf[n++] = ',';
+
+		if (nr_cpu_ids == total_cpus-1)
+			n += snprintf(&buf[n], len - n, "%d", nr_cpu_ids);
+		else
+			n += snprintf(&buf[n], len - n, "%d-%d",
+						      nr_cpu_ids, total_cpus-1);
+	}
+
+	n += snprintf(&buf[n], len - n, "\n");
+	return n;
+}
+static SYSDEV_CLASS_ATTR(offline, 0444, print_cpus_offline, NULL);
+
 static struct sysdev_class_attribute *cpu_state_attr[] = {
 	&attr_online_map,
 	&attr_possible_map,
 	&attr_present_map,
+	&attr_kernel_max,
+	&attr_offline,
 };
 
 static int cpu_states_init(void)
--- linux-2.6-for-ingo.orig/include/linux/smp.h
+++ linux-2.6-for-ingo/include/linux/smp.h
@@ -21,6 +21,9 @@ struct call_single_data {
 	u16 priv;
 };
 
+/* total number of cpus in this system (may exceed NR_CPUS) */
+extern unsigned int total_cpus;
+
 #ifdef CONFIG_SMP
 
 #include <linux/preempt.h>

-- 

Re: [PATCH 3/3] cpumask: add sysfs displays for configured and disabled cpu maps

[Rusty Russell]

On Tuesday 16 December 2008 14:56:48 Mike Travis wrote:
> Impact: add new functionality.
> 
> Add sysfs files "kernel_max" and "offline" to display the max CPU index
> allowed (NR_CPUS-1), and the map of cpus that are offline. 

I've applied this to the cpumask series, but Greg is sysfs maestro, so
I'll await his ack (I've quoted whole thing for easy reading).

Thanks,
Rusty.

> Cpus can be offlined via HOTPLUG, disabled by the BIOS ACPI tables, or
> if they exceed the number of cpus allowed by the NR_CPUS config option,
> or the "maxcpus=NUM" kernel start parameter.
> 
> The "possible_cpus=NUM" parameter can also extend the number of possible
> cpus allowed, in which case the cpus not present at startup will be
> in the offline state.  (These cpus can be HOTPLUGGED ON after system
> startup [pending a follow-on patch to provide the capability via the
> /sys/devices/sys/cpu/cpuN/online mechanism to bring them online.])
> 
> By design, the "offlined cpus > possible cpus" display will always
> use the following formats:
> 
>   * all possible cpus online:   "x$"    or "x-y$" 
>   * some possible cpus offline: ".*,x$" or ".*,x-y$"
> 
> where:
>   x == number of possible cpus (nr_cpu_ids); and
>   y == number of cpus >= NR_CPUS or maxcpus (if y > x).
> 
> One use of this feature is for distros to select (or configure) the
> appropriate kernel to install for the resident system.
> 
> Notes:
>   * cpus offlined <= possible cpus will be printed for all architectures.
>   * cpus offlined >  possible cpus will only be printed for arches that
>   	set 'total_cpus' [X86 only in this patch].
> 
> Based on tip/cpus4096 + .../rusty/linux-2.6-for-ingo.git/master +
> 	 x86-only-patches sent 12/15.
> 
> Signed-off-by: Mike Travis <travis@sgi.com>
> Acked-by: Rusty Russell <rusty@rustcorp.com.au>
> ---
>  arch/x86/kernel/smpboot.c |    2 ++
>  drivers/base/cpu.c        |   45 +++++++++++++++++++++++++++++++++++++++++++++
>  include/linux/smp.h       |    3 +++
>  3 files changed, 50 insertions(+)
> 
> --- linux-2.6-for-ingo.orig/arch/x86/kernel/smpboot.c
> +++ linux-2.6-for-ingo/arch/x86/kernel/smpboot.c
> @@ -1293,6 +1293,8 @@ __init void prefill_possible_map(void)
>  	else
>  		possible = setup_possible_cpus;
>  
> +	total_cpus = max_t(int, possible, num_processors + disabled_cpus);
> +
>  	if (possible > CONFIG_NR_CPUS) {
>  		printk(KERN_WARNING
>  			"%d Processors exceeds NR_CPUS limit of %d\n",
> --- linux-2.6-for-ingo.orig/drivers/base/cpu.c
> +++ linux-2.6-for-ingo/drivers/base/cpu.c
> @@ -128,10 +128,55 @@ print_cpus_func(online);
>  print_cpus_func(possible);
>  print_cpus_func(present);
>  
> +/*
> + * Print values for NR_CPUS and offlined cpus
> + */
> +static ssize_t print_cpus_kernel_max(struct sysdev_class *class, char *buf)
> +{
> +	int n = snprintf(buf, PAGE_SIZE-2, "%d\n", CONFIG_NR_CPUS - 1);
> +	return n;
> +}
> +static SYSDEV_CLASS_ATTR(kernel_max, 0444, print_cpus_kernel_max, NULL);
> +
> +/* arch-optional setting to enable display of offline cpus >= nr_cpu_ids */
> +unsigned int total_cpus;
> +
> +static ssize_t print_cpus_offline(struct sysdev_class *class, char *buf)
> +{
> +	int n = 0, len = PAGE_SIZE-2;
> +	cpumask_var_t offline;
> +
> +	/* display offline cpus < nr_cpu_ids */
> +	if (!alloc_cpumask_var(&offline, GFP_KERNEL))
> +		goto next;
> +	cpumask_complement(offline, cpu_online_mask);
> +	n = cpulist_scnprintf(buf, len, offline);
> +	free_cpumask_var(offline);
> +
> +next:
> +	/* display offline cpus >= nr_cpu_ids */
> +	if (total_cpus && nr_cpu_ids < total_cpus) {
> +		if (n && n < len)
> +			buf[n++] = ',';
> +
> +		if (nr_cpu_ids == total_cpus-1)
> +			n += snprintf(&buf[n], len - n, "%d", nr_cpu_ids);
> +		else
> +			n += snprintf(&buf[n], len - n, "%d-%d",
> +						      nr_cpu_ids, total_cpus-1);
> +	}
> +
> +	n += snprintf(&buf[n], len - n, "\n");
> +	return n;
> +}
> +static SYSDEV_CLASS_ATTR(offline, 0444, print_cpus_offline, NULL);
> +
>  static struct sysdev_class_attribute *cpu_state_attr[] = {
>  	&attr_online_map,
>  	&attr_possible_map,
>  	&attr_present_map,
> +	&attr_kernel_max,
> +	&attr_offline,
>  };
>  
>  static int cpu_states_init(void)
> --- linux-2.6-for-ingo.orig/include/linux/smp.h
> +++ linux-2.6-for-ingo/include/linux/smp.h
> @@ -21,6 +21,9 @@ struct call_single_data {
>  	u16 priv;
>  };
>  
> +/* total number of cpus in this system (may exceed NR_CPUS) */
> +extern unsigned int total_cpus;
> +
>  #ifdef CONFIG_SMP
>  
>  #include <linux/preempt.h>
> 

Re: [PATCH 3/3] cpumask: add sysfs displays for configured and disabled cpu maps

[Greg KH]

On Wed, Dec 17, 2008 at 10:23:52PM +1030, Rusty Russell wrote:
> On Tuesday 16 December 2008 14:56:48 Mike Travis wrote:
> > Impact: add new functionality.
> > 
> > Add sysfs files "kernel_max" and "offline" to display the max CPU index
> > allowed (NR_CPUS-1), and the map of cpus that are offline. 
> 
> I've applied this to the cpumask series, but Greg is sysfs maestro, so
> I'll await his ack (I've quoted whole thing for easy reading).

All new sysfs files should have corrisponding Documentation/ABI files
added as well, containing all of the information in the body of this
email so that others can find it in the future in an easy manner.

Mike, can you make this change please?

> > Cpus can be offlined via HOTPLUG, disabled by the BIOS ACPI tables, or
> > if they exceed the number of cpus allowed by the NR_CPUS config option,
> > or the "maxcpus=NUM" kernel start parameter.
> > 
> > The "possible_cpus=NUM" parameter can also extend the number of possible
> > cpus allowed, in which case the cpus not present at startup will be
> > in the offline state.  (These cpus can be HOTPLUGGED ON after system
> > startup [pending a follow-on patch to provide the capability via the
> > /sys/devices/sys/cpu/cpuN/online mechanism to bring them online.])
> > 
> > By design, the "offlined cpus > possible cpus" display will always
> > use the following formats:
> > 
> >   * all possible cpus online:   "x$"    or "x-y$" 
> >   * some possible cpus offline: ".*,x$" or ".*,x-y$"

Care to provide an example of what these sysfs files will actually hold?
It seems like you are craming more information than needed into a single
sysfs file.

Oh, and state why this is really needed, as I thought we already showed
this kind of information today in the existing sysfs files, but I'm
probably wrong :)

thanks,

greg k-h

Re: [PATCH 3/3] cpumask: add sysfs displays for configured and disabled cpu maps

[Mike Travis]

Greg KH wrote:
> On Wed, Dec 17, 2008 at 10:23:52PM +1030, Rusty Russell wrote:
>> On Tuesday 16 December 2008 14:56:48 Mike Travis wrote:
>>> Impact: add new functionality.
>>>
>>> Add sysfs files "kernel_max" and "offline" to display the max CPU index
>>> allowed (NR_CPUS-1), and the map of cpus that are offline. 
>> I've applied this to the cpumask series, but Greg is sysfs maestro, so
>> I'll await his ack (I've quoted whole thing for easy reading).
> 
> All new sysfs files should have corrisponding Documentation/ABI files
> added as well, containing all of the information in the body of this
> email so that others can find it in the future in an easy manner.
> 
> Mike, can you make this change please?

Yes, I will.
> 
>>> Cpus can be offlined via HOTPLUG, disabled by the BIOS ACPI tables, or
>>> if they exceed the number of cpus allowed by the NR_CPUS config option,
>>> or the "maxcpus=NUM" kernel start parameter.
>>>
>>> The "possible_cpus=NUM" parameter can also extend the number of possible
>>> cpus allowed, in which case the cpus not present at startup will be
>>> in the offline state.  (These cpus can be HOTPLUGGED ON after system
>>> startup [pending a follow-on patch to provide the capability via the
>>> /sys/devices/sys/cpu/cpuN/online mechanism to bring them online.])
>>>
>>> By design, the "offlined cpus > possible cpus" display will always
>>> use the following formats:
>>>
>>>   * all possible cpus online:   "x$"    or "x-y$" 
>>>   * some possible cpus offline: ".*,x$" or ".*,x-y$"
> 
> Care to provide an example of what these sysfs files will actually hold?
> It seems like you are craming more information than needed into a single
> sysfs file.

Well, it's somewhat of a cop out as well :*).  The cpus within 0 <= cpu < nr_cpu_ids
are printed with a cpulist_scnprintf.  This is printed for all architectures
and shows primarily cpus that either have been offlined via CPU HOTPLUG or
made possible by extending the number of "possible" cpus via a kernel start
parameter.

Those cpus that may actually be present in the system but their index exceeds
NR_CPUS/maxcpus are printed following the above and are appended to the list. 
If the list is non-empty, a comma is inserted first.  This keeps it all within
the syntax needed by cpulist_parse().  This is only done for x86 right now.

I'll put this info in the doc file.
> 
> Oh, and state why this is really needed, as I thought we already showed
> this kind of information today in the existing sysfs files, but I'm
> probably wrong :)

Almost, but not quite.  You can mine the syslog file to find if there's a
warning message but otherwise, "extra" cpus are silently ignored.

> 
> thanks,
> 
> greg k-h

Thanks!
Mike

[PATCH 1/1]: sysfs: add documentation to cputopology.txt for system cpumasks

[Mike Travis]

Subject: sysfs: add documentation to cputopology.txt for system cpumasks

Add information to cputopology.txt explaining the output of various
system cpumask's.

Signed-off-by: Mike Travis <travis@sgi.com>
---
 Documentation/cputopology.txt |   48 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

--- linux-2.6-for-ingo.orig/Documentation/cputopology.txt
+++ linux-2.6-for-ingo/Documentation/cputopology.txt
@@ -31,3 +31,51 @@ not defined by include/asm-XXX/topology.
 2) core_id: 0
 3) thread_siblings: just the given CPU
 4) core_siblings: just the given CPU
+
+Additionally, cpu topology information is provided under
+/sys/devices/system/cpu and includes these files.  The internal
+source for the output is in brackets ("[]").
+
+    kernel_max: the maximum cpu index allowed by the kernel configuration.
+		[NR_CPUS-1]
+
+    offline:	cpus that are not online because they have been
+		HOTPLUGGED off (see cpu-hotplug.txt) or exceed the limit
+		of cpus allowed by the kernel configuration (kernel_max
+		above). [~cpu_online_mask + cpus >= NR_CPUS]
+
+    online:	cpus that are online and being scheduled [cpu_online_mask]
+
+    possible:	cpus that have been allocated resources and can be
+		brought online if they are present. [cpu_possible_mask]
+
+    present:	cpus that have been identified as being present in the
+		system. [cpu_present_mask]
+
+The format for the above output is compatible with cpulist_parse()
+[see <linux/cpumask.h>].  Some examples follow.
+
+In this example, there are 64 cpus in the system but cpus 32-63 exceed
+the kernel max which is limited to 0..31 by the NR_CPUS config option
+being 32.  Note also that cpus 2 and 4-31 are not online but could be
+brought online as they are both present and possible.
+
+     kernel_max: 31
+        offline: 2,4-31,32-63
+         online: 0-1,3
+       possible: 0-31
+        present: 0-31
+
+In this example, the NR_CPUS config option is 128, but the kernel was
+started with possible_cpus=144.  There are 4 cpus in the system and cpu2
+was manually taken offline (and is the only cpu that can be brought
+online.)
+
+     kernel_max: 127
+        offline: 2,4-127,128-143
+         online: 0-1,3
+       possible: 0-127
+        present: 0-3
+
+See cpu-hotplug.txt for the possible_cpus=NUM kernel start parameter
+as well as more information on the various cpumask's.

Re: [PATCH 0/3] x86/cpumask: fixups and additions

[Mike Travis]

Ingo Molnar wrote:
...
> 
> Since this will all go into tip/cpus4096, would you mind to do a git tree 
> for this stuff?

Hi Ingo,

Please pull the following.

Thanks!
Mike
--- 
The following changes since commit c8cae544bba6aee0f5cb0756dbab1a71d2c68737:
  Mike Travis (1):
        x86: fix build error with post-merge of tip/cpus4096 and rr-for-ingo/master.

are available in the git repository at:

  git://git.kernel.org/pub/scm/linux/kernel/git/travis/linux-2.6-cpus4096-for-ingo.git master

Mike Travis (5):
      x86: enable MAXSMP
      x86 smp: modify send_IPI_mask interface to accept cpumask_t pointers
      x86: move and enhance debug printk for nr_cpu_ids etc.
      x86: Add cpu_mask_to_apicid_and
      x86: Update io_apic.c to use new cpumask API

Rusty Russell (11):
      x86: update add-cpu_mask_to_apicid_and to use struct cpumask*
      xen: convert to cpumask_var_t and new cpumask primitives.
      x86: fixup_irqs() doesnt need an argument.
      x86: cosmetic changes apic-related files.
      x86: Set CONFIG_NR_CPUS even on UP
      x86: prepare for cpumask iterators to only go to nr_cpu_ids
      x86: Use cpumask accessors code for possible/present maps.
      x86: Remove cpumask games in x86/kernel/cpu/intel_cacheinfo.c
      x86: use work_on_cpu in x86/kernel/cpu/mcheck/mce_amd_64.c
      x86: xen: use smp_call_function_many()
      x86: Introduce topology_core_cpumask()/topology_thread_cpumask()

 arch/x86/Kconfig                              |   13 +-
 arch/x86/include/asm/bigsmp/apic.h            |   30 ++-
 arch/x86/include/asm/bigsmp/ipi.h             |   13 +-
 arch/x86/include/asm/es7000/apic.h            |   86 +++++--
 arch/x86/include/asm/es7000/ipi.h             |   12 +-
 arch/x86/include/asm/genapic_32.h             |   13 +-
 arch/x86/include/asm/genapic_64.h             |   14 +-
 arch/x86/include/asm/ipi.h                    |   23 ++-
 arch/x86/include/asm/irq.h                    |    2 +-
 arch/x86/include/asm/mach-default/mach_apic.h |   27 ++-
 arch/x86/include/asm/mach-default/mach_ipi.h  |   18 +-
 arch/x86/include/asm/mach-generic/mach_apic.h |    1 +
 arch/x86/include/asm/numaq/apic.h             |   12 +-
 arch/x86/include/asm/numaq/ipi.h              |   13 +-
 arch/x86/include/asm/smp.h                    |    6 +-
 arch/x86/include/asm/summit/apic.h            |   51 ++++-
 arch/x86/include/asm/summit/ipi.h             |    9 +-
 arch/x86/include/asm/topology.h               |    2 +
 arch/x86/kernel/apic.c                        |   12 +-
 arch/x86/kernel/cpu/intel_cacheinfo.c         |   41 ++--
 arch/x86/kernel/cpu/mcheck/mce_amd_64.c       |  108 ++++----
 arch/x86/kernel/crash.c                       |    5 +-
 arch/x86/kernel/genapic_flat_64.c             |  105 ++++++--
 arch/x86/kernel/genx2apic_cluster.c           |   77 ++++--
 arch/x86/kernel/genx2apic_phys.c              |   72 ++++--
 arch/x86/kernel/genx2apic_uv_x.c              |   59 +++--
 arch/x86/kernel/io_apic.c                     |  341 ++++++++++++-------------
 arch/x86/kernel/ipi.c                         |   28 ++-
 arch/x86/kernel/irq_32.c                      |   13 +-
 arch/x86/kernel/irq_64.c                      |   15 +-
 arch/x86/kernel/setup_percpu.c                |   17 +-
 arch/x86/kernel/smp.c                         |    8 +-
 arch/x86/kernel/smpboot.c                     |    2 +-
 arch/x86/kernel/tlb_32.c                      |    2 +-
 arch/x86/kernel/tlb_64.c                      |    2 +-
 arch/x86/mach-generic/bigsmp.c                |    5 +-
 arch/x86/mach-generic/es7000.c                |    5 +-
 arch/x86/mach-generic/numaq.c                 |    5 +-
 arch/x86/mach-generic/summit.c                |    5 +-
 arch/x86/mach-voyager/voyager_smp.c           |    2 +-
 arch/x86/mm/numa_64.c                         |    4 +-
 arch/x86/mm/srat_64.c                         |    2 +-
 arch/x86/xen/mmu.c                            |   20 +-
 arch/x86/xen/smp.c                            |   27 +-
 arch/x86/xen/suspend.c                        |    3 +-
 arch/x86/xen/xen-ops.h                        |    2 +-
 46 files changed, 824 insertions(+), 508 deletions(-)

Re: [PATCH 1/1]: sysfs: add documentation to cputopology.txt for system cpumasks

[Greg KH]

On Wed, Dec 17, 2008 at 02:14:30PM -0800, Mike Travis wrote:
> Subject: sysfs: add documentation to cputopology.txt for system cpumasks
> 
> Add information to cputopology.txt explaining the output of various
> system cpumask's.
> 
> Signed-off-by: Mike Travis <travis@sgi.com>

Looks good to me, thanks for doing this.

Acked-by: Greg Kroah-Hartman <gregkh@suse.de>

Re: [PATCH 3/3] cpumask: add sysfs displays for configured and disabled cpu maps

[Rusty Russell]

On Tuesday 16 December 2008 14:56:48 Mike Travis wrote:
> Impact: add new functionality.
> 
> Add sysfs files "kernel_max" and "offline" to display the max CPU index
> allowed (NR_CPUS-1), and the map of cpus that are offline. 

OK, I removed the x86 part and applied this with the following fix.

Greg Acked the documentation patch, so I'll assume he had no problem
with this patch either (appended below in case he wanted to re-review).

Cheers,
Rusty.

diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c
--- a/drivers/base/cpu.c
+++ b/drivers/base/cpu.c
@@ -148,12 +148,11 @@ static ssize_t print_cpus_offline(struct
 
 	/* display offline cpus < nr_cpu_ids */
 	if (!alloc_cpumask_var(&offline, GFP_KERNEL))
-		goto next;
+		return -ENOMEM;
 	cpumask_complement(offline, cpu_online_mask);
 	n = cpulist_scnprintf(buf, len, offline);
 	free_cpumask_var(offline);
 
-next:
 	/* display offline cpus >= nr_cpu_ids */
 	if (total_cpus && nr_cpu_ids < total_cpus) {
 		if (n && n < len)

Subject: cpumask: add sysfs displays for configured and disabled cpu maps
Date: Mon, 15 Dec 2008 20:26:48 -0800
From: Mike Travis <travis@sgi.com>

Impact: add new sysfs files.

Add sysfs files "kernel_max" and "offline" to display the max CPU index
allowed (NR_CPUS-1), and the map of cpus that are offline. 

Cpus can be offlined via HOTPLUG, disabled by the BIOS ACPI tables, or
if they exceed the number of cpus allowed by the NR_CPUS config option,
or the "maxcpus=NUM" kernel start parameter.

The "possible_cpus=NUM" parameter can also extend the number of possible
cpus allowed, in which case the cpus not present at startup will be
in the offline state.  (These cpus can be HOTPLUGGED ON after system
startup [pending a follow-on patch to provide the capability via the
/sys/devices/sys/cpu/cpuN/online mechanism to bring them online.])

By design, the "offlined cpus > possible cpus" display will always
use the following formats:

  * all possible cpus online:   "x$"    or "x-y$" 
  * some possible cpus offline: ".*,x$" or ".*,x-y$"

where:
  x == number of possible cpus (nr_cpu_ids); and
  y == number of cpus >= NR_CPUS or maxcpus (if y > x).

One use of this feature is for distros to select (or configure) the
appropriate kernel to install for the resident system.

Notes:
  * cpus offlined <= possible cpus will be printed for all architectures.
  * cpus offlined >  possible cpus will only be printed for arches that
  	set 'total_cpus' [X86 only in this patch].

Based on tip/cpus4096 + .../rusty/linux-2.6-for-ingo.git/master +
	 x86-only-patches sent 12/15.

Signed-off-by: Mike Travis <travis@sgi.com>
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 arch/x86/kernel/smpboot.c |    2 ++
 drivers/base/cpu.c        |   45 +++++++++++++++++++++++++++++++++++++++++++++
 include/linux/smp.h       |    3 +++
 3 files changed, 50 insertions(+)

--- linux-2.6-for-ingo.orig/drivers/base/cpu.c
+++ linux-2.6-for-ingo/drivers/base/cpu.c
@@ -128,10 +128,55 @@ print_cpus_func(online);
 print_cpus_func(possible);
 print_cpus_func(present);
 
+/*
+ * Print values for NR_CPUS and offlined cpus
+ */
+static ssize_t print_cpus_kernel_max(struct sysdev_class *class, char *buf)
+{
+	int n = snprintf(buf, PAGE_SIZE-2, "%d\n", CONFIG_NR_CPUS - 1);
+	return n;
+}
+static SYSDEV_CLASS_ATTR(kernel_max, 0444, print_cpus_kernel_max, NULL);
+
+/* arch-optional setting to enable display of offline cpus >= nr_cpu_ids */
+unsigned int total_cpus;
+
+static ssize_t print_cpus_offline(struct sysdev_class *class, char *buf)
+{
+	int n = 0, len = PAGE_SIZE-2;
+	cpumask_var_t offline;
+
+	/* display offline cpus < nr_cpu_ids */
+	if (!alloc_cpumask_var(&offline, GFP_KERNEL))
+		goto next;
+	cpumask_complement(offline, cpu_online_mask);
+	n = cpulist_scnprintf(buf, len, offline);
+	free_cpumask_var(offline);
+
+next:
+	/* display offline cpus >= nr_cpu_ids */
+	if (total_cpus && nr_cpu_ids < total_cpus) {
+		if (n && n < len)
+			buf[n++] = ',';
+
+		if (nr_cpu_ids == total_cpus-1)
+			n += snprintf(&buf[n], len - n, "%d", nr_cpu_ids);
+		else
+			n += snprintf(&buf[n], len - n, "%d-%d",
+						      nr_cpu_ids, total_cpus-1);
+	}
+
+	n += snprintf(&buf[n], len - n, "\n");
+	return n;
+}
+static SYSDEV_CLASS_ATTR(offline, 0444, print_cpus_offline, NULL);
+
 static struct sysdev_class_attribute *cpu_state_attr[] = {
 	&attr_online_map,
 	&attr_possible_map,
 	&attr_present_map,
+	&attr_kernel_max,
+	&attr_offline,
 };
 
 static int cpu_states_init(void)
--- linux-2.6-for-ingo.orig/include/linux/smp.h
+++ linux-2.6-for-ingo/include/linux/smp.h
@@ -21,6 +21,9 @@ struct call_single_data {
 	u16 priv;
 };
 
+/* total number of cpus in this system (may exceed NR_CPUS) */
+extern unsigned int total_cpus;
+
 #ifdef CONFIG_SMP
 
 #include <linux/preempt.h>

Re: [PATCH 3/3] cpumask: add sysfs displays for configured and disabled cpu maps

[Greg KH]

On Fri, Dec 19, 2008 at 05:16:25PM +1030, Rusty Russell wrote:
> Based on tip/cpus4096 + .../rusty/linux-2.6-for-ingo.git/master +
> 	 x86-only-patches sent 12/15.
> 
> Signed-off-by: Mike Travis <travis@sgi.com>
> Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

Acked-by: Greg Kroah-Hartman