(resend) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

(resend) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

[Ashok Raj]

Hi Andrew/DavidM

as i mentioned earlier, i broke the smp build for i386 in the take3 patches i sent this morning.
This patch is tested on i386 and ia64 as well.


thanks to Anil Keshavamurthy for taking initiative to test and bringup the issue.

Cheers,
Ashok


Name: cpu_present_map.patch
Author: Ashok Raj (Intel Corporation)
D: This patch adds cpu_present_map, cpu_present() and for_each_cpu_present()
D: to distinguish between possible cpu's in a system and cpu's physically
D: present in a system. Before cpu hotplug was introduced cpu_possible()
D: represented cpu's physically present in the system. With hotplug capable
D: Kernel, there is a requirement to distinguish a cpu as possible verses a 
D: CPU physically present in the system. This is required so thta when
D: smp_init() attempts to start all cpu's it should now only attempt
D: to start cpu's present in the system. When a hotplug cpu is
D: physically inserted cpu_present_map will have bits updated
D: dynamically.


---

 linux-2.6.6-rc3-root/arch/ia64/kernel/smpboot.c |   22 +++++++++++++---------
 linux-2.6.6-rc3-root/fs/buffer.c                |    2 +-
 linux-2.6.6-rc3-root/fs/proc/proc_misc.c        |    4 ++--
 linux-2.6.6-rc3-root/include/asm-ia64/smp.h     |    3 ---
 linux-2.6.6-rc3-root/include/linux/cpumask.h    |   11 +++++++++++
 linux-2.6.6-rc3-root/init/main.c                |    4 ++--
 linux-2.6.6-rc3-root/kernel/cpu.c               |   10 +++++++++-
 linux-2.6.6-rc3-root/kernel/fork.c              |    2 +-
 linux-2.6.6-rc3-root/kernel/sched.c             |   17 ++++++++++++++---
 linux-2.6.6-rc3-root/kernel/timer.c             |    2 +-
 10 files changed, 54 insertions(+), 23 deletions(-)

diff -puN arch/ia64/kernel/smpboot.c~cpu_present arch/ia64/kernel/smpboot.c
--- linux-2.6.6-rc3/arch/ia64/kernel/smpboot.c~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/arch/ia64/kernel/smpboot.c	2004-05-04 15:00:35.302427934 -0700
@@ -75,11 +75,11 @@ extern unsigned long ia64_iobase;
 
 task_t *task_for_booting_cpu;
 
-/* Bitmask of currently online CPUs */
+/* Bitmasks of currently online, and possible CPUs */
 cpumask_t cpu_online_map;
 EXPORT_SYMBOL(cpu_online_map);
-cpumask_t phys_cpu_present_map;
-EXPORT_SYMBOL(phys_cpu_present_map);
+cpumask_t cpu_possible_map;
+EXPORT_SYMBOL(cpu_possible_map);
 
 /* which logical CPU number maps to which CPU (physical APIC ID) */
 volatile int ia64_cpu_to_sapicid[NR_CPUS];
@@ -99,6 +99,7 @@ static int __init
 nointroute (char *str)
 {
 	no_int_routing = 1;
+	printk ("no_int_routing on\n");
 	return 1;
 }
 
@@ -441,14 +442,15 @@ smp_build_cpu_map (void)
 		ia64_cpu_to_sapicid[cpu] = -1;
 
 	ia64_cpu_to_sapicid[0] = boot_cpu_id;
-	cpus_clear(phys_cpu_present_map);
-	cpu_set(0, phys_cpu_present_map);
-
+	cpus_clear(cpu_present_map);
+	cpu_set(0, cpu_present_map);
+	cpu_set(0, cpu_possible_map);
 	for (cpu = 1, i = 0; i < smp_boot_data.cpu_count; i++) {
 		sapicid = smp_boot_data.cpu_phys_id[i];
 		if (sapicid == boot_cpu_id)
 			continue;
-		cpu_set(cpu, phys_cpu_present_map);
+		cpu_set(cpu, cpu_present_map);
+		cpu_set(cpu, cpu_possible_map);
 		ia64_cpu_to_sapicid[cpu] = sapicid;
 		cpu++;
 	}
@@ -529,9 +531,11 @@ smp_prepare_cpus (unsigned int max_cpus)
 	if (!max_cpus) {
 		printk(KERN_INFO "SMP mode deactivated.\n");
 		cpus_clear(cpu_online_map);
-		cpus_clear(phys_cpu_present_map);
+		cpus_clear(cpu_present_map);
+		cpus_clear(cpu_possible_map);
 		cpu_set(0, cpu_online_map);
-		cpu_set(0, phys_cpu_present_map);
+		cpu_set(0, cpu_present_map);
+		cpu_set(0, cpu_possible_map);
 		return;
 	}
 }
diff -puN include/linux/cpumask.h~cpu_present include/linux/cpumask.h
--- linux-2.6.6-rc3/include/linux/cpumask.h~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/include/linux/cpumask.h	2004-05-04 13:33:45.000000000 -0700
@@ -10,11 +10,15 @@
 
 extern cpumask_t cpu_online_map;
 extern cpumask_t cpu_possible_map;
+extern cpumask_t cpu_present_map;
 
 #define num_online_cpus()		cpus_weight(cpu_online_map)
 #define num_possible_cpus()		cpus_weight(cpu_possible_map)
+#define num_present_cpus()		cpus_weight(cpu_present_map)
+
 #define cpu_online(cpu)			cpu_isset(cpu, cpu_online_map)
 #define cpu_possible(cpu)		cpu_isset(cpu, cpu_possible_map)
+#define cpu_present(cpu)		cpu_isset(cpu, cpu_present_map)
 
 #define for_each_cpu_mask(cpu, mask)					\
 	for (cpu = first_cpu_const(mk_cpumask_const(mask));		\
@@ -23,16 +27,23 @@ extern cpumask_t cpu_possible_map;
 
 #define for_each_cpu(cpu) for_each_cpu_mask(cpu, cpu_possible_map)
 #define for_each_online_cpu(cpu) for_each_cpu_mask(cpu, cpu_online_map)
+#define for_each_present_cpu(cpu) for_each_cpu_mask(cpu, cpu_present_map)
 #else
 #define	cpu_online_map			cpumask_of_cpu(0)
 #define	cpu_possible_map		cpumask_of_cpu(0)
+#define	cpu_present_map			cpumask_of_cpu(0)
+
 #define num_online_cpus()		1
 #define num_possible_cpus()		1
+#define num_present_cpus()		1
+
 #define cpu_online(cpu)			({ BUG_ON((cpu) != 0); 1; })
 #define cpu_possible(cpu)		({ BUG_ON((cpu) != 0); 1; })
+#define cpu_present(cpu)		({ BUG_ON((cpu) != 0); 1; })
 
 #define for_each_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
 #define for_each_online_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
+#define for_each_present_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
 #endif
 
 #define cpumask_scnprintf(buf, buflen, map)				\
diff -puN init/main.c~cpu_present init/main.c
--- linux-2.6.6-rc3/init/main.c~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/init/main.c	2004-05-04 13:33:45.000000000 -0700
@@ -354,10 +354,10 @@ static void __init smp_init(void)
 	unsigned j = 1;
 
 	/* FIXME: This should be done in userspace --RR */
-	for (i = 0; i < NR_CPUS; i++) {
+	for_each_present_cpu(i) {
 		if (num_online_cpus() >= max_cpus)
 			break;
-		if (cpu_possible(i) && !cpu_online(i)) {
+		if (!cpu_online(i)) {
 			cpu_up(i);
 			j++;
 		}
diff -puN kernel/cpu.c~cpu_present kernel/cpu.c
--- linux-2.6.6-rc3/kernel/cpu.c~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/kernel/cpu.c	2004-05-04 13:33:45.000000000 -0700
@@ -20,6 +20,14 @@
 DECLARE_MUTEX(cpucontrol);
 
 static struct notifier_block *cpu_chain;
+/*
+ * Represents all cpu's present in the system
+ * In systems capable of hotplug, this map could dynamically grow
+ * as new cpu's are detected in the system via any platform specific
+ * method, such as ACPI for e.g.
+ */
+cpumask_t	cpu_present_map;
+EXPORT_SYMBOL(cpu_present_map);
 
 /* Need to know about CPUs going up/down? */
 int register_cpu_notifier(struct notifier_block *nb)
@@ -169,7 +177,7 @@ int __devinit cpu_up(unsigned int cpu)
 	if ((ret = down_interruptible(&cpucontrol)) != 0)
 		return ret;
 
-	if (cpu_online(cpu)) {
+	if (cpu_online(cpu) || !cpu_present(cpu)) {
 		ret = -EINVAL;
 		goto out;
 	}
diff -puN fs/proc/proc_misc.c~cpu_present fs/proc/proc_misc.c
--- linux-2.6.6-rc3/fs/proc/proc_misc.c~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/fs/proc/proc_misc.c	2004-05-04 13:33:45.000000000 -0700
@@ -368,7 +368,7 @@ int show_stat(struct seq_file *p, void *
 	if (wall_to_monotonic.tv_nsec)
 		--jif;
 
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 		int j;
 
 		user += kstat_cpu(i).cpustat.user;
@@ -390,7 +390,7 @@ int show_stat(struct seq_file *p, void *
 		(unsigned long long)jiffies_64_to_clock_t(iowait),
 		(unsigned long long)jiffies_64_to_clock_t(irq),
 		(unsigned long long)jiffies_64_to_clock_t(softirq));
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 
 		/* Copy values here to work around gcc-2.95.3, gcc-2.96 */
 		user = kstat_cpu(i).cpustat.user;
diff -puN kernel/sched.c~cpu_present kernel/sched.c
--- linux-2.6.6-rc3/kernel/sched.c~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/kernel/sched.c	2004-05-04 13:41:48.000000000 -0700
@@ -945,7 +945,7 @@ unsigned long nr_uninterruptible(void)
 {
 	unsigned long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += cpu_rq(i)->nr_uninterruptible;
 
 	return sum;
@@ -955,7 +955,7 @@ unsigned long long nr_context_switches(v
 {
 	unsigned long long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += cpu_rq(i)->nr_switches;
 
 	return sum;
@@ -965,7 +965,7 @@ unsigned long nr_iowait(void)
 {
 	unsigned long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += atomic_read(&cpu_rq(i)->nr_iowait);
 
 	return sum;
@@ -2944,6 +2944,17 @@ void __init sched_init(void)
 	runqueue_t *rq;
 	int i, j, k;
 
+	/*
+	 * If arch is not hotplug ready and did not populate
+	 * cpu_present_map, just make cpu_present_map same as cpu_possible_map
+	 * for other cpu bringup code to function as normal. e.g smp_init() etc.
+	 */
+	if (cpus_empty(cpu_present_map)) {
+		for_each_cpu(i) {
+			cpu_set(i, cpu_present_map);
+		}
+	}
+
 	for (i = 0; i < NR_CPUS; i++) {
 		prio_array_t *array;
 
diff -puN fs/buffer.c~cpu_present fs/buffer.c
--- linux-2.6.6-rc3/fs/buffer.c~cpu_present	2004-05-04 13:38:00.000000000 -0700
+++ linux-2.6.6-rc3-root/fs/buffer.c	2004-05-04 13:39:05.000000000 -0700
@@ -2966,7 +2966,7 @@ static void recalc_bh_state(void)
 	if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
 		return;
 	__get_cpu_var(bh_accounting).ratelimit = 0;
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		tot += per_cpu(bh_accounting, i).nr;
 	buffer_heads_over_limit = (tot > max_buffer_heads);
 }
diff -puN kernel/fork.c~cpu_present kernel/fork.c
--- linux-2.6.6-rc3/kernel/fork.c~cpu_present	2004-05-04 13:40:01.000000000 -0700
+++ linux-2.6.6-rc3-root/kernel/fork.c	2004-05-04 13:40:11.000000000 -0700
@@ -60,7 +60,7 @@ int nr_processes(void)
 	int cpu;
 	int total = 0;
 
-	for_each_cpu(cpu)
+	for_each_online_cpu(cpu)
 		total += per_cpu(process_counts, cpu);
 
 	return total;
diff -puN kernel/timer.c~cpu_present kernel/timer.c
--- linux-2.6.6-rc3/kernel/timer.c~cpu_present	2004-05-04 13:42:55.000000000 -0700
+++ linux-2.6.6-rc3-root/kernel/timer.c	2004-05-04 13:43:09.000000000 -0700
@@ -332,7 +332,7 @@ int del_timer_sync(struct timer_list *ti
 del_again:
 	ret += del_timer(timer);
 
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 		base = &per_cpu(tvec_bases, i);
 		if (base->running_timer == timer) {
 			while (base->running_timer == timer) {
diff -puN include/asm-ia64/smp.h~cpu_present include/asm-ia64/smp.h
--- linux-2.6.6-rc3/include/asm-ia64/smp.h~cpu_present	2004-05-04 15:01:31.992886647 -0700
+++ linux-2.6.6-rc3-root/include/asm-ia64/smp.h	2004-05-04 15:02:04.318098510 -0700
@@ -38,7 +38,6 @@ extern struct smp_boot_data {
 
 extern char no_int_routing __devinitdata;
 
-extern cpumask_t phys_cpu_present_map;
 extern cpumask_t cpu_online_map;
 extern unsigned long ipi_base_addr;
 extern unsigned char smp_int_redirect;
@@ -48,8 +47,6 @@ extern volatile int ia64_cpu_to_sapicid[
 
 extern unsigned long ap_wakeup_vector;
 
-#define cpu_possible_map phys_cpu_present_map
-
 /*
  * Function to map hard smp processor id to logical id.  Slow, so don't use this in
  * performance-critical code.

_

Re: (resend) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

[Andrew Morton]

Ashok Raj <ashok.raj@intel.com> wrote:
>
>  Name: cpu_present_map.patch

This does not play happily with the sched_domains patches.


Program received signal SIGEMT, Emulation trap.
task_rq_lock (p=0x0, flags=0xcff98ee8) at include/linux/sched.h:1087
1087            return p->thread_info->cpu;
(gdb) bt
#0  task_rq_lock (p=0x0, flags=0xcff98ee8) at include/linux/sched.h:1087
#1  0xc01187cd in try_to_wake_up (p=0x0, state=7, sync=0) at kernel/sched.c:844
#2  0xc0118a1d in wake_up_process (p=0x0) at kernel/sched.c:981
#3  0xc011b688 in cpu_attach_domain (sd=0xc120b060, cpu=-805728492) at kernel/sched.c:3804
#4  0xc05db60b in arch_init_sched_domains () at arch/i386/kernel/smpboot.c:1327
#5  0xc05dfb44 in sched_init_smp () at kernel/sched.c:4013
#6  0xc010045d in init (unused=0x0) at init/main.c:643
#7  0xc010428d in kernel_thread_helper () at arch/i386/kernel/process.c:246
(gdb) p cpu_online_map
$1 = 1
(gdb) f 3
#3  0xc011b688 in cpu_attach_domain (sd=0xc120b060, cpu=-805728492) at kernel/sched.c:3804
3804                    wake_up_process(rq->migration_thread);
(gdb) p cpu
$2 = -805728492
(gdb) up
#4  0xc05db60b in arch_init_sched_domains () at arch/i386/kernel/smpboot.c:1327
1327                    cpu_attach_domain(cpu_domain, i);
(gdb) p i
$3 = 1
(gdb) info thr
  5 Thread 32769 (swapper)  start_secondary (unused=0x0) at arch/i386/kernel/smpboot.c:446
* 4 Thread 32768 (swapper)  0xc0398ae1 in schedule () at kernel/sched.c:1198
  3 Thread 3 (ksoftirqd/0)  0xc0398ae1 in schedule () at kernel/sched.c:1198
  2 Thread 2 (migration/0)  migration_thread (data=0x0) at include/asm/current.h:10
  1 Thread 1 (swapper)  task_rq_lock (p=0x0, flags=0xcff98ee8) at include/linux/sched.h:1087
(gdb) 


It appears that x86's arch_init_sched_domains() is blindly assuming that
each CPU's migration thread is running and is pointed to by
rq->migration_thread.  But from the above you'll see that CPU0's migration
thread is running, but CPU1's has never been created, hence the
null-pointer deref.

I'll drop this final patch.  Please reissue it against next -mm.  Thanks.

various cpu patches [was: (resend) take3: Updated CPU Hotplug patches]

[Paul Jackson]

Hmmm .. we're getting backed up here.  Sure glad Andrew's got the job of
conducting this, not me ...

The purpose of this message is to list several patches that are
interacting, and the order that I'm guessing Andrew will end up taking
them.  I claim no authority, and at best very limited forecasting
ability.

This is just to put a source of possible confusions on the table,
by running it up the flagpole, and inviting the shooting to begin.

The following patches are colliding or interacting, listed in the order
that I'm guessing they will end up going into *-mm, and with my guess of
their current status:

  1. Nick Piggin's sched_domains patches (in *-mm now)
  2. Ashok Raj's CPU Hotplug patches for IA64 (sent back to author for repair)
  3. Paul Jackson's bitmap/cpumask cleanup (in my workarea ready to submit)
  4. John Reiser's bssprot (unrelated, except for ia64 friendly fire damage)
  5. Matthew Dobson's nodemask_t (in Matthew's workarea, ready to submit)
  6. Andi Kleen's numa placement (being reviewed now, I think)

Actually, bssprot is independent, once it resolves some ia64 issue, and
the last two (nodemask and numa) are sufficiently independent to go
in parallel or reverse order.

If you _wanted_, Andrew, to add to your current patch load, I'd be
delighted to submit my final bitmap/cpumask cleanup now - I'm just
guessing that you will politely ignore my offer.

-- 
                          I won't rest till it's the best ...
                          Programmer, Linux Scalability
                          Paul Jackson <pj@sgi.com> 1.650.933.1373

Re: various cpu patches [was: (resend) take3: Updated CPU Hotplug patches]

[Andrew Morton]

Paul Jackson <pj@sgi.com> wrote:
>
> Hmmm .. we're getting backed up here.  Sure glad Andrew's got the job of
> conducting this, not me ...
> 
> The purpose of this message is to list several patches that are
> interacting, and the order that I'm guessing Andrew will end up taking
> them.  I claim no authority, and at best very limited forecasting
> ability.

I try to keep patches in the expected merge-up order.  The patching order
is in the series file - see the `patch-series' symlink in the kerne.org
directory.

> This is just to put a source of possible confusions on the table,
> by running it up the flagpole, and inviting the shooting to begin.
> 
> The following patches are colliding or interacting, listed in the order
> that I'm guessing they will end up going into *-mm, and with my guess of
> their current status:
> 
>   1. Nick Piggin's sched_domains patches (in *-mm now)
>   2. Ashok Raj's CPU Hotplug patches for IA64 (sent back to author for repair)
>   3. Paul Jackson's bitmap/cpumask cleanup (in my workarea ready to submit)
>   4. John Reiser's bssprot (unrelated, except for ia64 friendly fire damage)
>   5. Matthew Dobson's nodemask_t (in Matthew's workarea, ready to submit)
>   6. Andi Kleen's numa placement (being reviewed now, I think)

Planned for shortly after 2.6.6 are:

sched-domains
rmap 7-14		(15-23 probably a week later)
numa api

> Actually, bssprot is independent, once it resolves some ia64 issue, and
> the last two (nodemask and numa) are sufficiently independent to go
> in parallel or reverse order.

I dropped bssprot - it was causing too much grief.

> If you _wanted_, Andrew, to add to your current patch load, I'd be
> delighted to submit my final bitmap/cpumask cleanup now - I'm just
> guessing that you will politely ignore my offer.

I confess to being vaguely stunned that it is possible to generate a ~20
patch series againt the bitmap code.  I'll pay more attention next time it
flies past.

Re: various cpu patches [was: (resend) take3: Updated CPU Hotplug patches]

[Paul Jackson]

> The patching order is in the series file ...

Yes - I am aware of your series file.  So far as I know, only
Nick's patch is still in it, of the patches I listed.

> I confess to being vaguely stunned that it is possible to
> generate a ~20 patch series againt the bitmap code.

Perhaps I sliced and diced it too finely ;).

-- 
                          I won't rest till it's the best ...
                          Programmer, Linux Scalability
                          Paul Jackson <pj@sgi.com> 1.650.933.1373

Re: various cpu patches [was: (resend) take3: Updated CPU Hotplug patches]

[William Lee Irwin III]

On Wed, May 05, 2004 at 12:03:48AM -0700, Paul Jackson wrote:
> Hmmm .. we're getting backed up here.  Sure glad Andrew's got the job of
> conducting this, not me ...
> The purpose of this message is to list several patches that are
> interacting, and the order that I'm guessing Andrew will end up taking
> them.  I claim no authority, and at best very limited forecasting
> ability.
> This is just to put a source of possible confusions on the table,
> by running it up the flagpole, and inviting the shooting to begin.
> The following patches are colliding or interacting, listed in the order
> that I'm guessing they will end up going into *-mm, and with my guess of
> their current status:
>   1. Nick Piggin's sched_domains patches (in *-mm now)
>   2. Ashok Raj's CPU Hotplug patches for IA64 (sent back to author for repair)
>   3. Paul Jackson's bitmap/cpumask cleanup (in my workarea ready to submit)
>   4. John Reiser's bssprot (unrelated, except for ia64 friendly fire damage)
>   5. Matthew Dobson's nodemask_t (in Matthew's workarea, ready to submit)
>   6. Andi Kleen's numa placement (being reviewed now, I think)

I don't see any essential interaction between these patches. This appears
to be 100% mergewerk. I don't think there's any essential conflict, just
potential PITA rediffing for the (re)senders if/when they touch the same
lines of code and things get accepted in a different order from what ppl
merged their stuff against. As far as I'm concerned, business as usual.

-- wli

Re: various cpu patches [was: (resend) take3: Updated CPU Hotplug patches]

[Paul Jackson]

> I don't see any essential interaction between these patches.

Essentially, yes.  The interactions were:

 Ashok's Hotplug added an essential arch-specific ifdef to a
  arch-specific cpumask file that my cleanup removes - so I had
  to work with Ashok to remove that arch dependency.

 Reiser's bssprot broke ia64, so I couldn't test with that patch.

 Matthew has recoded his nodemask patch to presume my cpumask cleanup.

 Andi's numa patch is so far as we know independent, but close enough
  to all this other activity to be at risk for something.

The interactions were enough that I probably couldn't expect Andrew to
wade through them to adapt my cleanup.  Rather I pretty much have to
deliver my cleanup ready to go, with whatever Andrew thinks is his
current *-mm series.

I'll be doing well just to get Andrew to look at it, despite what I take
to be endorsements from Matthew, Rusty and Joe Korty, and conditional
acceptance from yourself.  Andrew would probably drop it in a heartbeat
if it collided non-trivially.

So merge work - yes.  But merge work that mostly I need to do, which
makes me very sensitive to the order in which things happen.

-- 
                          I won't rest till it's the best ...
                          Programmer, Linux Scalability
                          Paul Jackson <pj@sgi.com> 1.650.933.1373

Re: various cpu patches [was: (resend) take3: Updated CPU Hotplug patches]

[William Lee Irwin III]

On Wed, May 05, 2004 at 12:44:56AM -0700, Paul Jackson wrote:
> The interactions were enough that I probably couldn't expect Andrew to
> wade through them to adapt my cleanup.  Rather I pretty much have to
> deliver my cleanup ready to go, with whatever Andrew thinks is his
> current *-mm series.
> I'll be doing well just to get Andrew to look at it, despite what I take
> to be endorsements from Matthew, Rusty and Joe Korty, and conditional
> acceptance from yourself.  Andrew would probably drop it in a heartbeat
> if it collided non-trivially.
> So merge work - yes.  But merge work that mostly I need to do, which
> makes me very sensitive to the order in which things happen.

Yeah, merging is a pain in the butt. I make vague attempts to merge on a
minute-to-minute basis, but that has me, well, basically livelocked. So
it is with Linux in general, I suppose.


-- wli

Re: (resend) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

[Nick Piggin]

Andrew Morton wrote:
> Ashok Raj <ashok.raj@intel.com> wrote:

> It appears that x86's arch_init_sched_domains() is blindly assuming that
> each CPU's migration thread is running and is pointed to by
> rq->migration_thread.  But from the above you'll see that CPU0's migration
> thread is running, but CPU1's has never been created, hence the
> null-pointer deref.
> 

The assumption is that the migration thread is running unless
cpu_is_offline is true. I think this is still OK, I can have
another look at it if anyone thinks otherwise.

As far as arch_init_sched_domains goes, it can be run as late
as you like if it needs to be moved.

Re: (resend) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

[Ashok Raj]

On Tue, May 04, 2004 at 10:59:07PM -0700, Andrew Morton wrote:
> Ashok Raj <ashok.raj@intel.com> wrote:
> >
> >  Name: cpu_present_map.patch
> 
> This does not play happily with the sched_domains patches.
> 
> 
> I'll drop this final patch.  Please reissue it against next -mm.  Thanks.

Thanks, i will rework the patch and resend soon.

Cheers,
Ashok
-- Linux Core Software Group

Re: (resend-2) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

[Ashok Raj]

On Tue, May 04, 2004 at 10:59:07PM -0700, Andrew Morton wrote:
> Ashok Raj <ashok.raj@intel.com> wrote:
> >
> >  Name: cpu_present_map.patch
> 
> This does not play happily with the sched_domains patches.
> 
> I'll drop this final patch.  Please reissue it against next -mm.  Thanks.

Ooopsy Daisy...

apparently i picked up the patches from a different directory. This was the exact problem
i fixed with the (supposed resend patch)... this one should work with sched domain, and 
i also tested with that on i386 MP as well.

below is patch 6 resend-2

cheers,
ashok


Name: cpu_present_map.patch
Author: Ashok Raj (Intel Corporation)
D: This patch adds cpu_present_map, cpu_present() and for_each_cpu_present()
D: to distinguish between possible cpu's in a system and cpu's physically
D: present in a system. Before cpu hotplug was introduced cpu_possible()
D: represented cpu's physically present in the system. With hotplug capable
D: Kernel, there is a requirement to distinguish a cpu as possible verses a 
D: CPU physically present in the system. This is required so thta when
D: smp_init() attempts to start all cpu's it should now only attempt
D: to start cpu's present in the system. When a hotplug cpu is
D: physically inserted cpu_present_map will have bits updated
D: dynamically.


---

 linux-2.6.6-rc3-root/arch/ia64/kernel/smpboot.c |   22 +++++++++++++---------
 linux-2.6.6-rc3-root/fs/buffer.c                |    2 +-
 linux-2.6.6-rc3-root/fs/proc/proc_misc.c        |    4 ++--
 linux-2.6.6-rc3-root/include/asm-ia64/smp.h     |    3 ---
 linux-2.6.6-rc3-root/include/linux/cpumask.h    |   11 +++++++++++
 linux-2.6.6-rc3-root/init/main.c                |   21 +++++++++++++++++++--
 linux-2.6.6-rc3-root/kernel/cpu.c               |   10 +++++++++-
 linux-2.6.6-rc3-root/kernel/fork.c              |    2 +-
 linux-2.6.6-rc3-root/kernel/sched.c             |    6 +++---
 linux-2.6.6-rc3-root/kernel/timer.c             |    2 +-
 10 files changed, 60 insertions(+), 23 deletions(-)

diff -puN arch/ia64/kernel/smpboot.c~cpu_present arch/ia64/kernel/smpboot.c
--- linux-2.6.6-rc3/arch/ia64/kernel/smpboot.c~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/arch/ia64/kernel/smpboot.c	2004-05-04 20:46:48.966085414 -0700
@@ -75,11 +75,11 @@ extern unsigned long ia64_iobase;
 
 task_t *task_for_booting_cpu;
 
-/* Bitmask of currently online CPUs */
+/* Bitmasks of currently online, and possible CPUs */
 cpumask_t cpu_online_map;
 EXPORT_SYMBOL(cpu_online_map);
-cpumask_t phys_cpu_present_map;
-EXPORT_SYMBOL(phys_cpu_present_map);
+cpumask_t cpu_possible_map;
+EXPORT_SYMBOL(cpu_possible_map);
 
 /* which logical CPU number maps to which CPU (physical APIC ID) */
 volatile int ia64_cpu_to_sapicid[NR_CPUS];
@@ -99,6 +99,7 @@ static int __init
 nointroute (char *str)
 {
 	no_int_routing = 1;
+	printk ("no_int_routing on\n");
 	return 1;
 }
 
@@ -441,14 +442,15 @@ smp_build_cpu_map (void)
 		ia64_cpu_to_sapicid[cpu] = -1;
 
 	ia64_cpu_to_sapicid[0] = boot_cpu_id;
-	cpus_clear(phys_cpu_present_map);
-	cpu_set(0, phys_cpu_present_map);
-
+	cpus_clear(cpu_present_map);
+	cpu_set(0, cpu_present_map);
+	cpu_set(0, cpu_possible_map);
 	for (cpu = 1, i = 0; i < smp_boot_data.cpu_count; i++) {
 		sapicid = smp_boot_data.cpu_phys_id[i];
 		if (sapicid == boot_cpu_id)
 			continue;
-		cpu_set(cpu, phys_cpu_present_map);
+		cpu_set(cpu, cpu_present_map);
+		cpu_set(cpu, cpu_possible_map);
 		ia64_cpu_to_sapicid[cpu] = sapicid;
 		cpu++;
 	}
@@ -529,9 +531,11 @@ smp_prepare_cpus (unsigned int max_cpus)
 	if (!max_cpus) {
 		printk(KERN_INFO "SMP mode deactivated.\n");
 		cpus_clear(cpu_online_map);
-		cpus_clear(phys_cpu_present_map);
+		cpus_clear(cpu_present_map);
+		cpus_clear(cpu_possible_map);
 		cpu_set(0, cpu_online_map);
-		cpu_set(0, phys_cpu_present_map);
+		cpu_set(0, cpu_present_map);
+		cpu_set(0, cpu_possible_map);
 		return;
 	}
 }
diff -puN include/linux/cpumask.h~cpu_present include/linux/cpumask.h
--- linux-2.6.6-rc3/include/linux/cpumask.h~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/include/linux/cpumask.h	2004-05-04 13:33:45.000000000 -0700
@@ -10,11 +10,15 @@
 
 extern cpumask_t cpu_online_map;
 extern cpumask_t cpu_possible_map;
+extern cpumask_t cpu_present_map;
 
 #define num_online_cpus()		cpus_weight(cpu_online_map)
 #define num_possible_cpus()		cpus_weight(cpu_possible_map)
+#define num_present_cpus()		cpus_weight(cpu_present_map)
+
 #define cpu_online(cpu)			cpu_isset(cpu, cpu_online_map)
 #define cpu_possible(cpu)		cpu_isset(cpu, cpu_possible_map)
+#define cpu_present(cpu)		cpu_isset(cpu, cpu_present_map)
 
 #define for_each_cpu_mask(cpu, mask)					\
 	for (cpu = first_cpu_const(mk_cpumask_const(mask));		\
@@ -23,16 +27,23 @@ extern cpumask_t cpu_possible_map;
 
 #define for_each_cpu(cpu) for_each_cpu_mask(cpu, cpu_possible_map)
 #define for_each_online_cpu(cpu) for_each_cpu_mask(cpu, cpu_online_map)
+#define for_each_present_cpu(cpu) for_each_cpu_mask(cpu, cpu_present_map)
 #else
 #define	cpu_online_map			cpumask_of_cpu(0)
 #define	cpu_possible_map		cpumask_of_cpu(0)
+#define	cpu_present_map			cpumask_of_cpu(0)
+
 #define num_online_cpus()		1
 #define num_possible_cpus()		1
+#define num_present_cpus()		1
+
 #define cpu_online(cpu)			({ BUG_ON((cpu) != 0); 1; })
 #define cpu_possible(cpu)		({ BUG_ON((cpu) != 0); 1; })
+#define cpu_present(cpu)		({ BUG_ON((cpu) != 0); 1; })
 
 #define for_each_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
 #define for_each_online_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
+#define for_each_present_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
 #endif
 
 #define cpumask_scnprintf(buf, buflen, map)				\
diff -puN init/main.c~cpu_present init/main.c
--- linux-2.6.6-rc3/init/main.c~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/init/main.c	2004-05-04 20:51:33.582442075 -0700
@@ -354,10 +354,10 @@ static void __init smp_init(void)
 	unsigned j = 1;
 
 	/* FIXME: This should be done in userspace --RR */
-	for (i = 0; i < NR_CPUS; i++) {
+	for_each_present_cpu(i) {
 		if (num_online_cpus() >= max_cpus)
 			break;
-		if (cpu_possible(i) && !cpu_online(i)) {
+		if (!cpu_online(i)) {
 			cpu_up(i);
 			j++;
 		}
@@ -577,6 +577,22 @@ static void run_init_process(char *init_
 	execve(init_filename, argv_init, envp_init);
 }
 
+static void fixup_cpu_present_map(void)
+{
+	int i;
+
+	/*
+	 * If arch is not hotplug ready and did not populate
+	 * cpu_present_map, just make cpu_present_map same as cpu_possible_map
+	 * for other cpu bringup code to function as normal. e.g smp_init() etc.
+	 */
+	if (cpus_empty(cpu_present_map)) {
+		for_each_cpu(i) {
+			cpu_set(i, cpu_present_map);
+		}
+	}
+}
+
 static int init(void * unused)
 {
 	lock_kernel();
@@ -595,6 +611,7 @@ static int init(void * unused)
 
 	do_pre_smp_initcalls();
 
+	fixup_cpu_present_map();
 	smp_init();
 	do_basic_setup();
 
diff -puN kernel/cpu.c~cpu_present kernel/cpu.c
--- linux-2.6.6-rc3/kernel/cpu.c~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/kernel/cpu.c	2004-05-04 13:33:45.000000000 -0700
@@ -20,6 +20,14 @@
 DECLARE_MUTEX(cpucontrol);
 
 static struct notifier_block *cpu_chain;
+/*
+ * Represents all cpu's present in the system
+ * In systems capable of hotplug, this map could dynamically grow
+ * as new cpu's are detected in the system via any platform specific
+ * method, such as ACPI for e.g.
+ */
+cpumask_t	cpu_present_map;
+EXPORT_SYMBOL(cpu_present_map);
 
 /* Need to know about CPUs going up/down? */
 int register_cpu_notifier(struct notifier_block *nb)
@@ -169,7 +177,7 @@ int __devinit cpu_up(unsigned int cpu)
 	if ((ret = down_interruptible(&cpucontrol)) != 0)
 		return ret;
 
-	if (cpu_online(cpu)) {
+	if (cpu_online(cpu) || !cpu_present(cpu)) {
 		ret = -EINVAL;
 		goto out;
 	}
diff -puN fs/proc/proc_misc.c~cpu_present fs/proc/proc_misc.c
--- linux-2.6.6-rc3/fs/proc/proc_misc.c~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/fs/proc/proc_misc.c	2004-05-04 13:33:45.000000000 -0700
@@ -368,7 +368,7 @@ int show_stat(struct seq_file *p, void *
 	if (wall_to_monotonic.tv_nsec)
 		--jif;
 
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 		int j;
 
 		user += kstat_cpu(i).cpustat.user;
@@ -390,7 +390,7 @@ int show_stat(struct seq_file *p, void *
 		(unsigned long long)jiffies_64_to_clock_t(iowait),
 		(unsigned long long)jiffies_64_to_clock_t(irq),
 		(unsigned long long)jiffies_64_to_clock_t(softirq));
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 
 		/* Copy values here to work around gcc-2.95.3, gcc-2.96 */
 		user = kstat_cpu(i).cpustat.user;
diff -puN kernel/sched.c~cpu_present kernel/sched.c
--- linux-2.6.6-rc3/kernel/sched.c~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/kernel/sched.c	2004-05-04 20:49:19.431982765 -0700
@@ -945,7 +945,7 @@ unsigned long nr_uninterruptible(void)
 {
 	unsigned long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += cpu_rq(i)->nr_uninterruptible;
 
 	return sum;
@@ -955,7 +955,7 @@ unsigned long long nr_context_switches(v
 {
 	unsigned long long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += cpu_rq(i)->nr_switches;
 
 	return sum;
@@ -965,7 +965,7 @@ unsigned long nr_iowait(void)
 {
 	unsigned long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += atomic_read(&cpu_rq(i)->nr_iowait);
 
 	return sum;
diff -puN fs/buffer.c~cpu_present fs/buffer.c
--- linux-2.6.6-rc3/fs/buffer.c~cpu_present	2004-05-04 13:38:00.000000000 -0700
+++ linux-2.6.6-rc3-root/fs/buffer.c	2004-05-04 13:39:05.000000000 -0700
@@ -2966,7 +2966,7 @@ static void recalc_bh_state(void)
 	if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
 		return;
 	__get_cpu_var(bh_accounting).ratelimit = 0;
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		tot += per_cpu(bh_accounting, i).nr;
 	buffer_heads_over_limit = (tot > max_buffer_heads);
 }
diff -puN kernel/fork.c~cpu_present kernel/fork.c
--- linux-2.6.6-rc3/kernel/fork.c~cpu_present	2004-05-04 13:40:01.000000000 -0700
+++ linux-2.6.6-rc3-root/kernel/fork.c	2004-05-04 13:40:11.000000000 -0700
@@ -60,7 +60,7 @@ int nr_processes(void)
 	int cpu;
 	int total = 0;
 
-	for_each_cpu(cpu)
+	for_each_online_cpu(cpu)
 		total += per_cpu(process_counts, cpu);
 
 	return total;
diff -puN kernel/timer.c~cpu_present kernel/timer.c
--- linux-2.6.6-rc3/kernel/timer.c~cpu_present	2004-05-04 13:42:55.000000000 -0700
+++ linux-2.6.6-rc3-root/kernel/timer.c	2004-05-04 13:43:09.000000000 -0700
@@ -332,7 +332,7 @@ int del_timer_sync(struct timer_list *ti
 del_again:
 	ret += del_timer(timer);
 
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 		base = &per_cpu(tvec_bases, i);
 		if (base->running_timer == timer) {
 			while (base->running_timer == timer) {
diff -puN include/asm-ia64/smp.h~cpu_present include/asm-ia64/smp.h
--- linux-2.6.6-rc3/include/asm-ia64/smp.h~cpu_present	2004-05-04 15:01:31.000000000 -0700
+++ linux-2.6.6-rc3-root/include/asm-ia64/smp.h	2004-05-04 20:46:48.967061977 -0700
@@ -38,7 +38,6 @@ extern struct smp_boot_data {
 
 extern char no_int_routing __devinitdata;
 
-extern cpumask_t phys_cpu_present_map;
 extern cpumask_t cpu_online_map;
 extern unsigned long ipi_base_addr;
 extern unsigned char smp_int_redirect;
@@ -48,8 +47,6 @@ extern volatile int ia64_cpu_to_sapicid[
 
 extern unsigned long ap_wakeup_vector;
 
-#define cpu_possible_map phys_cpu_present_map
-
 /*
  * Function to map hard smp processor id to logical id.  Slow, so don't use this in
  * performance-critical code.

_

Re: (resend-2) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

[Andrew Morton]

Ashok Raj <ashok.raj@intel.com> wrote:
>
> Name: cpu_present_map.patch

Ho-hum.  Please at least compile-test non-trivial patches with
CONFIG_SMP=n, especially when they dink with bitmasks, bitmaps and
SMP-specific features.

init/main.c: In function `fixup_cpu_present_map':
init/main.c:636: warning: use of compound expressions as lvalues is deprecated
init/main.c:636: error: invalid lvalue in assignment

Due to:

			cpu_set(i, cpu_present_map);

It appears that cpu_set() is simply broken on UP:


#define	cpu_present_map			cpumask_of_cpu(0)

#define cpumask_of_cpu(cpu)		({ ((cpumask_t)1) << (cpu); })

#define cpu_set(cpu, map)		do { (void)(cpu); cpus_coerce(map) = 1UL; } while (0)

Put those things together and there's no way it can work.  It's not even
conceptually right: cpu_present_map is a "constant" on UP and we have no
business trying to modify it.  So perhaps a build error is the appropriate
response.

I'll stick a CONFIG_SMP in the caller, let the bitmap beavers worry about
the more general details.

Re: (resend-3) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

[Ashok Raj]

On Wed, May 05, 2004 at 11:13:50PM -0700, Andrew Morton wrote:
> Ashok Raj <ashok.raj@intel.com> wrote:
> >
> > Name: cpu_present_map.patch
> 
> Ho-hum.  Please at least compile-test non-trivial patches with
> CONFIG_SMP=n, especially when they dink with bitmasks, bitmaps and
> SMP-specific features.
> 
> 
> I'll stick a CONFIG_SMP in the caller, let the bitmap beavers worry about
> the more general details.
Now under the protective covers of CONFIG_SMP, init/main.c compiles now
but ran into another link error, seems like some workaround required in kthread_bind as well
as kthread_wait_task_inactive is defined in sched.c only for CONFIG_SMP case.

btw; maybe i should rename this patch and all the problems will go away... :-)

kernel/built-in.o(.text+0x144e4): In function `kthread_bind':
: undefined reference to `kthread_wait_task_inactive'
make: *** [.tmp_vmlinux1] Error 1




Name: cpu_present_map.patch
Author: Ashok Raj (Intel Corporation)
D: This patch adds cpu_present_map, cpu_present() and for_each_cpu_present()
D: to distinguish between possible cpu's in a system and cpu's physically
D: present in a system. Before cpu hotplug was introduced cpu_possible()
D: represented cpu's physically present in the system. With hotplug capable
D: Kernel, there is a requirement to distinguish a cpu as possible verses a 
D: CPU physically present in the system. This is required so thta when
D: smp_init() attempts to start all cpu's it should now only attempt
D: to start cpu's present in the system. When a hotplug cpu is
D: physically inserted cpu_present_map will have bits updated
D: dynamically.


---

 linux-2.6.6-rc3-root/arch/ia64/kernel/smpboot.c |   22 +++++++++++-------
 linux-2.6.6-rc3-root/fs/buffer.c                |    2 -
 linux-2.6.6-rc3-root/fs/proc/proc_misc.c        |    4 +--
 linux-2.6.6-rc3-root/include/asm-ia64/smp.h     |    3 --
 linux-2.6.6-rc3-root/include/linux/cpumask.h    |   11 +++++++++
 linux-2.6.6-rc3-root/init/main.c                |   28 ++++++++++++++++++++++--
 linux-2.6.6-rc3-root/kernel/cpu.c               |   10 +++++++-
 linux-2.6.6-rc3-root/kernel/fork.c              |    2 -
 linux-2.6.6-rc3-root/kernel/sched.c             |    6 ++---
 linux-2.6.6-rc3-root/kernel/timer.c             |    2 -
 10 files changed, 67 insertions(+), 23 deletions(-)

diff -puN arch/ia64/kernel/smpboot.c~cpu_present arch/ia64/kernel/smpboot.c
--- linux-2.6.6-rc3/arch/ia64/kernel/smpboot.c~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/arch/ia64/kernel/smpboot.c	2004-05-06 02:14:43.197080868 -0700
@@ -75,11 +75,11 @@ extern unsigned long ia64_iobase;
 
 task_t *task_for_booting_cpu;
 
-/* Bitmask of currently online CPUs */
+/* Bitmasks of currently online, and possible CPUs */
 cpumask_t cpu_online_map;
 EXPORT_SYMBOL(cpu_online_map);
-cpumask_t phys_cpu_present_map;
-EXPORT_SYMBOL(phys_cpu_present_map);
+cpumask_t cpu_possible_map;
+EXPORT_SYMBOL(cpu_possible_map);
 
 /* which logical CPU number maps to which CPU (physical APIC ID) */
 volatile int ia64_cpu_to_sapicid[NR_CPUS];
@@ -99,6 +99,7 @@ static int __init
 nointroute (char *str)
 {
 	no_int_routing = 1;
+	printk ("no_int_routing on\n");
 	return 1;
 }
 
@@ -441,14 +442,15 @@ smp_build_cpu_map (void)
 		ia64_cpu_to_sapicid[cpu] = -1;
 
 	ia64_cpu_to_sapicid[0] = boot_cpu_id;
-	cpus_clear(phys_cpu_present_map);
-	cpu_set(0, phys_cpu_present_map);
-
+	cpus_clear(cpu_present_map);
+	cpu_set(0, cpu_present_map);
+	cpu_set(0, cpu_possible_map);
 	for (cpu = 1, i = 0; i < smp_boot_data.cpu_count; i++) {
 		sapicid = smp_boot_data.cpu_phys_id[i];
 		if (sapicid == boot_cpu_id)
 			continue;
-		cpu_set(cpu, phys_cpu_present_map);
+		cpu_set(cpu, cpu_present_map);
+		cpu_set(cpu, cpu_possible_map);
 		ia64_cpu_to_sapicid[cpu] = sapicid;
 		cpu++;
 	}
@@ -529,9 +531,11 @@ smp_prepare_cpus (unsigned int max_cpus)
 	if (!max_cpus) {
 		printk(KERN_INFO "SMP mode deactivated.\n");
 		cpus_clear(cpu_online_map);
-		cpus_clear(phys_cpu_present_map);
+		cpus_clear(cpu_present_map);
+		cpus_clear(cpu_possible_map);
 		cpu_set(0, cpu_online_map);
-		cpu_set(0, phys_cpu_present_map);
+		cpu_set(0, cpu_present_map);
+		cpu_set(0, cpu_possible_map);
 		return;
 	}
 }
diff -puN include/linux/cpumask.h~cpu_present include/linux/cpumask.h
--- linux-2.6.6-rc3/include/linux/cpumask.h~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/include/linux/cpumask.h	2004-05-04 13:33:45.000000000 -0700
@@ -10,11 +10,15 @@
 
 extern cpumask_t cpu_online_map;
 extern cpumask_t cpu_possible_map;
+extern cpumask_t cpu_present_map;
 
 #define num_online_cpus()		cpus_weight(cpu_online_map)
 #define num_possible_cpus()		cpus_weight(cpu_possible_map)
+#define num_present_cpus()		cpus_weight(cpu_present_map)
+
 #define cpu_online(cpu)			cpu_isset(cpu, cpu_online_map)
 #define cpu_possible(cpu)		cpu_isset(cpu, cpu_possible_map)
+#define cpu_present(cpu)		cpu_isset(cpu, cpu_present_map)
 
 #define for_each_cpu_mask(cpu, mask)					\
 	for (cpu = first_cpu_const(mk_cpumask_const(mask));		\
@@ -23,16 +27,23 @@ extern cpumask_t cpu_possible_map;
 
 #define for_each_cpu(cpu) for_each_cpu_mask(cpu, cpu_possible_map)
 #define for_each_online_cpu(cpu) for_each_cpu_mask(cpu, cpu_online_map)
+#define for_each_present_cpu(cpu) for_each_cpu_mask(cpu, cpu_present_map)
 #else
 #define	cpu_online_map			cpumask_of_cpu(0)
 #define	cpu_possible_map		cpumask_of_cpu(0)
+#define	cpu_present_map			cpumask_of_cpu(0)
+
 #define num_online_cpus()		1
 #define num_possible_cpus()		1
+#define num_present_cpus()		1
+
 #define cpu_online(cpu)			({ BUG_ON((cpu) != 0); 1; })
 #define cpu_possible(cpu)		({ BUG_ON((cpu) != 0); 1; })
+#define cpu_present(cpu)		({ BUG_ON((cpu) != 0); 1; })
 
 #define for_each_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
 #define for_each_online_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
+#define for_each_present_cpu(cpu) for (cpu = 0; cpu < 1; cpu++)
 #endif
 
 #define cpumask_scnprintf(buf, buflen, map)				\
diff -puN init/main.c~cpu_present init/main.c
--- linux-2.6.6-rc3/init/main.c~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/init/main.c	2004-05-06 02:16:41.591672736 -0700
@@ -354,10 +354,10 @@ static void __init smp_init(void)
 	unsigned j = 1;
 
 	/* FIXME: This should be done in userspace --RR */
-	for (i = 0; i < NR_CPUS; i++) {
+	for_each_present_cpu(i) {
 		if (num_online_cpus() >= max_cpus)
 			break;
-		if (cpu_possible(i) && !cpu_online(i)) {
+		if (!cpu_online(i)) {
 			cpu_up(i);
 			j++;
 		}
@@ -577,6 +577,29 @@ static void run_init_process(char *init_
 	execve(init_filename, argv_init, envp_init);
 }
 
+#ifdef CONFIG_SMP
+/*
+ * Dirty Hack now, maybe cpu_set for UP needs to be redefined for UP??
+ */
+static void fixup_cpu_present_map(void)
+{
+	int i;
+
+	/*
+	 * If arch is not hotplug ready and did not populate
+	 * cpu_present_map, just make cpu_present_map same as cpu_possible_map
+	 * for other cpu bringup code to function as normal. e.g smp_init() etc.
+	 */
+	if (cpus_empty(cpu_present_map)) {
+		for_each_cpu(i) {
+			cpu_set(i, cpu_present_map);
+		}
+	}
+}
+#else
+#define fixup_cpu_present_map()
+#endif
+
 static int init(void * unused)
 {
 	lock_kernel();
@@ -595,6 +618,7 @@ static int init(void * unused)
 
 	do_pre_smp_initcalls();
 
+	fixup_cpu_present_map();
 	smp_init();
 	do_basic_setup();
 
diff -puN kernel/cpu.c~cpu_present kernel/cpu.c
--- linux-2.6.6-rc3/kernel/cpu.c~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/kernel/cpu.c	2004-05-04 13:33:45.000000000 -0700
@@ -20,6 +20,14 @@
 DECLARE_MUTEX(cpucontrol);
 
 static struct notifier_block *cpu_chain;
+/*
+ * Represents all cpu's present in the system
+ * In systems capable of hotplug, this map could dynamically grow
+ * as new cpu's are detected in the system via any platform specific
+ * method, such as ACPI for e.g.
+ */
+cpumask_t	cpu_present_map;
+EXPORT_SYMBOL(cpu_present_map);
 
 /* Need to know about CPUs going up/down? */
 int register_cpu_notifier(struct notifier_block *nb)
@@ -169,7 +177,7 @@ int __devinit cpu_up(unsigned int cpu)
 	if ((ret = down_interruptible(&cpucontrol)) != 0)
 		return ret;
 
-	if (cpu_online(cpu)) {
+	if (cpu_online(cpu) || !cpu_present(cpu)) {
 		ret = -EINVAL;
 		goto out;
 	}
diff -puN fs/proc/proc_misc.c~cpu_present fs/proc/proc_misc.c
--- linux-2.6.6-rc3/fs/proc/proc_misc.c~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/fs/proc/proc_misc.c	2004-05-04 13:33:45.000000000 -0700
@@ -368,7 +368,7 @@ int show_stat(struct seq_file *p, void *
 	if (wall_to_monotonic.tv_nsec)
 		--jif;
 
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 		int j;
 
 		user += kstat_cpu(i).cpustat.user;
@@ -390,7 +390,7 @@ int show_stat(struct seq_file *p, void *
 		(unsigned long long)jiffies_64_to_clock_t(iowait),
 		(unsigned long long)jiffies_64_to_clock_t(irq),
 		(unsigned long long)jiffies_64_to_clock_t(softirq));
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 
 		/* Copy values here to work around gcc-2.95.3, gcc-2.96 */
 		user = kstat_cpu(i).cpustat.user;
diff -puN kernel/sched.c~cpu_present kernel/sched.c
--- linux-2.6.6-rc3/kernel/sched.c~cpu_present	2004-05-04 13:33:45.000000000 -0700
+++ linux-2.6.6-rc3-root/kernel/sched.c	2004-05-04 20:49:19.000000000 -0700
@@ -945,7 +945,7 @@ unsigned long nr_uninterruptible(void)
 {
 	unsigned long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += cpu_rq(i)->nr_uninterruptible;
 
 	return sum;
@@ -955,7 +955,7 @@ unsigned long long nr_context_switches(v
 {
 	unsigned long long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += cpu_rq(i)->nr_switches;
 
 	return sum;
@@ -965,7 +965,7 @@ unsigned long nr_iowait(void)
 {
 	unsigned long i, sum = 0;
 
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		sum += atomic_read(&cpu_rq(i)->nr_iowait);
 
 	return sum;
diff -puN fs/buffer.c~cpu_present fs/buffer.c
--- linux-2.6.6-rc3/fs/buffer.c~cpu_present	2004-05-04 13:38:00.000000000 -0700
+++ linux-2.6.6-rc3-root/fs/buffer.c	2004-05-04 13:39:05.000000000 -0700
@@ -2966,7 +2966,7 @@ static void recalc_bh_state(void)
 	if (__get_cpu_var(bh_accounting).ratelimit++ < 4096)
 		return;
 	__get_cpu_var(bh_accounting).ratelimit = 0;
-	for_each_cpu(i)
+	for_each_online_cpu(i)
 		tot += per_cpu(bh_accounting, i).nr;
 	buffer_heads_over_limit = (tot > max_buffer_heads);
 }
diff -puN kernel/fork.c~cpu_present kernel/fork.c
--- linux-2.6.6-rc3/kernel/fork.c~cpu_present	2004-05-04 13:40:01.000000000 -0700
+++ linux-2.6.6-rc3-root/kernel/fork.c	2004-05-04 13:40:11.000000000 -0700
@@ -60,7 +60,7 @@ int nr_processes(void)
 	int cpu;
 	int total = 0;
 
-	for_each_cpu(cpu)
+	for_each_online_cpu(cpu)
 		total += per_cpu(process_counts, cpu);
 
 	return total;
diff -puN kernel/timer.c~cpu_present kernel/timer.c
--- linux-2.6.6-rc3/kernel/timer.c~cpu_present	2004-05-04 13:42:55.000000000 -0700
+++ linux-2.6.6-rc3-root/kernel/timer.c	2004-05-04 13:43:09.000000000 -0700
@@ -332,7 +332,7 @@ int del_timer_sync(struct timer_list *ti
 del_again:
 	ret += del_timer(timer);
 
-	for_each_cpu(i) {
+	for_each_online_cpu(i) {
 		base = &per_cpu(tvec_bases, i);
 		if (base->running_timer == timer) {
 			while (base->running_timer == timer) {
diff -puN include/asm-ia64/smp.h~cpu_present include/asm-ia64/smp.h
--- linux-2.6.6-rc3/include/asm-ia64/smp.h~cpu_present	2004-05-04 15:01:31.000000000 -0700
+++ linux-2.6.6-rc3-root/include/asm-ia64/smp.h	2004-05-06 02:14:43.198057431 -0700
@@ -38,7 +38,6 @@ extern struct smp_boot_data {
 
 extern char no_int_routing __devinitdata;
 
-extern cpumask_t phys_cpu_present_map;
 extern cpumask_t cpu_online_map;
 extern unsigned long ipi_base_addr;
 extern unsigned char smp_int_redirect;
@@ -48,8 +47,6 @@ extern volatile int ia64_cpu_to_sapicid[
 
 extern unsigned long ap_wakeup_vector;
 
-#define cpu_possible_map phys_cpu_present_map
-
 /*
  * Function to map hard smp processor id to logical id.  Slow, so don't use this in
  * performance-critical code.

_

Re: (resend-3) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

[Andrew Morton]

Ashok Raj <ashok.raj@intel.com> wrote:
>
> > I'll stick a CONFIG_SMP in the caller, let the bitmap beavers worry about
> > the more general details.
> Now under the protective covers of CONFIG_SMP, init/main.c compiles now

I already fixed it up, thanks.  With an incremental patch, which is very
much preferred over a wholesale replacement.

> kernel/built-in.o(.text+0x144e4): In function `kthread_bind':
> : undefined reference to `kthread_wait_task_inactive'
> make: *** [.tmp_vmlinux1] Error 1

There is no symbol `kthread_wait_task_inactive' in the tree.

Re: (resend-3) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

[Ashok Raj]

On Thu, May 06, 2004 at 02:45:30AM -0700, Andrew Morton wrote:
> Ashok Raj <ashok.raj@intel.com> wrote:
> >
> > > I'll stick a CONFIG_SMP in the caller, let the bitmap beavers worry about
> > > the more general details.
> > Now under the protective covers of CONFIG_SMP, init/main.c compiles now
> 
> I already fixed it up, thanks.  With an incremental patch, which is very
> much preferred over a wholesale replacement.
> 
I will in future.

> > kernel/built-in.o(.text+0x144e4): In function `kthread_bind':
> > : undefined reference to `kthread_wait_task_inactive'
> > make: *** [.tmp_vmlinux1] Error 1
> 
> There is no symbol `kthread_wait_task_inactive' in the tree.
> 

I think i put the experimental i386 hotplug patch, and that might have brought something 
with it.. i see the -mm compile clean now.

Cheers,
ashok

Re: (resend-2) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [7/7]

[Ashok Raj]

On Tue, May 04, 2004 at 10:59:07PM -0700, Andrew Morton wrote:
> Ashok Raj <ashok.raj@intel.com> wrote:
> I'll drop this final patch.  Please reissue it against next -mm.  Thanks.

Below is resend of IA64 arch hotplug sections.. tested with 2.6.6-rc3-mm2



Name: hotcpu_ia64.patch
Author: Ashok Raj (Intel Corporation)
D: Supports basic ability to enable hotplug functions for IA64.
D: Code is just evolving, and there are several loose ends to tie up.
D:
D: What this code drop does
D: - Support logical online and offline
D: - Handles interrupt migration without loss of interrupts.
D: - Handles stress fine > 24+ hrs with make -j/ftp/rcp workloads
D: - Handles irq migration from a dying cpu without loss of interrupts.
D: What needs to be done
D: - Boot CPU removal support, with platform level authentication
D: - Putting cpu being removed in BOOT_RENDEZ mode.


---

 linux-2.6.6-rc3-root/arch/ia64/Kconfig           |    9 +
 linux-2.6.6-rc3-root/arch/ia64/kernel/iosapic.c  |    7 -
 linux-2.6.6-rc3-root/arch/ia64/kernel/irq.c      |  103 +++++++++++++++++
 linux-2.6.6-rc3-root/arch/ia64/kernel/irq_ia64.c |   60 +++++++++-
 linux-2.6.6-rc3-root/arch/ia64/kernel/process.c  |   43 +++++++
 linux-2.6.6-rc3-root/arch/ia64/kernel/sal.c      |   13 ++
 linux-2.6.6-rc3-root/arch/ia64/kernel/smp.c      |   26 ++++
 linux-2.6.6-rc3-root/arch/ia64/kernel/smpboot.c  |  136 +++++++++++++++++++++--
 linux-2.6.6-rc3-root/arch/ia64/kernel/time.c     |    5 
 linux-2.6.6-rc3-root/include/asm-ia64/smp.h      |    2 
 10 files changed, 386 insertions(+), 18 deletions(-)

diff -puN arch/ia64/Kconfig~hotcpu_ia64 arch/ia64/Kconfig
--- linux-2.6.6-rc3/arch/ia64/Kconfig~hotcpu_ia64	2004-05-04 20:52:19.527778099 -0700
+++ linux-2.6.6-rc3-root/arch/ia64/Kconfig	2004-05-04 20:52:19.553168737 -0700
@@ -78,6 +78,15 @@ config IA64_HP_SIM
 
 endchoice
 
+config HOTPLUG_CPU
+    bool "Support for hot-pluggable CPUs (EXPERIMENTAL)"
+    depends on SMP && HOTPLUG && EXPERIMENTAL
+	default n
+    ---help---
+      Say Y here to experiment with turning CPUs off and on.  CPUs
+      can be controlled through /sys/devices/system/cpu/cpu#.
+      Say N if you want to disable cpu hotplug.
+
 choice
 	prompt "Processor type"
 	default ITANIUM
diff -puN arch/ia64/kernel/irq.c~hotcpu_ia64 arch/ia64/kernel/irq.c
--- linux-2.6.6-rc3/arch/ia64/kernel/irq.c~hotcpu_ia64	2004-05-04 20:52:19.529731225 -0700
+++ linux-2.6.6-rc3-root/arch/ia64/kernel/irq.c	2004-05-04 20:52:19.554145300 -0700
@@ -8,6 +8,12 @@
  * instead of just grabbing them. Thus setups with different IRQ numbers
  * shouldn't result in any weird surprises, and installing new handlers
  * should be easier.
+ *
+ * Copyright (C) Ashok Raj<ashok.raj@intel.com>, Intel Corporation 2004
+ *
+ * 4/14/2004: Added code to handle cpu migration and do safe irq
+ *			migration without lossing interrupts for iosapic
+ *			architecture.
  */
 
 /*
@@ -27,6 +33,7 @@
 #include <linux/timex.h>
 #include <linux/slab.h>
 #include <linux/random.h>
+#include <linux/cpu.h>
 #include <linux/ctype.h>
 #include <linux/smp_lock.h>
 #include <linux/init.h>
@@ -35,14 +42,17 @@
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
 #include <linux/kallsyms.h>
+#include <linux/notifier.h>
 
 #include <asm/atomic.h>
+#include <asm/cpu.h>
 #include <asm/io.h>
 #include <asm/smp.h>
 #include <asm/system.h>
 #include <asm/bitops.h>
 #include <asm/uaccess.h>
 #include <asm/pgalloc.h>
+#include <asm/tlbflush.h>
 #include <asm/delay.h>
 #include <asm/irq.h>
 
@@ -1006,6 +1016,99 @@ static int irq_affinity_write_proc (stru
 
 #endif /* CONFIG_SMP */
 
+#ifdef CONFIG_HOTPLUG_CPU
+unsigned int vectors_in_migration[NR_IRQS];
+
+/*
+ * Since cpu_online_map is already updated, we just need to check for
+ * affinity that has zeros
+ */
+static void migrate_irqs(void)
+{
+	cpumask_t	mask;
+	irq_desc_t *desc;
+	int 		irq, new_cpu;
+
+	for (irq=0; irq < NR_IRQS; irq++) {
+		desc = irq_descp(irq);
+
+		/*
+		 * No handling for now.
+		 * TBD: Implement a disable function so we can now
+		 * tell CPU not to respond to these local intr sources.
+		 * such as ITV,CPEI,MCA etc.
+		 */
+		if (desc->status == IRQ_PER_CPU)
+			continue;
+
+		cpus_and(mask, irq_affinity[irq], cpu_online_map);
+		if (any_online_cpu(mask) == NR_CPUS) {
+			/*
+			 * Save it for phase 2 processing
+			 */
+			vectors_in_migration[irq] = irq;
+
+			new_cpu = any_online_cpu(cpu_online_map);
+			mask = cpumask_of_cpu(new_cpu);
+
+			/*
+			 * Al three are essential, currently WARN_ON.. maybe panic?
+			 */
+			if (desc->handler && desc->handler->disable &&
+				desc->handler->enable && desc->handler->set_affinity) {
+				desc->handler->disable(irq);
+				desc->handler->set_affinity(irq, mask);
+				desc->handler->enable(irq);
+			} else {
+				WARN_ON((!(desc->handler) || !(desc->handler->disable) ||
+						!(desc->handler->enable) ||
+						!(desc->handler->set_affinity)));
+			}
+		}
+	}
+}
+
+void fixup_irqs(void)
+{
+	unsigned int irq;
+	extern void ia64_process_pending_intr(void);
+
+	ia64_set_itv(1<<16);
+	/*
+	 * Phase 1: Locate irq's bound to this cpu and
+	 * relocate them for cpu removal.
+	 */
+	migrate_irqs();
+
+	/*
+	 * Phase 2: Perform interrupt processing for all entries reported in
+	 * local APIC.
+	 */
+	ia64_process_pending_intr();
+
+	/*
+	 * Phase 3: Now handle any interrupts not captured in local APIC.
+	 * This is to account for cases that device interrupted during the time the
+	 * rte was being disabled and re-programmed.
+	 */
+	for (irq=0; irq < NR_IRQS; irq++) {
+		if (vectors_in_migration[irq]) {
+			vectors_in_migration[irq]=0;
+			do_IRQ(irq, NULL);
+		}
+	}
+
+	/*
+	 * Now let processor die. We do irq disable and max_xtp() to
+	 * ensure there is no more interrupts routed to this processor.
+	 * But the local timer interrupt can have 1 pending which we
+	 * take care in timer_interrupt().
+	 */
+	max_xtp();
+	local_irq_disable();
+}
+#endif
+
 static int prof_cpu_mask_read_proc (char *page, char **start, off_t off,
 			int count, int *eof, void *data)
 {
diff -puN arch/ia64/kernel/process.c~hotcpu_ia64 arch/ia64/kernel/process.c
--- linux-2.6.6-rc3/arch/ia64/kernel/process.c~hotcpu_ia64	2004-05-04 20:52:19.530707788 -0700
+++ linux-2.6.6-rc3-root/arch/ia64/kernel/process.c	2004-05-04 20:52:19.554145300 -0700
@@ -7,6 +7,7 @@
 #define __KERNEL_SYSCALLS__	/* see <asm/unistd.h> */
 #include <linux/config.h>
 
+#include <linux/cpu.h>
 #include <linux/pm.h>
 #include <linux/elf.h>
 #include <linux/errno.h>
@@ -14,6 +15,7 @@
 #include <linux/kernel.h>
 #include <linux/mm.h>
 #include <linux/module.h>
+#include <linux/notifier.h>
 #include <linux/personality.h>
 #include <linux/sched.h>
 #include <linux/slab.h>
@@ -22,13 +24,17 @@
 #include <linux/thread_info.h>
 #include <linux/unistd.h>
 #include <linux/efi.h>
+#include <linux/interrupt.h>
 
+#include <asm/cpu.h>
 #include <asm/delay.h>
 #include <asm/elf.h>
 #include <asm/ia32.h>
+#include <asm/irq.h>
 #include <asm/pgalloc.h>
 #include <asm/processor.h>
 #include <asm/sal.h>
+#include <asm/tlbflush.h>
 #include <asm/uaccess.h>
 #include <asm/unwind.h>
 #include <asm/user.h>
@@ -180,6 +186,40 @@ default_idle (void)
 			safe_halt();
 }
 
+#ifdef CONFIG_HOTPLUG_CPU
+/* We don't actually take CPU down, just spin without interrupts. */
+static inline void play_dead(void)
+{
+	extern void ia64_cpu_local_tick (void);
+	/* Ack it */
+	__get_cpu_var(cpu_state) = CPU_DEAD;
+
+	/* We shouldn't have to disable interrupts while dead, but
+	 * some interrupts just don't seem to go away, and this makes
+	 * it "work" for testing purposes. */
+	max_xtp();
+	local_irq_disable();
+	/* Death loop */
+	while (__get_cpu_var(cpu_state) != CPU_UP_PREPARE)
+		cpu_relax();
+
+	/*
+	 * Enable timer interrupts from now on
+	 * Not required if we put processor in SAL_BOOT_RENDEZ mode.
+	 */
+	local_flush_tlb_all();
+	cpu_set(smp_processor_id(), cpu_online_map);
+	wmb();
+	ia64_cpu_local_tick ();
+	local_irq_enable();
+}
+#else
+static inline void play_dead(void)
+{
+	BUG();
+}
+#endif /* CONFIG_HOTPLUG_CPU */
+
 void __attribute__((noreturn))
 cpu_idle (void *unused)
 {
@@ -195,7 +235,6 @@ cpu_idle (void *unused)
 		if (!need_resched())
 			min_xtp();
 #endif
-
 		while (!need_resched()) {
 			if (mark_idle)
 				(*mark_idle)(1);
@@ -210,6 +249,8 @@ cpu_idle (void *unused)
 #endif
 		schedule();
 		check_pgt_cache();
+		if (cpu_is_offline(smp_processor_id()))
+			play_dead();
 	}
 }
 
diff -puN arch/ia64/kernel/smpboot.c~hotcpu_ia64 arch/ia64/kernel/smpboot.c
--- linux-2.6.6-rc3/arch/ia64/kernel/smpboot.c~hotcpu_ia64	2004-05-04 20:52:19.532660914 -0700
+++ linux-2.6.6-rc3-root/arch/ia64/kernel/smpboot.c	2004-05-04 20:52:19.555121863 -0700
@@ -15,6 +15,7 @@
 #include <linux/module.h>
 #include <linux/acpi.h>
 #include <linux/bootmem.h>
+#include <linux/cpu.h>
 #include <linux/delay.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
@@ -22,10 +23,12 @@
 #include <linux/kernel.h>
 #include <linux/kernel_stat.h>
 #include <linux/mm.h>
+#include <linux/notifier.h>
 #include <linux/smp.h>
 #include <linux/smp_lock.h>
 #include <linux/spinlock.h>
 #include <linux/efi.h>
+#include <linux/percpu.h>
 
 #include <asm/atomic.h>
 #include <asm/bitops.h>
@@ -44,6 +47,7 @@
 #include <asm/ptrace.h>
 #include <asm/sal.h>
 #include <asm/system.h>
+#include <asm/tlbflush.h>
 #include <asm/unistd.h>
 
 #define SMP_DEBUG 0
@@ -75,6 +79,11 @@ extern unsigned long ia64_iobase;
 
 task_t *task_for_booting_cpu;
 
+/*
+ * State for each CPU
+ */
+DEFINE_PER_CPU(int, cpu_state);
+
 /* Bitmasks of currently online, and possible CPUs */
 cpumask_t cpu_online_map;
 EXPORT_SYMBOL(cpu_online_map);
@@ -281,12 +290,16 @@ smp_callin (void)
 	cpuid = smp_processor_id();
 	phys_id = hard_smp_processor_id();
 
-	if (cpu_test_and_set(cpuid, cpu_online_map)) {
+	if (cpu_online(cpuid)) {
 		printk(KERN_ERR "huh, phys CPU#0x%x, CPU#0x%x already present??\n",
 		       phys_id, cpuid);
 		BUG();
 	}
 
+	lock_ipi_calllock();
+	cpu_set(cpuid, cpu_online_map);
+	unlock_ipi_calllock();
+
 	smp_setup_percpu_timer();
 
 	/*
@@ -357,29 +370,51 @@ fork_by_hand (void)
 	return copy_process(CLONE_VM|CLONE_IDLETASK, 0, 0, 0, NULL, NULL);
 }
 
+struct create_idle {
+	struct task_struct *idle;
+	struct completion done;
+};
+
+void
+do_fork_idle(void *_c_idle)
+{
+	struct create_idle *c_idle = _c_idle;
+
+	c_idle->idle = fork_by_hand();
+	complete(&c_idle->done);
+}
+
 static int __devinit
 do_boot_cpu (int sapicid, int cpu)
 {
-	struct task_struct *idle;
 	int timeout;
+	struct create_idle c_idle;
+	DECLARE_WORK(work, do_fork_idle, &c_idle);
 
+	init_completion(&c_idle.done);
 	/*
 	 * We can't use kernel_thread since we must avoid to reschedule the child.
 	 */
-	idle = fork_by_hand();
-	if (IS_ERR(idle))
+	if (!keventd_up() || current_is_keventd())
+		work.func(work.data);
+	else {
+		schedule_work(&work);
+		wait_for_completion(&c_idle.done);
+	}
+
+	if (IS_ERR(c_idle.idle))
 		panic("failed fork for CPU %d", cpu);
-	wake_up_forked_process(idle);
+	wake_up_forked_process(c_idle.idle);
 
 	/*
 	 * We remove it from the pidhash and the runqueue
 	 * once we got the process:
 	 */
-	init_idle(idle, cpu);
+	init_idle(c_idle.idle, cpu);
 
-	unhash_process(idle);
+	unhash_process(c_idle.idle);
 
-	task_for_booting_cpu = idle;
+	task_for_booting_cpu = c_idle.idle;
 
 	Dprintk("Sending wakeup vector %lu to AP 0x%x/0x%x.\n", ap_wakeup_vector, cpu, sapicid);
 
@@ -438,8 +473,12 @@ smp_build_cpu_map (void)
 	int sapicid, cpu, i;
 	int boot_cpu_id = hard_smp_processor_id();
 
-	for (cpu = 0; cpu < NR_CPUS; cpu++)
+	for (cpu = 0; cpu < NR_CPUS; cpu++) {
 		ia64_cpu_to_sapicid[cpu] = -1;
+#ifdef CONFIG_HOTPLUG_CPU
+		cpu_set(cpu, cpu_possible_map);
+#endif
+	}
 
 	ia64_cpu_to_sapicid[0] = boot_cpu_id;
 	cpus_clear(cpu_present_map);
@@ -546,6 +585,74 @@ void __devinit smp_prepare_boot_cpu(void
 	cpu_set(smp_processor_id(), cpu_callin_map);
 }
 
+#ifdef CONFIG_HOTPLUG_CPU
+extern void fixup_irqs(void);
+/* must be called with cpucontrol mutex held */
+static int __devinit cpu_enable(unsigned int cpu)
+{
+	per_cpu(cpu_state,cpu) = CPU_UP_PREPARE;
+	wmb();
+
+	while (!cpu_online(cpu))
+		cpu_relax();
+	return 0;
+}
+
+int __cpu_disable(void)
+{
+	int cpu = smp_processor_id();
+
+	/*
+	 * dont permit boot processor for now
+	 */
+	if (cpu == 0)
+		return -EBUSY;
+
+	fixup_irqs();
+	local_flush_tlb_all();
+	printk ("Disabled cpu %u\n", smp_processor_id());
+	return 0;
+}
+
+void __cpu_die(unsigned int cpu)
+{
+	unsigned int i;
+
+	for (i = 0; i < 100; i++) {
+		/* They ack this in play_dead by setting CPU_DEAD */
+		if (per_cpu(cpu_state, cpu) == CPU_DEAD)
+		{
+			/*
+			 * TBD: Enable this when physical removal
+			 * or when we put the processor is put in
+			 * SAL_BOOT_RENDEZ mode
+			 * cpu_clear(cpu, cpu_callin_map);
+			 */
+			return;
+		}
+		current->state = TASK_UNINTERRUPTIBLE;
+		schedule_timeout(HZ/10);
+	}
+ 	printk(KERN_ERR "CPU %u didn't die...\n", cpu);
+}
+#else /* !CONFIG_HOTPLUG_CPU */
+static int __devinit cpu_enable(unsigned int cpu)
+{
+	return 0;
+}
+
+int __cpu_disable(void)
+{
+	return -ENOSYS;
+}
+
+void __cpu_die(unsigned int cpu)
+{
+	/* We said "no" in __cpu_disable */
+	BUG();
+}
+#endif /* CONFIG_HOTPLUG_CPU */
+
 void
 smp_cpus_done (unsigned int dummy)
 {
@@ -574,6 +681,17 @@ __cpu_up (unsigned int cpu)
 	if (sapicid == -1)
 		return -EINVAL;
 
+	/*
+	 * Already booted.. just enable and get outa idle lool
+	 */
+	if (cpu_isset(cpu, cpu_callin_map))
+	{
+		cpu_enable(cpu);
+		local_irq_enable();
+		while (!cpu_isset(cpu, cpu_online_map))
+			mb();
+		return 0;
+	}
 	/* Processor goes to start_secondary(), sets online flag */
 	ret = do_boot_cpu(sapicid, cpu);
 	if (ret < 0)
diff -puN arch/ia64/kernel/smp.c~hotcpu_ia64 arch/ia64/kernel/smp.c
--- linux-2.6.6-rc3/arch/ia64/kernel/smp.c~hotcpu_ia64	2004-05-04 20:52:19.533637477 -0700
+++ linux-2.6.6-rc3-root/arch/ia64/kernel/smp.c	2004-05-04 20:52:19.556098426 -0700
@@ -71,10 +71,23 @@ static volatile struct call_data_struct 
 /* This needs to be cacheline aligned because it is written to by *other* CPUs.  */
 static DEFINE_PER_CPU(u64, ipi_operation) ____cacheline_aligned;
 
+extern void cpu_halt (void);
+
+void
+lock_ipi_calllock(void)
+{
+	spin_lock_irq(&call_lock);
+}
+
+void
+unlock_ipi_calllock(void)
+{
+	spin_unlock_irq(&call_lock);
+}
+
 static void
 stop_this_cpu (void)
 {
-	extern void cpu_halt (void);
 	/*
 	 * Remove this CPU:
 	 */
@@ -84,6 +97,17 @@ stop_this_cpu (void)
 	cpu_halt();
 }
 
+void
+cpu_die(void)
+{
+	max_xtp();
+	local_irq_disable();
+	cpu_halt();
+	/* Should never be here */
+	BUG();
+	for (;;);
+}
+
 irqreturn_t
 handle_IPI (int irq, void *dev_id, struct pt_regs *regs)
 {
diff -puN arch/ia64/kernel/time.c~hotcpu_ia64 arch/ia64/kernel/time.c
--- linux-2.6.6-rc3/arch/ia64/kernel/time.c~hotcpu_ia64	2004-05-04 20:52:19.534614040 -0700
+++ linux-2.6.6-rc3-root/arch/ia64/kernel/time.c	2004-05-04 20:52:19.556098426 -0700
@@ -10,6 +10,7 @@
  */
 #include <linux/config.h>
 
+#include <linux/cpu.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -244,6 +245,10 @@ timer_interrupt (int irq, void *dev_id, 
 {
 	unsigned long new_itm;
 
+	if (unlikely(cpu_is_offline(smp_processor_id()))) {
+		return IRQ_HANDLED;
+	}
+
 	platform_timer_interrupt(irq, dev_id, regs);
 
 	new_itm = local_cpu_data->itm_next;
diff -puN include/asm-ia64/smp.h~hotcpu_ia64 include/asm-ia64/smp.h
--- linux-2.6.6-rc3/include/asm-ia64/smp.h~hotcpu_ia64	2004-05-04 20:52:19.536567166 -0700
+++ linux-2.6.6-rc3-root/include/asm-ia64/smp.h	2004-05-04 20:52:19.556098426 -0700
@@ -120,6 +120,8 @@ extern void smp_do_timer (struct pt_regs
 extern int smp_call_function_single (int cpuid, void (*func) (void *info), void *info,
 				     int retry, int wait);
 extern void smp_send_reschedule (int cpu);
+extern void lock_ipi_calllock(void);
+extern void unlock_ipi_calllock(void);
 
 #endif /* CONFIG_SMP */
 #endif /* _ASM_IA64_SMP_H */
diff -puN arch/ia64/kernel/iosapic.c~hotcpu_ia64 arch/ia64/kernel/iosapic.c
--- linux-2.6.6-rc3/arch/ia64/kernel/iosapic.c~hotcpu_ia64	2004-05-04 20:52:19.537543729 -0700
+++ linux-2.6.6-rc3-root/arch/ia64/kernel/iosapic.c	2004-05-04 20:52:19.557074989 -0700
@@ -32,6 +32,8 @@
  * 03/02/19	B. Helgaas	Make pcat_compat system-wide, not per-IOSAPIC.
  *				Remove iosapic_address & gsi_base from external interfaces.
  *				Rationalize __init/__devinit attributes.
+ * 04/12/04 Ashok Raj	<ashok.raj@intel.com> Intel Corporation 2004
+ *				Updated to work with irq migration necessary for CPU Hotplug
  */
 /*
  * Here is what the interrupt logic between a PCI device and the kernel looks like:
@@ -189,8 +191,10 @@ set_rte (unsigned int vector, unsigned i
 	pol     = iosapic_intr_info[vector].polarity;
 	trigger = iosapic_intr_info[vector].trigger;
 	dmode   = iosapic_intr_info[vector].dmode;
+	vector &= (~IA64_IRQ_REDIRECTED);
 
 	redir = (dmode == IOSAPIC_LOWEST_PRIORITY) ? 1 : 0;
+
 #ifdef CONFIG_SMP
 	{
 		unsigned int irq;
@@ -312,9 +316,8 @@ iosapic_set_affinity (unsigned int irq, 
 
 	spin_lock_irqsave(&iosapic_lock, flags);
 	{
-		/* get current delivery mode by reading the low32 */
-		writel(IOSAPIC_RTE_LOW(rte_index), addr + IOSAPIC_REG_SELECT);
 		low32 = iosapic_intr_info[vec].low32 & ~(7 << IOSAPIC_DELIVERY_SHIFT);
+
 		if (redir)
 		        /* change delivery mode to lowest priority */
 			low32 |= (IOSAPIC_LOWEST_PRIORITY << IOSAPIC_DELIVERY_SHIFT);
diff -puN arch/ia64/kernel/irq_ia64.c~hotcpu_ia64 arch/ia64/kernel/irq_ia64.c
--- linux-2.6.6-rc3/arch/ia64/kernel/irq_ia64.c~hotcpu_ia64	2004-05-04 20:52:19.550239048 -0700
+++ linux-2.6.6-rc3-root/arch/ia64/kernel/irq_ia64.c	2004-05-04 20:52:19.557074989 -0700
@@ -10,6 +10,8 @@
  *
  * 09/15/00 Goutham Rao <goutham.rao@intel.com> Implemented pci_irq_to_vector
  *                      PCI to vector allocation routine.
+ * 04/14/2004 Ashok Raj <ashok.raj@intel.com>
+ *						Added CPU Hotplug handling for IPF.
  */
 
 #include <linux/config.h>
@@ -85,6 +87,11 @@ assign_irq_vector (int irq)
 
 extern unsigned int do_IRQ(unsigned long irq, struct pt_regs *regs);
 
+#ifdef CONFIG_SMP
+#	define IS_RESCHEDULE(vec)	(vec == IA64_IPI_RESCHEDULE)
+#else
+#	define IS_RESCHEDULE(vec)	(0)
+#endif
 /*
  * That's where the IVT branches when we get an external
  * interrupt. This branches to the correct hardware IRQ handler via
@@ -94,11 +101,6 @@ void
 ia64_handle_irq (ia64_vector vector, struct pt_regs *regs)
 {
 	unsigned long saved_tpr;
-#ifdef CONFIG_SMP
-#	define IS_RESCHEDULE(vec)	(vec == IA64_IPI_RESCHEDULE)
-#else
-#	define IS_RESCHEDULE(vec)	(0)
-#endif
 
 #if IRQ_DEBUG
 	{
@@ -162,6 +164,54 @@ ia64_handle_irq (ia64_vector vector, str
 	irq_exit();
 }
 
+#ifdef CONFIG_HOTPLUG_CPU
+/*
+ * This function emulates a interrupt processing when a cpu is about to be
+ * brought down.
+ */
+void ia64_process_pending_intr(void)
+{
+	ia64_vector vector;
+	unsigned long saved_tpr;
+	extern unsigned int vectors_in_migration[NR_IRQS];
+
+	vector = ia64_get_ivr();
+
+	 irq_enter();
+	 saved_tpr = ia64_getreg(_IA64_REG_CR_TPR);
+	 ia64_srlz_d();
+
+	 /*
+	  * Perform normal interrupt style processing
+	  */
+	while (vector != IA64_SPURIOUS_INT_VECTOR) {
+		if (!IS_RESCHEDULE(vector)) {
+			ia64_setreg(_IA64_REG_CR_TPR, vector);
+			ia64_srlz_d();
+
+			/*
+			 * Now try calling normal ia64_handle_irq as it would have got called
+			 * from a real intr handler. Try passing null for pt_regs, hopefully
+			 * it will work. I hope it works!.
+			 * Probably could shared code.
+			 */
+			vectors_in_migration[local_vector_to_irq(vector)]=0;
+			do_IRQ(local_vector_to_irq(vector), NULL);
+
+			/*
+			 * Disable interrupts and send EOI
+			 */
+			local_irq_disable();
+			ia64_setreg(_IA64_REG_CR_TPR, saved_tpr);
+		}
+		ia64_eoi();
+		vector = ia64_get_ivr();
+	}
+	irq_exit();
+}
+#endif
+
+
 #ifdef CONFIG_SMP
 extern irqreturn_t handle_IPI (int irq, void *dev_id, struct pt_regs *regs);
 
diff -puN arch/ia64/kernel/sal.c~hotcpu_ia64 arch/ia64/kernel/sal.c
--- linux-2.6.6-rc3/arch/ia64/kernel/sal.c~hotcpu_ia64	2004-05-04 20:52:19.551215611 -0700
+++ linux-2.6.6-rc3-root/arch/ia64/kernel/sal.c	2004-05-04 20:52:19.557074989 -0700
@@ -122,10 +122,23 @@ sal_desc_entry_point (void *p)
 static void __init
 set_smp_redirect (int flag)
 {
+#ifndef CONFIG_HOTPLUG_CPU
 	if (no_int_routing)
 		smp_int_redirect &= ~flag;
 	else
 		smp_int_redirect |= flag;
+#else
+	/*
+	 * For CPU Hotplug we dont want to do any chipset supported
+	 * interrupt redirection. The reason is this would require that
+	 * All interrupts be stopped and hard bind the irq to a cpu.
+	 * Later when the interrupt is fired we need to set the redir hint
+	 * on again in the vector. This is combersome for something that the
+	 * user mode irq balancer will solve anyways.
+	 */
+	no_int_routing=1;
+	smp_int_redirect &= ~flag;
+#endif
 }
 #else
 #define set_smp_redirect(flag)	do { } while (0)

_

Re: (resend) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

[Paul Jackson]

Being somewhat thick in the head at times, it took me a lengthy private
email thread with Ashok to understand why he needed to add a
cpu_present_map for his cpu hotplug work, and just how that new map
related to the existing ones.

After patient and repeated explanations from Ashok, I worked out the
following explanation that he found accurate.  On the off chance that
others find this way of phrasing it helpful, I post it here for the
record.

===

Ashok's cpu hot plug patch adds a cpu_present_map, resulting in the
following cpu maps being available.  All the following maps are fixed
size bitmaps of size NR_CPUS.

#ifdef CONFIG_HOTPLUG_CPU
	cpu_possible_map - map with all NR_CPUS bits set
	cpu_present_map - map with bit 'cpu' set iff cpu is populated
	cpu_online_map - map with bit 'cpu' set iff cpu available to scheduler
#else
	cpu_possible_map - map with bit 'cpu' set iff cpu is populated
	cpu_present_map - copy of cpu_possible_map
	cpu_online_map - map with bit 'cpu' set iff cpu available to scheduler
#endif

In either case, NR_CPUS is fixed at compile time, as the static size of
these bitmaps.  The cpu_possible_map is fixed at boot time, as the set
of CPU id's that it is possible might ever be plugged in at anytime
during the life of that system boot.  The cpu_present_map is dynamic(*),
representing which CPUs are currently plugged in.  And cpu_online_map is
the dynamic subset of cpu_present_map, indicating those CPUs available
for scheduling.

If HOTPLUG is enabled, then cpu_possible_map is forced to have all
NR_CPUS bits set, otherwise it is just the set of CPUs that ACPI reports
present at boot.

If HOTPLUG is enabled, then cpu_present_map varies dynamically,
depending on what ACPI reports as currently plugged in, otherwise
cpu_present_map is just a copy of cpu_possible_map.

(*) Well, cpu_present_map is dynamic in the hotplug case.
    If not hotplug, it's the same as cpu_possible_map, hence
    fixed at boot.

-- 
                          I won't rest till it's the best ...
                          Programmer, Linux Scalability
                          Paul Jackson <pj@sgi.com> 1.650.933.1373

Re: (resend) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

[Andrew Morton]

Paul Jackson <pj@sgi.com> wrote:
>
> On the off chance that
> others find this way of phrasing it helpful, I post it here for the
> record.

Thanks, I added that to the changelog.  If you think additional code
commentary is needed, please send patches.

I guess I'm not doing anything useful with these patches.  Is it OK with
everyone if I scoot them over to davidm?

Re: (resend) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

[Paul Jackson]

> Thanks, I added that to the changelog.

Good idea - thanks.

> If you think additional code commentary is needed,
> please send patches.

Ok - I've no plans to add any code comments at this time to this code.
Patches make sense if I did.

> Is it OK with everyone if I scoot them over to davidm?

Either way ... doesn't matter to me.

-- 
                          I won't rest till it's the best ...
                          Programmer, Linux Scalability
                          Paul Jackson <pj@sgi.com> 1.650.933.1373

Re: (resend) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

[Ashok Raj]

On Tue, May 11, 2004 at 04:38:01PM -0700, Andrew Morton wrote:
> Paul Jackson <pj@sgi.com> wrote:
> >
> > On the off chance that
> > others find this way of phrasing it helpful, I post it here for the
> > record.
> 
> Thanks, I added that to the changelog.  If you think additional code
> commentary is needed, please send patches.
> 
> I guess I'm not doing anything useful with these patches.  Is it OK with
> everyone if I scoot them over to davidm?


I had sent a private mail to davidm for inclusion in ia64 tree, he mentioned 
that it would be preferable to cook akpm tree to make sure no build breaks, 
and then he would consider those for inclusion in his tree. especially -mm gets
more testing, but iam not sure how may would really try this hotplug :-)

anytime now seems appropriate...

until David pickup's these it would be great to leave the here for anyone 
interested in testing...

Cheers,
Ashok

Re: (resend) take3: Updated CPU Hotplug patches for IA64 (pj blessed) Patch [6/7]

[Andrew Morton]

Ashok Raj <ashok.raj@intel.com> wrote:
>
> I had sent a private mail to davidm for inclusion in ia64 tree, he mentioned 
> that it would be preferable to cook akpm tree to make sure no build breaks, 
> and then he would consider those for inclusion in his tree. especially -mm gets
> more testing, but iam not sure how may would really try this hotplug :-)

Oh the patches are fine wrt side-effects.  I've had them for a couple of
weeks, they're tested on at least x86, ppc64, ppc, ia64, x86_64 and
sparc64.

It's just a question of whether David would prefer to handle them?

Affected files are:

arch/ia64/dig/Makefile
arch/ia64/dig/topology.c
arch/ia64/Kconfig
arch/ia64/kernel/iosapic.c
arch/ia64/kernel/irq.c
arch/ia64/kernel/irq_ia64.c
arch/ia64/kernel/palinfo.c
arch/ia64/kernel/process.c
arch/ia64/kernel/sal.c
arch/ia64/kernel/setup.c
arch/ia64/kernel/smpboot.c
arch/ia64/kernel/smp.c
arch/ia64/kernel/time.c
arch/ia64/mm/init.c
arch/ia64/mm/tlb.c
fs/proc/proc_misc.c
include/asm-ia64/cpu.h
include/asm-ia64/cpumask.h
include/asm-ia64/smp.h
include/linux/cpumask.h
init/main.c
kernel/cpu.c
kernel/sched.c