[PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c with flush_tlb_all()

[PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c with flush_tlb_all()

[Thomas Schlichter]

[-- Attachment #1.1: body text --]
[-- Type: text/plain, Size: 333 bytes --]

This patch replaces the flush_map() function in the arch/i386/mm/pageattr.c file with flush_tlb_all() calls, as the flush_map() function wants to do the same, but just forgot the preempt_disable() and preempt_enable() calls.

To minimize future inconsistency I think this patch should be applied...

Best regards
  Thomas Schlichter

[-- Attachment #1.2: remove_flush_map.patch --]
[-- Type: text/x-diff, Size: 1389 bytes --]

--- linux-2.5.62/arch/i386/mm/pageattr.c.orig	Wed Feb 19 16:44:56 2003
+++ linux-2.5.62/arch/i386/mm/pageattr.c	Wed Feb 19 16:46:11 2003
@@ -45,17 +45,6 @@
 	return base;
 } 
 
-static void flush_kernel_map(void *dummy) 
-{ 
-	/* Could use CLFLUSH here if the CPU supports it (Hammer,P4) */
-	if (boot_cpu_data.x86_model >= 4) 
-		asm volatile("wbinvd":::"memory"); 
-	/* Flush all to work around Errata in early athlons regarding 
-	 * large page flushing. 
-	 */
-	__flush_tlb_all(); 	
-}
-
 static void set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) 
 { 
 	set_pte_atomic(kpte, pte); 	/* change init_mm */
@@ -129,14 +118,6 @@
 	return 0;
 } 
 
-static inline void flush_map(void)
-{	
-#ifdef CONFIG_SMP 
-	smp_call_function(flush_kernel_map, NULL, 1, 1);
-#endif	
-	flush_kernel_map(NULL);
-}
-
 struct deferred_page { 
 	struct deferred_page *next; 
 	struct page *fpage;
@@ -172,7 +153,7 @@
 			struct deferred_page *df;
 			df = kmalloc(sizeof(struct deferred_page), GFP_KERNEL); 
 			if (!df) {
-				flush_map();
+				flush_tlb_all();
 				__free_page(fpage);
 			} else { 
 				df->next = df_list;
@@ -192,7 +173,7 @@
 	down_read(&init_mm.mmap_sem);
 	df = xchg(&df_list, NULL);
 	up_read(&init_mm.mmap_sem);
-	flush_map();
+	flush_tlb_all();
 	for (; df; df = next_df) { 
 		next_df = df->next;
 		if (df->fpage) 

[-- Attachment #2: signature --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c with flush_tlb_all()

[Andrew Morton]

Thomas Schlichter <schlicht@uni-mannheim.de> wrote:
>
> This patch replaces the flush_map() function in the arch/i386/mm/pageattr.c
> file with flush_tlb_all() calls, as the flush_map() function wants to do
> the same, but just forgot the preempt_disable() and preempt_enable() calls.
> 

flush_map() is doing a wbinvd:

> -static void flush_kernel_map(void *dummy) 
> -{ 
> -	/* Could use CLFLUSH here if the CPU supports it (Hammer,P4) */
> -	if (boot_cpu_data.x86_model >= 4) 
> -		asm volatile("wbinvd":::"memory"); 

It appears that by replacing flush_map() with flush_tlb_all(), you are no
longer doing this?

Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c with flush_tlb_all()

[Dave Jones]

On Thu, Feb 20, 2003 at 09:00:05PM +0100, Thomas Schlichter wrote:

 > This patch replaces the flush_map() function in the arch/i386/mm/pageattr.c file with flush_tlb_all() calls, as the flush_map() function wants to do the same, but just forgot the preempt_disable() and preempt_enable() calls.
 > 
 > To minimize future inconsistency I think this patch should be applied...

This looks bogus. You're killing the wbinvd() in flush_kernel_map() which
is needed.

		Dave

-- 
| Dave Jones.        http://www.codemonkey.org.uk
| SuSE Labs

Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c with flush_tlb_all()

[Thomas Schlichter]

[-- Attachment #1: signed data --]
[-- Type: text/plain, Size: 351 bytes --]

On Thu, Feb 20, 2003 at 21:36, Dave Jones wrote:
> This looks bogus. You're killing the wbinvd() in flush_kernel_map() which
> is needed.

I must admit I don't exactly know the wbinvd() command, but as the comment 
says:
  /* Could use CLFLUSH here if the CPU supports it (Hammer,P4) */

I thought it is not NEEDED, just a COULD...

  Thomas

[-- Attachment #2: signature --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c with flush_tlb_all()

[Dave Jones]

On Thu, Feb 20, 2003 at 09:30:55PM +0100, Thomas Schlichter wrote:

 > > This looks bogus. You're killing the wbinvd() in flush_kernel_map() which
 > > is needed.
 > I must admit I don't exactly know the wbinvd() command, but as the comment 
 > says:
 >   /* Could use CLFLUSH here if the CPU supports it (Hammer,P4) */
 > 
 > I thought it is not NEEDED, just a COULD...

Its hinting at a possible optimisation, not saying
that it is unneeded.

		Dave

-- 
| Dave Jones.        http://www.codemonkey.org.uk
| SuSE Labs

Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c with flush_tlb_all()

[Thomas Schlichter]

[-- Attachment #1.1: body text --]
[-- Type: text/plain, Size: 469 bytes --]

On Thu, Feb 20, 2003 21:50, Dave Jones wrote:
> Its hinting at a possible optimisation, not saying
> that it is unneeded.

OK, sorry, than I just misunderstood the comment...

So here is a minimal change patch that should solve the preempt issue in 
flush_map().

Instead of just doing a preempt_disable() before and a preempt_enable() after 
the flush_kernel_map() calls I just changed the order so that the preempt 
point is not between them...

  Thomas

[-- Attachment #1.2: flush_map_preempt.patch --]
[-- Type: text/x-diff, Size: 808 bytes --]

--- linux-2.5.62/arch/i386/mm/pageattr.c.orig	Fri Feb 21 11:47:19 2003
+++ linux-2.5.62/arch/i386/mm/pageattr.c	Fri Feb 21 12:12:15 2003
@@ -131,10 +131,10 @@
 
 static inline void flush_map(void)
 {	
+	flush_kernel_map(NULL);
 #ifdef CONFIG_SMP 
 	smp_call_function(flush_kernel_map, NULL, 1, 1);
 #endif	
-	flush_kernel_map(NULL);
 }
 
 struct deferred_page { 
--- linux-2.5.62/arch/x86_64/mm/pageattr.c.orig	Fri Feb 21 12:14:25 2003
+++ linux-2.5.62/arch/x86_64/mm/pageattr.c	Fri Feb 21 12:14:30 2003
@@ -123,10 +123,10 @@
 
 static inline void flush_map(unsigned long address)
 {	
+	flush_kernel_map((void *)address);
 #ifdef CONFIG_SMP 
 	smp_call_function(flush_kernel_map, (void *)address, 1, 1);
 #endif	
-	flush_kernel_map((void *)address);
 }
 
 struct deferred_page { 

[-- Attachment #2: signature --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c w ith flush_tlb_all()

[Hugh Dickins]

On Fri, 21 Feb 2003, Thomas Schlichter wrote:
> 
> So here is a minimal change patch that should solve the preempt issue in 
> flush_map().
> 
> Instead of just doing a preempt_disable() before and a preempt_enable()
> after 
> the flush_kernel_map() calls I just changed the order so that the preempt 
> point is not between them...

No.  All that does is make sure that the cpu you start out on is
flushed, once or twice, and the cpu you end up on may be missed.
Use preempt_disable and preempt_enable.

Hugh

Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c w ith flush_tlb_all()

[Thomas Schlichter]

[-- Attachment #1.1: body text --]
[-- Type: text/plain, Size: 350 bytes --]

On Fri, 21 Feb 2003, Hugh Dickins wrote:
> No.  All that does is make sure that the cpu you start out on is
> flushed, once or twice, and the cpu you end up on may be missed.
> Use preempt_disable and preempt_enable.

Oh, you are right! I think I am totally stupid this morning...!
Now finally I hope this is the correct patch...

  Thomas Schlichter

[-- Attachment #1.2: flush_map_preempt.patch --]
[-- Type: text/x-diff, Size: 968 bytes --]

--- linux-2.5.62/arch/i386/mm/pageattr.c.orig	Fri Feb 21 13:27:59 2003
+++ linux-2.5.62/arch/i386/mm/pageattr.c	Fri Feb 21 13:32:39 2003
@@ -131,10 +131,14 @@
 
 static inline void flush_map(void)
 {	
-#ifdef CONFIG_SMP 
+#ifdef CONFIG_SMP
+	preempt_disable();
 	smp_call_function(flush_kernel_map, NULL, 1, 1);
-#endif	
 	flush_kernel_map(NULL);
+	preempt_enable();
+#else
+	flush_kernel_map(NULL);
+#endif
 }
 
 struct deferred_page { 
--- linux-2.5.62/arch/x86_64/mm/pageattr.c.orig	Fri Feb 21 13:33:38 2003
+++ linux-2.5.62/arch/x86_64/mm/pageattr.c	Fri Feb 21 13:34:40 2003
@@ -123,10 +123,14 @@
 
 static inline void flush_map(unsigned long address)
 {	
-#ifdef CONFIG_SMP 
+#ifdef CONFIG_SMP
+	preempt_disable();
 	smp_call_function(flush_kernel_map, (void *)address, 1, 1);
-#endif	
 	flush_kernel_map((void *)address);
+	preempt_enable();
+#else
+	flush_kernel_map((void *)address);
+#endif
 }
 
 struct deferred_page { 

[-- Attachment #2: signature --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c w ith flush_tlb_all()

[Dave Jones]

On Fri, Feb 21, 2003 at 01:42:12PM +0100, Thomas Schlichter wrote:

 > > No.  All that does is make sure that the cpu you start out on is
 > > flushed, once or twice, and the cpu you end up on may be missed.
 > > Use preempt_disable and preempt_enable.
 > 
 > Oh, you are right! I think I am totally stupid this morning...!
 > Now finally I hope this is the correct patch...

That would appear to do what you want, but its an ugly construct to
be repeating everywhere that wants to call a function on all CPUs.
It would probably clean things up a lot if we had a function to do..

static inline void on_each_cpu(void *func)
{      
#ifdef CONFIG_SMP
	preempt_disable();
	smp_call_function(func, NULL, 1, 1);
	func(NULL);
	preempt_enable();
#else
	func(NULL);
#endif
}

Bluesmoke and agpgart could both use this to cleanup some mess,
and no doubt there are others

Comments?

		Dave

-- 
| Dave Jones.        http://www.codemonkey.org.uk

Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c w ith flush_tlb_all()

[Thomas Schlichter]

[-- Attachment #1: signed data --]
[-- Type: text/plain, Size: 1773 bytes --]

Yes, you are right. I was just looking for this preempt-problem where a 
flush_tlb* was done, but there are many other places where this problem 
occours, too... Nearly everywhere where smp_call_function() is used!

I found a function in the file mm/slab.c called smp_call_function_all_cpus() 
which tries to do the thing we want, but I think not even this function is 
preempt-safe...!

But here I think I better get off my fingers, I just wanted to help a bit with 
this issue, but I don't think I have the time and knowledge to solve it 
completely in the mentioned good way...

Perhaps even the semantic of the function smp_call_function() could be changed 
to call the function on every CPU? Just an idea...

Best regards
  Thomas Schlichter


On Fri, 21 Feb 2003 15:20, Dave Jones wrote:
> On Fri, Feb 21, 2003 at 01:42:12PM +0100, Thomas Schlichter wrote:
>  > > No.  All that does is make sure that the cpu you start out on is
>  > > flushed, once or twice, and the cpu you end up on may be missed.
>  > > Use preempt_disable and preempt_enable.
>  >
>  > Oh, you are right! I think I am totally stupid this morning...!
>  > Now finally I hope this is the correct patch...
>
> That would appear to do what you want, but its an ugly construct to
> be repeating everywhere that wants to call a function on all CPUs.
> It would probably clean things up a lot if we had a function to do..
>
> static inline void on_each_cpu(void *func)
> {
> #ifdef CONFIG_SMP
> 	preempt_disable();
> 	smp_call_function(func, NULL, 1, 1);
> 	func(NULL);
> 	preempt_enable();
> #else
> 	func(NULL);
> #endif
> }
>
> Bluesmoke and agpgart could both use this to cleanup some mess,
> and no doubt there are others
>
> Comments?
>
> 		Dave

[-- Attachment #2: signature --]
[-- Type: application/pgp-signature, Size: 189 bytes --]

Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c w ith flush_tlb_all()

[Dave Jones]

On Fri, Feb 21, 2003 at 03:25:01PM +0100, Thomas Schlichter wrote:

 > I found a function in the file mm/slab.c called smp_call_function_all_cpus() 
 > which tries to do the thing we want, but I think not even this function is 
 > preempt-safe...!

That certainly looks erm, odd.

 > Perhaps even the semantic of the function smp_call_function() could be changed 
 > to call the function on every CPU? Just an idea...

Some places that use it actually rely on it not executing on the local cpu.
It would also break backwards compatabiltiy for no real good reason.

		Dave

-- 
| Dave Jones.        http://www.codemonkey.org.uk
| SuSE Labs

Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c w ith flush_tlb_all()

[Hugh Dickins]

On Fri, 21 Feb 2003, Dave Jones wrote:
> It would probably clean things up a lot if we had a function to do..
> 
> static inline void on_each_cpu(void *func)
> {      
> #ifdef CONFIG_SMP
> 	preempt_disable();
> 	smp_call_function(func, NULL, 1, 1);
> 	func(NULL);
> 	preempt_enable();
> #else
> 	func(NULL);
> #endif
> }

Of course that's much much better.  But I think rather better as
static inline void on_each_cpu(void (*func) (void *info), void *info)
passing info to func instead of assuming NULL.  inline? maybe.

Hugh

Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c w ith flush_tlb_all()

[Dave Jones]

On Fri, Feb 21, 2003 at 02:52:38PM +0000, Hugh Dickins wrote:

 > Of course that's much much better.  But I think rather better as
 > static inline void on_each_cpu(void (*func) (void *info), void *info)
 > passing info to func instead of assuming NULL.  inline? maybe.

Sounds good to me. As for inline - This thing will be generating lots
of IPIs on SMP, so its hardly going to be performance critical, so
we could leave it out of line IMO.

		Dave

-- 
| Dave Jones.        http://www.codemonkey.org.uk
| SuSE Labs

Re: [PATCH][2.5] replace flush_map() in arch/i386/mm/pageattr.c w ith flush_tlb_all()

[Dave Jones]

On Fri, Feb 21, 2003 at 02:20:39PM +0000, Dave Jones wrote:

 > That would appear to do what you want, but its an ugly construct to
 > be repeating everywhere that wants to call a function on all CPUs.
 > It would probably clean things up a lot if we had a function to do..

Ok, here's a first stab at an implementation. Compiles, but is untested..
Fixes up a few preemption races Thomas highlighted, and converts
a few smp_call_function() users over to on_each_cpu(), which
saves quite a bit of code.

		Dave


diff -Nru a/arch/i386/kernel/cpuid.c b/arch/i386/kernel/cpuid.c
--- a/arch/i386/kernel/cpuid.c	Fri Feb 21 17:16:00 2003
+++ b/arch/i386/kernel/cpuid.c	Fri Feb 21 17:16:00 2003
@@ -64,6 +64,7 @@
 {
   struct cpuid_command cmd;
   
+  preempt_disable();
   if ( cpu == smp_processor_id() ) {
     cpuid(reg, &data[0], &data[1], &data[2], &data[3]);
   } else {
@@ -73,6 +74,7 @@
     
     smp_call_function(cpuid_smp_cpuid, &cmd, 1, 1);
   }
+  preempt_enable();
 }
 #else /* ! CONFIG_SMP */
 
diff -Nru a/arch/i386/kernel/microcode.c b/arch/i386/kernel/microcode.c
--- a/arch/i386/kernel/microcode.c	Fri Feb 21 17:16:00 2003
+++ b/arch/i386/kernel/microcode.c	Fri Feb 21 17:16:00 2003
@@ -183,11 +183,8 @@
 	int i, error = 0, err;
 	struct microcode *m;
 
-	if (smp_call_function(do_update_one, NULL, 1, 1) != 0) {
-		printk(KERN_ERR "microcode: IPI timeout, giving up\n");
+	if (on_each_cpu(do_update_one, NULL)==-1)
 		return -EIO;
-	}
-	do_update_one(NULL);
 
 	for (i=0; i<NR_CPUS; i++) {
 		err = update_req[i].err;
diff -Nru a/arch/i386/kernel/msr.c b/arch/i386/kernel/msr.c
--- a/arch/i386/kernel/msr.c	Fri Feb 21 17:16:00 2003
+++ b/arch/i386/kernel/msr.c	Fri Feb 21 17:16:00 2003
@@ -115,9 +115,13 @@
 static inline int do_wrmsr(int cpu, u32 reg, u32 eax, u32 edx)
 {
   struct msr_command cmd;
+  int ret;
 
+  preempt_disable();
   if ( cpu == smp_processor_id() ) {
-    return wrmsr_eio(reg, eax, edx);
+    ret = wrmsr_eio(reg, eax, edx);
+    preempt_enable();
+    return ret;
   } else {
     cmd.cpu = cpu;
     cmd.reg = reg;
@@ -125,6 +129,7 @@
     cmd.data[1] = edx;
     
     smp_call_function(msr_smp_wrmsr, &cmd, 1, 1);
+    preempt_enable();
     return cmd.err;
   }
 }
diff -Nru a/arch/i386/kernel/smp.c b/arch/i386/kernel/smp.c
--- a/arch/i386/kernel/smp.c	Fri Feb 21 17:16:00 2003
+++ b/arch/i386/kernel/smp.c	Fri Feb 21 17:16:00 2003
@@ -547,6 +547,41 @@
 	return 0;
 }
 
+/*
+ * [SUMMARY] Run a function on every CPU.
+ * <func> The function to run. This must be fast and non-blocking.
+ * <info> An arbitrary pointer to pass to the function.
+ * <nonatomic> currently unused.
+ * [RETURNS] 0 on success, else a negative status code. Does not return until
+ * remote CPUs are nearly ready to execute <<func>> or are or have executed.
+ *
+ * You must not call this function with disabled interrupts or from a
+ * hardware interrupt handler or from a bottom half handler.
+ */
+int on_each_cpu(void (*func) (void *info), void *info)
+{
+#ifdef CONFIG_SMP
+	preempt_disable();
+
+	if (num_online_cpus() == 1)
+		goto only_one;
+
+	if (smp_call_function(func, info, 1, 1) != 0) {
+		printk (KERN_ERR "%p: IPI timeout, giving up\n",
+			__builtin_return_address(0));
+		preempt_enable();
+		return -1;
+	}
+
+only_one:
+	func(info);
+	preempt_enable();
+#else
+	func(info);
+#endif
+	return 0;
+}
+
 static void stop_this_cpu (void * dummy)
 {
 	/*
diff -Nru a/arch/i386/kernel/sysenter.c b/arch/i386/kernel/sysenter.c
--- a/arch/i386/kernel/sysenter.c	Fri Feb 21 17:16:00 2003
+++ b/arch/i386/kernel/sysenter.c	Fri Feb 21 17:16:00 2003
@@ -17,6 +17,7 @@
 #include <asm/msr.h>
 #include <asm/pgtable.h>
 #include <asm/unistd.h>
+#include <asm/smp.h>
 
 extern asmlinkage void sysenter_entry(void);
 
@@ -97,8 +98,7 @@
 		return 0;
 
 	memcpy((void *) page, sysent, sizeof(sysent));
-	enable_sep_cpu(NULL);
-	smp_call_function(enable_sep_cpu, NULL, 1, 1);
+	on_each_cpu (enable_sep_cpu, NULL);
 	return 0;
 }
 
diff -Nru a/arch/i386/mm/pageattr.c b/arch/i386/mm/pageattr.c
--- a/arch/i386/mm/pageattr.c	Fri Feb 21 17:16:00 2003
+++ b/arch/i386/mm/pageattr.c	Fri Feb 21 17:16:00 2003
@@ -131,10 +131,7 @@
 
 static inline void flush_map(void)
 {	
-#ifdef CONFIG_SMP 
-	smp_call_function(flush_kernel_map, NULL, 1, 1);
-#endif	
-	flush_kernel_map(NULL);
+	on_each_cpu(flush_kernel_map, NULL);
 }
 
 struct deferred_page { 
diff -Nru a/arch/x86_64/mm/pageattr.c b/arch/x86_64/mm/pageattr.c
--- a/arch/x86_64/mm/pageattr.c	Fri Feb 21 17:16:00 2003
+++ b/arch/x86_64/mm/pageattr.c	Fri Feb 21 17:16:00 2003
@@ -123,10 +123,7 @@
 
 static inline void flush_map(unsigned long address)
 {	
-#ifdef CONFIG_SMP 
-	smp_call_function(flush_kernel_map, (void *)address, 1, 1);
-#endif	
-	flush_kernel_map((void *)address);
+	on_each_cpu(flush_kernel_map,(void *)address);
 }
 
 struct deferred_page { 
diff -Nru a/drivers/char/agp/agp.h b/drivers/char/agp/agp.h
--- a/drivers/char/agp/agp.h	Fri Feb 21 17:16:00 2003
+++ b/drivers/char/agp/agp.h	Fri Feb 21 17:16:00 2003
@@ -34,24 +34,10 @@
 
 #define PFX "agpgart: "
 
-#ifdef CONFIG_SMP
-static void ipi_handler(void *null)
-{
-	flush_agp_cache();
-}
-
-static void __attribute__((unused)) global_cache_flush(void)
-{
-	if (smp_call_function(ipi_handler, NULL, 1, 1) != 0)
-		panic(PFX "timed out waiting for the other CPUs!\n");
-	flush_agp_cache();
-}
-#else
 static inline void global_cache_flush(void)
 {
-	flush_agp_cache();
+	on_each_cpu(flush_agp_cache, NULL);
 }
-#endif	/* !CONFIG_SMP */
 
 enum aper_size_type {
 	U8_APER_SIZE,
diff -Nru a/include/asm-i386/agp.h b/include/asm-i386/agp.h
--- a/include/asm-i386/agp.h	Fri Feb 21 17:16:00 2003
+++ b/include/asm-i386/agp.h	Fri Feb 21 17:16:00 2003
@@ -18,6 +18,8 @@
 /* Could use CLFLUSH here if the cpu supports it. But then it would
    need to be called for each cacheline of the whole page so it may not be 
    worth it. Would need a page for it. */
-#define flush_agp_cache() asm volatile("wbinvd":::"memory")
-
+static void flush_agp_cache(void *info)
+{
+	__asm__ __volatile__ ("wbinvd": : :"memory");
+}
 #endif
diff -Nru a/include/asm-i386/smp.h b/include/asm-i386/smp.h
--- a/include/asm-i386/smp.h	Fri Feb 21 17:16:00 2003
+++ b/include/asm-i386/smp.h	Fri Feb 21 17:16:00 2003
@@ -46,6 +46,8 @@
 extern void (*mtrr_hook) (void);
 extern void zap_low_mappings (void);
 
+extern int on_each_cpu(void (*func) (void *info), void *info);
+
 #define MAX_APICID 256
 
 /*
diff -Nru a/include/asm-x86_64/agp.h b/include/asm-x86_64/agp.h
--- a/include/asm-x86_64/agp.h	Fri Feb 21 17:16:00 2003
+++ b/include/asm-x86_64/agp.h	Fri Feb 21 17:16:00 2003
@@ -18,6 +18,8 @@
 /* Could use CLFLUSH here if the cpu supports it. But then it would
    need to be called for each cacheline of the whole page so it may not be 
    worth it. Would need a page for it. */
-#define flush_agp_cache() asm volatile("wbinvd":::"memory")
-
+static void flush_agp_cache(void *info)
+{
+	__asm__ __volatile__ ("wbinvd": : :"memory");
+}
 #endif
diff -Nru a/include/asm-x86_64/smp.h b/include/asm-x86_64/smp.h
--- a/include/asm-x86_64/smp.h	Fri Feb 21 17:16:00 2003
+++ b/include/asm-x86_64/smp.h	Fri Feb 21 17:16:00 2003
@@ -46,6 +46,8 @@
 extern void (*mtrr_hook) (void);
 extern void zap_low_mappings(void);
 
+extern int on_each_cpu(void (*func) (void *info), void *info);
+
 #define SMP_TRAMPOLINE_BASE 0x6000
 
 /*
-- 
| Dave Jones.        http://www.codemonkey.org.uk
| SuSE Labs

[PATCH][2.5] fix preempt-issues with smp_call_function()

[Thomas Schlichter]

[-- Attachment #1.1: body text --]
[-- Type: text/plain, Size: 567 bytes --]

On Fri, Feb 21, 2003 at 18:36, Dave Jones wrote:
> Ok, here's a first stab at an implementation. Compiles, but is untested..
> Fixes up a few preemption races Thomas highlighted, and converts
> a few smp_call_function() users over to on_each_cpu(), which
> saves quite a bit of code.

Your patch was really fine, I just modified it a bit and fixed some more 
preempt-issues with the smp_call_function() calls. It compiles and workes 
with no problems so far... I hope I did not make any big mistakes... ;-)

I hope you like this one, too...

     Thomas

[-- Attachment #1.2: preempt_fix.patch --]
[-- Type: text/x-diff, Size: 35354 bytes --]

diff -urP linux-2.5.62/arch/alpha/kernel/process.c linux-2.5.62_patched/arch/alpha/kernel/process.c
--- linux-2.5.62/arch/alpha/kernel/process.c	Mon Feb 17 23:56:54 2003
+++ linux-2.5.62_patched/arch/alpha/kernel/process.c	Sat Feb 22 02:02:17 2003
@@ -155,10 +155,7 @@
 	struct halt_info args;
 	args.mode = mode;
 	args.restart_cmd = restart_cmd;
-#ifdef CONFIG_SMP
-	smp_call_function(common_shutdown_1, &args, 1, 0);
-#endif
-	common_shutdown_1(&args);
+	on_each_cpu(common_shutdown_1, &args, 1, 0);
 }
 
 void
diff -urP linux-2.5.62/arch/alpha/kernel/smp.c linux-2.5.62_patched/arch/alpha/kernel/smp.c
--- linux-2.5.62/arch/alpha/kernel/smp.c	Mon Feb 17 23:56:10 2003
+++ linux-2.5.62_patched/arch/alpha/kernel/smp.c	Sat Feb 22 02:02:17 2003
@@ -899,10 +899,8 @@
 smp_imb(void)
 {
 	/* Must wait other processors to flush their icache before continue. */
-	if (smp_call_function(ipi_imb, NULL, 1, 1))
+	if (on_each_cpu(ipi_imb, NULL, 1, 1))
 		printk(KERN_CRIT "smp_imb: timed out\n");
-
-	imb();
 }
 
 static void
@@ -916,11 +914,9 @@
 {
 	/* Although we don't have any data to pass, we do want to
 	   synchronize with the other processors.  */
-	if (smp_call_function(ipi_flush_tlb_all, NULL, 1, 1)) {
+	if (on_each_cpu(ipi_flush_tlb_all, NULL, 1, 1)) {
 		printk(KERN_CRIT "flush_tlb_all: timed out\n");
 	}
-
-	tbia();
 }
 
 #define asn_locked() (cpu_data[smp_processor_id()].asn_lock)
@@ -938,6 +934,8 @@
 void
 flush_tlb_mm(struct mm_struct *mm)
 {
+	preempt_disable();
+
 	if (mm == current->active_mm) {
 		flush_tlb_current(mm);
 		if (atomic_read(&mm->mm_users) <= 1) {
@@ -948,6 +946,7 @@
 				if (mm->context[cpu])
 					mm->context[cpu] = 0;
 			}
+			preempt_enable();
 			return;
 		}
 	}
@@ -955,6 +954,8 @@
 	if (smp_call_function(ipi_flush_tlb_mm, mm, 1, 1)) {
 		printk(KERN_CRIT "flush_tlb_mm: timed out\n");
 	}
+
+	preempt_enable();
 }
 
 struct flush_tlb_page_struct {
@@ -981,6 +982,8 @@
 	struct flush_tlb_page_struct data;
 	struct mm_struct *mm = vma->vm_mm;
 
+	preempt_disable();
+
 	if (mm == current->active_mm) {
 		flush_tlb_current_page(mm, vma, addr);
 		if (atomic_read(&mm->mm_users) <= 1) {
@@ -991,6 +994,7 @@
 				if (mm->context[cpu])
 					mm->context[cpu] = 0;
 			}
+			preempt_enable();
 			return;
 		}
 	}
@@ -1002,6 +1006,8 @@
 	if (smp_call_function(ipi_flush_tlb_page, &data, 1, 1)) {
 		printk(KERN_CRIT "flush_tlb_page: timed out\n");
 	}
+
+	preempt_enable();
 }
 
 void
@@ -1030,6 +1036,8 @@
 	if ((vma->vm_flags & VM_EXEC) == 0)
 		return;
 
+	preempt_disable();
+
 	if (mm == current->active_mm) {
 		__load_new_mm_context(mm);
 		if (atomic_read(&mm->mm_users) <= 1) {
@@ -1040,6 +1048,7 @@
 				if (mm->context[cpu])
 					mm->context[cpu] = 0;
 			}
+			preempt_enable();
 			return;
 		}
 	}
@@ -1047,6 +1056,8 @@
 	if (smp_call_function(ipi_flush_icache_page, mm, 1, 1)) {
 		printk(KERN_CRIT "flush_icache_page: timed out\n");
 	}
+
+	preempt_enable();
 }
 \f
 #ifdef CONFIG_DEBUG_SPINLOCK
diff -urP linux-2.5.62/arch/i386/kernel/cpuid.c linux-2.5.62_patched/arch/i386/kernel/cpuid.c
--- linux-2.5.62/arch/i386/kernel/cpuid.c	Mon Feb 17 23:55:49 2003
+++ linux-2.5.62_patched/arch/i386/kernel/cpuid.c	Sat Feb 22 02:02:17 2003
@@ -44,8 +44,6 @@
 #include <asm/uaccess.h>
 #include <asm/system.h>
 
-#ifdef CONFIG_SMP
-
 struct cpuid_command {
   int cpu;
   u32 reg;
@@ -64,24 +62,12 @@
 {
   struct cpuid_command cmd;
   
-  if ( cpu == smp_processor_id() ) {
-    cpuid(reg, &data[0], &data[1], &data[2], &data[3]);
-  } else {
-    cmd.cpu  = cpu;
-    cmd.reg  = reg;
-    cmd.data = data;
-    
-    smp_call_function(cpuid_smp_cpuid, &cmd, 1, 1);
-  }
-}
-#else /* ! CONFIG_SMP */
-
-static inline void do_cpuid(int cpu, u32 reg, u32 *data)
-{
-  cpuid(reg, &data[0], &data[1], &data[2], &data[3]);
+  cmd.cpu  = cpu;
+  cmd.reg  = reg;
+  cmd.data = data;
+  
+  on_each_cpu(cpuid_smp_cpuid, &cmd, 1, 1);
 }
-
-#endif /* ! CONFIG_SMP */
 
 static loff_t cpuid_seek(struct file *file, loff_t offset, int orig)
 {
diff -urP linux-2.5.62/arch/i386/kernel/io_apic.c linux-2.5.62_patched/arch/i386/kernel/io_apic.c
--- linux-2.5.62/arch/i386/kernel/io_apic.c	Mon Feb 17 23:56:10 2003
+++ linux-2.5.62_patched/arch/i386/kernel/io_apic.c	Sat Feb 22 02:02:17 2003
@@ -1376,8 +1376,7 @@
 
 void print_all_local_APICs (void)
 {
-	smp_call_function(print_local_APIC, NULL, 1, 1);
-	print_local_APIC(NULL);
+	on_each_cpu(print_local_APIC, NULL, 1, 1);
 }
 
 void /*__init*/ print_PIC(void)
@@ -1855,8 +1854,7 @@
 	 */ 
 	printk(KERN_INFO "activating NMI Watchdog ...");
 
-	smp_call_function(enable_NMI_through_LVT0, NULL, 1, 1);
-	enable_NMI_through_LVT0(NULL);
+	on_each_cpu(enable_NMI_through_LVT0, NULL, 1, 1);
 
 	printk(" done.\n");
 }
diff -urP linux-2.5.62/arch/i386/kernel/ldt.c linux-2.5.62_patched/arch/i386/kernel/ldt.c
--- linux-2.5.62/arch/i386/kernel/ldt.c	Mon Feb 17 23:56:25 2003
+++ linux-2.5.62_patched/arch/i386/kernel/ldt.c	Sat Feb 22 02:02:17 2003
@@ -55,13 +55,13 @@
 	wmb();
 
 	if (reload) {
+		preempt_disable();
 		load_LDT(pc);
 #ifdef CONFIG_SMP
-		preempt_disable();
 		if (current->mm->cpu_vm_mask != (1 << smp_processor_id()))
 			smp_call_function(flush_ldt, 0, 1, 1);
-		preempt_enable();
 #endif
+		preempt_enable();
 	}
 	if (oldsize) {
 		if (oldsize*LDT_ENTRY_SIZE > PAGE_SIZE)
diff -urP linux-2.5.62/arch/i386/kernel/microcode.c linux-2.5.62_patched/arch/i386/kernel/microcode.c
--- linux-2.5.62/arch/i386/kernel/microcode.c	Mon Feb 17 23:56:02 2003
+++ linux-2.5.62_patched/arch/i386/kernel/microcode.c	Sat Feb 22 02:02:17 2003
@@ -183,11 +183,8 @@
 	int i, error = 0, err;
 	struct microcode *m;
 
-	if (smp_call_function(do_update_one, NULL, 1, 1) != 0) {
-		printk(KERN_ERR "microcode: IPI timeout, giving up\n");
+	if (on_each_cpu(do_update_one, NULL, 1, 1) != 0)
 		return -EIO;
-	}
-	do_update_one(NULL);
 
 	for (i=0; i<NR_CPUS; i++) {
 		err = update_req[i].err;
diff -urP linux-2.5.62/arch/i386/kernel/msr.c linux-2.5.62_patched/arch/i386/kernel/msr.c
--- linux-2.5.62/arch/i386/kernel/msr.c	Mon Feb 17 23:56:13 2003
+++ linux-2.5.62_patched/arch/i386/kernel/msr.c	Sat Feb 22 02:02:17 2003
@@ -116,36 +116,28 @@
 {
   struct msr_command cmd;
 
-  if ( cpu == smp_processor_id() ) {
-    return wrmsr_eio(reg, eax, edx);
-  } else {
-    cmd.cpu = cpu;
-    cmd.reg = reg;
-    cmd.data[0] = eax;
-    cmd.data[1] = edx;
-    
-    smp_call_function(msr_smp_wrmsr, &cmd, 1, 1);
-    return cmd.err;
-  }
+  cmd.cpu = cpu;
+  cmd.reg = reg;
+  cmd.data[0] = eax;
+  cmd.data[1] = edx;
+
+  on_each_cpu(msr_smp_wrmsr, &cmd, 1, 1);
+  return cmd.err;
 }
 
 static inline int do_rdmsr(int cpu, u32 reg, u32 *eax, u32 *edx)
 {
   struct msr_command cmd;
 
-  if ( cpu == smp_processor_id() ) {
-    return rdmsr_eio(reg, eax, edx);
-  } else {
-    cmd.cpu = cpu;
-    cmd.reg = reg;
-
-    smp_call_function(msr_smp_rdmsr, &cmd, 1, 1);
-    
-    *eax = cmd.data[0];
-    *edx = cmd.data[1];
+  cmd.cpu = cpu;
+  cmd.reg = reg;
+
+  on_each_cpu(msr_smp_rdmsr, &cmd, 1, 1);
+
+  *eax = cmd.data[0];
+  *edx = cmd.data[1];
 
-    return cmd.err;
-  }
+  return cmd.err;
 }
 
 #else /* ! CONFIG_SMP */
diff -urP linux-2.5.62/arch/i386/kernel/smp.c linux-2.5.62_patched/arch/i386/kernel/smp.c
--- linux-2.5.62/arch/i386/kernel/smp.c	Mon Feb 17 23:55:52 2003
+++ linux-2.5.62_patched/arch/i386/kernel/smp.c	Sat Feb 22 02:02:17 2003
@@ -436,7 +436,7 @@
 	preempt_enable();
 }
 
-static inline void do_flush_tlb_all_local(void)
+void do_flush_tlb_all(void* info)
 {
 	unsigned long cpu = smp_processor_id();
 
@@ -445,18 +445,9 @@
 		leave_mm(cpu);
 }
 
-static void flush_tlb_all_ipi(void* info)
-{
-	do_flush_tlb_all_local();
-}
-
 void flush_tlb_all(void)
 {
-	preempt_disable();
-	smp_call_function (flush_tlb_all_ipi,0,1,1);
-
-	do_flush_tlb_all_local();
-	preempt_enable();
+	on_each_cpu(do_flush_tlb_all, 0, 1, 1);
 }
 
 /*
diff -urP linux-2.5.62/arch/i386/kernel/sysenter.c linux-2.5.62_patched/arch/i386/kernel/sysenter.c
--- linux-2.5.62/arch/i386/kernel/sysenter.c	Mon Feb 17 23:57:19 2003
+++ linux-2.5.62_patched/arch/i386/kernel/sysenter.c	Sat Feb 22 02:02:17 2003
@@ -95,8 +95,7 @@
 		return 0;
 
 	memcpy((void *) page, sysent, sizeof(sysent));
-	enable_sep_cpu(NULL);
-	smp_call_function(enable_sep_cpu, NULL, 1, 1);
+	on_each_cpu(enable_sep_cpu, NULL, 1, 1);
 	return 0;
 }
 
diff -urP linux-2.5.62/arch/i386/mach-voyager/voyager_smp.c linux-2.5.62_patched/arch/i386/mach-voyager/voyager_smp.c
--- linux-2.5.62/arch/i386/mach-voyager/voyager_smp.c	Mon Feb 17 23:56:12 2003
+++ linux-2.5.62_patched/arch/i386/mach-voyager/voyager_smp.c	Sat Feb 22 02:02:17 2003
@@ -1209,8 +1209,8 @@
 		smp_call_function_interrupt();
 }
 
-static inline void
-do_flush_tlb_all_local(void)
+void
+do_flush_tlb_all(void* info)
 {
 	unsigned long cpu = smp_processor_id();
 
@@ -1220,19 +1220,11 @@
 }
 
 
-static void
-flush_tlb_all_function(void* info)
-{
-	do_flush_tlb_all_local();
-}
-
 /* flush the TLB of every active CPU in the system */
 void
 flush_tlb_all(void)
 {
-	smp_call_function (flush_tlb_all_function, 0, 1, 1);
-
-	do_flush_tlb_all_local();
+	on_each_cpu(do_flush_tlb_all, 0, 1, 1);
 }
 
 /* used to set up the trampoline for other CPUs when the memory manager
diff -urP linux-2.5.62/arch/i386/mm/pageattr.c linux-2.5.62_patched/arch/i386/mm/pageattr.c
--- linux-2.5.62/arch/i386/mm/pageattr.c	Mon Feb 17 23:56:13 2003
+++ linux-2.5.62_patched/arch/i386/mm/pageattr.c	Sat Feb 22 02:02:17 2003
@@ -130,11 +130,8 @@
 } 
 
 static inline void flush_map(void)
-{	
-#ifdef CONFIG_SMP 
-	smp_call_function(flush_kernel_map, NULL, 1, 1);
-#endif	
-	flush_kernel_map(NULL);
+{
+	on_each_cpu(flush_kernel_map, NULL, 1, 1);
 }
 
 struct deferred_page { 
diff -urP linux-2.5.62/arch/i386/oprofile/nmi_int.c linux-2.5.62_patched/arch/i386/oprofile/nmi_int.c
--- linux-2.5.62/arch/i386/oprofile/nmi_int.c	Mon Feb 17 23:56:59 2003
+++ linux-2.5.62_patched/arch/i386/oprofile/nmi_int.c	Sat Feb 22 02:02:17 2003
@@ -95,8 +95,7 @@
 	 * without actually triggering any NMIs as this will
 	 * break the core code horrifically.
 	 */
-	smp_call_function(nmi_cpu_setup, NULL, 0, 1);
-	nmi_cpu_setup(0);
+	on_each_cpu(nmi_cpu_setup, NULL, 0, 1);
 	set_nmi_callback(nmi_callback);
 	oprofile_pmdev = set_nmi_pm_callback(oprofile_pm_callback);
 	return 0;
@@ -148,8 +147,7 @@
 {
 	unset_nmi_pm_callback(oprofile_pmdev);
 	unset_nmi_callback();
-	smp_call_function(nmi_cpu_shutdown, NULL, 0, 1);
-	nmi_cpu_shutdown(0);
+	on_each_cpu(nmi_cpu_shutdown, NULL, 0, 1);
 }
 
  
@@ -162,8 +160,7 @@
 
 static int nmi_start(void)
 {
-	smp_call_function(nmi_cpu_start, NULL, 0, 1);
-	nmi_cpu_start(0);
+	on_each_cpu(nmi_cpu_start, NULL, 0, 1);
 	return 0;
 }
  
@@ -177,8 +174,7 @@
  
 static void nmi_stop(void)
 {
-	smp_call_function(nmi_cpu_stop, NULL, 0, 1);
-	nmi_cpu_stop(0);
+	on_each_cpu(nmi_cpu_stop, NULL, 0, 1);
 }
 
 
diff -urP linux-2.5.62/arch/ia64/kernel/smp.c linux-2.5.62_patched/arch/ia64/kernel/smp.c
--- linux-2.5.62/arch/ia64/kernel/smp.c	Mon Feb 17 23:57:19 2003
+++ linux-2.5.62_patched/arch/ia64/kernel/smp.c	Sat Feb 22 02:02:17 2003
@@ -206,18 +206,18 @@
 void
 smp_flush_tlb_all (void)
 {
-	smp_call_function((void (*)(void *))local_flush_tlb_all, 0, 1, 1);
-	local_flush_tlb_all();
+	on_each_cpu((void (*)(void *))local_flush_tlb_all, 0, 1, 1);
 }
 
 void
 smp_flush_tlb_mm (struct mm_struct *mm)
 {
-	local_finish_flush_tlb_mm(mm);
-
 	/* this happens for the common case of a single-threaded fork():  */
 	if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1))
+	{
+		local_finish_flush_tlb_mm(mm);
 		return;
+	}
 
 	/*
 	 * We could optimize this further by using mm->cpu_vm_mask to track which CPUs
@@ -226,7 +226,7 @@
 	 * anyhow, and once a CPU is interrupted, the cost of local_flush_tlb_all() is
 	 * rather trivial.
 	 */
-	smp_call_function((void (*)(void *))local_finish_flush_tlb_mm, mm, 1, 1);
+	on_each_cpu((void (*)(void *))local_finish_flush_tlb_mm, mm, 1, 1);
 }
 
 /*
diff -urP linux-2.5.62/arch/mips64/kernel/smp.c linux-2.5.62_patched/arch/mips64/kernel/smp.c
--- linux-2.5.62/arch/mips64/kernel/smp.c	Mon Feb 17 23:56:27 2003
+++ linux-2.5.62_patched/arch/mips64/kernel/smp.c	Sat Feb 22 02:02:17 2003
@@ -195,8 +195,7 @@
 
 void flush_tlb_all(void)
 {
-	smp_call_function(flush_tlb_all_ipi, 0, 1, 1);
-	_flush_tlb_all();
+	on_each_cpu(flush_tlb_all_ipi, 0, 1, 1);
 }
 
 static void flush_tlb_mm_ipi(void *mm)
@@ -219,6 +218,8 @@
 
 void flush_tlb_mm(struct mm_struct *mm)
 {
+	preempt_disable();
+
 	if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
 		smp_call_function(flush_tlb_mm_ipi, (void *)mm, 1, 1);
 	} else {
@@ -228,6 +229,8 @@
 				CPU_CONTEXT(i, mm) = 0;
 	}
 	_flush_tlb_mm(mm);
+
+	preempt_enable();
 }
 
 struct flush_tlb_data {
@@ -246,6 +249,8 @@
 
 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
 {
+	preempt_disable();
+
 	if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
 		struct flush_tlb_data fd;
 
@@ -260,6 +265,8 @@
 				CPU_CONTEXT(i, mm) = 0;
 	}
 	_flush_tlb_range(mm, start, end);
+
+	preempt_enable();
 }
 
 static void flush_tlb_page_ipi(void *info)
@@ -271,6 +278,8 @@
 
 void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 {
+	preempt_disable();
+
 	if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) {
 		struct flush_tlb_data fd;
 
@@ -284,5 +293,7 @@
 				CPU_CONTEXT(i, vma->vm_mm) = 0;
 	}
 	_flush_tlb_page(vma, page);
+
+	preempt_enable();
 }
 
diff -urP linux-2.5.62/arch/parisc/kernel/cache.c linux-2.5.62_patched/arch/parisc/kernel/cache.c
--- linux-2.5.62/arch/parisc/kernel/cache.c	Mon Feb 17 23:56:13 2003
+++ linux-2.5.62_patched/arch/parisc/kernel/cache.c	Sat Feb 22 02:02:17 2003
@@ -39,8 +39,7 @@
 void
 flush_data_cache(void)
 {
-	smp_call_function((void (*)(void *))flush_data_cache_local, NULL, 1, 1);
-	flush_data_cache_local();
+	on_each_cpu((void (*)(void *))flush_data_cache_local, NULL, 1, 1);
 }
 #endif
 
diff -urP linux-2.5.62/arch/parisc/kernel/irq.c linux-2.5.62_patched/arch/parisc/kernel/irq.c
--- linux-2.5.62/arch/parisc/kernel/irq.c	Mon Feb 17 23:56:13 2003
+++ linux-2.5.62_patched/arch/parisc/kernel/irq.c	Sat Feb 22 02:02:17 2003
@@ -61,20 +61,17 @@
 
 static spinlock_t irq_lock = SPIN_LOCK_UNLOCKED;  /* protect IRQ regions */
 
-#ifdef CONFIG_SMP
 static void cpu_set_eiem(void *info)
 {
 	set_eiem((unsigned long) info);
 }
-#endif
 
 static inline void disable_cpu_irq(void *unused, int irq)
 {
 	unsigned long eirr_bit = EIEM_MASK(irq);
 
 	cpu_eiem &= ~eirr_bit;
-	set_eiem(cpu_eiem);
-        smp_call_function(cpu_set_eiem, (void *) cpu_eiem, 1, 1);
+        on_each_cpu(cpu_set_eiem, (void *) cpu_eiem, 1, 1);
 }
 
 static void enable_cpu_irq(void *unused, int irq)
@@ -83,8 +80,7 @@
 
 	mtctl(eirr_bit, 23);	/* clear EIRR bit before unmasking */
 	cpu_eiem |= eirr_bit;
-        smp_call_function(cpu_set_eiem, (void *) cpu_eiem, 1, 1);
-	set_eiem(cpu_eiem);
+        on_each_cpu(cpu_set_eiem, (void *) cpu_eiem, 1, 1);
 }
 
 /* mask and disable are the same at the CPU level
@@ -100,8 +96,7 @@
 	** handle *any* unmasked pending interrupts.
 	** ie We don't need to check for pending interrupts here.
 	*/
-        smp_call_function(cpu_set_eiem, (void *) cpu_eiem, 1, 1);
-	set_eiem(cpu_eiem);
+        on_each_cpu(cpu_set_eiem, (void *) cpu_eiem, 1, 1);
 }
 
 /*
diff -urP linux-2.5.62/arch/parisc/kernel/smp.c linux-2.5.62_patched/arch/parisc/kernel/smp.c
--- linux-2.5.62/arch/parisc/kernel/smp.c	Mon Feb 17 23:57:18 2003
+++ linux-2.5.62_patched/arch/parisc/kernel/smp.c	Sat Feb 22 02:02:17 2003
@@ -401,7 +401,7 @@
 __setup("maxcpus=", maxcpus);
 
 /*
- * Flush all other CPU's tlb and then mine.  Do this with smp_call_function()
+ * Flush all other CPU's tlb and then mine.  Do this with on_each_cpu()
  * as we want to ensure all TLB's flushed before proceeding.
  */
 
@@ -410,8 +410,7 @@
 void
 smp_flush_tlb_all(void)
 {
-	smp_call_function((void (*)(void *))flush_tlb_all_local, NULL, 1, 1);
-	flush_tlb_all_local();
+	on_each_cpu((void (*)(void *))flush_tlb_all_local, NULL, 1, 1);
 }
 
 
diff -urP linux-2.5.62/arch/parisc/mm/init.c linux-2.5.62_patched/arch/parisc/mm/init.c
--- linux-2.5.62/arch/parisc/mm/init.c	Mon Feb 17 23:56:14 2003
+++ linux-2.5.62_patched/arch/parisc/mm/init.c	Sat Feb 22 02:02:17 2003
@@ -974,8 +974,7 @@
 	    do_recycle++;
 	}
 	spin_unlock(&sid_lock);
-	smp_call_function((void (*)(void *))flush_tlb_all_local, NULL, 1, 1);
-	flush_tlb_all_local();
+	on_each_cpu((void (*)(void *))flush_tlb_all_local, NULL, 1, 1);
 	if (do_recycle) {
 	    spin_lock(&sid_lock);
 	    recycle_sids(recycle_ndirty,recycle_dirty_array);
diff -urP linux-2.5.62/arch/ppc/kernel/temp.c linux-2.5.62_patched/arch/ppc/kernel/temp.c
--- linux-2.5.62/arch/ppc/kernel/temp.c	Mon Feb 17 23:56:16 2003
+++ linux-2.5.62_patched/arch/ppc/kernel/temp.c	Sat Feb 22 02:02:17 2003
@@ -194,10 +194,7 @@
 
 	/* schedule ourselves to be run again */
 	mod_timer(&tau_timer, jiffies + shrink_timer) ;
-#ifdef CONFIG_SMP
-	smp_call_function(tau_timeout, NULL, 1, 0);
-#endif
-	tau_timeout(NULL);
+	on_each_cpu(tau_timeout, NULL, 1, 0);
 }
 
 /*
@@ -239,10 +236,7 @@
 	tau_timer.expires = jiffies + shrink_timer;
 	add_timer(&tau_timer);
 	
-#ifdef CONFIG_SMP
-	smp_call_function(TAU_init_smp, NULL, 1, 0);
-#endif
-	TAU_init_smp(NULL);
+	on_each_cpu(TAU_init_smp, NULL, 1, 0);
 	
 	printk("Thermal assist unit ");
 #ifdef CONFIG_TAU_INT
diff -urP linux-2.5.62/arch/s390/kernel/smp.c linux-2.5.62_patched/arch/s390/kernel/smp.c
--- linux-2.5.62/arch/s390/kernel/smp.c	Mon Feb 17 23:55:49 2003
+++ linux-2.5.62_patched/arch/s390/kernel/smp.c	Sat Feb 22 02:02:17 2003
@@ -228,8 +228,7 @@
 void machine_restart_smp(char * __unused) 
 {
 	cpu_restart_map = cpu_online_map;
-        smp_call_function(do_machine_restart, NULL, 0, 0);
-	do_machine_restart(NULL);
+        on_each_cpu(do_machine_restart, NULL, 0, 0);
 }
 
 static void do_machine_halt(void * __unused)
@@ -247,8 +246,7 @@
 
 void machine_halt_smp(void)
 {
-        smp_call_function(do_machine_halt, NULL, 0, 0);
-	do_machine_halt(NULL);
+        on_each_cpu(do_machine_halt, NULL, 0, 0);
 }
 
 static void do_machine_power_off(void * __unused)
@@ -266,8 +264,7 @@
 
 void machine_power_off_smp(void)
 {
-        smp_call_function(do_machine_power_off, NULL, 0, 0);
-	do_machine_power_off(NULL);
+        on_each_cpu(do_machine_power_off, NULL, 0, 0);
 }
 
 /*
@@ -339,8 +336,7 @@
 
 void smp_ptlb_all(void)
 {
-        smp_call_function(smp_ptlb_callback, NULL, 0, 1);
-	local_flush_tlb();
+        on_each_cpu(smp_ptlb_callback, NULL, 0, 1);
 }
 
 /*
@@ -400,8 +396,7 @@
 	parms.end_ctl = cr;
 	parms.orvals[cr] = 1 << bit;
 	parms.andvals[cr] = 0xFFFFFFFF;
-	smp_call_function(smp_ctl_bit_callback, &parms, 0, 1);
-        __ctl_set_bit(cr, bit);
+	on_each_cpu(smp_ctl_bit_callback, &parms, 0, 1);
 }
 
 /*
@@ -414,8 +409,7 @@
 	parms.end_ctl = cr;
 	parms.orvals[cr] = 0x00000000;
 	parms.andvals[cr] = ~(1 << bit);
-	smp_call_function(smp_ctl_bit_callback, &parms, 0, 1);
-        __ctl_clear_bit(cr, bit);
+	on_each_cpu(smp_ctl_bit_callback, &parms, 0, 1);
 }
 
 /*
diff -urP linux-2.5.62/arch/s390x/kernel/smp.c linux-2.5.62_patched/arch/s390x/kernel/smp.c
--- linux-2.5.62/arch/s390x/kernel/smp.c	Mon Feb 17 23:56:15 2003
+++ linux-2.5.62_patched/arch/s390x/kernel/smp.c	Sat Feb 22 02:02:17 2003
@@ -227,8 +227,7 @@
 void machine_restart_smp(char * __unused) 
 {
 	cpu_restart_map = cpu_online_map;
-        smp_call_function(do_machine_restart, NULL, 0, 0);
-	do_machine_restart(NULL);
+        on_each_cpu(do_machine_restart, NULL, 0, 0);
 }
 
 static void do_machine_halt(void * __unused)
@@ -246,8 +245,7 @@
 
 void machine_halt_smp(void)
 {
-        smp_call_function(do_machine_halt, NULL, 0, 0);
-	do_machine_halt(NULL);
+        on_each_cpu(do_machine_halt, NULL, 0, 0);
 }
 
 static void do_machine_power_off(void * __unused)
@@ -265,8 +263,7 @@
 
 void machine_power_off_smp(void)
 {
-        smp_call_function(do_machine_power_off, NULL, 0, 0);
-	do_machine_power_off(NULL);
+        on_each_cpu(do_machine_power_off, NULL, 0, 0);
 }
 
 /*
@@ -383,8 +380,7 @@
 	parms.end_ctl = cr;
 	parms.orvals[cr] = 1 << bit;
 	parms.andvals[cr] = -1L;
-	smp_call_function(smp_ctl_bit_callback, &parms, 0, 1);
-        __ctl_set_bit(cr, bit);
+	on_each_cpu(smp_ctl_bit_callback, &parms, 0, 1);
 }
 
 /*
@@ -397,8 +393,7 @@
 	parms.end_ctl = cr;
 	parms.orvals[cr] = 0;
 	parms.andvals[cr] = ~(1L << bit);
-	smp_call_function(smp_ctl_bit_callback, &parms, 0, 1);
-        __ctl_clear_bit(cr, bit);
+	on_each_cpu(smp_ctl_bit_callback, &parms, 0, 1);
 }
 
 
diff -urP linux-2.5.62/arch/x86_64/kernel/bluesmoke.c linux-2.5.62_patched/arch/x86_64/kernel/bluesmoke.c
--- linux-2.5.62/arch/x86_64/kernel/bluesmoke.c	Mon Feb 17 23:56:55 2003
+++ linux-2.5.62_patched/arch/x86_64/kernel/bluesmoke.c	Sat Feb 22 02:02:18 2003
@@ -111,11 +111,7 @@
 {
 	u32 low, high;
 	int i;
-	unsigned int *cpu = info;
 
-	BUG_ON (*cpu != smp_processor_id());
-
-	preempt_disable();
 	for (i=0; i<banks; i++) {
 		rdmsr(MSR_IA32_MC0_STATUS+i*4, low, high);
 
@@ -130,20 +126,12 @@
 			wmb();
 		}
 	}
-	preempt_enable();
 }
 
 
 static void mce_timerfunc (unsigned long data)
 {
-	unsigned int i;
-
-	for (i=0; i<smp_num_cpus; i++) {
-		if (i == smp_processor_id())
-			mce_checkregs(&i);
-		else
-			smp_call_function (mce_checkregs, &i, 1, 1);
-	}
+	on_each_cpu (mce_checkregs, NULL, 1, 1);
 
 	/* Refresh the timer. */
 	mce_timer.expires = jiffies + MCE_RATE;
diff -urP linux-2.5.62/arch/x86_64/kernel/cpuid.c linux-2.5.62_patched/arch/x86_64/kernel/cpuid.c
--- linux-2.5.62/arch/x86_64/kernel/cpuid.c	Mon Feb 17 23:56:02 2003
+++ linux-2.5.62_patched/arch/x86_64/kernel/cpuid.c	Sat Feb 22 02:02:18 2003
@@ -63,16 +63,12 @@
 static inline void do_cpuid(int cpu, u32 reg, u32 *data)
 {
   struct cpuid_command cmd;
-  
-  if ( cpu == smp_processor_id() ) {
-    cpuid(reg, &data[0], &data[1], &data[2], &data[3]);
-  } else {
-    cmd.cpu  = cpu;
-    cmd.reg  = reg;
-    cmd.data = data;
-    
-    smp_call_function(cpuid_smp_cpuid, &cmd, 1, 1);
-  }
+
+  cmd.cpu  = cpu;
+  cmd.reg  = reg;
+  cmd.data = data;
+
+  on_each_cpu(cpuid_smp_cpuid, &cmd, 1, 1);
 }
 #else /* ! CONFIG_SMP */
 
diff -urP linux-2.5.62/arch/x86_64/kernel/io_apic.c linux-2.5.62_patched/arch/x86_64/kernel/io_apic.c
--- linux-2.5.62/arch/x86_64/kernel/io_apic.c	Mon Feb 17 23:56:15 2003
+++ linux-2.5.62_patched/arch/x86_64/kernel/io_apic.c	Sat Feb 22 02:02:18 2003
@@ -926,8 +926,7 @@
 
 void print_all_local_APICs (void)
 {
-	smp_call_function(print_local_APIC, NULL, 1, 1);
-	print_local_APIC(NULL);
+	on_each_cpu(print_local_APIC, NULL, 1, 1);
 }
 
 void /*__init*/ print_PIC(void)
diff -urP linux-2.5.62/arch/x86_64/kernel/ldt.c linux-2.5.62_patched/arch/x86_64/kernel/ldt.c
--- linux-2.5.62/arch/x86_64/kernel/ldt.c	Mon Feb 17 23:56:58 2003
+++ linux-2.5.62_patched/arch/x86_64/kernel/ldt.c	Sat Feb 22 02:02:18 2003
@@ -60,13 +60,13 @@
 	pc->size = mincount;
 	wmb();
 	if (reload) {
+		preempt_disable();
 		load_LDT(pc);
 #ifdef CONFIG_SMP
-		preempt_disable();
 		if (current->mm->cpu_vm_mask != (1<<smp_processor_id()))
 			smp_call_function(flush_ldt, 0, 1, 1);
-		preempt_enable();
 #endif
+		preempt_enable();
 	}
 	if (oldsize) {
 		if (oldsize*LDT_ENTRY_SIZE > PAGE_SIZE)
diff -urP linux-2.5.62/arch/x86_64/kernel/msr.c linux-2.5.62_patched/arch/x86_64/kernel/msr.c
--- linux-2.5.62/arch/x86_64/kernel/msr.c	Mon Feb 17 23:55:57 2003
+++ linux-2.5.62_patched/arch/x86_64/kernel/msr.c	Sat Feb 22 02:02:18 2003
@@ -120,36 +120,28 @@
 {
   struct msr_command cmd;
 
-  if ( cpu == smp_processor_id() ) {
-    return wrmsr_eio(reg, eax, edx);
-  } else {
-    cmd.cpu = cpu;
-    cmd.reg = reg;
-    cmd.data[0] = eax;
-    cmd.data[1] = edx;
-    
-    smp_call_function(msr_smp_wrmsr, &cmd, 1, 1);
-    return cmd.err;
-  }
+  cmd.cpu = cpu;
+  cmd.reg = reg;
+  cmd.data[0] = eax;
+  cmd.data[1] = edx;
+
+  on_each_cpu(msr_smp_wrmsr, &cmd, 1, 1);
+  return cmd.err;
 }
 
 static inline int do_rdmsr(int cpu, u32 reg, u32 *eax, u32 *edx)
 {
   struct msr_command cmd;
 
-  if ( cpu == smp_processor_id() ) {
-    return rdmsr_eio(reg, eax, edx);
-  } else {
-    cmd.cpu = cpu;
-    cmd.reg = reg;
-
-    smp_call_function(msr_smp_rdmsr, &cmd, 1, 1);
-    
-    *eax = cmd.data[0];
-    *edx = cmd.data[1];
+  cmd.cpu = cpu;
+  cmd.reg = reg;
+
+  on_each_cpu(msr_smp_rdmsr, &cmd, 1, 1);
+
+  *eax = cmd.data[0];
+  *edx = cmd.data[1];
 
-    return cmd.err;
-  }
+  return cmd.err;
 }
 
 #else /* ! CONFIG_SMP */
diff -urP linux-2.5.62/arch/x86_64/kernel/smp.c linux-2.5.62_patched/arch/x86_64/kernel/smp.c
--- linux-2.5.62/arch/x86_64/kernel/smp.c	Mon Feb 17 23:56:09 2003
+++ linux-2.5.62_patched/arch/x86_64/kernel/smp.c	Sat Feb 22 02:02:18 2003
@@ -328,7 +328,7 @@
 	preempt_enable();
 }
 
-static inline void do_flush_tlb_all_local(void)
+void do_flush_tlb_all(void* info)
 {
 	unsigned long cpu = smp_processor_id();
 
@@ -337,16 +337,9 @@
 		leave_mm(cpu);
 }
 
-static void flush_tlb_all_ipi(void* info)
-{
-	do_flush_tlb_all_local();
-}
-
 void flush_tlb_all(void)
 {
-	smp_call_function (flush_tlb_all_ipi,0,1,1);
-
-	do_flush_tlb_all_local();
+	on_each_cpu(do_flush_tlb_all, 0, 1, 1);
 }
 
 void smp_kdb_stop(void)
diff -urP linux-2.5.62/arch/x86_64/mm/pageattr.c linux-2.5.62_patched/arch/x86_64/mm/pageattr.c
--- linux-2.5.62/arch/x86_64/mm/pageattr.c	Mon Feb 17 23:56:14 2003
+++ linux-2.5.62_patched/arch/x86_64/mm/pageattr.c	Sat Feb 22 02:02:18 2003
@@ -122,11 +122,8 @@
 } 
 
 static inline void flush_map(unsigned long address)
-{	
-#ifdef CONFIG_SMP 
-	smp_call_function(flush_kernel_map, (void *)address, 1, 1);
-#endif	
-	flush_kernel_map((void *)address);
+{
+	on_each_cpu(flush_kernel_map, (void *)address, 1, 1);
 }
 
 struct deferred_page { 
diff -urP linux-2.5.62/drivers/char/agp/agp.h linux-2.5.62_patched/drivers/char/agp/agp.h
--- linux-2.5.62/drivers/char/agp/agp.h	Mon Feb 17 23:56:13 2003
+++ linux-2.5.62_patched/drivers/char/agp/agp.h	Sat Feb 22 02:02:18 2003
@@ -34,24 +34,10 @@
 
 #define PFX "agpgart: "
 
-#ifdef CONFIG_SMP
-static void ipi_handler(void *null)
-{
-	flush_agp_cache();
-}
-
-static void __attribute__((unused)) global_cache_flush(void)
-{
-	if (smp_call_function(ipi_handler, NULL, 1, 1) != 0)
-		panic(PFX "timed out waiting for the other CPUs!\n");
-	flush_agp_cache();
-}
-#else
 static inline void global_cache_flush(void)
 {
-	flush_agp_cache();
+	on_each_cpu(flush_agp_cache, NULL, 1, 1);
 }
-#endif	/* !CONFIG_SMP */
 
 enum aper_size_type {
 	U8_APER_SIZE,
diff -urP linux-2.5.62/drivers/s390/char/sclp.c linux-2.5.62_patched/drivers/s390/char/sclp.c
--- linux-2.5.62/drivers/s390/char/sclp.c	Mon Feb 17 23:56:20 2003
+++ linux-2.5.62_patched/drivers/s390/char/sclp.c	Sat Feb 22 02:02:18 2003
@@ -481,8 +481,7 @@
 do_machine_quiesce(void)
 {
 	cpu_quiesce_map = cpu_online_map;
-	smp_call_function(do_load_quiesce_psw, NULL, 0, 0);
-	do_load_quiesce_psw(NULL);
+	on_each_cpu(do_load_quiesce_psw, NULL, 0, 0);
 }
 #else
 static void
diff -urP linux-2.5.62/drivers/s390/net/iucv.c linux-2.5.62_patched/drivers/s390/net/iucv.c
--- linux-2.5.62/drivers/s390/net/iucv.c	Mon Feb 17 23:55:52 2003
+++ linux-2.5.62_patched/drivers/s390/net/iucv.c	Sat Feb 22 02:02:18 2003
@@ -617,10 +617,7 @@
 	ulong b2f0_result = 0x0deadbeef;
 
 	iucv_debug(1, "entering");
-	if (smp_processor_id() == 0)
-		iucv_declare_buffer_cpu0(&b2f0_result);
-	else
-		smp_call_function(iucv_declare_buffer_cpu0, &b2f0_result, 0, 1);
+	on_each_cpu(iucv_declare_buffer_cpu0, &b2f0_result, 0, 1);
 	iucv_debug(1, "Address of EIB = %p", iucv_external_int_buffer);
 	if (b2f0_result == 0x0deadbeef)
 	    b2f0_result = 0xaa;
@@ -639,10 +636,7 @@
 {
 	iucv_debug(1, "entering");
 	if (declare_flag) {
-		if (smp_processor_id() == 0)
-			iucv_retrieve_buffer_cpu0(0);
-		else
-			smp_call_function(iucv_retrieve_buffer_cpu0, 0, 0, 1);
+		on_each_cpu(iucv_retrieve_buffer_cpu0, 0, 0, 1);
 		declare_flag = 0;
 	}
 	iucv_debug(1, "exiting");
diff -urP linux-2.5.62/fs/buffer.c linux-2.5.62_patched/fs/buffer.c
--- linux-2.5.62/fs/buffer.c	Mon Feb 17 23:56:17 2003
+++ linux-2.5.62_patched/fs/buffer.c	Sat Feb 22 02:02:18 2003
@@ -1404,10 +1404,7 @@
 	
 static void invalidate_bh_lrus(void)
 {
-	preempt_disable();
-	invalidate_bh_lru(NULL);
-	smp_call_function(invalidate_bh_lru, NULL, 1, 1);
-	preempt_enable();
+	on_each_cpu(invalidate_bh_lru, NULL, 1, 1);
 }
 
 void set_bh_page(struct buffer_head *bh,
diff -urP linux-2.5.62/include/asm-alpha/agp.h linux-2.5.62_patched/include/asm-alpha/agp.h
--- linux-2.5.62/include/asm-alpha/agp.h	Mon Feb 17 23:55:50 2003
+++ linux-2.5.62_patched/include/asm-alpha/agp.h	Sat Feb 22 02:02:18 2003
@@ -8,6 +8,10 @@
 #define map_page_into_agp(page) 
 #define unmap_page_from_agp(page) 
 #define flush_agp_mappings() 
-#define flush_agp_cache() mb()
+
+static void flush_agp_cache(void* info)
+{
+	mb();
+}
 
 #endif
diff -urP linux-2.5.62/include/asm-i386/agp.h linux-2.5.62_patched/include/asm-i386/agp.h
--- linux-2.5.62/include/asm-i386/agp.h	Mon Feb 17 23:55:49 2003
+++ linux-2.5.62_patched/include/asm-i386/agp.h	Sat Feb 22 02:02:18 2003
@@ -18,6 +18,9 @@
 /* Could use CLFLUSH here if the cpu supports it. But then it would
    need to be called for each cacheline of the whole page so it may not be 
    worth it. Would need a page for it. */
-#define flush_agp_cache() asm volatile("wbinvd":::"memory")
+static void flush_agp_cache(void *info)
+{
+	__asm__ __volatile__ ("wbinvd": : :"memory");
+}
 
 #endif
diff -urP linux-2.5.62/include/asm-ia64/agp.h linux-2.5.62_patched/include/asm-ia64/agp.h
--- linux-2.5.62/include/asm-ia64/agp.h	Mon Feb 17 23:56:15 2003
+++ linux-2.5.62_patched/include/asm-ia64/agp.h	Sat Feb 22 02:02:18 2003
@@ -16,7 +16,14 @@
 #define map_page_into_agp(page)		/* nothing */
 #define unmap_page_from_agp(page)	/* nothing */
 #define flush_agp_mappings()		/* nothing */
-#define flush_agp_cache()		mb()
+
+/* Could use CLFLUSH here if the cpu supports it. But then it would
+   need to be called for each cacheline of the whole page so it may not be 
+   worth it. Would need a page for it. */
+static void flush_agp_cache(void* info)
+{
+	mb();
+}
 
 /* Page-protection value to be used for AGP memory mapped into kernel space.  */
 #define PAGE_AGP			PAGE_KERNEL
diff -urP linux-2.5.62/include/asm-parisc/cacheflush.h linux-2.5.62_patched/include/asm-parisc/cacheflush.h
--- linux-2.5.62/include/asm-parisc/cacheflush.h	Mon Feb 17 23:57:01 2003
+++ linux-2.5.62_patched/include/asm-parisc/cacheflush.h	Sat Feb 22 02:02:18 2003
@@ -25,16 +25,10 @@
 
 extern void flush_cache_all_local(void);
 
-#ifdef CONFIG_SMP
 static inline void flush_cache_all(void)
 {
-	smp_call_function((void (*)(void *))flush_cache_all_local, NULL, 1, 1);
-	flush_cache_all_local();
+	on_each_cpu((void (*)(void *))flush_cache_all_local, NULL, 1, 1);
 }
-#else
-#define flush_cache_all flush_cache_all_local
-#endif
-
 
 /* The following value needs to be tuned and probably scaled with the
  * cache size.
diff -urP linux-2.5.62/include/asm-sparc64/agp.h linux-2.5.62_patched/include/asm-sparc64/agp.h
--- linux-2.5.62/include/asm-sparc64/agp.h	Mon Feb 17 23:55:55 2003
+++ linux-2.5.62_patched/include/asm-sparc64/agp.h	Sat Feb 22 02:02:18 2003
@@ -6,6 +6,13 @@
 #define map_page_into_agp(page) 
 #define unmap_page_from_agp(page) 
 #define flush_agp_mappings() 
-#define flush_agp_cache() mb()
+
+/* Could use CLFLUSH here if the cpu supports it. But then it would
+   need to be called for each cacheline of the whole page so it may not be 
+   worth it. Would need a page for it. */
+static void flush_agp_cache(void* info)
+{
+	mb();
+}
 
 #endif
diff -urP linux-2.5.62/include/asm-x86_64/agp.h linux-2.5.62_patched/include/asm-x86_64/agp.h
--- linux-2.5.62/include/asm-x86_64/agp.h	Mon Feb 17 23:55:49 2003
+++ linux-2.5.62_patched/include/asm-x86_64/agp.h	Sat Feb 22 02:02:18 2003
@@ -18,6 +18,9 @@
 /* Could use CLFLUSH here if the cpu supports it. But then it would
    need to be called for each cacheline of the whole page so it may not be 
    worth it. Would need a page for it. */
-#define flush_agp_cache() asm volatile("wbinvd":::"memory")
+static void flush_agp_cache(void* info)
+{
+	__asm__ __volatile__ ("wbinvd": : :"memory");
+}
 
 #endif
diff -urP linux-2.5.62/include/linux/smp.h linux-2.5.62_patched/include/linux/smp.h
--- linux-2.5.62/include/linux/smp.h	Mon Feb 17 23:56:16 2003
+++ linux-2.5.62_patched/include/linux/smp.h	Sat Feb 22 02:06:57 2003
@@ -10,9 +10,10 @@
 
 #ifdef CONFIG_SMP
 
+#include <linux/preempt.h>
 #include <linux/kernel.h>
 #include <linux/compiler.h>
-#include <linux/threads.h>
+#include <linux/thread_info.h>
 #include <asm/smp.h>
 #include <asm/bug.h>
 
@@ -54,6 +55,31 @@
 			      int retry, int wait);
 
 /*
+ * Call a function on all processors
+ */
+static inline int on_each_cpu(void (*func) (void *info), void *info,
+			      int retry, int wait)
+{
+	int ret = 0;
+
+	preempt_disable();
+
+	if(num_online_cpus() == 1)
+		goto only_one;
+
+	ret = smp_call_function(func, info, retry, wait);
+	if(ret != 0)
+		printk(KERN_ERR "%p: IPI timeout, giving up\n",
+			__builtin_return_address(0));
+
+only_one:
+	func(info);
+	preempt_enable();
+
+	return ret;
+}
+
+/*
  * True once the per process idle is forked
  */
 extern int smp_threads_ready;
@@ -96,6 +122,7 @@
 #define hard_smp_processor_id()			0
 #define smp_threads_ready			1
 #define smp_call_function(func,info,retry,wait)	({ 0; })
+#define on_each_cpu(func,info,retry,wait)	({ func(info); 0; })
 static inline void smp_send_reschedule(int cpu) { }
 static inline void smp_send_reschedule_all(void) { }
 #define cpu_online_map				1
diff -urP linux-2.5.62/mm/slab.c linux-2.5.62_patched/mm/slab.c
--- linux-2.5.62/mm/slab.c	Mon Feb 17 23:56:45 2003
+++ linux-2.5.62_patched/mm/slab.c	Sat Feb 22 02:02:18 2003
@@ -1116,12 +1116,16 @@
 static void smp_call_function_all_cpus(void (*func) (void *arg), void *arg)
 {
 	check_irq_on();
+	preempt_disable();
+
 	local_irq_disable();
 	func(arg);
 	local_irq_enable();
 
 	if (smp_call_function(func, arg, 1, 1))
 		BUG();
+
+	preempt_enable();
 }
 
 static void free_block (kmem_cache_t* cachep, void** objpp, int len);

[-- Attachment #2: signature --]
[-- Type: application/pgp-signature, Size: 189 bytes --]