[PATCH 1/4] SGI Altix cross partition functionality

[PATCH 1/4] SGI Altix cross partition functionality

[Dean Nelson]

This patch exports the symbols needed by XP[C|NET].


Index: linux/arch/ia64/kernel/sal.c
=================================--- linux.orig/arch/ia64/kernel/sal.c	Thu Jun 10 14:47:44 2004
+++ linux/arch/ia64/kernel/sal.c	Thu Jun 10 16:31:55 2004
@@ -12,12 +12,15 @@
 #include <linux/init.h>
 #include <linux/spinlock.h>
 #include <linux/string.h>
+#include <linux/module.h>
 
 #include <asm/page.h>
 #include <asm/sal.h>
 #include <asm/pal.h>
 
 spinlock_t sal_lock __cacheline_aligned = SPIN_LOCK_UNLOCKED;
+EXPORT_SYMBOL(sal_lock);
+
 unsigned long sal_platform_features;
 
 unsigned short sal_revision;
@@ -38,6 +41,8 @@ default_handler (void)
 }
 
 ia64_sal_handler ia64_sal = (ia64_sal_handler) default_handler;
+EXPORT_SYMBOL(ia64_sal);
+
 ia64_sal_desc_ptc_t *ia64_ptc_domain_info;
 
 const char *
Index: linux/arch/ia64/kernel/smpboot.c
=================================--- linux.orig/arch/ia64/kernel/smpboot.c	Thu Jun 10 14:47:53 2004
+++ linux/arch/ia64/kernel/smpboot.c	Thu Jun 10 16:31:55 2004
@@ -498,6 +498,7 @@ u8 cpu_to_node_map[NR_CPUS] __cacheline_
 EXPORT_SYMBOL(cpu_to_node_map);
 /* which logical CPUs are on which nodes */
 cpumask_t node_to_cpu_mask[MAX_NUMNODES] __cacheline_aligned;
+EXPORT_SYMBOL(node_to_cpu_mask);
 
 /*
  * Build cpu to node mapping and initialize the per node cpu masks.
Index: linux/arch/ia64/sn/kernel/bte.c
=================================--- linux.orig/arch/ia64/sn/kernel/bte.c	Mon May 10 12:32:38 2004
+++ linux/arch/ia64/sn/kernel/bte.c	Thu Jun 10 16:31:55 2004
@@ -7,6 +7,7 @@
  */
 
 #include <linux/config.h>
+#include <linux/module.h>
 #include <asm/sn/sgi.h>
 #include <asm/sn/nodepda.h>
 #include <asm/sn/addrs.h>
@@ -169,6 +170,7 @@ bte_copy(u64 src, u64 dest, u64 len, u64
 
 	return bte_status;
 }
+EXPORT_SYMBOL(bte_copy);
 
 
 /*
@@ -405,7 +407,7 @@ void
 bte_init_cpu(void)
 {
 	/* Called by setup.c as each cpu is being added to the nodepda */
-	if (local_node_data->active_cpu_count & 0x1) {
+	if (get_slice() = 0) {
 		pda->cpu_bte_if[0] = &(nodepda->bte_if[0]);
 		pda->cpu_bte_if[1] = &(nodepda->bte_if[1]);
 	} else {
Index: linux/arch/ia64/sn/kernel/setup.c
=================================--- linux.orig/arch/ia64/sn/kernel/setup.c	Thu Jun 10 16:29:44 2004
+++ linux/arch/ia64/sn/kernel/setup.c	Thu Jun 10 16:31:55 2004
@@ -50,6 +50,7 @@
 #include <asm/sn/sn2/shub.h>
 
 DEFINE_PER_CPU(struct pda_s, pda_percpu);
+EXPORT_SYMBOL(per_cpu__pda_percpu);
 
 #define MAX_PHYS_MEMORY		(1UL << 49)     /* 1 TB */
 
@@ -68,8 +69,11 @@ unsigned long sn_rtc_cycles_per_second; 
 EXPORT_SYMBOL(sn_rtc_cycles_per_second);
 
 partid_t sn_partid = -1;
+EXPORT_SYMBOL(sn_partid);
 char sn_system_serial_number_string[128];
+EXPORT_SYMBOL(sn_system_serial_number_string);
 u64 sn_partition_serial_number;
+EXPORT_SYMBOL(sn_partition_serial_number);
 
 short physical_node_map[MAX_PHYSNODE_ID];
 
Index: linux/include/asm-ia64/sn/sn_sal.h
=================================--- linux.orig/include/asm-ia64/sn/sn_sal.h	Mon May 10 12:33:21 2004
+++ linux/include/asm-ia64/sn/sn_sal.h	Thu Jun 10 16:31:55 2004
@@ -13,6 +13,7 @@
 
 
 #include <linux/config.h>
+#include <asm/sn/sgi.h>
 #include <asm/sal.h>
 #include <asm/sn/sn_cpuid.h>
 #include <asm/sn/arch.h>
Index: linux/kernel/exit.c
=================================--- linux.orig/kernel/exit.c	Thu Jun 10 15:04:09 2004
+++ linux/kernel/exit.c	Thu Jun 10 16:31:55 2004
@@ -1184,6 +1184,7 @@ end_wait4:
 	remove_wait_queue(&current->wait_chldexit,&wait);
 	return retval;
 }
+EXPORT_SYMBOL(sys_wait4);
 
 #ifdef __ARCH_WANT_SYS_WAITPID
 
Index: linux/kernel/sched.c
=================================--- linux.orig/kernel/sched.c	Thu Jun 10 16:30:39 2004
+++ linux/kernel/sched.c	Thu Jun 10 16:31:55 2004
@@ -2814,6 +2814,7 @@ asmlinkage long sys_sched_setscheduler(p
 {
 	return setscheduler(pid, policy, param);
 }
+EXPORT_SYMBOL(sys_sched_setscheduler);
 
 /**
  * sys_sched_setparam - set/change the RT priority of a thread
Index: linux/mm/page_alloc.c
=================================--- linux.orig/mm/page_alloc.c	Thu Jun 10 14:47:51 2004
+++ linux/mm/page_alloc.c	Thu Jun 10 16:31:55 2004
@@ -40,6 +40,7 @@ unsigned long totalram_pages;
 unsigned long totalhigh_pages;
 int nr_swap_pages;
 int numnodes = 1;
+EXPORT_SYMBOL(numnodes);
 int sysctl_lower_zone_protection = 0;
 
 EXPORT_SYMBOL(totalram_pages);

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Christoph Hellwig]

On Wed, Jun 16, 2004 at 11:33:39AM -0500, Dean Nelson wrote:
>  #include <asm/page.h>
>  #include <asm/sal.h>
>  #include <asm/pal.h>
>  
>  spinlock_t sal_lock __cacheline_aligned = SPIN_LOCK_UNLOCKED;
> +EXPORT_SYMBOL(sal_lock);

No.  Please don't make random sal calls from modules.

> +
>  unsigned long sal_platform_features;
>  
>  unsigned short sal_revision;
> @@ -38,6 +41,8 @@ default_handler (void)
>  }
>  
>  ia64_sal_handler ia64_sal = (ia64_sal_handler) default_handler;
> +EXPORT_SYMBOL(ia64_sal);

Dito.  I think you need to find a better abstraction.

> Index: linux/arch/ia64/kernel/smpboot.c
> =================================> --- linux.orig/arch/ia64/kernel/smpboot.c	Thu Jun 10 14:47:53 2004
> +++ linux/arch/ia64/kernel/smpboot.c	Thu Jun 10 16:31:55 2004
> @@ -498,6 +498,7 @@ u8 cpu_to_node_map[NR_CPUS] __cacheline_
>  EXPORT_SYMBOL(cpu_to_node_map);
>  /* which logical CPUs are on which nodes */
>  cpumask_t node_to_cpu_mask[MAX_NUMNODES] __cacheline_aligned;
> +EXPORT_SYMBOL(node_to_cpu_mask);

Isn't there a higher level interface for this?

> =================================> --- linux.orig/arch/ia64/sn/kernel/setup.c	Thu Jun 10 16:29:44 2004
> +++ linux/arch/ia64/sn/kernel/setup.c	Thu Jun 10 16:31:55 2004
> @@ -50,6 +50,7 @@
>  #include <asm/sn/sn2/shub.h>
>  
>  DEFINE_PER_CPU(struct pda_s, pda_percpu);
> +EXPORT_SYMBOL(per_cpu__pda_percpu);

Umm, no.  Poking into kernel per-cpu data like the pda from modules sounds
like a bad idea.  And if you absolutely must use the proper helpers for
exporting it.

> Index: linux/kernel/exit.c
> =================================> --- linux.orig/kernel/exit.c	Thu Jun 10 15:04:09 2004
> +++ linux/kernel/exit.c	Thu Jun 10 16:31:55 2004
> @@ -1184,6 +1184,7 @@ end_wait4:
>  	remove_wait_queue(&current->wait_chldexit,&wait);
>  	return retval;
>  }
> +EXPORT_SYMBOL(sys_wait4);

No way.  Calling syscalls from kernelspace is always a bad idea.  (And this
is not linux-ia64 material anyway)

> Index: linux/kernel/sched.c
> =================================> --- linux.orig/kernel/sched.c	Thu Jun 10 16:30:39 2004
> +++ linux/kernel/sched.c	Thu Jun 10 16:31:55 2004
> @@ -2814,6 +2814,7 @@ asmlinkage long sys_sched_setscheduler(p
>  {
>  	return setscheduler(pid, policy, param);
>  }
> +EXPORT_SYMBOL(sys_sched_setscheduler);

Again, don't mess with scheduler paramters from your modules.

>  /**
>   * sys_sched_setparam - set/change the RT priority of a thread
> Index: linux/mm/page_alloc.c
> =================================> --- linux.orig/mm/page_alloc.c	Thu Jun 10 14:47:51 2004
> +++ linux/mm/page_alloc.c	Thu Jun 10 16:31:55 2004
> @@ -40,6 +40,7 @@ unsigned long totalram_pages;
>  unsigned long totalhigh_pages;
>  int nr_swap_pages;
>  int numnodes = 1;
> +EXPORT_SYMBOL(numnodes);

bad idea.  I have patches to completely kill numnodes, don't try to
mess with it.  So far it's also completly an architecture-thing.

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Jesse Barnes]

On Wednesday, June 16, 2004 12:39 pm, Christoph Hellwig wrote:
> >  cpumask_t node_to_cpu_mask[MAX_NUMNODES] __cacheline_aligned;
> > +EXPORT_SYMBOL(node_to_cpu_mask);
>
> Isn't there a higher level interface for this?

Yeah, but node_to_cpumask() is just a wrapper around this array.

Jesse

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Christoph Hellwig]

On Wed, Jun 16, 2004 at 01:20:25PM -0400, Jesse Barnes wrote:
> On Wednesday, June 16, 2004 12:39 pm, Christoph Hellwig wrote:
> > >  cpumask_t node_to_cpu_mask[MAX_NUMNODES] __cacheline_aligned;
> > > +EXPORT_SYMBOL(node_to_cpu_mask);
> >
> > Isn't there a higher level interface for this?
> 
> Yeah, but node_to_cpumask() is just a wrapper around this array.

Ok, then it's fine with me.

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Robin Holt]

> > Index: linux/kernel/exit.c
> > =================================> > --- linux.orig/kernel/exit.c	Thu Jun 10 15:04:09 2004
> > +++ linux/kernel/exit.c	Thu Jun 10 16:31:55 2004
> > @@ -1184,6 +1184,7 @@ end_wait4:
> >  	remove_wait_queue(&current->wait_chldexit,&wait);
> >  	return retval;
> >  }
> > +EXPORT_SYMBOL(sys_wait4);
> 
> No way.  Calling syscalls from kernelspace is always a bad idea.  (And this
> is not linux-ia64 material anyway)

How should a parent thread be reaping children?

> 
> > Index: linux/kernel/sched.c
> > =================================> > --- linux.orig/kernel/sched.c	Thu Jun 10 16:30:39 2004
> > +++ linux/kernel/sched.c	Thu Jun 10 16:31:55 2004
> > @@ -2814,6 +2814,7 @@ asmlinkage long sys_sched_setscheduler(p
> >  {
> >  	return setscheduler(pid, policy, param);
> >  }
> > +EXPORT_SYMBOL(sys_sched_setscheduler);
> 
> Again, don't mess with scheduler paramters from your modules.

How should a kernel thread raise itself to real-time priority?

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Christoph Hellwig]

On Wed, Jun 16, 2004 at 12:40:53PM -0500, Robin Holt wrote:
> > > Index: linux/kernel/exit.c
> > > =================================> > > --- linux.orig/kernel/exit.c	Thu Jun 10 15:04:09 2004
> > > +++ linux/kernel/exit.c	Thu Jun 10 16:31:55 2004
> > > @@ -1184,6 +1184,7 @@ end_wait4:
> > >  	remove_wait_queue(&current->wait_chldexit,&wait);
> > >  	return retval;
> > >  }
> > > +EXPORT_SYMBOL(sys_wait4);
> > 
> > No way.  Calling syscalls from kernelspace is always a bad idea.  (And this
> > is not linux-ia64 material anyway)
> 
> How should a parent thread be reaping children?
> 
> > 
> > > Index: linux/kernel/sched.c
> > > =================================> > > --- linux.orig/kernel/sched.c	Thu Jun 10 16:30:39 2004
> > > +++ linux/kernel/sched.c	Thu Jun 10 16:31:55 2004
> > > @@ -2814,6 +2814,7 @@ asmlinkage long sys_sched_setscheduler(p
> > >  {
> > >  	return setscheduler(pid, policy, param);
> > >  }
> > > +EXPORT_SYMBOL(sys_sched_setscheduler);
> > 
> > Again, don't mess with scheduler paramters from your modules.
> 
> How should a kernel thread raise itself to real-time priority?

Answer to both:  it shouldn't

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Robin Holt]

On Wed, Jun 16, 2004 at 06:43:47PM +0100, Christoph Hellwig wrote:
> On Wed, Jun 16, 2004 at 12:40:53PM -0500, Robin Holt wrote:
> > > > Index: linux/kernel/exit.c
> > > > =================================> > > > --- linux.orig/kernel/exit.c	Thu Jun 10 15:04:09 2004
> > > > +++ linux/kernel/exit.c	Thu Jun 10 16:31:55 2004
> > > > @@ -1184,6 +1184,7 @@ end_wait4:
> > > >  	remove_wait_queue(&current->wait_chldexit,&wait);
> > > >  	return retval;
> > > >  }
> > > > +EXPORT_SYMBOL(sys_wait4);
> > > 
> > > No way.  Calling syscalls from kernelspace is always a bad idea.  (And this
> > > is not linux-ia64 material anyway)
> > 
> > How should a parent thread be reaping children?
> > 
> > > 
> > > > Index: linux/kernel/sched.c
> > > > =================================> > > > --- linux.orig/kernel/sched.c	Thu Jun 10 16:30:39 2004
> > > > +++ linux/kernel/sched.c	Thu Jun 10 16:31:55 2004
> > > > @@ -2814,6 +2814,7 @@ asmlinkage long sys_sched_setscheduler(p
> > > >  {
> > > >  	return setscheduler(pid, policy, param);
> > > >  }
> > > > +EXPORT_SYMBOL(sys_sched_setscheduler);
> > > 
> > > Again, don't mess with scheduler paramters from your modules.
> > 
> > How should a kernel thread raise itself to real-time priority?
> 
> Answer to both:  it shouldn't

To the second, we found that contention would result in very high
latency without raising the priority to real-time levels.  What is
the proper way to handle having a user thread at the same priority
as a kernel thread causing this holdoff?

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Dean Nelson]

On Wed, Jun 16, 2004 at 05:39:44PM +0100, Christoph Hellwig wrote:
> On Wed, Jun 16, 2004 at 11:33:39AM -0500, Dean Nelson wrote:
> >  #include <asm/page.h>
> >  #include <asm/sal.h>
> >  #include <asm/pal.h>
> >  
> >  spinlock_t sal_lock __cacheline_aligned = SPIN_LOCK_UNLOCKED;
> > +EXPORT_SYMBOL(sal_lock);
> 
> No.  Please don't make random sal calls from modules.

We're only making one sal call (i.e., SN_SAL_GET_PARTITION_ADDR) to get the
address of a partition's reserved page.

> > +
> >  unsigned long sal_platform_features;
> >  
> >  unsigned short sal_revision;
> > @@ -38,6 +41,8 @@ default_handler (void)
> >  }
> >  
> >  ia64_sal_handler ia64_sal = (ia64_sal_handler) default_handler;
> > +EXPORT_SYMBOL(ia64_sal);
> 
> Dito.  I think you need to find a better abstraction.

What kind of abstraction are you looking for?

In include/asm-ia64/sn/sn_sal.h, there are inline functions that are wrappers
for the SAL_CALL() macro, (for example, ia64_sn_get_console_nasid()). Is this
what you are looking for? (But even if we did create such a function, we'd
still need to export sal_lock and ia64_sal because both are referenced by the
SAL_CALL() macro.) I'd hate to put such an abstraction into sal.c, since the
functionality of my sal call is ia64/sn specific, and sal.c is generic
ia64.

Thanks,
Dean

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Dean Nelson]

On Wed, Jun 16, 2004 at 05:39:44PM +0100, Christoph Hellwig wrote:
> On Wed, Jun 16, 2004 at 11:33:39AM -0500, Dean Nelson wrote:
> > --- linux.orig/mm/page_alloc.c	Thu Jun 10 14:47:51 2004
> > +++ linux/mm/page_alloc.c	Thu Jun 10 16:31:55 2004
> > @@ -40,6 +40,7 @@ unsigned long totalram_pages;
> >  unsigned long totalhigh_pages;
> >  int nr_swap_pages;
> >  int numnodes = 1;
> > +EXPORT_SYMBOL(numnodes);
> 
> bad idea.  I have patches to completely kill numnodes, don't try to
> mess with it.  So far it's also completly an architecture-thing.

What are you planning to replace numnodes with?

XPC has two functions (xpc_allow_IPI_ops() and xpc_restrict_IPI_ops()) which
need to change the protections governing IPI operations and AMO (memory)
operations for each node in a partition. What mechanism are you proposing
to allow one to iterate through all of the nodes in a partition?

Thanks,
Dean

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Dean Nelson]

On Wed, Jul 14, 2004 at 11:01:08AM -0500, Dean Nelson wrote:
> On Wed, Jun 16, 2004 at 05:39:44PM +0100, Christoph Hellwig wrote:
> > On Wed, Jun 16, 2004 at 11:33:39AM -0500, Dean Nelson wrote:
> > >  #include <asm/page.h>
> > >  #include <asm/sal.h>
> > >  #include <asm/pal.h>
> > >  
> > >  spinlock_t sal_lock __cacheline_aligned = SPIN_LOCK_UNLOCKED;
> > > +EXPORT_SYMBOL(sal_lock);
> > 
> > No.  Please don't make random sal calls from modules.
> 
> We're only making one sal call (i.e., SN_SAL_GET_PARTITION_ADDR) to get the
> address of a partition's reserved page.

To be more precise, this is the only direct sal call we are making (i.e.,
using the SAL_CALL() macro directly). We are also making the following 
indirect sal calls via the following wrappers, all of which are defined as
inline functions in include/asm-ia64/sn/sn_sal.h:

	sn_register_nofault_code()
	sn_change_memprotect()
	sn_register_xp_addr_region()
	sn_local_partid()
	sn_partition_serial_number_val()

So I'll create a wrapper like these for the one remaining direct SAL_CALL()
we are making.

Being that they are all inline functions that call SAL_CALL() (one calls
SAL_CALL_NOLOCK()) still requires the exporting of sal_lock and ia64_sal.

Does all of this sound okay to you?

Thanks,
Dean

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Christoph Hellwig]

On Wed, Jul 14, 2004 at 11:01:08AM -0500, Dean Nelson wrote:
> > No.  Please don't make random sal calls from modules.
> 
> We're only making one sal call (i.e., SN_SAL_GET_PARTITION_ADDR) to get the
> address of a partition's reserved page.

Then please provide some accessor from core SN2 code.  If that partition addr
is constant after bootup a global variable should do it, else a small wrapper
function.

> In include/asm-ia64/sn/sn_sal.h, there are inline functions that are wrappers
> for the SAL_CALL() macro, (for example, ia64_sn_get_console_nasid()). Is this
> what you are looking for? (But even if we did create such a function, we'd
> still need to export sal_lock and ia64_sal because both are referenced by the
> SAL_CALL() macro.) I'd hate to put such an abstraction into sal.c, since the
> functionality of my sal call is ia64/sn specific, and sal.c is generic
> ia64.

similar to that, just not inline.

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Christoph Hellwig]

On Wed, Jul 14, 2004 at 11:04:48AM -0500, Dean Nelson wrote:
> > bad idea.  I have patches to completely kill numnodes, don't try to
> > mess with it.  So far it's also completly an architecture-thing.
> 
> What are you planning to replace numnodes with?

A bitmap of online nodes once pj's nodemask_t is in.  That way the silly
distinctinon of nasid_t and cnodeid_t values in IP27 and SN2 code can go
away.

> XPC has two functions (xpc_allow_IPI_ops() and xpc_restrict_IPI_ops()) which
> need to change the protections governing IPI operations and AMO (memory)
> operations for each node in a partition. What mechanism are you proposing
> to allow one to iterate through all of the nodes in a partition?

I'm not exactly sure.  I'll look through once I get a little time.

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Jack Steiner]

On Thu, Jul 15, 2004 at 10:00:04PM +0100, Christoph Hellwig wrote:
> On Wed, Jul 14, 2004 at 11:04:48AM -0500, Dean Nelson wrote:
> > > bad idea.  I have patches to completely kill numnodes, don't try to
> > > mess with it.  So far it's also completly an architecture-thing.
> > 
> > What are you planning to replace numnodes with?
> 
> A bitmap of online nodes once pj's nodemask_t is in.  That way the silly
> distinctinon of nasid_t and cnodeid_t values in IP27 and SN2 code can go
> away.


Can you explain your proposal a little more. 

The concept of nasid_t & cnodeid_t are different. A system can have a maximum
of 256 nodes (cnodeid_t 0..255), but the nasid values could
be 1536..2048. 

I'm not sure what you are proposing. Dont increase NR_NODES to 2048 -
some tables will get too big. Also, scanning bitmaps of 2048 bits is
more costly than scanning bitmaps of 256 elements. I havent checked
if this is a frequent operation, though....



> 
> > XPC has two functions (xpc_allow_IPI_ops() and xpc_restrict_IPI_ops()) which
> > need to change the protections governing IPI operations and AMO (memory)
> > operations for each node in a partition. What mechanism are you proposing
> > to allow one to iterate through all of the nodes in a partition?
> 
> I'm not exactly sure.  I'll look through once I get a little time.
> 
> -
> To unsubscribe from this list: send the line "unsubscribe linux-ia64" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

-- 
Thanks

Jack Steiner (steiner@sgi.com)          651-683-5302
Principal Engineer                      SGI - Silicon Graphics, Inc.

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Dean Nelson]

On Thu, Jul 15, 2004 at 09:58:27PM +0100, Christoph Hellwig wrote:
> On Wed, Jul 14, 2004 at 11:01:08AM -0500, Dean Nelson wrote:
> > > No.  Please don't make random sal calls from modules.
> > 
> > We're only making one sal call (i.e., SN_SAL_GET_PARTITION_ADDR) to get the
> > address of a partition's reserved page.
> 
> Then please provide some accessor from core SN2 code.  If that partition addr
> is constant after bootup a global variable should do it, else a small wrapper
> function.
> 
> > In include/asm-ia64/sn/sn_sal.h, there are inline functions that are wrappers
> > for the SAL_CALL() macro, (for example, ia64_sn_get_console_nasid()). Is this
> > what you are looking for? (But even if we did create such a function, we'd
> > still need to export sal_lock and ia64_sal because both are referenced by the
> > SAL_CALL() macro.) I'd hate to put such an abstraction into sal.c, since the
> > functionality of my sal call is ia64/sn specific, and sal.c is generic
> > ia64.
> 
> similar to that, just not inline.


David and Tony,

The modules that are part of this series of patches need to be able to call
into SAL. They do this by way of the following inline functions, which are
currently defined (or will be) in sn_sal.h.

	sn_partition_reserved_page_pa() calls
		SAL_CALL(, SN_SAL_GET_PARTITION_ADDR, ...)

	sn_register_xp_addr_region() calls
		SAL_CALL(, SN_SAL_XP_ADDR_REGION, ...)

	sn_register_nofault_code() calls
		SAL_CALL(, SN_SAL_NO_FAULT_ZONE_VIRTUAL, ...)

	sn_partition_serial_number_val() calls
		ia64_sn_partition_serial_get() calls
			SAL_CALL(, SN_SAL_PARTITION_SERIAL_GET, ...)

	sn_local_partid() calls
		ia64_sn_sysctl_partition_get() calls
			SAL_CALL(, SN_SAL_SYSCTL_PARTITION_GET, ...)

	sn_change_coherence() calls
		SAL_CALL(, SN_SAL_COHERENCE, ...)

	sn_change_memprotect() calls
		SAL_CALL_NOLOCK(, SN_SAL_MEMPROTECT, ...)

The question is whether we export ia64_sal and sal_lock? Or change these
inline functions to no longer be inline, move them to a new file (sn_sal.c),
and export the individual function names?

From my point of view the first solution is better from the standpoint of
the diagnostics folks who write tests that need to make SAL calls. It doesn't
seem appropriate that the linux kernel be the repository of a bunch of ad hoc
and miscellaneous SAL wrapper functions that no one but the diagnostics folks
use.

I would like to know your opinions on the direction to go with this.

Thanks,
Dean

RE: [PATCH 1/4] SGI Altix cross partition functionality

[Luck, Tony]

Dean> The question is whether we export ia64_sal and sal_lock? Or change these
Dean> inline functions to no longer be inline, move them to a new file (sn_sal.c),
Dean> and export the individual function names?

Dean> From my point of view the first solution is better from the standpoint of
Dean> the diagnostics folks who write tests that need to make SAL calls. It doesn't
Dean> seem appropriate that the linux kernel be the repository of a bunch of ad hoc
Dean> and miscellaneous SAL wrapper functions that no one but the diagnostics folks
Dean> use.

Dean> I would like to know your opinions on the direction to go with this.

This looks like a lesser of two evils decision.  On balance I think
that I'd prefer not to open up the floodgates for modules to make
arbitrary SAL calls ... so I'd prefer to see you un-inline and
export the functions that you need. "sn_sal.c" sounds a plausible
name for a file to house these functions.

-Tony

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Robin Holt]

On Fri, Jul 30, 2004 at 03:15:35PM -0700, Luck, Tony wrote:
> Dean> The question is whether we export ia64_sal and sal_lock? Or change these
> Dean> inline functions to no longer be inline, move them to a new file (sn_sal.c),
> Dean> and export the individual function names?
> 
> Dean> From my point of view the first solution is better from the standpoint of
> Dean> the diagnostics folks who write tests that need to make SAL calls. It doesn't
> Dean> seem appropriate that the linux kernel be the repository of a bunch of ad hoc
> Dean> and miscellaneous SAL wrapper functions that no one but the diagnostics folks
> Dean> use.
> 
> Dean> I would like to know your opinions on the direction to go with this.
> 
> This looks like a lesser of two evils decision.  On balance I think
> that I'd prefer not to open up the floodgates for modules to make
> arbitrary SAL calls ... so I'd prefer to see you un-inline and
> export the functions that you need. "sn_sal.c" sounds a plausible
> name for a file to house these functions.
> 

This leaves our diags people with a fairly difficult position.
They have some online diags which need to excercise parts of the shub.
Those activities are currently coded in the PROM (make online and offline
diags a lot more consistent) executed via SAL calls.

Would you be alright with changing the inlines into real functions,
but still exporting the ia64_sal symbol as well?

Will you accept the online diags people sending sal call additions that
no kernel component will ever use?  Will that policy remain consistent
into the future?  This makes our job much more difficult as we move
toward not having our own kernel at all, but using standard distribution
kernels as they will always lag the community in merging back changes
made to the ia64 tree.  Again, that argues for having the ia64_sal
exported.  Please consider those needs when making your decision.

Thanks,
Robin

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Dean Nelson]

On Sat, Jul 31, 2004 at 07:46:23AM -0500, Robin Holt wrote:
> On Fri, Jul 30, 2004 at 03:15:35PM -0700, Luck, Tony wrote:
> > Dean> The question is whether we export ia64_sal and sal_lock? Or change these
> > Dean> inline functions to no longer be inline, move them to a new file (sn_sal.c),
> > Dean> and export the individual function names?
> > 
> > Dean> From my point of view the first solution is better from the standpoint of
> > Dean> the diagnostics folks who write tests that need to make SAL calls. It doesn't
> > Dean> seem appropriate that the linux kernel be the repository of a bunch of ad hoc
> > Dean> and miscellaneous SAL wrapper functions that no one but the diagnostics folks
> > Dean> use.
> > 
> > Dean> I would like to know your opinions on the direction to go with this.
> > 
> > This looks like a lesser of two evils decision.  On balance I think
> > that I'd prefer not to open up the floodgates for modules to make
> > arbitrary SAL calls ... so I'd prefer to see you un-inline and
> > export the functions that you need. "sn_sal.c" sounds a plausible
> > name for a file to house these functions.
> 
> This leaves our diags people with a fairly difficult position.
> They have some online diags which need to excercise parts of the shub.
> Those activities are currently coded in the PROM (make online and offline
> diags a lot more consistent) executed via SAL calls.
> 
> Would you be alright with changing the inlines into real functions,
> but still exporting the ia64_sal symbol as well?
> 
> Will you accept the online diags people sending sal call additions that
> no kernel component will ever use?  Will that policy remain consistent
> into the future?  This makes our job much more difficult as we move
> toward not having our own kernel at all, but using standard distribution
> kernels as they will always lag the community in merging back changes
> made to the ia64 tree.  Again, that argues for having the ia64_sal
> exported.  Please consider those needs when making your decision.

Tony,

I'm interested in your response to what Robin wrote. Care to comment?

And just another data point: we no longer need the sal_lock exported,
just ia64_sal, does this affect your thoughts on this matter?

Thanks,
Dean

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Dean Nelson]

On Tue, Aug 10, 2004 at 02:54:00PM -0500, Dean Nelson wrote:
>
> Tony,
> 
> I'm interested in your response to what Robin wrote. Care to comment?
> 
> And just another data point: we no longer need the sal_lock exported,
> just ia64_sal, does this affect your thoughts on this matter?

Just wanted to add a couple of additional data points to the
discussion of whether or not to export the SAL call itself.

It was pointed out to me that the PAL call is exported. You
can find PAL_CALL() in include/asm-ia64/pal.h and its export
in arch/ia64/kernel/ia64_ksyms.c, where the necessary symbols
are exported:
                                                                                
        ia64_pal_call_static
        ia64_load_scratch_fpregs
        ia64_save_scratch_fpregs
                                                                                
(The latter two are needed by SAL_CALL(), which also requires
the exporting of ia64_sal and sal_lock.)
                                                                                
Is there a fundamental difference between PAL and SAL that would
justify exporting one and not the other?
                                                                                
Also, what makes it okay for 'CONFIG_IA64_SGI_SN_XPC=y' to make a
SAL_CALL() and not okay for 'CONFIG_IA64_SGI_SN_XPC=m'? It's the
same code, either compiled into the kernel or into a kernel module.
What makes the first one 'trustworthy' and the second one not?

Thanks,
Dean

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Dean Nelson]

On Wed, Jun 16, 2004 at 05:39:44PM +0100, Christoph Hellwig wrote:
> On Wed, Jun 16, 2004 at 11:33:39AM -0500, Dean Nelson wrote:
> > --- linux.orig/arch/ia64/sn/kernel/setup.c	Thu Jun 10 16:29:44 2004
> > +++ linux/arch/ia64/sn/kernel/setup.c	Thu Jun 10 16:31:55 2004
> > @@ -50,6 +50,7 @@
> >  #include <asm/sn/sn2/shub.h>
> >  
> >  DEFINE_PER_CPU(struct pda_s, pda_percpu);
> > +EXPORT_SYMBOL(per_cpu__pda_percpu);
> 
> Umm, no.  Poking into kernel per-cpu data like the pda from modules sounds
> like a bad idea.  And if you absolutely must use the proper helpers for
> exporting it.

Just an FYI, XPC needs to use the enable_shub_wars_1_1() macro which
accesses pda->shub_1_1_found. So we do need to export per_cpu__pda_percpu.
I've switched EXPORT_SYMBOL() to EXPORT_PER_CPU_SYMBOL() instead.

Dean

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Dean Nelson]

On Thu, Jul 15, 2004 at 10:00:04PM +0100, Christoph Hellwig wrote:
> On Wed, Jul 14, 2004 at 11:04:48AM -0500, Dean Nelson wrote:
> > > bad idea.  I have patches to completely kill numnodes, don't try to
> > > mess with it.  So far it's also completly an architecture-thing.
> > 
> > What are you planning to replace numnodes with?
> 
> A bitmap of online nodes once pj's nodemask_t is in.  That way the silly
> distinctinon of nasid_t and cnodeid_t values in IP27 and SN2 code can go
> away.
> 
> > XPC has two functions (xpc_allow_IPI_ops() and xpc_restrict_IPI_ops()) which
> > need to change the protections governing IPI operations and AMO (memory)
> > operations for each node in a partition. What mechanism are you proposing
> > to allow one to iterate through all of the nodes in a partition?
> 
> I'm not exactly sure.  I'll look through once I get a little time.

Although I agree that the nodemask_t patch is the proper solution to the
numnodes issue, I don't feel that waiting for its arrival should prevent
my continuing on with my XP[C|NET] patch. So I'm going to resubmit my
proposed patch with numnodes exported. This is with the understanding
that once the nodemask_t patch is accepted by the community (resulting
in the 'EXPORT_SYMBOL(numnodes);' line being deleted), I'll be glade to
rework the two XPC functions to use the 'for_each_node()' macro instead.

Dean

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Christoph Hellwig]

On Mon, Aug 23, 2004 at 11:50:59AM -0500, Dean Nelson wrote:
> Just an FYI, XPC needs to use the enable_shub_wars_1_1() macro which
> accesses pda->shub_1_1_found. So we do need to export per_cpu__pda_percpu.
> I've switched EXPORT_SYMBOL() to EXPORT_PER_CPU_SYMBOL() instead.

What about making enable_shub_wars_1_1() a function instead and exporting
that one?  That's the export a sensible interface instead of random symbols
thing, it shouldn't be _that_ hard to get..

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Christoph Hellwig]

On Mon, Aug 23, 2004 at 12:10:08PM -0500, Dean Nelson wrote:
> Although I agree that the nodemask_t patch is the proper solution to the
> numnodes issue, I don't feel that waiting for its arrival should prevent
> my continuing on with my XP[C|NET] patch. So I'm going to resubmit my
> proposed patch with numnodes exported. This is with the understanding
> that once the nodemask_t patch is accepted by the community (resulting
> in the 'EXPORT_SYMBOL(numnodes);' line being deleted), I'll be glade to
> rework the two XPC functions to use the 'for_each_node()' macro instead.

Can you post a current snapshot of your work?  As in the last mail numnodes
is not the kind if 'interface' we'd like to export, especially with things
like node hotplug in the pipe.

p.s. and linux-ia64 is the wrong list for discussing that

Re: [PATCH 1/4] SGI Altix cross partition functionality

[Dean Nelson]

On Mon, Aug 23, 2004 at 06:52:39PM +0100, Christoph Hellwig wrote:
> On Mon, Aug 23, 2004 at 12:10:08PM -0500, Dean Nelson wrote:
> > Although I agree that the nodemask_t patch is the proper solution to the
> > numnodes issue, I don't feel that waiting for its arrival should prevent
> > my continuing on with my XP[C|NET] patch. So I'm going to resubmit my
> > proposed patch with numnodes exported. This is with the understanding
> > that once the nodemask_t patch is accepted by the community (resulting
> > in the 'EXPORT_SYMBOL(numnodes);' line being deleted), I'll be glade to
> > rework the two XPC functions to use the 'for_each_node()' macro instead.
> 
> Can you post a current snapshot of your work?  As in the last mail numnodes
> is not the kind if 'interface' we'd like to export, especially with things
> like node hotplug in the pipe.

Yeah, I'll submit my latest version (probably tommorrow) for round two of
the review process. Keep in mind that it will not reflect your comments
from your latest two emails.

[PATCH 1/4] SGI Altix cross partition functionality (1st revision)

[Dean Nelson]

This patch exports the symbols needed by XP[C|NET].

Signed-off-by: Dean Nelson <dcn@sgi.com>


Index: bk-linux-2.6/arch/ia64/kernel/smpboot.c
=================================--- bk-linux-2.6.orig/arch/ia64/kernel/smpboot.c	2004-08-23 14:38:36.000000000 -0500
+++ bk-linux-2.6/arch/ia64/kernel/smpboot.c	2004-08-24 07:29:11.000000000 -0500
@@ -498,6 +498,7 @@
 EXPORT_SYMBOL(cpu_to_node_map);
 /* which logical CPUs are on which nodes */
 cpumask_t node_to_cpu_mask[MAX_NUMNODES] __cacheline_aligned;
+EXPORT_SYMBOL(node_to_cpu_mask);
 
 /*
  * Build cpu to node mapping and initialize the per node cpu masks.
Index: bk-linux-2.6/arch/ia64/sn/kernel/setup.c
=================================--- bk-linux-2.6.orig/arch/ia64/sn/kernel/setup.c	2004-08-23 14:38:37.000000000 -0500
+++ bk-linux-2.6/arch/ia64/sn/kernel/setup.c	2004-08-24 07:29:11.000000000 -0500
@@ -50,6 +50,7 @@
 #include <asm/sn/sn2/shub.h>
 
 DEFINE_PER_CPU(struct pda_s, pda_percpu);
+EXPORT_PER_CPU_SYMBOL(pda_percpu);
 
 #define MAX_PHYS_MEMORY		(1UL << 49)     /* 1 TB */
 
@@ -65,8 +66,11 @@
 unsigned long sn_rtc_cycles_per_second;   
 
 partid_t sn_partid = -1;
+EXPORT_SYMBOL(sn_partid);
 char sn_system_serial_number_string[128];
+EXPORT_SYMBOL(sn_system_serial_number_string);
 u64 sn_partition_serial_number;
+EXPORT_SYMBOL(sn_partition_serial_number);
 
 short physical_node_map[MAX_PHYSNODE_ID];
 
Index: bk-linux-2.6/kernel/sched.c
=================================--- bk-linux-2.6.orig/kernel/sched.c	2004-08-23 14:39:35.000000000 -0500
+++ bk-linux-2.6/kernel/sched.c	2004-08-24 07:29:11.000000000 -0500
@@ -2814,6 +2814,7 @@
 {
 	return setscheduler(pid, policy, param);
 }
+EXPORT_SYMBOL(sys_sched_setscheduler);
 
 /**
  * sys_sched_setparam - set/change the RT priority of a thread
Index: bk-linux-2.6/mm/page_alloc.c
=================================--- bk-linux-2.6.orig/mm/page_alloc.c	2004-08-23 14:38:16.000000000 -0500
+++ bk-linux-2.6/mm/page_alloc.c	2004-08-24 07:29:11.000000000 -0500
@@ -40,6 +40,7 @@
 unsigned long totalhigh_pages;
 long nr_swap_pages;
 int numnodes = 1;
+EXPORT_SYMBOL(numnodes);
 int sysctl_lower_zone_protection = 0;
 
 EXPORT_SYMBOL(totalram_pages);
Index: bk-linux-2.6/include/asm-ia64/sn/sn_sal.h
=================================--- bk-linux-2.6.orig/include/asm-ia64/sn/sn_sal.h	2004-08-23 14:40:00.000000000 -0500
+++ bk-linux-2.6/include/asm-ia64/sn/sn_sal.h	2004-08-24 07:29:11.000000000 -0500
@@ -494,7 +494,8 @@
 ia64_sn_partition_serial_get(void)
 {
 	struct ia64_sal_retval ret_stuff;
-	SAL_CALL(ret_stuff, SN_SAL_PARTITION_SERIAL_GET, 0, 0, 0, 0, 0, 0, 0);
+	ia64_sal_oemcall_reentrant(&ret_stuff, SN_SAL_PARTITION_SERIAL_GET, 0,
+				   0, 0, 0, 0, 0, 0);
 	if (ret_stuff.status != 0)
 	    return 0;
 	return ret_stuff.v0;
@@ -502,11 +503,10 @@
 
 static inline u64
 sn_partition_serial_number_val(void) {
-	if (sn_partition_serial_number) {
-		return(sn_partition_serial_number);
-	} else {
-		return(sn_partition_serial_number = ia64_sn_partition_serial_get());
+	if (unlikely(sn_partition_serial_number = 0)) {
+		sn_partition_serial_number = ia64_sn_partition_serial_get();
 	}
+	return sn_partition_serial_number;
 }
 
 /*
@@ -517,8 +517,8 @@
 ia64_sn_sysctl_partition_get(nasid_t nasid)
 {
 	struct ia64_sal_retval ret_stuff;
-	SAL_CALL(ret_stuff, SN_SAL_SYSCTL_PARTITION_GET, nasid,
-		 0, 0, 0, 0, 0, 0);
+	ia64_sal_oemcall_nolock(&ret_stuff, SN_SAL_SYSCTL_PARTITION_GET, nasid,
+				0, 0, 0, 0, 0, 0);
 	if (ret_stuff.status != 0)
 	    return INVALID_PARTID;
 	return ((partid_t)ret_stuff.v0);
@@ -532,11 +532,38 @@
 
 static inline partid_t
 sn_local_partid(void) {
-	if (sn_partid < 0) {
-		return (sn_partid = ia64_sn_sysctl_partition_get(cpuid_to_nasid(smp_processor_id())));
-	} else {
-		return sn_partid;
+	if (unlikely(sn_partid < 0)) {
+		sn_partid = ia64_sn_sysctl_partition_get(cpuid_to_nasid(smp_processor_id()));
 	}
+	return sn_partid;
+}
+
+/*
+ * Returns the physical address of the partition's reserved page through
+ * an iterative number of calls.
+ *
+ * On first call, 'cookie' and 'len' should be set to 0, and 'addr'
+ * set to the nasid of the partition whose reserved page's address is
+ * being sought.
+ * On subsequent calls, pass the values, that were passed back on the
+ * previous call.
+ *
+ * While the return status equals SALRET_MORE_PASSES, keep calling
+ * this function after first copying 'len' bytes starting at 'addr'
+ * into 'buf'. Once the return status equals SALRET_OK, 'addr' will
+ * be the physical address of the partition's reserved page. If the
+ * return status equals neither of these, an error as occurred.
+ */
+static inline s64
+sn_partition_reserved_page_pa(u64 buf, u64 *cookie, u64 *addr, u64 *len)
+{
+	struct ia64_sal_retval rv;
+	ia64_sal_oemcall_reentrant(&rv, SN_SAL_GET_PARTITION_ADDR, *cookie,
+				   *addr, buf, *len, 0, 0, 0);
+	*cookie = rv.v0;
+	*addr = rv.v1;
+	*len = rv.v2;
+	return rv.status;
 }
 
 /*
@@ -558,8 +585,8 @@
 sn_register_xp_addr_region(u64 paddr, u64 len, int operation)
 {
 	struct ia64_sal_retval ret_stuff;
-	SAL_CALL(ret_stuff, SN_SAL_XP_ADDR_REGION, paddr, len, (u64)operation,
-		 0, 0, 0, 0);
+	ia64_sal_oemcall(&ret_stuff, SN_SAL_XP_ADDR_REGION, paddr, len,
+			 (u64)operation, 0, 0, 0, 0);
 	return ret_stuff.status;
 }
 
@@ -583,8 +610,8 @@
 	} else {
 		call = SN_SAL_NO_FAULT_ZONE_PHYSICAL;
 	}
-	SAL_CALL(ret_stuff, call, start_addr, end_addr, return_addr, (u64)1,
-		 0, 0, 0);
+	ia64_sal_oemcall(&ret_stuff, call, start_addr, end_addr, return_addr,
+			 (u64)1, 0, 0, 0);
 	return ret_stuff.status;
 }
 
@@ -605,8 +632,8 @@
 sn_change_coherence(u64 *new_domain, u64 *old_domain)
 {
 	struct ia64_sal_retval ret_stuff;
-	SAL_CALL(ret_stuff, SN_SAL_COHERENCE, new_domain, old_domain, 0, 0,
-		 0, 0, 0);
+	ia64_sal_oemcall(&ret_stuff, SN_SAL_COHERENCE, (u64)new_domain,
+			 (u64)old_domain, 0, 0, 0, 0, 0);
 	return ret_stuff.status;
 }
 
@@ -625,8 +652,8 @@
 	cnodeid = nasid_to_cnodeid(get_node_number(paddr));
 	spin_lock(&NODEPDA(cnodeid)->bist_lock);
 	local_irq_save(irq_flags);
-	SAL_CALL_NOLOCK(ret_stuff, SN_SAL_MEMPROTECT, paddr, len, nasid_array,
-		 perms, 0, 0, 0);
+	ia64_sal_oemcall_nolock(&ret_stuff, SN_SAL_MEMPROTECT, paddr, len,
+				(u64)nasid_array, perms, 0, 0, 0);
 	local_irq_restore(irq_flags);
 	spin_unlock(&NODEPDA(cnodeid)->bist_lock);
 	return ret_stuff.status;

Re: [PATCH 1/4] SGI Altix cross partition functionality (1st revision)

[Christoph Hellwig]

> --- bk-linux-2.6.orig/kernel/sched.c	2004-08-23 14:39:35.000000000 -0500
> +++ bk-linux-2.6/kernel/sched.c	2004-08-24 07:29:11.000000000 -0500
> @@ -2814,6 +2814,7 @@
>  {
>  	return setscheduler(pid, policy, param);
>  }
> +EXPORT_SYMBOL(sys_sched_setscheduler);

As said previously you're not supposed to mess with this one.

>  long nr_swap_pages;
>  int numnodes = 1;
> +EXPORT_SYMBOL(numnodes);

Looing at the two routines that use this I think those two should always
go into the core kernel code.  That way you also get rid of the
pda_percpu export.

Re: [PATCH 1/4] SGI Altix cross partition functionality (1st revision)

[Dean Nelson]

On Tue, Aug 24, 2004 at 08:13:43PM +0100, Christoph Hellwig wrote:
> > +++ bk-linux-2.6/kernel/sched.c	2004-08-24 07:29:11.000000000 -0500
> > @@ -2814,6 +2814,7 @@
> >  {
> >  	return setscheduler(pid, policy, param);
> >  }
> > +EXPORT_SYMBOL(sys_sched_setscheduler);
> 
> As said previously you're not supposed to mess with this one.

Yeah, I know, but how is one suppose to deal with the following
issue raised by Robin? (Never did get a response from you.)

On Wed, Jun 16, 2004 at 02:36:22PM -0500, Robin Holt wrote:
> On Wed, Jun 16, 2004 at 06:43:47PM +0100, Christoph Hellwig wrote:
> > On Wed, Jun 16, 2004 at 12:40:53PM -0500, Robin Holt wrote:
> > > > > +EXPORT_SYMBOL(sys_sched_setscheduler);
> > > >
> > > > Again, don't mess with scheduler paramters from your modules.
> > >
> > > How should a kernel thread raise itself to real-time priority?
> >
> > Answer to both:  it shouldn't
>
> To the second, we found that contention would result in very high
> latency without raising the priority to real-time levels.  What is
> the proper way to handle having a user thread at the same priority
> as a kernel thread causing this holdoff?

The problem arises when enough user processes, which have the same
priority as XPC's kthreads, are spinning doing a bit of work mixed
with sleeping. Because of the sleep, these processes get a bonus
which gives them a higher effective priority than the XPC kthreads.
As a result, when cross partition interrupts come in, the XPC kthreads
do not get scheduled immediately, but are held off until the end of
the user processes' time slice.

This problem was encountered running a legitimate user land work load.
I concocted the following program to reproduce the behavior we were
seeing. I did this to see if the problem still exists on the 2.6
kernel (we originally saw the problem running on 2.4).

I ran the program on one of two partitions that were connected via XPC
and XPNET. A ping on each partition was pinging the other partition. With
XPC's priority left to default, the ping times shot way up (300 times the
latency). But with XPC's priority set to realtime, there was no change.

So I would ask again, how are we to deal with this issue if we're
not allowed to change the priority of XPC's kthreads to realtime?

Should we be exporting setscheduler() instead?

Thanks,
Dean


 \\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\

#include <stdio.h>
#include <unistd.h>
#include <errno.h>
#include <time.h>
#include <linux/types.h>
#include <sys/types.h>
#include <sys/wait.h>
// #include <asm/ia64regs.h>

/*
 *	choke [ -n <nchildren> ]
 *
 *	-n  number of children to fork (defaults to '#of CPUs * 4')
 */

#define STR_BUF_SIZE	80

#define _IA64_REG_AR_KR0	3072
#define _IA64_REG_AR_ITC	3116
                                                                                
__u64 current_itc;
// long long current_itc;
                                                                                
__u64
get_itc()
{
	asm volatile ("mov %0=ar%1" : "=r" (current_itc) :
			"i"(_IA64_REG_AR_ITC - _IA64_REG_AR_KR0));
	return current_itc;
}

int
get_nCPUs(void)
{
	char str_buf[STR_BUF_SIZE];
	FILE *str_fd;


	/* figure out the number of CPUs on this system */
	str_fd = popen("cat /proc/cpuinfo | grep processor | wc -l", "r");
	if (str_fd = NULL || fgets(str_buf, STR_BUF_SIZE, str_fd) = NULL) {
		fprintf(stderr, "couldn't determine number of CPUs\n");
		return 0;
	}
	(void) pclose(str_fd);
	return atoi(str_buf);
}

void
child_work_and_sleep(void)
{
	int i;
	__u64 t1, t2;
	struct timespec req_time;

	
	req_time.tv_sec = 0;
	req_time.tv_nsec = 4000000;	/* 4msec */

	while (1) {
		/* work for a bit (just less than 1msec) */
		t2 = get_itc();
		do {
			t1 = t2;
			for (i = 1123000; i > 0; i--) {
				i--;
			}
			t2 = get_itc();
		} while (t2 - t1 < 99999);	/* < 1msec */

		/* sleep for a bit (about 4msec) */
		if (nanosleep(&req_time, NULL) = -1) {
			fprintf(stderr, "nanosleep() failed, errno=%d\n",
									errno);
			exit(1);
		}
	}
}

int
main(int argc, char *argv[])
{
	pid_t pid;
	int c, i;
	int nChildren = -1;


	while ((c = getopt(argc, argv, "n:")) != EOF) {
		switch (c) {
		case 'n':
			nChildren = atoi(optarg);
			break;
		case '?':
			fprintf(stderr, "choke [-n nchildren]\n");
			exit(1);
		}
	}

	if (optind != argc) {
		fprintf(stderr, "choke [-n nchildren]\n");
		exit(1);
	}

	if (nChildren = -1) {
		nChildren = get_nCPUs() * 4;
	}


	/* fork nChildren worth of children who work a msec and sleep 4msec */
	printf("forking %d children\n", nChildren);


	for (i = 0; i < nChildren; i++) {

		if ((pid = fork()) = -1) {
			fprintf(stderr, "fork() failed, errno=%d\n", errno);
			exit(1);
		}
		if (pid != 0) {
			printf("child %d: %d\n", i+1, pid);
			continue;
		}

		child_work_and_sleep();
		exit(1);	/* should never get here */
	}

	while ((pid = wait(NULL)) != -1) { /* spin */ }
	if (errno != ECHILD) {
		fprintf(stderr, "wait() failed, errno=%d\n", errno);
		exit(1);
	}
	exit(0);
}

Re: [PATCH 1/4] SGI Altix cross partition functionality (1st revision)

[Christoph Hellwig]

On Tue, Sep 14, 2004 at 01:58:37PM -0500, Dean Nelson wrote:
> > As said previously you're not supposed to mess with this one.
> 
> Yeah, I know, but how is one suppose to deal with the following
> issue raised by Robin? (Never did get a response from you.)
> 
> On Wed, Jun 16, 2004 at 02:36:22PM -0500, Robin Holt wrote:
> > On Wed, Jun 16, 2004 at 06:43:47PM +0100, Christoph Hellwig wrote:
> > > On Wed, Jun 16, 2004 at 12:40:53PM -0500, Robin Holt wrote:
> > > > > > +EXPORT_SYMBOL(sys_sched_setscheduler);
> > > > >
> > > > > Again, don't mess with scheduler paramters from your modules.
> > > >
> > > > How should a kernel thread raise itself to real-time priority?
> > >
> > > Answer to both:  it shouldn't
> >
> > To the second, we found that contention would result in very high
> > latency without raising the priority to real-time levels.  What is
> > the proper way to handle having a user thread at the same priority
> > as a kernel thread causing this holdoff?
> 
> The problem arises when enough user processes, which have the same
> priority as XPC's kthreads, are spinning doing a bit of work mixed
> with sleeping. Because of the sleep, these processes get a bonus
> which gives them a higher effective priority than the XPC kthreads.
> As a result, when cross partition interrupts come in, the XPC kthreads
> do not get scheduled immediately, but are held off until the end of
> the user processes' time slice.
> 
> This problem was encountered running a legitimate user land work load.
> I concocted the following program to reproduce the behavior we were
> seeing. I did this to see if the problem still exists on the 2.6
> kernel (we originally saw the problem running on 2.4).

Any reason you're doing that work from thread and not tasklets that
get scheduled ASAP?

Re: [PATCH 1/4] SGI Altix cross partition functionality (1st revision)

[Dean Nelson]

On Thu, Oct 21, 2004 at 09:59:27AM +0100, Christoph Hellwig wrote:
> On Tue, Sep 14, 2004 at 01:58:37PM -0500, Dean Nelson wrote:
> > > As said previously you're not supposed to mess with this one.
> > 
> > Yeah, I know, but how is one suppose to deal with the following
> > issue raised by Robin? (Never did get a response from you.)
> > 
> > On Wed, Jun 16, 2004 at 02:36:22PM -0500, Robin Holt wrote:
> > > On Wed, Jun 16, 2004 at 06:43:47PM +0100, Christoph Hellwig wrote:
> > > > On Wed, Jun 16, 2004 at 12:40:53PM -0500, Robin Holt wrote:
> > > > > > > +EXPORT_SYMBOL(sys_sched_setscheduler);
> > > > > >
> > > > > > Again, don't mess with scheduler paramters from your modules.
> > > > >
> > > > > How should a kernel thread raise itself to real-time priority?
> > > >
> > > > Answer to both:  it shouldn't
> > >
> > > To the second, we found that contention would result in very high
> > > latency without raising the priority to real-time levels.  What is
> > > the proper way to handle having a user thread at the same priority
> > > as a kernel thread causing this holdoff?
> > 
> > The problem arises when enough user processes, which have the same
> > priority as XPC's kthreads, are spinning doing a bit of work mixed
> > with sleeping. Because of the sleep, these processes get a bonus
> > which gives them a higher effective priority than the XPC kthreads.
> > As a result, when cross partition interrupts come in, the XPC kthreads
> > do not get scheduled immediately, but are held off until the end of
> > the user processes' time slice.
> > 
> > This problem was encountered running a legitimate user land work load.
> > I concocted the following program to reproduce the behavior we were
> > seeing. I did this to see if the problem still exists on the 2.6
> > kernel (we originally saw the problem running on 2.4).
> 
> Any reason you're doing that work from thread and not tasklets that
> get scheduled ASAP?

Yeah, these threads may block for an indefinite period of time.