[PATCH 0/2] non-contiguous NUMA nodes handling in numactl and libnuma

[PATCH 0/2] non-contiguous NUMA nodes handling in numactl and libnuma

[Amit K. Arora]

PROBLEM: If the NUMA nodes in a system are not contiguous, numactl shows
incorrect output. Here is an example from one such system:

------------------------------------------------------
# ls /sys/devices/system/node
has_cpu  has_normal_memory  node0  node1  node4  node5  online  possible
# numactl --hardware
available: 6 nodes (0-5)
node 0 cpus: 0 1 2 3
node 0 size: 15232 MB
node 0 free: 2265 MB
node 1 cpus: 4 5 6 7
node 1 size: 16256 MB
node 1 free: 3108 MB
libnuma: Warning: /sys not mounted or invalid. Assuming one node: No
such file or directory
node 2 cpus:
node 2 size: <not available>
node 2 free: <not available>
node 3 cpus:
node 3 size: <not available>
node 3 free: <not available>
node 4 cpus: 8 9 10 11
node 4 size: 16000 MB
node 4 free: 9292 MB
node 5 cpus: 12 13 14 15
node 5 size: 16128 MB
node 5 free: 9479 MB
node distances:
node   0   1   2   3   4   5 
  0:  10  20  20  20  20  10 
  1:  20  10  20  20   0   0 
  2:   0   0   0   0   0   0 
  3:   0   0   0   0   0   0 
  4:   0   0   0  1112145  1024  2607344 
  5:  1024  2607344  1024  2607344   0   0 
------------------------------------------------------


As we see above, there are three problems with above output:
o it is showing wrong number of available nodes
	Actually there are 4 nodes available, but it says "6".
o it tries to access information for non-exsistent nodes from /sys
	Thus displaying a wrong warning message too
o the NUMA distances it shows above are clearly incorrect.

Following two patches fixes these problems:
1) numactl-handle-NUMA-dist-properly.patch
2) numactl-ignore-nonavailable-nodes.patch


--
Regards,
Amit Arora

[PATCH 1/2] handle NUMA distances properly for non-contiguous nodes

[Amit K. Arora]

Patch: numactl-handle-NUMA-dist-properly.patch

This patch fixes the problem of displaying incorrect NUMA distances. It
increases the table size to accomodate node distance information for
nodes from 0 to nodemax. parse_numbers() makes sure not to add distances
for nodes which are not existing.

Signed-off-by: Amit K Arora <aarora@linux.vnet.ibm.com>

diff -Nuarp numactl-2.0.2.ORG/distance.c numactl-2.0.2/distance.c
--- numactl-2.0.2.ORG/distance.c	2009-03-02 02:34:12.000000000 -0600
+++ numactl-2.0.2/distance.c	2009-03-02 05:29:48.000000000 -0600
@@ -26,25 +26,16 @@
 static int distance_numnodes;
 static int *distance_table; 
 
-static int count_numbers(char *s)
-{
-	int n = 0;
-	for (;;) {
-		char *end;  
-		strtoul(s, &end, 0);
-		if (s == end)
-			return n;
-		s = end;
-		n++; 
-	}
-}
-
 static void parse_numbers(char *s, int *iptr, int n)
 {
-	int i;
+	int i, d, j;
 	char *end;
-	for (i = 0; i < n; i++) { 
-		*iptr++ = strtoul(s, &end, 0);
+	for (i = 0, j = 0; i < n; i++, j++) { 
+		d = strtoul(s, &end, 0);
+		/* Skip unavailable nodes */
+		while (j<n &&  !numa_bitmask_isbitset(numa_all_nodes_ptr, j))
+			j++;
+		*(iptr+j) = d;
 		if (s == end)
 			break;
 		s = end; 
@@ -68,7 +59,10 @@ static int read_distance_table(void)
 		if (!dfh) { 
 			if (errno == ENOENT && nd > 0)
 				err = 0;
-			break;
+			if (!err && nd<numa_num_configured_nodes())
+				continue;
+			else
+				break;
 		}
 		len = getdelim(&line, &linelen, '\n', dfh);
 		fclose(dfh);
@@ -76,7 +70,7 @@ static int read_distance_table(void)
 			break;
 
 		if (!table) { 
-			numnodes = count_numbers(line); 
+			numnodes = numa_num_configured_nodes();
 			table = calloc(numnodes * numnodes, sizeof(int)); 
 			if (!table) {
 				errno = ENOMEM; 

--
Regards,
Amit Arora

[PATCH 2/2] ignore unavailable NUMA nodes

[Amit K. Arora]

Patch: numactl-ignore-nonavailable-nodes.patch

This patch fixes the problem of displaying wrong number of "available"
nodes. It also ignores non-existent nodes and thus avoids the wrong
warning message also.

Signed-off-by: Amit K Arora <aarora@linux.vnet.ibm.com>

diff -Nuarp numactl-2.0.3-rc2.ORG/libnuma.c numactl-2.0.3-rc2/libnuma.c
--- numactl-2.0.3-rc2.ORG/libnuma.c	2009-03-02 05:10:10.000000000 -0600
+++ numactl-2.0.3-rc2/libnuma.c	2009-03-02 05:11:01.000000000 -0600
@@ -54,6 +54,8 @@ struct bitmask *numa_all_cpus_ptr = NULL
 static unsigned long *node_cpu_mask_v1[NUMA_NUM_NODES];
 struct bitmask **node_cpu_mask_v2;
 
+char *nodes_allowed_list = NULL;
+
 WEAK void numa_error(char *where);
 
 #ifdef __thread
@@ -452,6 +454,12 @@ set_thread_constraints(void)
 			maxprocnode =
 				read_mask(buffer + 15, numa_all_nodes_ptr);
 		}
+		if (strncmp(buffer,"Mems_allowed_list:",18) == 0) {
+			nodes_allowed_list = malloc(strlen(buffer)-18);
+			strncpy(nodes_allowed_list, buffer + 19,
+				strlen(buffer) - 19);
+			nodes_allowed_list[strlen(nodes_allowed_list)-1] = '\0';
+		}
 	}
 	fclose(f);
 	free(buffer);
diff -Nuarp numactl-2.0.3-rc2.ORG/numactl.c numactl-2.0.3-rc2/numactl.c
--- numactl-2.0.3-rc2.ORG/numactl.c	2009-03-02 05:10:10.000000000 -0600
+++ numactl-2.0.3-rc2/numactl.c	2009-03-02 05:11:01.000000000 -0600
@@ -214,13 +214,24 @@ void print_node_cpus(int node)
 
 void hardware(void)
 { 
-	int i;
+	int i, numconfigurednodes=0;
 	int maxnode = numa_num_configured_nodes()-1;
-	printf("available: %d nodes (0-%d)\n", 1+maxnode, maxnode); 	
+
+	for (i = 0; i<=maxnode; i++)
+		if (numa_bitmask_isbitset(numa_all_nodes_ptr, i))
+			numconfigurednodes++;
+	if (nodes_allowed_list)
+		printf("available: %d nodes (%s)\n", numconfigurednodes, nodes_allowed_list);
+	else
+		printf("available: %d nodes (0-%d)\n", maxnode+1, maxnode); 	
+		
 	for (i = 0; i <= maxnode; i++) { 
 		char buf[64];
 		long long fr;
 		unsigned long long sz = numa_node_size64(i, &fr); 
+		if (!numa_bitmask_isbitset(numa_all_nodes_ptr, i))
+			continue;
+
 		printf("node %d cpus:", i);
 		print_node_cpus(i);
 		printf("node %d size: %s\n", i, fmt_mem(sz, buf));
diff -Nuarp numactl-2.0.3-rc2.ORG/numa.h numactl-2.0.3-rc2/numa.h
--- numactl-2.0.3-rc2.ORG/numa.h	2009-03-02 05:10:10.000000000 -0600
+++ numactl-2.0.3-rc2/numa.h	2009-03-02 05:11:01.000000000 -0600
@@ -161,6 +161,8 @@ extern struct bitmask *numa_no_nodes_ptr
 /* Source compatibility */
 extern nodemask_t numa_no_nodes;
 
+extern char *nodes_allowed_list;
+
 /* Only run and allocate memory from a specific set of nodes. */
 void numa_bind(struct bitmask *nodes);
 
diff -Nuarp numactl-2.0.3-rc2.ORG/versions.ldscript numactl-2.0.3-rc2/versions.ldscript
--- numactl-2.0.3-rc2.ORG/versions.ldscript	2009-03-02 05:10:10.000000000 -0600
+++ numactl-2.0.3-rc2/versions.ldscript	2009-03-02 05:11:01.000000000 -0600
@@ -143,6 +143,7 @@ libnuma_1.2 {
     numa_tonode_memory;
     numa_tonodemask_memory;
     numa_warn;
+    nodes_allowed_list;
   local:
     *;
 } libnuma_1.1;

--
Regards,
Amit Arora

Results after applying the patch

[Amit K. Arora]

On Mon, Mar 02, 2009 at 06:35:46PM +0530, Amit K. Arora wrote:
> # ls /sys/devices/system/node
> has_cpu  has_normal_memory  node0  node1  node4  node5  online  possible
> # numactl --hardware
> available: 6 nodes (0-5)
> node 0 cpus: 0 1 2 3
> node 0 size: 15232 MB
> node 0 free: 2265 MB
> node 1 cpus: 4 5 6 7
> node 1 size: 16256 MB
> node 1 free: 3108 MB
> libnuma: Warning: /sys not mounted or invalid. Assuming one node: No
> such file or directory
> node 2 cpus:
> node 2 size: <not available>
> node 2 free: <not available>
> node 3 cpus:
> node 3 size: <not available>
> node 3 free: <not available>
> node 4 cpus: 8 9 10 11
> node 4 size: 16000 MB
> node 4 free: 9292 MB
> node 5 cpus: 12 13 14 15
> node 5 size: 16128 MB
> node 5 free: 9479 MB
> node distances:
> node   0   1   2   3   4   5 
>   0:  10  20  20  20  20  10 
>   1:  20  10  20  20   0   0 
>   2:   0   0   0   0   0   0 
>   3:   0   0   0   0   0   0 
>   4:   0   0   0  1112145  1024  2607344 
>   5:  1024  2607344  1024  2607344   0   0 

After applying the two patches, we see following result on the same
system:

# export LD_LIBRARY_PATH=`pwd`
# ./numactl --hardware
available: 4 nodes (0-1,4-5)
node 0 cpus: 0 1 2 3
node 0 size: 15232 MB
node 0 free: 2273 MB
node 1 cpus: 4 5 6 7
node 1 size: 16256 MB
node 1 free: 3108 MB
node 4 cpus: 8 9 10 11
node 4 size: 16000 MB
node 4 free: 9292 MB
node 5 cpus: 12 13 14 15
node 5 size: 16128 MB
node 5 free: 9479 MB
node distances:
node   0   1   2   3   4   5 
  0:  10  20   0   0  20  20 
  1:  20  10   0   0  20  20 
  2:   0   0   0   0   0   0 
  3:   0   0   0   0   0   0 
  4:  20  20   0   0  10  20 
  5:  20  20   0   0  20  10 


--
Regards,
Amit Arora

Re: [PATCH 0/2] non-contiguous NUMA nodes handling in numactl and libnuma

[Cliff Wickman]

Hi Amit,

Thanks much for the patches.

I tested them this morning.  The standard tests pass on ia64 (8 nodes).

The patches are included in the numactl-2.0.3-rc2.tar.gz tarball at
ftp://oss.sgi.com/www/projects/libnuma/download/

Any other review and testing is welcome.

-Cliff

On Mon, Mar 02, 2009 at 06:35:46PM +0530, Amit K. Arora wrote:
> PROBLEM: If the NUMA nodes in a system are not contiguous, numactl shows
> incorrect output. Here is an example from one such system:
> 
> ------------------------------------------------------
> # ls /sys/devices/system/node
> has_cpu  has_normal_memory  node0  node1  node4  node5  online  possible
> # numactl --hardware
> available: 6 nodes (0-5)
> node 0 cpus: 0 1 2 3
> node 0 size: 15232 MB
> node 0 free: 2265 MB
> node 1 cpus: 4 5 6 7
> node 1 size: 16256 MB
> node 1 free: 3108 MB
> libnuma: Warning: /sys not mounted or invalid. Assuming one node: No
> such file or directory
> node 2 cpus:
> node 2 size: <not available>
> node 2 free: <not available>
> node 3 cpus:
> node 3 size: <not available>
> node 3 free: <not available>
> node 4 cpus: 8 9 10 11
> node 4 size: 16000 MB
> node 4 free: 9292 MB
> node 5 cpus: 12 13 14 15
> node 5 size: 16128 MB
> node 5 free: 9479 MB
> node distances:
> node   0   1   2   3   4   5 
>   0:  10  20  20  20  20  10 
>   1:  20  10  20  20   0   0 
>   2:   0   0   0   0   0   0 
>   3:   0   0   0   0   0   0 
>   4:   0   0   0  1112145  1024  2607344 
>   5:  1024  2607344  1024  2607344   0   0 
> ------------------------------------------------------
> 
> 
> As we see above, there are three problems with above output:
> o it is showing wrong number of available nodes
> 	Actually there are 4 nodes available, but it says "6".
> o it tries to access information for non-exsistent nodes from /sys
> 	Thus displaying a wrong warning message too
> o the NUMA distances it shows above are clearly incorrect.
> 
> Following two patches fixes these problems:
> 1) numactl-handle-NUMA-dist-properly.patch
> 2) numactl-ignore-nonavailable-nodes.patch
> 
> 
> --
> Regards,
> Amit Arora
> --
> To unsubscribe from this list: send the line "unsubscribe linux-numa" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

-- 
Cliff Wickman
Silicon Graphics, Inc.
cpw@sgi.com
(651) 683-3824

[PATCH] distance table for non-contiguous NUMA nodes

[Amit K. Arora]

Hi Cliff,

I am suggesting this new patch, which applies on top of the previous two
(which I think you have already added to 2.0.3-rc2). This patch avoids
showing those NUMA nodes which are unavailable when showing the NUMA
distances too.

So, currently (with previous two patches applied) the NUMA distance
table for non-contiguous NODES looks something like this (assuming
nodes 2 and 3 are not existing) :

node   0   1   2   3   4   5
  0:  10  20   0   0  20  20
  1:  20  10   0   0  20  20
  2:   0   0   0   0   0   0
  3:   0   0   0   0   0   0
  4:  20  20   0   0  10  20
  5:  20  20   0   0  20  10

Please note that for non-existing NODES 2 and 3, a NUMA distance of "0"
is being shown, which doesn't make sense.

Hence I propose this patch which will skip the unavailable nodes (in
this case NODE 2 and 3), while printing the NUMA distances and the
output on the same machine will look something like:

node   0   1   4   5 
  0:  10  20  20  20 
  1:  20  10  20  20 
  4:  20  20  10  20 
  5:  20  20  20  10 


Please find the patch below. Thanks!


Signed-off-by: Amit K Arora <aarora@linux.vnet.ibm.com>

diff -Nuarp numactl-2.0.3-rc2.ORG/numactl.c numactl-2.0.3-rc2/numactl.c
--- numactl-2.0.3-rc2.ORG/numactl.c	2009-03-09 18:43:05.000000000 +0530
+++ numactl-2.0.3-rc2/numactl.c	2009-03-09 18:43:14.000000000 +0530
@@ -187,12 +187,16 @@ static void print_distances(int maxnode)
 	printf("node distances:\n"); 
 	printf("node ");
 	for (i = 0; i <= maxnode; i++) 
-		printf("% 3d ", i);
+		if (numa_bitmask_isbitset(numa_all_nodes_ptr, i))
+			printf("% 3d ", i);
 	printf("\n");
 	for (i = 0; i <= maxnode; i++) {
+		if (!numa_bitmask_isbitset(numa_all_nodes_ptr, i))
+			continue;
 		printf("% 3d: ", i);
-		for (k = 0; k <= maxnode; k++) 
-			printf("% 3d ", numa_distance(i,k)); 
+		for (k = 0; k <= maxnode; k++)
+			if (numa_bitmask_isbitset(numa_all_nodes_ptr, k))
+				printf("% 3d ", numa_distance(i,k)); 
 		printf("\n");
 	}			
 }

--
Regards,
Amit Arora