[PATCH V2 1/2] powerpc/numa: Update CPU topology when VPHN enabled

[PATCH V2 1/2] powerpc/numa: Update CPU topology when VPHN enabled

[Michael Bringmann]

powerpc/numa: Correct the currently broken capability to set the
topology for shared CPUs in LPARs.  At boot time for shared CPU
lpars, the topology for each shared CPU is set to node zero, however,
this is now updated correctly using the Virtual Processor Home Node
(VPHN) capabilities information provided by the pHyp. The VPHN handling
in Linux is disabled, if PRRN handling is present.

Signed-off-by: Michael Bringmann <mwb@linux.vnet.ibm.com>
---
 arch/powerpc/mm/numa.c                       |   19 ++++++++++++++++++-
 arch/powerpc/platforms/pseries/dlpar.c       |    2 ++
 arch/powerpc/platforms/pseries/hotplug-cpu.c |    3 ++-
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c
index 371792e..15c2dd5 100644
--- a/arch/powerpc/mm/numa.c
+++ b/arch/powerpc/mm/numa.c
@@ -29,6 +29,7 @@
 #include <linux/seq_file.h>
 #include <linux/uaccess.h>
 #include <linux/slab.h>
+#include <linux/sched.h>
 #include <asm/cputhreads.h>
 #include <asm/sparsemem.h>
 #include <asm/prom.h>
@@ -42,6 +43,8 @@
 #include <asm/vdso.h>
 
 static int numa_enabled = 1;
+static int topology_inited;
+static int topology_update_needed;
 
 static char *cmdline __initdata;
 
@@ -1321,8 +1324,11 @@ int arch_update_cpu_topology(void)
 	struct device *dev;
 	int weight, new_nid, i = 0;
 
-	if (!prrn_enabled && !vphn_enabled)
+	if (!prrn_enabled && !vphn_enabled) {
+		if (!topology_inited)
+			topology_update_needed = 1;
 		return 0;
+	}
 
 	weight = cpumask_weight(&cpu_associativity_changes_mask);
 	if (!weight)
@@ -1361,6 +1367,8 @@ int arch_update_cpu_topology(void)
 			cpumask_andnot(&cpu_associativity_changes_mask,
 					&cpu_associativity_changes_mask,
 					cpu_sibling_mask(cpu));
+			pr_info("Assoc chg gives same node %d for cpu%d\n",
+					new_nid, cpu);
 			cpu = cpu_last_thread_sibling(cpu);
 			continue;
 		}
@@ -1377,6 +1385,9 @@ int arch_update_cpu_topology(void)
 		cpu = cpu_last_thread_sibling(cpu);
 	}
 
+	if (i)
+		updates[i-1].next = NULL;
+
 	pr_debug("Topology update for the following CPUs:\n");
 	if (cpumask_weight(&updated_cpus)) {
 		for (ud = &updates[0]; ud; ud = ud->next) {
@@ -1423,6 +1434,7 @@ int arch_update_cpu_topology(void)
 
 out:
 	kfree(updates);
+	topology_update_needed = 0;
 	return changed;
 }
 
@@ -1600,6 +1612,11 @@ static int topology_update_init(void)
 	if (!proc_create("powerpc/topology_updates", 0644, NULL, &topology_ops))
 		return -ENOMEM;
 
+	topology_inited = 1;
+	if (topology_update_needed)
+		bitmap_fill(cpumask_bits(&cpu_associativity_changes_mask),
+					nr_cpumask_bits);
+
 	return 0;
 }
 device_initcall(topology_update_init);
diff --git a/arch/powerpc/platforms/pseries/dlpar.c b/arch/powerpc/platforms/pseries/dlpar.c
index bda18d8..5106263 100644
--- a/arch/powerpc/platforms/pseries/dlpar.c
+++ b/arch/powerpc/platforms/pseries/dlpar.c
@@ -592,6 +592,8 @@ static ssize_t dlpar_show(struct class *class, struct class_attribute *attr,
 
 static int __init pseries_dlpar_init(void)
 {
+	arch_update_cpu_topology();
+
 	pseries_hp_wq = alloc_workqueue("pseries hotplug workqueue",
 					WQ_UNBOUND, 1);
 	return sysfs_create_file(kernel_kobj, &class_attr_dlpar.attr);
diff --git a/arch/powerpc/platforms/pseries/hotplug-cpu.c b/arch/powerpc/platforms/pseries/hotplug-cpu.c
index 7bc0e91..b5eff35 100644
--- a/arch/powerpc/platforms/pseries/hotplug-cpu.c
+++ b/arch/powerpc/platforms/pseries/hotplug-cpu.c
@@ -619,7 +619,8 @@ static int dlpar_cpu_remove_by_index(u32 drc_index)
 	}
 
 	rc = dlpar_cpu_remove(dn, drc_index);
-	of_node_put(dn);
+	if (rc)
+		of_node_put(dn);
 	return rc;
 }
 

Re: [PATCH V2 1/2] powerpc/numa: Update CPU topology when VPHN enabled

[Tyrel Datwyler]

On 05/25/2017 10:35 AM, Michael Bringmann wrote:
> 
> powerpc/numa: Correct the currently broken capability to set the
> topology for shared CPUs in LPARs.  At boot time for shared CPU
> lpars, the topology for each shared CPU is set to node zero, however,
> this is now updated correctly using the Virtual Processor Home Node
> (VPHN) capabilities information provided by the pHyp. The VPHN handling
> in Linux is disabled, if PRRN handling is present.
> 
> Signed-off-by: Michael Bringmann <mwb@linux.vnet.ibm.com>
> ---
>  arch/powerpc/mm/numa.c                       |   19 ++++++++++++++++++-
>  arch/powerpc/platforms/pseries/dlpar.c       |    2 ++
>  arch/powerpc/platforms/pseries/hotplug-cpu.c |    3 ++-
>  3 files changed, 22 insertions(+), 2 deletions(-)
> 
> diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c
> index 371792e..15c2dd5 100644
> --- a/arch/powerpc/mm/numa.c
> +++ b/arch/powerpc/mm/numa.c
> @@ -29,6 +29,7 @@
>  #include <linux/seq_file.h>
>  #include <linux/uaccess.h>
>  #include <linux/slab.h>
> +#include <linux/sched.h>
>  #include <asm/cputhreads.h>
>  #include <asm/sparsemem.h>
>  #include <asm/prom.h>
> @@ -42,6 +43,8 @@
>  #include <asm/vdso.h>
>  
>  static int numa_enabled = 1;
> +static int topology_inited;
> +static int topology_update_needed;
>  
>  static char *cmdline __initdata;
>  
> @@ -1321,8 +1324,11 @@ int arch_update_cpu_topology(void)
>  	struct device *dev;
>  	int weight, new_nid, i = 0;
>  
> -	if (!prrn_enabled && !vphn_enabled)
> +	if (!prrn_enabled && !vphn_enabled) {
> +		if (!topology_inited)
> +			topology_update_needed = 1;
>  		return 0;
> +	}
>  
>  	weight = cpumask_weight(&cpu_associativity_changes_mask);
>  	if (!weight)
> @@ -1361,6 +1367,8 @@ int arch_update_cpu_topology(void)
>  			cpumask_andnot(&cpu_associativity_changes_mask,
>  					&cpu_associativity_changes_mask,
>  					cpu_sibling_mask(cpu));
> +			pr_info("Assoc chg gives same node %d for cpu%d\n",
> +					new_nid, cpu);
>  			cpu = cpu_last_thread_sibling(cpu);
>  			continue;
>  		}
> @@ -1377,6 +1385,9 @@ int arch_update_cpu_topology(void)
>  		cpu = cpu_last_thread_sibling(cpu);
>  	}
>  
> +	if (i)
> +		updates[i-1].next = NULL;
> +
>  	pr_debug("Topology update for the following CPUs:\n");
>  	if (cpumask_weight(&updated_cpus)) {
>  		for (ud = &updates[0]; ud; ud = ud->next) {
> @@ -1423,6 +1434,7 @@ int arch_update_cpu_topology(void)
>  
>  out:
>  	kfree(updates);
> +	topology_update_needed = 0;
>  	return changed;
>  }
>  
> @@ -1600,6 +1612,11 @@ static int topology_update_init(void)
>  	if (!proc_create("powerpc/topology_updates", 0644, NULL, &topology_ops))
>  		return -ENOMEM;
>  
> +	topology_inited = 1;
> +	if (topology_update_needed)
> +		bitmap_fill(cpumask_bits(&cpu_associativity_changes_mask),
> +					nr_cpumask_bits);
> +
>  	return 0;
>  }
>  device_initcall(topology_update_init);
> diff --git a/arch/powerpc/platforms/pseries/dlpar.c b/arch/powerpc/platforms/pseries/dlpar.c
> index bda18d8..5106263 100644
> --- a/arch/powerpc/platforms/pseries/dlpar.c
> +++ b/arch/powerpc/platforms/pseries/dlpar.c
> @@ -592,6 +592,8 @@ static ssize_t dlpar_show(struct class *class, struct class_attribute *attr,
>  
>  static int __init pseries_dlpar_init(void)
>  {
> +	arch_update_cpu_topology();
> +
>  	pseries_hp_wq = alloc_workqueue("pseries hotplug workqueue",
>  					WQ_UNBOUND, 1);
>  	return sysfs_create_file(kernel_kobj, &class_attr_dlpar.attr);
> diff --git a/arch/powerpc/platforms/pseries/hotplug-cpu.c b/arch/powerpc/platforms/pseries/hotplug-cpu.c
> index 7bc0e91..b5eff35 100644
> --- a/arch/powerpc/platforms/pseries/hotplug-cpu.c
> +++ b/arch/powerpc/platforms/pseries/hotplug-cpu.c
> @@ -619,7 +619,8 @@ static int dlpar_cpu_remove_by_index(u32 drc_index)
>  	}
>  
>  	rc = dlpar_cpu_remove(dn, drc_index);
> -	of_node_put(dn);
> +	if (rc)
> +		of_node_put(dn);

This if statement is unnecessary. A reference to dn is grabbed with cpu_drc_index_to_dn()
earlier in the function. So, regardless of whether dlpar_cpu_remove() succeeds or fails we
still need to release the reference we took. I suspect that maybe you threw this in there
to prevent the of_node underflow that was being seen during dlpar remove of cpus. A fix
for that should already be upstream. Either way this hunk doesn't seem to have anything to
do with the rest of this VPHN patch, and should have been a separate patch with its own
change log explaining the reasoning.

-Tyrel

>  	return rc;
>  }
>  
> 

Re: [PATCH V2 1/2] powerpc/numa: Update CPU topology when VPHN enabled

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 1736 bytes --]

Hi Michael,

[auto build test ERROR on powerpc/next]
[also build test ERROR on v4.12-rc2 next-20170526]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]

url:    https://github.com/0day-ci/linux/commits/Michael-Bringmann/powerpc-numa-Update-CPU-topology-when-VPHN-enabled/20170526-231219
base:   https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-powernv_defconfig (attached as .config)
compiler: powerpc64-linux-gnu-gcc (Debian 6.1.1-9) 6.1.1 20160705
reproduce:
        wget https://raw.githubusercontent.com/01org/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        make.cross ARCH=powerpc 

All errors (new ones prefixed by >>):

>> arch/powerpc/mm/numa.c:47:12: error: 'topology_update_needed' defined but not used [-Werror=unused-variable]
    static int topology_update_needed;
               ^~~~~~~~~~~~~~~~~~~~~~
>> arch/powerpc/mm/numa.c:46:12: error: 'topology_inited' defined but not used [-Werror=unused-variable]
    static int topology_inited;
               ^~~~~~~~~~~~~~~
   cc1: all warnings being treated as errors

vim +/topology_update_needed +47 arch/powerpc/mm/numa.c

    40	#include <asm/paca.h>
    41	#include <asm/hvcall.h>
    42	#include <asm/setup.h>
    43	#include <asm/vdso.h>
    44	
    45	static int numa_enabled = 1;
  > 46	static int topology_inited;
  > 47	static int topology_update_needed;
    48	
    49	static char *cmdline __initdata;
    50	

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 22459 bytes --]

[PATCH V2 2/2]: powerpc/hotplug/mm: Fix hot-add memory node assoc

[Michael Bringmann]

Removing or adding memory via the PowerPC hotplug interface shows
anomalies in the association between memory and nodes.  The code
was updated to ensure that all nodes found at boot are still available
to subsequent DLPAR hotplug-memory operations, even if they are not
needed at boot time.

Signed-off-by: Michael Bringmann <mwb@linux.vnet.ibm.com>
---
Changes in V2:
  -- Simplify patches to ensure more nodes in possible map, removing
     code from PowerPC numa.c that constrained possible map to size
     of online map.
---
 arch/powerpc/mm/numa.c |    7 -------
 1 file changed, 7 deletions(-)

diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c
index 15c2dd5..18f3038 100644
--- a/arch/powerpc/mm/numa.c
+++ b/arch/powerpc/mm/numa.c
@@ -907,13 +907,6 @@ void __init initmem_init(void)
 
 	memblock_dump_all();
 
-	/*
-	 * Reduce the possible NUMA nodes to the online NUMA nodes,
-	 * since we do not support node hotplug. This ensures that  we
-	 * lower the maximum NUMA node ID to what is actually present.
-	 */
-	nodes_and(node_possible_map, node_possible_map, node_online_map);
-
 	for_each_online_node(nid) {
 		unsigned long start_pfn, end_pfn;
 

Re: [PATCH V2 2/2]: powerpc/hotplug/mm: Fix hot-add memory node assoc

[Balbir Singh]

On Thu, 25 May 2017 12:37:40 -0500
Michael Bringmann <mwb@linux.vnet.ibm.com> wrote:

> Removing or adding memory via the PowerPC hotplug interface shows
> anomalies in the association between memory and nodes.  The code
> was updated to ensure that all nodes found at boot are still available
> to subsequent DLPAR hotplug-memory operations, even if they are not
> needed at boot time.
> 
> Signed-off-by: Michael Bringmann <mwb@linux.vnet.ibm.com>
> ---
> Changes in V2:
>   -- Simplify patches to ensure more nodes in possible map, removing
>      code from PowerPC numa.c that constrained possible map to size
>      of online map.
> ---
>  arch/powerpc/mm/numa.c |    7 -------
>  1 file changed, 7 deletions(-)
> 
> diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c
> index 15c2dd5..18f3038 100644
> --- a/arch/powerpc/mm/numa.c
> +++ b/arch/powerpc/mm/numa.c
> @@ -907,13 +907,6 @@ void __init initmem_init(void)
>  
>  	memblock_dump_all();
>  
> -	/*
> -	 * Reduce the possible NUMA nodes to the online NUMA nodes,
> -	 * since we do not support node hotplug. This ensures that  we
> -	 * lower the maximum NUMA node ID to what is actually present.
> -	 */
> -	nodes_and(node_possible_map, node_possible_map, node_online_map);
> -

There is an overhead with turning this off if you have too many cgroups
with the memory controller. I think this fix was added for a pathological
test case. On my system I see 84 cgroups with 1 node, so the probable
overhead is 84*255*sizeof(struct mem_cgroup_tree_per_node).

I tried some patches to reduce the overhead, but those need more overhauling
and rework.

Balbir Singh.

Re: [PATCH V2 2/2]: powerpc/hotplug/mm: Fix hot-add memory node assoc

[Michael Bringmann]

>>  arch/powerpc/mm/numa.c |    7 -------
>>  1 file changed, 7 deletions(-)
>>
>> diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c
>> index 15c2dd5..18f3038 100644
>> --- a/arch/powerpc/mm/numa.c
>> +++ b/arch/powerpc/mm/numa.c
>> @@ -907,13 +907,6 @@ void __init initmem_init(void)
>>  
>>  	memblock_dump_all();
>>  
>> -	/*
>> -	 * Reduce the possible NUMA nodes to the online NUMA nodes,
>> -	 * since we do not support node hotplug. This ensures that  we
>> -	 * lower the maximum NUMA node ID to what is actually present.
>> -	 */
>> -	nodes_and(node_possible_map, node_possible_map, node_online_map);
>> -
> 
> There is an overhead with turning this off if you have too many cgroups
> with the memory controller. I think this fix was added for a pathological
> test case. On my system I see 84 cgroups with 1 node, so the probable
> overhead is 84*255*sizeof(struct mem_cgroup_tree_per_node).
> 
> I tried some patches to reduce the overhead, but those need more overhauling
> and rework.

Is there some other way to add a node to a dynamic, running system without
crashing?  I have not encountered one as yet.

> Balbir Singh.

-- 
Michael W. Bringmann
Linux Technology Center
IBM Corporation
Tie-Line  363-5196
External: (512) 286-5196
Cell:       (512) 466-0650
mwb@linux.vnet.ibm.com

Re: [PATCH V2 2/2]: powerpc/hotplug/mm: Fix hot-add memory node assoc

[Michael Ellerman]

Michael Bringmann <mwb@linux.vnet.ibm.com> writes:

> Removing or adding memory via the PowerPC hotplug interface shows
> anomalies in the association between memory and nodes.

What anomalies? Please describe the actual problem you're seeing, with
details, and why you think this is the correct fix.

This is a revert of 3af229f2071f ("powerpc/numa: Reset node_possible_map
to only node_online_map"), so please explain why all the things
mentioned in the change log for that commit are either wrong or no
longer true.

cheers