[PATCH v9 0/6] arm64, numa: Add numa support for arm64 platforms

[PATCH v9 0/6] arm64, numa: Add numa support for arm64 platforms

[Ganapatrao Kulkarni]

v9:	- Added cleanup patch to reuse and avoid redefinition of cpumask_of_pcibus
	  as suggested from Will Deacon and Bjorn Helgaas.
	- Including patch to Make pci-host-generic driver numa aware.
	- Incorporated comment from Shannon Zhao.

v8:
	- Incorporated review comments of Mark Rutland and Will Deacon.
	- Added pci helper function and macro for numa.

v7:
	- managing numa memory mapping using memblock.
	- Incorporated review comments of Mark Rutland.

v6:
	- defined and implemented the numa dt binding using
	node property proximity and device node distance-map.
	- renamed dt_numa to of_numa

v5:
        - created base verion of numa.c which creates dummy numa without using dt
          on single socket platforms. Then added patches for dt support.
        - Incorporated review comments from Hanjun Guo.

v4:
done changes as per Arnd review comments.

v3:
Added changes to support numa on arm64 based platforms.
Tested these patches on cavium's multinode(2 node topology) platform.
In this patchset, defined and implemented dt bindings for numa mapping
for core and memory using device node property arm,associativity.

v2:
Defined and implemented numa map for memory, cores to node and
proximity distance matrix of nodes.

v1:
Initial patchset to support numa on arm64 platforms.

Note:
        1. This patchset is tested for numa with dt on
           thunderx single socket and dual socket boards.
        2. Numa DT booting needs the dt memory nodes, which are deleted in current efi-stub,
        hence to try numa with dt, you need to rebase with ard's patchset.
        http://git.linaro.org/people/ard.biesheuvel/linux-arm.git/shortlog/refs/heads/arm64-uefi-early-fdt-handling
	3. PATCH 6 is not tested for other architectures.

Ganapatrao Kulkarni (6):
  arm64, numa: adding numa support for arm64 platforms.
  Documentation, dt, arm64/arm: dt bindings for numa.
  arm64/arm, numa, dt: adding numa dt binding implementation for arm64
    platforms.
  arm64, dt, thunderx: Add initial dts for Cavium Thunderx in 2 node
    topology.
  PCI: generic: Make pci-host-generic driver numa aware
  topology, cleanup: Avoid redefinition of cpumask_of_pcibus in asm
    header files.

 Documentation/devicetree/bindings/arm/numa.txt  | 272 ++++++++
 arch/arm64/Kconfig                              |  35 +
 arch/arm64/boot/dts/cavium/Makefile             |   2 +-
 arch/arm64/boot/dts/cavium/thunder-88xx-2n.dts  |  83 +++
 arch/arm64/boot/dts/cavium/thunder-88xx-2n.dtsi | 806 ++++++++++++++++++++++++
 arch/arm64/include/asm/mmzone.h                 |  18 +
 arch/arm64/include/asm/numa.h                   |  50 ++
 arch/arm64/include/asm/topology.h               |   7 +
 arch/arm64/kernel/Makefile                      |   1 +
 arch/arm64/kernel/of_numa.c                     | 257 ++++++++
 arch/arm64/kernel/pci.c                         |   8 +
 arch/arm64/kernel/setup.c                       |   4 +
 arch/arm64/kernel/smp.c                         |   4 +
 arch/arm64/mm/Makefile                          |   1 +
 arch/arm64/mm/init.c                            |  31 +-
 arch/arm64/mm/mmu.c                             |   1 +
 arch/arm64/mm/numa.c                            | 395 ++++++++++++
 arch/ia64/include/asm/topology.h                |   4 -
 arch/metag/include/asm/topology.h               |   3 -
 arch/powerpc/include/asm/topology.h             |   4 -
 arch/s390/include/asm/pci.h                     |   2 +-
 arch/s390/include/asm/topology.h                |   1 +
 arch/sh/include/asm/topology.h                  |   3 -
 arch/x86/include/asm/pci.h                      |   2 +-
 arch/x86/include/asm/topology.h                 |   1 +
 drivers/pci/host/pci-host-generic.c             |   9 +-
 include/asm-generic/topology.h                  |   4 +-
 27 files changed, 1981 insertions(+), 27 deletions(-)
 create mode 100644 Documentation/devicetree/bindings/arm/numa.txt
 create mode 100644 arch/arm64/boot/dts/cavium/thunder-88xx-2n.dts
 create mode 100644 arch/arm64/boot/dts/cavium/thunder-88xx-2n.dtsi
 create mode 100644 arch/arm64/include/asm/mmzone.h
 create mode 100644 arch/arm64/include/asm/numa.h
 create mode 100644 arch/arm64/kernel/of_numa.c
 create mode 100644 arch/arm64/mm/numa.c

-- 
1.8.1.4

[PATCH v9 1/6] arm64, numa: adding numa support for arm64 platforms.

[Ganapatrao Kulkarni]

Adding numa support for arm64 based platforms.
This patch adds by default the dummy numa node and
maps all memory and cpus to node 0.
using this patch, numa can be simulated on single node arm64 platforms.

Tested-by: Shannon Zhao <shannon.zhao@linaro.org>
Reviewed-by: Robert Richter <rrichter@cavium.com>
Signed-off-by: Ganapatrao Kulkarni <gkulkarni@caviumnetworks.com>
---
 arch/arm64/Kconfig                |  25 +++
 arch/arm64/include/asm/mmzone.h   |  18 ++
 arch/arm64/include/asm/numa.h     |  40 ++++
 arch/arm64/include/asm/topology.h |  10 +
 arch/arm64/kernel/pci.c           |   8 +
 arch/arm64/kernel/setup.c         |   4 +
 arch/arm64/kernel/smp.c           |   2 +
 arch/arm64/mm/Makefile            |   1 +
 arch/arm64/mm/init.c              |  31 ++-
 arch/arm64/mm/mmu.c               |   1 +
 arch/arm64/mm/numa.c              | 387 ++++++++++++++++++++++++++++++++++++++
 11 files changed, 522 insertions(+), 5 deletions(-)
 create mode 100644 arch/arm64/include/asm/mmzone.h
 create mode 100644 arch/arm64/include/asm/numa.h
 create mode 100644 arch/arm64/mm/numa.c

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 871f217..74f5d73 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -71,6 +71,7 @@ config ARM64
 	select HAVE_GENERIC_DMA_COHERENT
 	select HAVE_HW_BREAKPOINT if PERF_EVENTS
 	select HAVE_MEMBLOCK
+	select HAVE_MEMBLOCK_NODE_MAP if NUMA
 	select HAVE_PATA_PLATFORM
 	select HAVE_PERF_EVENTS
 	select HAVE_PERF_REGS
@@ -503,6 +504,30 @@ config HOTPLUG_CPU
 	  Say Y here to experiment with turning CPUs off and on.  CPUs
 	  can be controlled through /sys/devices/system/cpu.
 
+# Common NUMA Features
+config NUMA
+	bool "Numa Memory Allocation and Scheduler Support"
+	depends on SMP
+	help
+	  Enable NUMA (Non Uniform Memory Access) support.
+
+	  The kernel will try to allocate memory used by a CPU on the
+	  local memory of the CPU and add some more
+	  NUMA awareness to the kernel.
+
+config NODES_SHIFT
+	int "Maximum NUMA Nodes (as a power of 2)"
+	range 1 10
+	default "2"
+	depends on NEED_MULTIPLE_NODES
+	help
+	  Specify the maximum number of NUMA Nodes available on the target
+	  system.  Increases memory reserved to accommodate various tables.
+
+config USE_PERCPU_NUMA_NODE_ID
+	def_bool y
+	depends on NUMA
+
 source kernel/Kconfig.preempt
 source kernel/Kconfig.hz
 
diff --git a/arch/arm64/include/asm/mmzone.h b/arch/arm64/include/asm/mmzone.h
new file mode 100644
index 0000000..2cd804d
--- /dev/null
+++ b/arch/arm64/include/asm/mmzone.h
@@ -0,0 +1,18 @@
+#ifndef __ASM_MMZONE_H
+#define __ASM_MMZONE_H
+
+#ifdef CONFIG_NUMA
+
+#include <linux/mmdebug.h>
+#include <linux/types.h>
+
+#include <asm/smp.h>
+#include <asm/numa.h>
+
+extern struct pglist_data *node_data[];
+
+#define NODE_DATA(nid)		(node_data[(nid)])
+
+#endif /* CONFIG_NUMA */
+
+#endif /* __ASM_MMZONE_H */
diff --git a/arch/arm64/include/asm/numa.h b/arch/arm64/include/asm/numa.h
new file mode 100644
index 0000000..f28f15b0
--- /dev/null
+++ b/arch/arm64/include/asm/numa.h
@@ -0,0 +1,40 @@
+#ifndef __ASM_NUMA_H
+#define __ASM_NUMA_H
+
+#include <linux/nodemask.h>
+#include <asm/topology.h>
+
+#ifdef CONFIG_NUMA
+
+/* currently, arm64 implements flat NUMA topology */
+#define parent_node(node)	(node)
+
+extern int __node_distance(int from, int to);
+#define node_distance(a, b) __node_distance(a, b)
+
+extern int cpu_to_node_map[NR_CPUS];
+extern nodemask_t numa_nodes_parsed __initdata;
+
+/* Mappings between node number and cpus on that node. */
+extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
+extern void numa_clear_node(unsigned int cpu);
+#ifdef CONFIG_DEBUG_PER_CPU_MAPS
+extern const struct cpumask *cpumask_of_node(int node);
+#else
+/* Returns a pointer to the cpumask of CPUs on Node 'node'. */
+static inline const struct cpumask *cpumask_of_node(int node)
+{
+	return node_to_cpumask_map[node];
+}
+#endif
+
+void __init arm64_numa_init(void);
+int __init numa_add_memblk(int nodeid, u64 start, u64 end);
+void __init numa_set_distance(int from, int to, int distance);
+void __init numa_reset_distance(void);
+void numa_store_cpu_info(unsigned int cpu);
+#else	/* CONFIG_NUMA */
+static inline void numa_store_cpu_info(unsigned int cpu)		{ }
+static inline void arm64_numa_init(void)		{ }
+#endif	/* CONFIG_NUMA */
+#endif	/* __ASM_NUMA_H */
diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h
index a3e9d6f..8b57339 100644
--- a/arch/arm64/include/asm/topology.h
+++ b/arch/arm64/include/asm/topology.h
@@ -22,6 +22,16 @@ void init_cpu_topology(void);
 void store_cpu_topology(unsigned int cpuid);
 const struct cpumask *cpu_coregroup_mask(int cpu);
 
+#ifdef CONFIG_NUMA
+
+struct pci_bus;
+int pcibus_to_node(struct pci_bus *bus);
+#define cpumask_of_pcibus(bus)	(pcibus_to_node(bus) == -1 ?		\
+				 cpu_all_mask :				\
+				 cpumask_of_node(pcibus_to_node(bus)))
+
+#endif /* CONFIG_NUMA */
+
 #include <asm-generic/topology.h>
 
 #endif /* _ASM_ARM_TOPOLOGY_H */
diff --git a/arch/arm64/kernel/pci.c b/arch/arm64/kernel/pci.c
index b3d098b..37fd8ef 100644
--- a/arch/arm64/kernel/pci.c
+++ b/arch/arm64/kernel/pci.c
@@ -76,6 +76,14 @@ int raw_pci_write(unsigned int domain, unsigned int bus,
 	return -ENXIO;
 }
 
+#ifdef CONFIG_NUMA
+int pcibus_to_node(struct pci_bus *bus)
+{
+	return dev_to_node(&bus->dev);
+}
+EXPORT_SYMBOL(pcibus_to_node);
+#endif
+
 #ifdef CONFIG_ACPI
 /* Root bridge scanning */
 struct pci_bus *pci_acpi_scan_root(struct acpi_pci_root *root)
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 8119479..d9b9761 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -53,6 +53,7 @@
 #include <asm/cpufeature.h>
 #include <asm/cpu_ops.h>
 #include <asm/kasan.h>
+#include <asm/numa.h>
 #include <asm/sections.h>
 #include <asm/setup.h>
 #include <asm/smp_plat.h>
@@ -372,6 +373,9 @@ static int __init topology_init(void)
 {
 	int i;
 
+	for_each_online_node(i)
+		register_one_node(i);
+
 	for_each_possible_cpu(i) {
 		struct cpu *cpu = &per_cpu(cpu_data.cpu, i);
 		cpu->hotpluggable = 1;
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index b1adc51..d6e7d6a 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -45,6 +45,7 @@
 #include <asm/cputype.h>
 #include <asm/cpu_ops.h>
 #include <asm/mmu_context.h>
+#include <asm/numa.h>
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
 #include <asm/processor.h>
@@ -125,6 +126,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle)
 static void smp_store_cpu_info(unsigned int cpuid)
 {
 	store_cpu_topology(cpuid);
+	numa_store_cpu_info(cpuid);
 }
 
 /*
diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
index 57f57fd..54bb209 100644
--- a/arch/arm64/mm/Makefile
+++ b/arch/arm64/mm/Makefile
@@ -4,6 +4,7 @@ obj-y				:= dma-mapping.o extable.o fault.o init.o \
 				   context.o proc.o pageattr.o
 obj-$(CONFIG_HUGETLB_PAGE)	+= hugetlbpage.o
 obj-$(CONFIG_ARM64_PTDUMP)	+= dump.o
+obj-$(CONFIG_NUMA)		+= numa.o
 
 obj-$(CONFIG_KASAN)		+= kasan_init.o
 KASAN_SANITIZE_kasan_init.o	:= n
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 17bf39a..2954599 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -37,6 +37,7 @@
 
 #include <asm/fixmap.h>
 #include <asm/memory.h>
+#include <asm/numa.h>
 #include <asm/sections.h>
 #include <asm/setup.h>
 #include <asm/sizes.h>
@@ -77,6 +78,19 @@ static phys_addr_t max_zone_dma_phys(void)
 	return min(offset + (1ULL << 32), memblock_end_of_DRAM());
 }
 
+#ifdef CONFIG_NUMA
+static void __init zone_sizes_init(unsigned long min, unsigned long max)
+{
+	unsigned long max_zone_pfns[MAX_NR_ZONES]  = {0};
+
+	if (IS_ENABLED(CONFIG_ZONE_DMA))
+		max_zone_pfns[ZONE_DMA] = PFN_DOWN(max_zone_dma_phys());
+	max_zone_pfns[ZONE_NORMAL] = max;
+
+	free_area_init_nodes(max_zone_pfns);
+}
+
+#else
 static void __init zone_sizes_init(unsigned long min, unsigned long max)
 {
 	struct memblock_region *reg;
@@ -116,6 +130,7 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
 
 	free_area_init_node(0, zone_size, min, zhole_size);
 }
+#endif /* CONFIG_NUMA */
 
 #ifdef CONFIG_HAVE_ARCH_PFN_VALID
 int pfn_valid(unsigned long pfn)
@@ -133,10 +148,15 @@ static void arm64_memory_present(void)
 static void arm64_memory_present(void)
 {
 	struct memblock_region *reg;
+	int nid = 0;
 
-	for_each_memblock(memory, reg)
-		memory_present(0, memblock_region_memory_base_pfn(reg),
-			       memblock_region_memory_end_pfn(reg));
+	for_each_memblock(memory, reg) {
+#ifdef CONFIG_NUMA
+		nid = reg->nid;
+#endif
+		memory_present(nid, memblock_region_memory_base_pfn(reg),
+				memblock_region_memory_end_pfn(reg));
+	}
 }
 #endif
 
@@ -181,7 +201,6 @@ void __init arm64_memblock_init(void)
 	dma_contiguous_reserve(arm64_dma_phys_limit);
 
 	memblock_allow_resize();
-	memblock_dump_all();
 }
 
 void __init bootmem_init(void)
@@ -193,6 +212,9 @@ void __init bootmem_init(void)
 
 	early_memtest(min << PAGE_SHIFT, max << PAGE_SHIFT);
 
+	max_pfn = max_low_pfn = max;
+
+	arm64_numa_init();
 	/*
 	 * Sparsemem tries to allocate bootmem in memory_present(), so must be
 	 * done after the fixed reservations.
@@ -203,7 +225,6 @@ void __init bootmem_init(void)
 	zone_sizes_init(min, max);
 
 	high_memory = __va((max << PAGE_SHIFT) - 1) + 1;
-	max_pfn = max_low_pfn = max;
 }
 
 #ifndef CONFIG_SPARSEMEM_VMEMMAP
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 873e363..6f68075 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -453,6 +453,7 @@ void __init paging_init(void)
 	zero_page = early_alloc(PAGE_SIZE);
 
 	bootmem_init();
+	memblock_dump_all();
 
 	empty_zero_page = virt_to_page(zero_page);
 
diff --git a/arch/arm64/mm/numa.c b/arch/arm64/mm/numa.c
new file mode 100644
index 0000000..050b083
--- /dev/null
+++ b/arch/arm64/mm/numa.c
@@ -0,0 +1,387 @@
+/*
+ * NUMA support, based on the x86 implementation.
+ *
+ * Copyright (C) 2015 Cavium Inc.
+ * Author: Ganapatrao Kulkarni <gkulkarni@cavium.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/bootmem.h>
+#include <linux/ctype.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/memblock.h>
+#include <linux/module.h>
+#include <linux/mmzone.h>
+#include <linux/nodemask.h>
+#include <linux/sched.h>
+#include <linux/string.h>
+#include <linux/topology.h>
+
+#include <asm/smp_plat.h>
+
+struct pglist_data *node_data[MAX_NUMNODES] __read_mostly;
+EXPORT_SYMBOL(node_data);
+nodemask_t numa_nodes_parsed __initdata;
+int cpu_to_node_map[NR_CPUS] = { [0 ... NR_CPUS-1] = NUMA_NO_NODE };
+
+static int numa_off;
+static int numa_distance_cnt;
+static u8 *numa_distance;
+
+static __init int numa_parse_early_param(char *opt)
+{
+	if (!opt)
+		return -EINVAL;
+	if (!strncmp(opt, "off", 3)) {
+		pr_info("%s\n", "NUMA turned off");
+		numa_off = 1;
+	}
+	return 0;
+}
+early_param("numa", numa_parse_early_param);
+
+cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
+EXPORT_SYMBOL(node_to_cpumask_map);
+
+#ifdef CONFIG_DEBUG_PER_CPU_MAPS
+/*
+ * Returns a pointer to the bitmask of CPUs on Node 'node'.
+ */
+const struct cpumask *cpumask_of_node(int node)
+{
+
+	if (WARN_ON(node >= nr_node_ids))
+		return cpu_none_mask;
+
+	if (WARN_ON(node_to_cpumask_map[node] == NULL))
+		return cpu_online_mask;
+
+	return node_to_cpumask_map[node];
+}
+EXPORT_SYMBOL(cpumask_of_node);
+#endif
+
+static void map_cpu_to_node(unsigned int cpu, int nid)
+{
+	set_cpu_numa_node(cpu, nid);
+	if (nid >= 0)
+		cpumask_set_cpu(cpu, node_to_cpumask_map[nid]);
+}
+
+static void unmap_cpu_to_node(unsigned int cpu)
+{
+	int nid = cpu_to_node(cpu);
+
+	if (nid >= 0)
+		cpumask_clear_cpu(cpu, node_to_cpumask_map[nid]);
+	set_cpu_numa_node(cpu, NUMA_NO_NODE);
+}
+
+void numa_clear_node(unsigned int cpu)
+{
+	unmap_cpu_to_node(cpu);
+}
+
+/*
+ * Allocate node_to_cpumask_map based on number of available nodes
+ * Requires node_possible_map to be valid.
+ *
+ * Note: cpumask_of_node() is not valid until after this is done.
+ * (Use CONFIG_DEBUG_PER_CPU_MAPS to check this.)
+ */
+static void __init setup_node_to_cpumask_map(void)
+{
+	unsigned int cpu;
+	int node;
+
+	/* setup nr_node_ids if not done yet */
+	if (nr_node_ids == MAX_NUMNODES)
+		setup_nr_node_ids();
+
+	/* allocate and clear the mapping */
+	for (node = 0; node < nr_node_ids; node++) {
+		alloc_bootmem_cpumask_var(&node_to_cpumask_map[node]);
+		cpumask_clear(node_to_cpumask_map[node]);
+	}
+
+	for_each_possible_cpu(cpu)
+		set_cpu_numa_node(cpu, NUMA_NO_NODE);
+
+	/* cpumask_of_node() will now work */
+	pr_debug("Node to cpumask map for %d nodes\n", nr_node_ids);
+}
+
+/*
+ *  Set the cpu to node and mem mapping
+ */
+void numa_store_cpu_info(unsigned int cpu)
+{
+	map_cpu_to_node(cpu, numa_off ? 0 : cpu_to_node_map[cpu]);
+}
+
+/**
+ * numa_add_memblk - Set node id to memblk
+ * @nid: NUMA node ID of the new memblk
+ * @start: Start address of the new memblk
+ * @size:  Size of the new memblk
+ *
+ * RETURNS:
+ * 0 on success, -errno on failure.
+ */
+int __init numa_add_memblk(int nid, u64 start, u64 size)
+{
+	int ret;
+
+	ret = memblock_set_node(start, size, &memblock.memory, nid);
+	if (ret < 0) {
+		pr_err("NUMA: memblock [0x%llx - 0x%llx] failed to add on node %d\n",
+			start, (start + size - 1), nid);
+		return ret;
+	}
+
+	node_set(nid, numa_nodes_parsed);
+	pr_info("NUMA: Adding memblock [0x%llx - 0x%llx] on node %d\n",
+			start, (start + size - 1), nid);
+	return ret;
+}
+EXPORT_SYMBOL(numa_add_memblk);
+
+/* Initialize NODE_DATA for a node on the local memory */
+static void __init setup_node_data(int nid, u64 start_pfn, u64 end_pfn)
+{
+	const size_t nd_size = roundup(sizeof(pg_data_t), SMP_CACHE_BYTES);
+	u64 nd_pa;
+	void *nd;
+	int tnid;
+
+	pr_info("Initmem setup node %d [mem %#010Lx-%#010Lx]\n",
+			nid, start_pfn << PAGE_SHIFT,
+			(end_pfn << PAGE_SHIFT) - 1);
+
+	nd_pa = memblock_alloc_try_nid(nd_size, SMP_CACHE_BYTES, nid);
+	nd = __va(nd_pa);
+
+	/* report and initialize */
+	pr_info("  NODE_DATA [mem %#010Lx-%#010Lx]\n",
+		nd_pa, nd_pa + nd_size - 1);
+	tnid = early_pfn_to_nid(nd_pa >> PAGE_SHIFT);
+	if (tnid != nid)
+		pr_info("    NODE_DATA(%d) on node %d\n", nid, tnid);
+
+	node_data[nid] = nd;
+	memset(NODE_DATA(nid), 0, sizeof(pg_data_t));
+	NODE_DATA(nid)->node_id = nid;
+	NODE_DATA(nid)->node_start_pfn = start_pfn;
+	NODE_DATA(nid)->node_spanned_pages = end_pfn - start_pfn;
+}
+
+/**
+ * numa_reset_distance - Reset NUMA distance table
+ *
+ * The current table is freed.
+ * The next numa_set_distance() call will create a new one.
+ */
+void __init numa_reset_distance(void)
+{
+	size_t size;
+
+	if (!numa_distance)
+		return;
+
+	size = numa_distance_cnt * numa_distance_cnt *
+		sizeof(numa_distance[0]);
+
+	memblock_free(__pa(numa_distance), size);
+	numa_distance_cnt = 0;
+	numa_distance = NULL;
+}
+
+static int __init numa_alloc_distance(void)
+{
+	size_t size;
+	u64 phys;
+	int i, j;
+
+	size = nr_node_ids * nr_node_ids * sizeof(numa_distance[0]);
+	phys = memblock_find_in_range(0, PFN_PHYS(max_pfn),
+				      size, PAGE_SIZE);
+	if (WARN_ON(!phys))
+		return -ENOMEM;
+
+	memblock_reserve(phys, size);
+
+	numa_distance = __va(phys);
+	numa_distance_cnt = nr_node_ids;
+
+	/* fill with the default distances */
+	for (i = 0; i < numa_distance_cnt; i++)
+		for (j = 0; j < numa_distance_cnt; j++)
+			numa_distance[i * numa_distance_cnt + j] = i == j ?
+				LOCAL_DISTANCE : REMOTE_DISTANCE;
+
+	pr_debug("NUMA: Initialized distance table, cnt=%d\n",
+			numa_distance_cnt);
+
+	return 0;
+}
+
+/**
+ * numa_set_distance - Set NUMA distance from one NUMA to another
+ * @from: the 'from' node to set distance
+ * @to: the 'to'  node to set distance
+ * @distance: NUMA distance
+ *
+ * Set the distance from node @from to @to to @distance.  If distance table
+ * doesn't exist, one which is large enough to accommodate all the currently
+ * known nodes will be created.
+ *
+ * If such table cannot be allocated, a warning is printed and further
+ * calls are ignored until the distance table is reset with
+ * numa_reset_distance().
+ *
+ * If @from or @to is higher than the highest known node or lower than zero
+ * at the time of table creation or @distance doesn't make sense, the call
+ * is ignored.
+ * This is to allow simplification of specific NUMA config implementations.
+ */
+void __init numa_set_distance(int from, int to, int distance)
+{
+	if (!numa_distance)
+		return;
+
+	if (from >= numa_distance_cnt || to >= numa_distance_cnt ||
+			from < 0 || to < 0) {
+		pr_warn_once("NUMA: Warning: node ids are out of bound, from=%d to=%d distance=%d\n",
+			    from, to, distance);
+		return;
+	}
+
+	if ((u8)distance != distance ||
+	    (from == to && distance != LOCAL_DISTANCE)) {
+		pr_warn_once("NUMA: Warning: invalid distance parameter, from=%d to=%d distance=%d\n",
+			     from, to, distance);
+		return;
+	}
+
+	numa_distance[from * numa_distance_cnt + to] = distance;
+}
+EXPORT_SYMBOL(numa_set_distance);
+
+int __node_distance(int from, int to)
+{
+	if (from >= numa_distance_cnt || to >= numa_distance_cnt)
+		return from == to ? LOCAL_DISTANCE : REMOTE_DISTANCE;
+	return numa_distance[from * numa_distance_cnt + to];
+}
+EXPORT_SYMBOL(__node_distance);
+
+static int __init numa_register_nodes(void)
+{
+	int nid;
+	struct memblock_region *mblk;
+
+	/* Check that valid nid is set to memblks */
+	for_each_memblock(memory, mblk)
+		if (mblk->nid == NUMA_NO_NODE || mblk->nid >= MAX_NUMNODES)
+			return -EINVAL;
+
+	/* Finally register nodes. */
+	for_each_node_mask(nid, numa_nodes_parsed) {
+		unsigned long start_pfn, end_pfn;
+
+		get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
+		setup_node_data(nid, start_pfn, end_pfn);
+		node_set_online(nid);
+	}
+
+	/* Setup online nodes to actual nodes*/
+	node_possible_map = numa_nodes_parsed;
+
+	return 0;
+}
+
+static int __init numa_init(int (*init_func)(void))
+{
+	int ret;
+
+	nodes_clear(numa_nodes_parsed);
+	nodes_clear(node_possible_map);
+	nodes_clear(node_online_map);
+	numa_reset_distance();
+
+	ret = init_func();
+	if (ret < 0)
+		return ret;
+
+	if (nodes_empty(numa_nodes_parsed))
+		return -EINVAL;
+
+	ret = numa_register_nodes();
+	if (ret < 0)
+		return ret;
+
+	ret = numa_alloc_distance();
+	if (ret < 0)
+		return ret;
+
+	setup_node_to_cpumask_map();
+
+	/* init boot processor */
+	cpu_to_node_map[0] = 0;
+	map_cpu_to_node(0, 0);
+
+	return 0;
+}
+
+/**
+ * dummy_numa_init - Fallback dummy NUMA init
+ *
+ * Used if there's no underlying NUMA architecture, NUMA initialization
+ * fails, or NUMA is disabled on the command line.
+ *
+ * Must online at least one node (node 0) and add memory blocks that cover all
+ * allowed memory. It is unlikely that this function fails.
+ */
+static int __init dummy_numa_init(void)
+{
+	int ret;
+	struct memblock_region *mblk;
+
+	pr_info("%s\n", "No NUMA configuration found");
+	pr_info("Faking a node at [mem %#018Lx-%#018Lx]\n",
+	       0LLU, PFN_PHYS(max_pfn) - 1);
+
+	for_each_memblock(memory, mblk) {
+		ret = numa_add_memblk(0, mblk->base, mblk->size);
+		if (unlikely(ret < 0))
+			return ret;
+	}
+
+	numa_off = 1;
+	return 0;
+}
+
+/**
+ * arm64_numa_init - Initialize NUMA
+ *
+ * Try each configured NUMA initialization method until one succeeds.  The
+ * last fallback is dummy single node config encomapssing whole memory and
+ * never fails.
+ */
+void __init arm64_numa_init(void)
+{
+	numa_init(dummy_numa_init);
+}
-- 
1.8.1.4

Re: [PATCH v9 1/6] arm64, numa: adding numa support for arm64 platforms.

[Bjorn Helgaas]

On Mon, Jan 18, 2016 at 10:06:00PM +0530, Ganapatrao Kulkarni wrote:
> Adding numa support for arm64 based platforms.
> This patch adds by default the dummy numa node and
> maps all memory and cpus to node 0.
> using this patch, numa can be simulated on single node arm64 platforms.

> diff --git a/arch/arm64/include/asm/mmzone.h b/arch/arm64/include/asm/mmzone.h
> new file mode 100644
> index 0000000..2cd804d
> --- /dev/null
> +++ b/arch/arm64/include/asm/mmzone.h
> @@ -0,0 +1,18 @@
> +#ifndef __ASM_MMZONE_H
> +#define __ASM_MMZONE_H
> +
> +#ifdef CONFIG_NUMA
> +
> +#include <linux/mmdebug.h>
> +#include <linux/types.h>
> +
> +#include <asm/smp.h>
> +#include <asm/numa.h>

Are all these #includes really necessary?  s390 looks very similar and
doesn't need the #includes.

> +extern struct pglist_data *node_data[];
> +
> +#define NODE_DATA(nid)		(node_data[(nid)])
> +
> +#endif /* CONFIG_NUMA */
> +
> +#endif /* __ASM_MMZONE_H */

> diff --git a/arch/arm64/include/asm/numa.h b/arch/arm64/include/asm/numa.h
> new file mode 100644
> index 0000000..f28f15b0
> --- /dev/null
> +++ b/arch/arm64/include/asm/numa.h
> @@ -0,0 +1,40 @@
> +#ifndef __ASM_NUMA_H
> +#define __ASM_NUMA_H
> +
> +#include <linux/nodemask.h>
> +#include <asm/topology.h>
> +
> +#ifdef CONFIG_NUMA
> +
> +/* currently, arm64 implements flat NUMA topology */
> +#define parent_node(node)	(node)
> +
> +extern int __node_distance(int from, int to);

Below you add some extern function declarations without the "extern"
keyword.  I don't know what the convention is for arm64 (I omit the
"extern" for PCI), but it seems like you should at least be
consistent.

> +#define node_distance(a, b) __node_distance(a, b)
> +
> +extern int cpu_to_node_map[NR_CPUS];
> +extern nodemask_t numa_nodes_parsed __initdata;
> +
> +/* Mappings between node number and cpus on that node. */
> +extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
> +extern void numa_clear_node(unsigned int cpu);
> +#ifdef CONFIG_DEBUG_PER_CPU_MAPS
> +extern const struct cpumask *cpumask_of_node(int node);
> +#else
> +/* Returns a pointer to the cpumask of CPUs on Node 'node'. */
> +static inline const struct cpumask *cpumask_of_node(int node)
> +{
> +	return node_to_cpumask_map[node];
> +}
> +#endif
> +
> +void __init arm64_numa_init(void);
> +int __init numa_add_memblk(int nodeid, u64 start, u64 end);
> +void __init numa_set_distance(int from, int to, int distance);
> +void __init numa_reset_distance(void);
> +void numa_store_cpu_info(unsigned int cpu);

[PATCH v9 2/6] Documentation, dt, arm64/arm: dt bindings for numa.

[Ganapatrao Kulkarni]

DT bindings for numa mapping of memory, cores and IOs.

Reviewed-by: Robert Richter <rrichter-YGCgFSpz5w/QT0dZR+AlfA@public.gmane.org>
Signed-off-by: Ganapatrao Kulkarni <gkulkarni-M3mlKVOIwJVv6pq1l3V1OdBPR1lH4CV8@public.gmane.org>
---
 Documentation/devicetree/bindings/arm/numa.txt | 272 +++++++++++++++++++++++++
 1 file changed, 272 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/arm/numa.txt

diff --git a/Documentation/devicetree/bindings/arm/numa.txt b/Documentation/devicetree/bindings/arm/numa.txt
new file mode 100644
index 0000000..734e5b4
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/numa.txt
@@ -0,0 +1,272 @@
+==============================================================================
+NUMA binding description.
+==============================================================================
+
+==============================================================================
+1 - Introduction
+==============================================================================
+
+Systems employing a Non Uniform Memory Access (NUMA) architecture contain
+collections of hardware resources including processors, memory, and I/O buses,
+that comprise what is commonly known as a NUMA node.
+Processor accesses to memory within the local NUMA node is generally faster
+than processor accesses to memory outside of the local NUMA node.
+DT defines interfaces that allow the platform to convey NUMA node
+topology information to OS.
+
+==============================================================================
+2 - numa-node-id
+==============================================================================
+
+For the purpose of identification, each NUMA node is associated with a unique
+token known as a node id. For the purpose of this binding
+a node id is a 32-bit integer.
+
+A device node is associated with a NUMA node by the presence of a
+numa-node-id property which contains the node id of the device.
+
+Example:
+	/* numa node 0 */
+	numa-node-id = <0>;
+
+	/* numa node 1 */
+	numa-node-id = <1>;
+
+==============================================================================
+3 - distance-map
+==============================================================================
+
+The device tree node distance-map describes the relative
+distance (memory latency) between all numa nodes.
+
+- compatible : Should at least contain "numa-distance-map-v1".
+
+- distance-matrix
+  This property defines a matrix to describe the relative distances
+  between all numa nodes.
+  It is represented as a list of node pairs and their relative distance.
+
+  Note:
+	1. Each entry represents distance from first node to second node.
+	The distances are equal in either direction.
+	2. The distance from a node to self (local distance) is represented
+	with value 10 and all internode distance should be represented with
+	a value greater than 10.
+	3. distance-matrix should have entries in lexicographical ascending
+	order of nodes.
+	4. There must be only one device node distance-map which must reside in the root node.
+
+Example:
+	4 nodes connected in mesh/ring topology as below,
+
+		0_______20______1
+		|               |
+		|               |
+		20             20
+		|               |
+		|               |
+		|_______________|
+		3       20      2
+
+	if relative distance for each hop is 20,
+	then internode distance would be,
+	      0 -> 1 = 20
+	      1 -> 2 = 20
+	      2 -> 3 = 20
+	      3 -> 0 = 20
+	      0 -> 2 = 40
+	      1 -> 3 = 40
+
+     and dt presentation for this distance matrix is,
+
+		distance-map {
+			 compatible = "numa-distance-map-v1";
+			 distance-matrix = <0 0  10>,
+					   <0 1  20>,
+					   <0 2  40>,
+					   <0 3  20>,
+					   <1 0  20>,
+					   <1 1  10>,
+					   <1 2  20>,
+					   <1 3  40>,
+					   <2 0  40>,
+					   <2 1  20>,
+					   <2 2  10>,
+					   <2 3  20>,
+					   <3 0  20>,
+					   <3 1  40>,
+					   <3 2  20>,
+					   <3 3  10>;
+		};
+
+==============================================================================
+4 - Example dts
+==============================================================================
+
+2 sockets system consists of 2 boards connected through ccn bus and
+each board having one socket/soc of 8 cpus, memory and pci bus.
+
+	memory@00c00000 {
+		device_type = "memory";
+		reg = <0x0 0x00c00000 0x0 0x80000000>;
+		/* node 0 */
+		numa-node-id = <0>;
+	};
+
+	memory@10000000000 {
+		device_type = "memory";
+		reg = <0x100 0x00000000 0x0 0x80000000>;
+		/* node 1 */
+		numa-node-id = <1>;
+	};
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		cpu@000 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x000>;
+			enable-method = "psci";
+			/* node 0 */
+			numa-node-id = <0>;
+		};
+		cpu@001 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x001>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@002 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x002>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@003 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x003>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@004 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x004>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@005 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x005>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@006 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x006>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@007 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x007>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@008 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x008>;
+			enable-method = "psci";
+			/* node 1 */
+			numa-node-id = <1>;
+		};
+		cpu@009 {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x009>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@00a {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x00a>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@00b {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x00b>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@00c {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x00c>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@00d {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x00d>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@00e {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x00e>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@00f {
+			device_type = "cpu";
+			compatible =  "arm,armv8";
+			reg = <0x0 0x00f>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+	};
+
+	pcie0: pcie0@0x8480,00000000 {
+		compatible = "arm,armv8";
+		device_type = "pci";
+		bus-range = <0 255>;
+		#size-cells = <2>;
+		#address-cells = <3>;
+		reg = <0x8480 0x00000000 0 0x10000000>;  /* Configuration space */
+		ranges = <0x03000000 0x8010 0x00000000 0x8010 0x00000000 0x70 0x00000000>;
+		/* node 0 */
+		numa-node-id = <0>;
+        };
+
+	pcie1: pcie1@0x9480,00000000 {
+		compatible = "arm,armv8";
+		device_type = "pci";
+		bus-range = <0 255>;
+		#size-cells = <2>;
+		#address-cells = <3>;
+		reg = <0x9480 0x00000000 0 0x10000000>;  /* Configuration space */
+		ranges = <0x03000000 0x9010 0x00000000 0x9010 0x00000000 0x70 0x00000000>;
+		/* node 1 */
+		numa-node-id = <1>;
+        };
+
+	distance-map {
+		compatible = "numa-distance-map-v1";
+		distance-matrix = <0 0 10>,
+				  <0 1 20>,
+				  <1 1 10>;
+	};
-- 
1.8.1.4

--
To unsubscribe from this list: send the line "unsubscribe devicetree" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v9 2/6] Documentation, dt, arm64/arm: dt bindings for numa.

[Rob Herring]

On Mon, Jan 18, 2016 at 10:06:01PM +0530, Ganapatrao Kulkarni wrote:
> DT bindings for numa mapping of memory, cores and IOs.
> 
> Reviewed-by: Robert Richter <rrichter@cavium.com>
> Signed-off-by: Ganapatrao Kulkarni <gkulkarni@caviumnetworks.com>
> ---
>  Documentation/devicetree/bindings/arm/numa.txt | 272 +++++++++++++++++++++++++
>  1 file changed, 272 insertions(+)
>  create mode 100644 Documentation/devicetree/bindings/arm/numa.txt

This is looks okay to me, but some cosmetic things on the example.

> +==============================================================================
> +4 - Example dts
> +==============================================================================
> +
> +2 sockets system consists of 2 boards connected through ccn bus and
> +each board having one socket/soc of 8 cpus, memory and pci bus.
> +
> +	memory@00c00000 {

Drop the leading 0s on unit addresses.

> +		device_type = "memory";
> +		reg = <0x0 0x00c00000 0x0 0x80000000>;
> +		/* node 0 */
> +		numa-node-id = <0>;
> +	};
> +
> +	memory@10000000000 {
> +		device_type = "memory";
> +		reg = <0x100 0x00000000 0x0 0x80000000>;
> +		/* node 1 */
> +		numa-node-id = <1>;
> +	};
> +
> +	cpus {
> +		#address-cells = <2>;
> +		#size-cells = <0>;
> +
> +		cpu@000 {

Same here (leaving one of course).

> +			device_type = "cpu";
> +			compatible =  "arm,armv8";
> +			reg = <0x0 0x000>;
> +			enable-method = "psci";
> +			/* node 0 */
> +			numa-node-id = <0>;
> +		};
> +		cpu@001 {

and so on...

> +			device_type = "cpu";
> +			compatible =  "arm,armv8";
> +			reg = <0x0 0x001>;

Either all leading 0s or none.

> +			reg = <0x0 0x008>;
> +			enable-method = "psci";
> +			/* node 1 */

Kind of a pointless comment.

Wouldn't each cluster of cpus for a given numa node be in a different 
cpu affinity? Certainly not required by the architecture, but the common 
case at least.

> +			numa-node-id = <1>;
> +		};

[...]

> +	pcie0: pcie0@0x8480,00000000 {

Drop the 0x and the comma.

> +		compatible = "arm,armv8";
> +		device_type = "pci";
> +		bus-range = <0 255>;
> +		#size-cells = <2>;
> +		#address-cells = <3>;
> +		reg = <0x8480 0x00000000 0 0x10000000>;  /* Configuration space */
> +		ranges = <0x03000000 0x8010 0x00000000 0x8010 0x00000000 0x70 0x00000000>;
> +		/* node 0 */
> +		numa-node-id = <0>;
> +        };
> +
> +	pcie1: pcie1@0x9480,00000000 {

ditto

> +		compatible = "arm,armv8";
> +		device_type = "pci";
> +		bus-range = <0 255>;
> +		#size-cells = <2>;
> +		#address-cells = <3>;
> +		reg = <0x9480 0x00000000 0 0x10000000>;  /* Configuration space */
> +		ranges = <0x03000000 0x9010 0x00000000 0x9010 0x00000000 0x70 0x00000000>;
> +		/* node 1 */
> +		numa-node-id = <1>;
> +        };
> +
> +	distance-map {
> +		compatible = "numa-distance-map-v1";
> +		distance-matrix = <0 0 10>,
> +				  <0 1 20>,
> +				  <1 1 10>;
> +	};
> -- 
> 1.8.1.4
> 
> --
> To unsubscribe from this list: send the line "unsubscribe devicetree" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v9 2/6] Documentation, dt, arm64/arm: dt bindings for numa.

[Ganapatrao Kulkarni]

Hi Rob, Mark,


On Wed, Jan 20, 2016 at 7:48 PM, Rob Herring <robh-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org> wrote:
> On Mon, Jan 18, 2016 at 10:06:01PM +0530, Ganapatrao Kulkarni wrote:
>> DT bindings for numa mapping of memory, cores and IOs.
>>
>> Reviewed-by: Robert Richter <rrichter-YGCgFSpz5w/QT0dZR+AlfA@public.gmane.org>
>> Signed-off-by: Ganapatrao Kulkarni <gkulkarni-M3mlKVOIwJVv6pq1l3V1OdBPR1lH4CV8@public.gmane.org>
>> ---
>>  Documentation/devicetree/bindings/arm/numa.txt | 272 +++++++++++++++++++++++++
>>  1 file changed, 272 insertions(+)
>>  create mode 100644 Documentation/devicetree/bindings/arm/numa.txt
>
> This is looks okay to me, but some cosmetic things on the example.
can i have your Ack please?
>
>> +==============================================================================
>> +4 - Example dts
>> +==============================================================================
>> +
>> +2 sockets system consists of 2 boards connected through ccn bus and
>> +each board having one socket/soc of 8 cpus, memory and pci bus.
>> +
>> +     memory@00c00000 {
>
> Drop the leading 0s on unit addresses.
i will correct these in next version.
>
>> +             device_type = "memory";
>> +             reg = <0x0 0x00c00000 0x0 0x80000000>;
>> +             /* node 0 */
>> +             numa-node-id = <0>;
>> +     };
>> +
>> +     memory@10000000000 {
>> +             device_type = "memory";
>> +             reg = <0x100 0x00000000 0x0 0x80000000>;
>> +             /* node 1 */
>> +             numa-node-id = <1>;
>> +     };
>> +
>> +     cpus {
>> +             #address-cells = <2>;
>> +             #size-cells = <0>;
>> +
>> +             cpu@000 {
>
> Same here (leaving one of course).
>
>> +                     device_type = "cpu";
>> +                     compatible =  "arm,armv8";
>> +                     reg = <0x0 0x000>;
>> +                     enable-method = "psci";
>> +                     /* node 0 */
>> +                     numa-node-id = <0>;
>> +             };
>> +             cpu@001 {
>
> and so on...
>
>> +                     device_type = "cpu";
>> +                     compatible =  "arm,armv8";
>> +                     reg = <0x0 0x001>;
>
> Either all leading 0s or none.
>
>> +                     reg = <0x0 0x008>;
>> +                     enable-method = "psci";
>> +                     /* node 1 */
>
> Kind of a pointless comment.
>
> Wouldn't each cluster of cpus for a given numa node be in a different
> cpu affinity? Certainly not required by the architecture, but the common
> case at least.
>
>> +                     numa-node-id = <1>;
>> +             };
>
> [...]
>
>> +     pcie0: pcie0@0x8480,00000000 {
>
> Drop the 0x and the comma.
>
>> +             compatible = "arm,armv8";
>> +             device_type = "pci";
>> +             bus-range = <0 255>;
>> +             #size-cells = <2>;
>> +             #address-cells = <3>;
>> +             reg = <0x8480 0x00000000 0 0x10000000>;  /* Configuration space */
>> +             ranges = <0x03000000 0x8010 0x00000000 0x8010 0x00000000 0x70 0x00000000>;
>> +             /* node 0 */
>> +             numa-node-id = <0>;
>> +        };
>> +
>> +     pcie1: pcie1@0x9480,00000000 {
>
> ditto
>
>> +             compatible = "arm,armv8";
>> +             device_type = "pci";
>> +             bus-range = <0 255>;
>> +             #size-cells = <2>;
>> +             #address-cells = <3>;
>> +             reg = <0x9480 0x00000000 0 0x10000000>;  /* Configuration space */
>> +             ranges = <0x03000000 0x9010 0x00000000 0x9010 0x00000000 0x70 0x00000000>;
>> +             /* node 1 */
>> +             numa-node-id = <1>;
>> +        };
>> +
>> +     distance-map {
>> +             compatible = "numa-distance-map-v1";
>> +             distance-matrix = <0 0 10>,
>> +                               <0 1 20>,
>> +                               <1 1 10>;
>> +     };
>> --
>> 1.8.1.4
thanks
Ganapat
>>
>> --
>> To unsubscribe from this list: send the line "unsubscribe devicetree" in
>> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe devicetree" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH v9 3/6] arm64/arm, numa, dt: adding numa dt binding implementation for arm64 platforms.

[Ganapatrao Kulkarni]

Adding numa dt binding support for arm64 based platforms.
dt node parsing for numa topology is done using device property
numa-node-id and device node distance-map.

Reviewed-by: Robert Richter <rrichter@cavium.com>
Signed-off-by: Ganapatrao Kulkarni <gkulkarni@caviumnetworks.com>
---
 arch/arm64/Kconfig            |  10 ++
 arch/arm64/include/asm/numa.h |  10 ++
 arch/arm64/kernel/Makefile    |   1 +
 arch/arm64/kernel/of_numa.c   | 257 ++++++++++++++++++++++++++++++++++++++++++
 arch/arm64/kernel/smp.c       |   2 +
 arch/arm64/mm/numa.c          |  10 +-
 6 files changed, 289 insertions(+), 1 deletion(-)
 create mode 100644 arch/arm64/kernel/of_numa.c

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 74f5d73..775cf4a 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -515,6 +515,16 @@ config NUMA
 	  local memory of the CPU and add some more
 	  NUMA awareness to the kernel.
 
+config OF_NUMA
+	bool "Device Tree NUMA support"
+	depends on NUMA
+	depends on OF
+	default y
+	help
+	  Enable Device Tree NUMA support.
+	  This enables the numa mapping of cpu, memory, io and
+	  inter node distances using dt bindings.
+
 config NODES_SHIFT
 	int "Maximum NUMA Nodes (as a power of 2)"
 	range 1 10
diff --git a/arch/arm64/include/asm/numa.h b/arch/arm64/include/asm/numa.h
index f28f15b0..54deb38 100644
--- a/arch/arm64/include/asm/numa.h
+++ b/arch/arm64/include/asm/numa.h
@@ -37,4 +37,14 @@ void numa_store_cpu_info(unsigned int cpu);
 static inline void numa_store_cpu_info(unsigned int cpu)		{ }
 static inline void arm64_numa_init(void)		{ }
 #endif	/* CONFIG_NUMA */
+
+struct device_node;
+#ifdef CONFIG_OF_NUMA
+int __init arm64_of_numa_init(void);
+void __init of_numa_set_node_info(unsigned int cpu, struct device_node *dn);
+#else
+static inline void of_numa_set_node_info(unsigned int cpu,
+		struct device_node *dn) { }
+#endif
+
 #endif	/* __ASM_NUMA_H */
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index 474691f..7987763 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -41,6 +41,7 @@ arm64-obj-$(CONFIG_EFI)			+= efi.o efi-entry.stub.o
 arm64-obj-$(CONFIG_PCI)			+= pci.o
 arm64-obj-$(CONFIG_ARMV8_DEPRECATED)	+= armv8_deprecated.o
 arm64-obj-$(CONFIG_ACPI)		+= acpi.o
+arm64-obj-$(CONFIG_OF_NUMA)		+= of_numa.o
 
 obj-y					+= $(arm64-obj-y) vdso/
 obj-m					+= $(arm64-obj-m)
diff --git a/arch/arm64/kernel/of_numa.c b/arch/arm64/kernel/of_numa.c
new file mode 100644
index 0000000..2f9e34b
--- /dev/null
+++ b/arch/arm64/kernel/of_numa.c
@@ -0,0 +1,257 @@
+/*
+ * OF NUMA Parsing support.
+ *
+ * Copyright (C) 2015 Cavium Inc.
+ * Author: Ganapatrao Kulkarni <gkulkarni@cavium.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/ctype.h>
+#include <linux/memblock.h>
+#include <linux/module.h>
+#include <linux/nodemask.h>
+#include <linux/of.h>
+#include <linux/of_fdt.h>
+
+#include <asm/smp_plat.h>
+
+/* define default numa node to 0 */
+#define DEFAULT_NODE 0
+#define OF_NUMA_PROP "numa-node-id"
+
+/* Returns nid in the range [0..MAX_NUMNODES-1],
+ * or NUMA_NO_NODE if no valid numa-node-id entry found
+ * or DEFAULT_NODE if no numa-node-id entry exists
+ */
+static int of_numa_prop_to_nid(const __be32 *of_numa_prop, int length)
+{
+	int nid;
+
+	if (!of_numa_prop)
+		return DEFAULT_NODE;
+
+	if (length != sizeof(*of_numa_prop)) {
+		pr_warn("NUMA: Invalid of_numa_prop length %d found.\n",
+				length);
+		return NUMA_NO_NODE;
+	}
+
+	nid = of_read_number(of_numa_prop, 1);
+	if (nid >= MAX_NUMNODES) {
+		pr_warn("NUMA: Invalid numa node %d found.\n", nid);
+		return NUMA_NO_NODE;
+	}
+
+	return nid;
+}
+
+/* Must hold reference to node during call */
+static int of_get_numa_nid(struct device_node *device)
+{
+	int length;
+	const __be32 *of_numa_prop;
+
+	of_numa_prop = of_get_property(device, OF_NUMA_PROP, &length);
+
+	return of_numa_prop_to_nid(of_numa_prop, length);
+}
+
+static int __init early_init_of_get_numa_nid(unsigned long node)
+{
+	int length;
+	const __be32 *of_numa_prop;
+
+	of_numa_prop = of_get_flat_dt_prop(node, OF_NUMA_PROP, &length);
+
+	return of_numa_prop_to_nid(of_numa_prop, length);
+}
+
+/* Walk the device tree upwards, looking for a numa-node-id property */
+int of_node_to_nid(struct device_node *device)
+{
+	struct device_node *parent;
+	int nid = NUMA_NO_NODE;
+
+	of_node_get(device);
+	while (device) {
+		const __be32 *of_numa_prop;
+		int length;
+
+		of_numa_prop = of_get_property(device, OF_NUMA_PROP, &length);
+		if (of_numa_prop) {
+			nid = of_numa_prop_to_nid(of_numa_prop, length);
+			break;
+		}
+
+		parent = device;
+		device = of_get_parent(parent);
+		of_node_put(parent);
+	}
+	of_node_put(device);
+
+	return nid;
+}
+
+void __init of_numa_set_node_info(unsigned int cpu, struct device_node *device)
+{
+	int nid = DEFAULT_NODE;
+
+	if (device)
+		nid = of_get_numa_nid(device);
+
+	cpu_to_node_map[cpu] = nid;
+}
+
+/*
+ * Even though we connect cpus to numa domains later in SMP
+ * init, we need to know the node ids now for all cpus.
+*/
+static int __init early_init_parse_cpu_node(unsigned long node)
+{
+	int nid;
+
+	const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
+
+	/* We are scanning "cpu" nodes only */
+	if (type == NULL)
+		return 0;
+	else if (strcmp(type, "cpu") != 0)
+		return 0;
+
+	nid = early_init_of_get_numa_nid(node);
+
+	if (nid == NUMA_NO_NODE)
+		return -EINVAL;
+
+	node_set(nid, numa_nodes_parsed);
+	return 0;
+}
+
+static int __init early_init_parse_memory_node(unsigned long node)
+{
+	const __be32 *reg, *endp;
+	int length;
+	int nid;
+
+	const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
+
+	/* We are scanning "memory" nodes only */
+	if (type == NULL)
+		return 0;
+	else if (strcmp(type, "memory") != 0)
+		return 0;
+
+	nid = early_init_of_get_numa_nid(node);
+
+	if (nid == NUMA_NO_NODE)
+		return -EINVAL;
+
+	reg = of_get_flat_dt_prop(node, "reg", &length);
+	endp = reg + (length / sizeof(__be32));
+
+	while ((endp - reg) >= (dt_root_addr_cells + dt_root_size_cells)) {
+		u64 base, size;
+
+		base = dt_mem_next_cell(dt_root_addr_cells, &reg);
+		size = dt_mem_next_cell(dt_root_size_cells, &reg);
+		pr_debug("NUMA-DT:  base = %llx , node = %u\n",
+				base, nid);
+
+		if (numa_add_memblk(nid, base, size) < 0)
+			return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int __init early_init_parse_distance_map_v1(unsigned long node,
+		const char *uname)
+{
+
+	const __be32 *prop_dist_matrix;
+	int length = 0, i, matrix_count;
+	int nr_size_cells = OF_ROOT_NODE_SIZE_CELLS_DEFAULT;
+
+	pr_info("NUMA: parsing numa-distance-map-v1\n");
+
+	prop_dist_matrix =
+		of_get_flat_dt_prop(node, "distance-matrix", &length);
+
+	if (!length) {
+		pr_err("NUMA: failed to parse distance-matrix\n");
+		return  -ENODEV;
+	}
+
+	matrix_count = ((length / sizeof(__be32)) / (3 * nr_size_cells));
+
+	if ((matrix_count * sizeof(__be32) * 3 * nr_size_cells) !=  length) {
+		pr_warn("NUMA: invalid distance-matrix length %d\n", length);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < matrix_count; i++) {
+		u32 nodea, nodeb, distance;
+
+		nodea = dt_mem_next_cell(nr_size_cells, &prop_dist_matrix);
+		nodeb = dt_mem_next_cell(nr_size_cells, &prop_dist_matrix);
+		distance = dt_mem_next_cell(nr_size_cells, &prop_dist_matrix);
+		numa_set_distance(nodea, nodeb, distance);
+		pr_debug("NUMA-DT:  distance[node%d -> node%d] = %d\n",
+				nodea, nodeb, distance);
+
+		/* Set default distance of node B->A same as A->B */
+		if (nodeb > nodea)
+			numa_set_distance(nodeb, nodea, distance);
+	}
+
+	return 0;
+}
+
+static int __init early_init_parse_distance_map(unsigned long node,
+		const char *uname)
+{
+
+	if (strcmp(uname, "distance-map") != 0)
+		return 0;
+
+	if (of_flat_dt_is_compatible(node, "numa-distance-map-v1"))
+		return early_init_parse_distance_map_v1(node, uname);
+
+	return -EINVAL;
+}
+
+/**
+ * early_init_of_scan_numa_map - parse memory node and map nid to memory range.
+ */
+int __init early_init_of_scan_numa_map(unsigned long node, const char *uname,
+				     int depth, void *data)
+{
+	int ret;
+
+	ret = early_init_parse_cpu_node(node);
+
+	if (!ret)
+		ret = early_init_parse_memory_node(node);
+
+	if (!ret)
+		ret = early_init_parse_distance_map(node, uname);
+
+	return ret;
+}
+
+/* DT node mapping is done already early_init_of_scan_memory */
+int __init arm64_of_numa_init(void)
+{
+	return of_scan_flat_dt(early_init_of_scan_numa_map, NULL);
+}
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index d6e7d6a..a2a8c2d 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -520,6 +520,8 @@ static void __init of_parse_and_init_cpus(void)
 
 		pr_debug("cpu logical map 0x%llx\n", hwid);
 		cpu_logical_map(cpu_count) = hwid;
+		/* map logical cpu to node */
+		of_numa_set_node_info(cpu_count, dn);
 next:
 		cpu_count++;
 	}
diff --git a/arch/arm64/mm/numa.c b/arch/arm64/mm/numa.c
index 050b083..9e8704b 100644
--- a/arch/arm64/mm/numa.c
+++ b/arch/arm64/mm/numa.c
@@ -383,5 +383,13 @@ static int __init dummy_numa_init(void)
  */
 void __init arm64_numa_init(void)
 {
-	numa_init(dummy_numa_init);
+	int ret = -ENODEV;
+
+#ifdef CONFIG_OF_NUMA
+	if (!numa_off)
+		ret = numa_init(arm64_of_numa_init);
+#endif
+
+	if (ret)
+		numa_init(dummy_numa_init);
 }
-- 
1.8.1.4

Re: [PATCH v9 3/6] arm64/arm, numa, dt: adding numa dt binding implementation for arm64 platforms.

[Bjorn Helgaas]

Subject is "arm64/arm, numa, dt: adding ..."  What is the significance
of the "arm" part?  The other patches only mention "arm64".

General comment: the code below has little, if anything, that is
actually arm64-specific.  Maybe this is the first DT-based NUMA
platform?  I don't see other similar code for other arches, so maybe
it's too early to try to generalize it, but we should try to avoid
adding duplicates of this code if/when other arches do show up.

On Mon, Jan 18, 2016 at 10:06:02PM +0530, Ganapatrao Kulkarni wrote:
> Adding numa dt binding support for arm64 based platforms.
> dt node parsing for numa topology is done using device property
> numa-node-id and device node distance-map.
> 
> Reviewed-by: Robert Richter <rrichter-YGCgFSpz5w/QT0dZR+AlfA@public.gmane.org>
> Signed-off-by: Ganapatrao Kulkarni <gkulkarni-M3mlKVOIwJVv6pq1l3V1OdBPR1lH4CV8@public.gmane.org>
> ---
>  arch/arm64/Kconfig            |  10 ++
>  arch/arm64/include/asm/numa.h |  10 ++
>  arch/arm64/kernel/Makefile    |   1 +
>  arch/arm64/kernel/of_numa.c   | 257 ++++++++++++++++++++++++++++++++++++++++++
>  arch/arm64/kernel/smp.c       |   2 +
>  arch/arm64/mm/numa.c          |  10 +-
>  6 files changed, 289 insertions(+), 1 deletion(-)
>  create mode 100644 arch/arm64/kernel/of_numa.c
> 
> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> index 74f5d73..775cf4a 100644
> --- a/arch/arm64/Kconfig
> +++ b/arch/arm64/Kconfig
> @@ -515,6 +515,16 @@ config NUMA
>  	  local memory of the CPU and add some more
>  	  NUMA awareness to the kernel.
>  
> +config OF_NUMA
> +	bool "Device Tree NUMA support"
> +	depends on NUMA
> +	depends on OF
> +	default y
> +	help
> +	  Enable Device Tree NUMA support.
> +	  This enables the numa mapping of cpu, memory, io and
> +	  inter node distances using dt bindings.
> +
>  config NODES_SHIFT
>  	int "Maximum NUMA Nodes (as a power of 2)"
>  	range 1 10
> diff --git a/arch/arm64/include/asm/numa.h b/arch/arm64/include/asm/numa.h
> index f28f15b0..54deb38 100644
> --- a/arch/arm64/include/asm/numa.h
> +++ b/arch/arm64/include/asm/numa.h
> @@ -37,4 +37,14 @@ void numa_store_cpu_info(unsigned int cpu);
>  static inline void numa_store_cpu_info(unsigned int cpu)		{ }
>  static inline void arm64_numa_init(void)		{ }
>  #endif	/* CONFIG_NUMA */
> +
> +struct device_node;
> +#ifdef CONFIG_OF_NUMA
> +int __init arm64_of_numa_init(void);
> +void __init of_numa_set_node_info(unsigned int cpu, struct device_node *dn);
> +#else
> +static inline void of_numa_set_node_info(unsigned int cpu,
> +		struct device_node *dn) { }
> +#endif
> +
>  #endif	/* __ASM_NUMA_H */
> diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
> index 474691f..7987763 100644
> --- a/arch/arm64/kernel/Makefile
> +++ b/arch/arm64/kernel/Makefile
> @@ -41,6 +41,7 @@ arm64-obj-$(CONFIG_EFI)			+= efi.o efi-entry.stub.o
>  arm64-obj-$(CONFIG_PCI)			+= pci.o
>  arm64-obj-$(CONFIG_ARMV8_DEPRECATED)	+= armv8_deprecated.o
>  arm64-obj-$(CONFIG_ACPI)		+= acpi.o
> +arm64-obj-$(CONFIG_OF_NUMA)		+= of_numa.o
>  
>  obj-y					+= $(arm64-obj-y) vdso/
>  obj-m					+= $(arm64-obj-m)
> diff --git a/arch/arm64/kernel/of_numa.c b/arch/arm64/kernel/of_numa.c
> new file mode 100644
> index 0000000..2f9e34b
> --- /dev/null
> +++ b/arch/arm64/kernel/of_numa.c
> @@ -0,0 +1,257 @@
> +/*
> + * OF NUMA Parsing support.
> + *
> + * Copyright (C) 2015 Cavium Inc.
> + * Author: Ganapatrao Kulkarni <gkulkarni-YGCgFSpz5w/QT0dZR+AlfA@public.gmane.org>
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License
> + * along with this program.  If not, see <http://www.gnu.org/licenses/>.
> + */
> +
> +#include <linux/ctype.h>
> +#include <linux/memblock.h>
> +#include <linux/module.h>
> +#include <linux/nodemask.h>
> +#include <linux/of.h>
> +#include <linux/of_fdt.h>
> +
> +#include <asm/smp_plat.h>
> +
> +/* define default numa node to 0 */
> +#define DEFAULT_NODE 0
> +#define OF_NUMA_PROP "numa-node-id"
> +
> +/* Returns nid in the range [0..MAX_NUMNODES-1],
> + * or NUMA_NO_NODE if no valid numa-node-id entry found
> + * or DEFAULT_NODE if no numa-node-id entry exists
> + */
> +static int of_numa_prop_to_nid(const __be32 *of_numa_prop, int length)
> +{
> +	int nid;
> +
> +	if (!of_numa_prop)
> +		return DEFAULT_NODE;
> +
> +	if (length != sizeof(*of_numa_prop)) {
> +		pr_warn("NUMA: Invalid of_numa_prop length %d found.\n",
> +				length);
> +		return NUMA_NO_NODE;
> +	}
> +
> +	nid = of_read_number(of_numa_prop, 1);
> +	if (nid >= MAX_NUMNODES) {
> +		pr_warn("NUMA: Invalid numa node %d found.\n", nid);
> +		return NUMA_NO_NODE;
> +	}
> +
> +	return nid;
> +}
> +
> +/* Must hold reference to node during call */
> +static int of_get_numa_nid(struct device_node *device)
> +{
> +	int length;
> +	const __be32 *of_numa_prop;
> +
> +	of_numa_prop = of_get_property(device, OF_NUMA_PROP, &length);
> +
> +	return of_numa_prop_to_nid(of_numa_prop, length);
> +}
> +
> +static int __init early_init_of_get_numa_nid(unsigned long node)
> +{
> +	int length;
> +	const __be32 *of_numa_prop;
> +
> +	of_numa_prop = of_get_flat_dt_prop(node, OF_NUMA_PROP, &length);
> +
> +	return of_numa_prop_to_nid(of_numa_prop, length);
> +}
> +
> +/* Walk the device tree upwards, looking for a numa-node-id property */
> +int of_node_to_nid(struct device_node *device)
> +{
> +	struct device_node *parent;
> +	int nid = NUMA_NO_NODE;
> +
> +	of_node_get(device);
> +	while (device) {
> +		const __be32 *of_numa_prop;
> +		int length;
> +
> +		of_numa_prop = of_get_property(device, OF_NUMA_PROP, &length);
> +		if (of_numa_prop) {
> +			nid = of_numa_prop_to_nid(of_numa_prop, length);
> +			break;
> +		}
> +
> +		parent = device;
> +		device = of_get_parent(parent);
> +		of_node_put(parent);
> +	}
> +	of_node_put(device);
> +
> +	return nid;
> +}
> +
> +void __init of_numa_set_node_info(unsigned int cpu, struct device_node *device)
> +{
> +	int nid = DEFAULT_NODE;
> +
> +	if (device)
> +		nid = of_get_numa_nid(device);
> +
> +	cpu_to_node_map[cpu] = nid;
> +}
> +
> +/*
> + * Even though we connect cpus to numa domains later in SMP
> + * init, we need to know the node ids now for all cpus.
> +*/
> +static int __init early_init_parse_cpu_node(unsigned long node)
> +{
> +	int nid;
> +
> +	const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
> +
> +	/* We are scanning "cpu" nodes only */
> +	if (type == NULL)
> +		return 0;
> +	else if (strcmp(type, "cpu") != 0)
> +		return 0;
> +
> +	nid = early_init_of_get_numa_nid(node);
> +
> +	if (nid == NUMA_NO_NODE)
> +		return -EINVAL;
> +
> +	node_set(nid, numa_nodes_parsed);
> +	return 0;
> +}
> +
> +static int __init early_init_parse_memory_node(unsigned long node)
> +{
> +	const __be32 *reg, *endp;
> +	int length;
> +	int nid;
> +
> +	const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
> +
> +	/* We are scanning "memory" nodes only */
> +	if (type == NULL)
> +		return 0;
> +	else if (strcmp(type, "memory") != 0)
> +		return 0;
> +
> +	nid = early_init_of_get_numa_nid(node);
> +
> +	if (nid == NUMA_NO_NODE)
> +		return -EINVAL;
> +
> +	reg = of_get_flat_dt_prop(node, "reg", &length);
> +	endp = reg + (length / sizeof(__be32));
> +
> +	while ((endp - reg) >= (dt_root_addr_cells + dt_root_size_cells)) {
> +		u64 base, size;
> +
> +		base = dt_mem_next_cell(dt_root_addr_cells, &reg);
> +		size = dt_mem_next_cell(dt_root_size_cells, &reg);
> +		pr_debug("NUMA-DT:  base = %llx , node = %u\n",
> +				base, nid);
> +
> +		if (numa_add_memblk(nid, base, size) < 0)
> +			return -EINVAL;
> +	}
> +
> +	return 0;
> +}
> +
> +static int __init early_init_parse_distance_map_v1(unsigned long node,
> +		const char *uname)
> +{
> +
> +	const __be32 *prop_dist_matrix;
> +	int length = 0, i, matrix_count;
> +	int nr_size_cells = OF_ROOT_NODE_SIZE_CELLS_DEFAULT;
> +
> +	pr_info("NUMA: parsing numa-distance-map-v1\n");
> +
> +	prop_dist_matrix =
> +		of_get_flat_dt_prop(node, "distance-matrix", &length);
> +
> +	if (!length) {
> +		pr_err("NUMA: failed to parse distance-matrix\n");
> +		return  -ENODEV;
> +	}
> +
> +	matrix_count = ((length / sizeof(__be32)) / (3 * nr_size_cells));
> +
> +	if ((matrix_count * sizeof(__be32) * 3 * nr_size_cells) !=  length) {
> +		pr_warn("NUMA: invalid distance-matrix length %d\n", length);
> +		return -EINVAL;
> +	}
> +
> +	for (i = 0; i < matrix_count; i++) {
> +		u32 nodea, nodeb, distance;
> +
> +		nodea = dt_mem_next_cell(nr_size_cells, &prop_dist_matrix);
> +		nodeb = dt_mem_next_cell(nr_size_cells, &prop_dist_matrix);
> +		distance = dt_mem_next_cell(nr_size_cells, &prop_dist_matrix);
> +		numa_set_distance(nodea, nodeb, distance);
> +		pr_debug("NUMA-DT:  distance[node%d -> node%d] = %d\n",
> +				nodea, nodeb, distance);
> +
> +		/* Set default distance of node B->A same as A->B */
> +		if (nodeb > nodea)
> +			numa_set_distance(nodeb, nodea, distance);
> +	}
> +
> +	return 0;
> +}
> +
> +static int __init early_init_parse_distance_map(unsigned long node,
> +		const char *uname)
> +{
> +
> +	if (strcmp(uname, "distance-map") != 0)
> +		return 0;
> +
> +	if (of_flat_dt_is_compatible(node, "numa-distance-map-v1"))
> +		return early_init_parse_distance_map_v1(node, uname);
> +
> +	return -EINVAL;
> +}
> +
> +/**
> + * early_init_of_scan_numa_map - parse memory node and map nid to memory range.
> + */
> +int __init early_init_of_scan_numa_map(unsigned long node, const char *uname,
> +				     int depth, void *data)
> +{
> +	int ret;
> +
> +	ret = early_init_parse_cpu_node(node);
> +
> +	if (!ret)
> +		ret = early_init_parse_memory_node(node);
> +
> +	if (!ret)
> +		ret = early_init_parse_distance_map(node, uname);
> +
> +	return ret;
> +}
> +
> +/* DT node mapping is done already early_init_of_scan_memory */
> +int __init arm64_of_numa_init(void)
> +{
> +	return of_scan_flat_dt(early_init_of_scan_numa_map, NULL);
> +}
> diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
> index d6e7d6a..a2a8c2d 100644
> --- a/arch/arm64/kernel/smp.c
> +++ b/arch/arm64/kernel/smp.c
> @@ -520,6 +520,8 @@ static void __init of_parse_and_init_cpus(void)
>  
>  		pr_debug("cpu logical map 0x%llx\n", hwid);
>  		cpu_logical_map(cpu_count) = hwid;
> +		/* map logical cpu to node */
> +		of_numa_set_node_info(cpu_count, dn);
>  next:
>  		cpu_count++;
>  	}
> diff --git a/arch/arm64/mm/numa.c b/arch/arm64/mm/numa.c
> index 050b083..9e8704b 100644
> --- a/arch/arm64/mm/numa.c
> +++ b/arch/arm64/mm/numa.c
> @@ -383,5 +383,13 @@ static int __init dummy_numa_init(void)
>   */
>  void __init arm64_numa_init(void)
>  {
> -	numa_init(dummy_numa_init);
> +	int ret = -ENODEV;
> +
> +#ifdef CONFIG_OF_NUMA
> +	if (!numa_off)
> +		ret = numa_init(arm64_of_numa_init);
> +#endif
> +
> +	if (ret)
> +		numa_init(dummy_numa_init);
>  }
> -- 
> 1.8.1.4
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-arch" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v9 3/6] arm64/arm, numa, dt: adding numa dt binding implementation for arm64 platforms.

[Will Deacon]

On Tue, Jan 26, 2016 at 02:36:04PM -0600, Bjorn Helgaas wrote:
> Subject is "arm64/arm, numa, dt: adding ..."  What is the significance
> of the "arm" part?  The other patches only mention "arm64".
> 
> General comment: the code below has little, if anything, that is
> actually arm64-specific.  Maybe this is the first DT-based NUMA
> platform?  I don't see other similar code for other arches, so maybe
> it's too early to try to generalize it, but we should try to avoid
> adding duplicates of this code if/when other arches do show up.

Having it in the core code would allow us to share it with arch/arm/
fairly straightforwardly.

Will

Re: [PATCH v9 3/6] arm64/arm, numa, dt: adding numa dt binding implementation for arm64 platforms.

[Ganapatrao Kulkarni]

Hi Will,


On Thu, Jan 28, 2016 at 8:09 PM, Will Deacon <will.deacon-5wv7dgnIgG8@public.gmane.org> wrote:
> On Tue, Jan 26, 2016 at 02:36:04PM -0600, Bjorn Helgaas wrote:
>> Subject is "arm64/arm, numa, dt: adding ..."  What is the significance
>> of the "arm" part?  The other patches only mention "arm64".
>>
>> General comment: the code below has little, if anything, that is
>> actually arm64-specific.  Maybe this is the first DT-based NUMA
>> platform?  I don't see other similar code for other arches, so maybe
>> it's too early to try to generalize it, but we should try to avoid
>> adding duplicates of this code if/when other arches do show up.
>
> Having it in the core code would allow us to share it with arch/arm/
> fairly straightforwardly.
This binding can be used for arm too.
however at this moment it is the need of arm64 platforms.
can we please keep this to arm64 as it's too early to try to
generalize it(as Bjorn suggested)
I prefer to keep it as it is, otherwise ok.
Please suggest.
>
> Will
thanks
Ganapat

Re: [PATCH v9 3/6] arm64/arm, numa, dt: adding numa dt binding implementation for arm64 platforms.

[Will Deacon]

On Thu, Jan 28, 2016 at 10:42:17PM +0530, Ganapatrao Kulkarni wrote:
> On Thu, Jan 28, 2016 at 8:09 PM, Will Deacon <will.deacon@arm.com> wrote:
> > On Tue, Jan 26, 2016 at 02:36:04PM -0600, Bjorn Helgaas wrote:
> >> Subject is "arm64/arm, numa, dt: adding ..."  What is the significance
> >> of the "arm" part?  The other patches only mention "arm64".
> >>
> >> General comment: the code below has little, if anything, that is
> >> actually arm64-specific.  Maybe this is the first DT-based NUMA
> >> platform?  I don't see other similar code for other arches, so maybe
> >> it's too early to try to generalize it, but we should try to avoid
> >> adding duplicates of this code if/when other arches do show up.
> >
> > Having it in the core code would allow us to share it with arch/arm/
> > fairly straightforwardly.
> This binding can be used for arm too.
> however at this moment it is the need of arm64 platforms.
> can we please keep this to arm64 as it's too early to try to
> generalize it(as Bjorn suggested)
> I prefer to keep it as it is, otherwise ok.
> Please suggest.

My suggestions time and time again on the NUMA patches from you have
consistently been around consolidation of existing code, or moving things
that aren't architecture-specific out of the architecture code.

Will

Re: [PATCH v9 3/6] arm64/arm, numa, dt: adding numa dt binding implementation for arm64 platforms.

[Ganapatrao Kulkarni]

On Thu, Jan 28, 2016 at 11:38 PM, Will Deacon <will.deacon@arm.com> wrote:
> On Thu, Jan 28, 2016 at 10:42:17PM +0530, Ganapatrao Kulkarni wrote:
>> On Thu, Jan 28, 2016 at 8:09 PM, Will Deacon <will.deacon@arm.com> wrote:
>> > On Tue, Jan 26, 2016 at 02:36:04PM -0600, Bjorn Helgaas wrote:
>> >> Subject is "arm64/arm, numa, dt: adding ..."  What is the significance
>> >> of the "arm" part?  The other patches only mention "arm64".
>> >>
>> >> General comment: the code below has little, if anything, that is
>> >> actually arm64-specific.  Maybe this is the first DT-based NUMA
>> >> platform?  I don't see other similar code for other arches, so maybe
>> >> it's too early to try to generalize it, but we should try to avoid
>> >> adding duplicates of this code if/when other arches do show up.
>> >
>> > Having it in the core code would allow us to share it with arch/arm/
>> > fairly straightforwardly.
>> This binding can be used for arm too.
>> however at this moment it is the need of arm64 platforms.
>> can we please keep this to arm64 as it's too early to try to
>> generalize it(as Bjorn suggested)
>> I prefer to keep it as it is, otherwise ok.
>> Please suggest.
>
> My suggestions time and time again on the NUMA patches from you have
> consistently been around consolidation of existing code, or moving things
> that aren't architecture-specific out of the architecture code.

thanks, i shall move this out to drivers/of
>
> Will

thanks
Ganapat

[PATCH v9 4/6] arm64, dt, thunderx: Add initial dts for Cavium Thunderx in 2 node topology.

[Ganapatrao Kulkarni]

Adding dt file for Cavium's Thunderx dual socket platform.

Signed-off-by: Ganapatrao Kulkarni <gkulkarni@caviumnetworks.com>
---
 arch/arm64/boot/dts/cavium/Makefile             |   2 +-
 arch/arm64/boot/dts/cavium/thunder-88xx-2n.dts  |  83 +++
 arch/arm64/boot/dts/cavium/thunder-88xx-2n.dtsi | 806 ++++++++++++++++++++++++
 3 files changed, 890 insertions(+), 1 deletion(-)
 create mode 100644 arch/arm64/boot/dts/cavium/thunder-88xx-2n.dts
 create mode 100644 arch/arm64/boot/dts/cavium/thunder-88xx-2n.dtsi

diff --git a/arch/arm64/boot/dts/cavium/Makefile b/arch/arm64/boot/dts/cavium/Makefile
index e34f89d..7fe7067 100644
--- a/arch/arm64/boot/dts/cavium/Makefile
+++ b/arch/arm64/boot/dts/cavium/Makefile
@@ -1,4 +1,4 @@
-dtb-$(CONFIG_ARCH_THUNDER) += thunder-88xx.dtb
+dtb-$(CONFIG_ARCH_THUNDER) += thunder-88xx.dtb thunder-88xx-2n.dtb
 
 always		:= $(dtb-y)
 subdir-y	:= $(dts-dirs)
diff --git a/arch/arm64/boot/dts/cavium/thunder-88xx-2n.dts b/arch/arm64/boot/dts/cavium/thunder-88xx-2n.dts
new file mode 100644
index 0000000..5601e87
--- /dev/null
+++ b/arch/arm64/boot/dts/cavium/thunder-88xx-2n.dts
@@ -0,0 +1,83 @@
+/*
+ * Cavium Thunder DTS file - Thunder board description
+ *
+ * Copyright (C) 2014, Cavium Inc.
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public
+ *     License along with this library; if not, write to the Free
+ *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
+ *     MA 02110-1301 USA
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/dts-v1/;
+
+/include/ "thunder-88xx-2n.dtsi"
+
+/ {
+	model = "Cavium ThunderX CN88XX board";
+	compatible = "cavium,thunder-88xx";
+
+	aliases {
+		serial0 = &uaa0;
+		serial1 = &uaa1;
+	};
+
+	memory@00000000 {
+		device_type = "memory";
+		reg = <0x0 0x01400000 0x3 0xFEC00000>;
+		/* socket 0 */
+		numa-node-id = <0>;
+	};
+
+	memory@10000000000 {
+		device_type = "memory";
+		reg = <0x100 0x00400000 0x3 0xFFC00000>;
+		 /* socket 1 */
+		numa-node-id = <1>;
+	};
+
+	distance-map {
+		compatible = "numa-distance-map-v1";
+		distance-matrix = <0 0  10>,
+				  <0 1  20>,
+				  <1 1  10>;
+	};
+};
diff --git a/arch/arm64/boot/dts/cavium/thunder-88xx-2n.dtsi b/arch/arm64/boot/dts/cavium/thunder-88xx-2n.dtsi
new file mode 100644
index 0000000..b58e5c7
--- /dev/null
+++ b/arch/arm64/boot/dts/cavium/thunder-88xx-2n.dtsi
@@ -0,0 +1,806 @@
+/*
+ * Cavium Thunder DTS file - Thunder SoC description
+ *
+ * Copyright (C) 2014, Cavium Inc.
+ *
+ * This file is dual-licensed: you can use it either under the terms
+ * of the GPL or the X11 license, at your option. Note that this dual
+ * licensing only applies to this file, and not this project as a
+ * whole.
+ *
+ *  a) This library is free software; you can redistribute it and/or
+ *     modify it under the terms of the GNU General Public License as
+ *     published by the Free Software Foundation; either version 2 of the
+ *     License, or (at your option) any later version.
+ *
+ *     This library is distributed in the hope that it will be useful,
+ *     but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *     GNU General Public License for more details.
+ *
+ *     You should have received a copy of the GNU General Public
+ *     License along with this library; if not, write to the Free
+ *     Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
+ *     MA 02110-1301 USA
+ *
+ * Or, alternatively,
+ *
+ *  b) Permission is hereby granted, free of charge, to any person
+ *     obtaining a copy of this software and associated documentation
+ *     files (the "Software"), to deal in the Software without
+ *     restriction, including without limitation the rights to use,
+ *     copy, modify, merge, publish, distribute, sublicense, and/or
+ *     sell copies of the Software, and to permit persons to whom the
+ *     Software is furnished to do so, subject to the following
+ *     conditions:
+ *
+ *     The above copyright notice and this permission notice shall be
+ *     included in all copies or substantial portions of the Software.
+ *
+ *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ *     OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/ {
+	compatible = "cavium,thunder-88xx";
+	interrupt-parent = <&gic0>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	psci {
+		compatible = "arm,psci-0.2";
+		method = "smc";
+	};
+
+	cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		cpu@000 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x000>;
+			enable-method = "psci";
+			/* socket 0 */
+			numa-node-id = <0>;
+		};
+		cpu@001 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x001>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@002 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x002>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@003 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x003>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@004 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x004>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@005 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x005>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@006 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x006>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@007 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x007>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@008 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x008>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@009 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x009>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@00a {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x00a>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@00b {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x00b>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@00c {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x00c>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@00d {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x00d>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@00e {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x00e>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@00f {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x00f>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@100 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x100>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@101 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x101>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@102 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x102>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@103 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x103>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@104 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x104>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@105 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x105>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@106 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x106>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@107 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x107>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@108 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x108>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@109 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x109>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@10a {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10a>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@10b {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10b>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@10c {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10c>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@10d {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10d>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@10e {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10e>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@10f {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10f>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@200 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x200>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@201 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x201>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@202 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x202>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@203 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x203>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@204 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x204>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@205 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x205>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@206 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x206>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@207 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x207>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@208 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x208>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@209 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x209>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@20a {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x20a>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@20b {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x20b>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@20c {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x20c>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@20d {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x20d>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@20e {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x20e>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@20f {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x20f>;
+			enable-method = "psci";
+			numa-node-id = <0>;
+		};
+		cpu@10000 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10000>;
+			enable-method = "psci";
+			/* socket 1 */
+			numa-node-id = <1>;
+		};
+		cpu@10001 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10001>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10002 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10002>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10003 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10003>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10004 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10004>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10005 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10005>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10006 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10006>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10007 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10007>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10008 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10008>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10009 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10009>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1000a {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1000a>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1000b {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1000b>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1000c {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1000c>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1000d {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1000d>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1000e {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1000e>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1000f {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1000f>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10100 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10100>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10101 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10101>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10102 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10102>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10103 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10103>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10104 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10104>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10105 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10105>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10106 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10106>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10107 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10107>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10108 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10108>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10109 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10109>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1010a {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1010a>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1010b {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1010b>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1010c {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1010c>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1010d {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1010d>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1010e {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1010e>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1010f {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1010f>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10200 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10200>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10201 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10201>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10202 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10202>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10203 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10203>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10204 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10204>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10205 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10205>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10206 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10206>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10207 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10207>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10208 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10208>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@10209 {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x10209>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1020a {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1020a>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1020b {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1020b>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1020c {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1020c>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1020d {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1020d>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1020e {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1020e>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+		cpu@1020f {
+			device_type = "cpu";
+			compatible = "cavium,thunder", "arm,armv8";
+			reg = <0x0 0x1020f>;
+			enable-method = "psci";
+			numa-node-id = <1>;
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <1 13 0xff01>,
+		             <1 14 0xff01>,
+		             <1 11 0xff01>,
+		             <1 10 0xff01>;
+	};
+
+	soc {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		refclk50mhz: refclk50mhz {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <50000000>;
+			clock-output-names = "refclk50mhz";
+		};
+
+		gic0: interrupt-controller@8010,00000000 {
+			compatible = "arm,gic-v3";
+			#interrupt-cells = <3>;
+			#address-cells = <2>;
+			#size-cells = <2>;
+			#redistributor-regions = <2>;
+			ranges;
+			interrupt-controller;
+			reg = <0x8010 0x00000000 0x0 0x010000>, /* GICD */
+			      <0x8010 0x80000000 0x0 0x600000>, /* GICR Node 0 */
+			      <0x9010 0x80000000 0x0 0x600000>; /* GICR Node 1 */
+			interrupts = <1 9 0xf04>;
+
+			its: gic-its@8010,00020000 {
+				compatible = "arm,gic-v3-its";
+				msi-controller;
+				reg = <0x8010 0x20000 0x0 0x200000>;
+				numa-node-id = <0>;
+			};
+
+			its1: gic-its@9010,00020000 {
+				compatible = "arm,gic-v3-its";
+				msi-controller;
+				reg = <0x9010 0x20000 0x0 0x200000>;
+				numa-node-id = <1>;
+			};
+		};
+
+		uaa0: serial@87e0,24000000 {
+			compatible = "arm,pl011", "arm,primecell";
+			reg = <0x87e0 0x24000000 0x0 0x1000>;
+			interrupts = <1 21 4>;
+			clocks = <&refclk50mhz>;
+			clock-names = "apb_pclk";
+		};
+
+		uaa1: serial@87e0,25000000 {
+			compatible = "arm,pl011", "arm,primecell";
+			reg = <0x87e0 0x25000000 0x0 0x1000>;
+			interrupts = <1 22 4>;
+			clocks = <&refclk50mhz>;
+			clock-names = "apb_pclk";
+		};
+	};
+};
-- 
1.8.1.4

Re: [PATCH v9 4/6] arm64, dt, thunderx: Add initial dts for Cavium Thunderx in 2 node topology.

[David Daney]

On 01/18/2016 08:36 AM, Ganapatrao Kulkarni wrote:
> Adding dt file for Cavium's Thunderx dual socket platform.
>
> Signed-off-by: Ganapatrao Kulkarni <gkulkarni@caviumnetworks.com>
> ---
>   arch/arm64/boot/dts/cavium/Makefile             |   2 +-
>   arch/arm64/boot/dts/cavium/thunder-88xx-2n.dts  |  83 +++
>   arch/arm64/boot/dts/cavium/thunder-88xx-2n.dtsi | 806 ++++++++++++++++++++++++
>   3 files changed, 890 insertions(+), 1 deletion(-)
>   create mode 100644 arch/arm64/boot/dts/cavium/thunder-88xx-2n.dts
>   create mode 100644 arch/arm64/boot/dts/cavium/thunder-88xx-2n.dtsi
>

These files are useless, they don't contain device nodes for anything 
other than the UARTs.  There is no existing system that has only UARTs 
and no other devices, so these files cannot be (and can never be) used 
anywhere.

Since the boot firmware in all real, and emulated, systems is supplying 
correct DTBs, we should consider not merging these files, and removing 
the likewise useless thunder-88xx.dts  thunder-88xx.dtsi

David Daney


[...]

[PATCH v9 5/6] PCI: generic: Make pci-host-generic driver numa aware

[Ganapatrao Kulkarni]

update numa_node of device associated with pci bus.
moved down devm_kzalloc to allocate from node memory.

Signed-off-by: Ganapatrao Kulkarni <gkulkarni@caviumnetworks.com>
---
 drivers/pci/host/pci-host-generic.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/drivers/pci/host/pci-host-generic.c b/drivers/pci/host/pci-host-generic.c
index 5434c90..0e1ce06 100644
--- a/drivers/pci/host/pci-host-generic.c
+++ b/drivers/pci/host/pci-host-generic.c
@@ -215,11 +215,9 @@ static int gen_pci_probe(struct platform_device *pdev)
 	const struct of_device_id *of_id;
 	struct device *dev = &pdev->dev;
 	struct device_node *np = dev->of_node;
-	struct gen_pci *pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
+	struct gen_pci *pci;
 	struct pci_bus *bus, *child;
 
-	if (!pci)
-		return -ENOMEM;
 
 	type = of_get_property(np, "device_type", NULL);
 	if (!type || strcmp(type, "pci")) {
@@ -230,6 +228,11 @@ static int gen_pci_probe(struct platform_device *pdev)
 	of_pci_check_probe_only();
 
 	of_id = of_match_node(gen_pci_of_match, np);
+	set_dev_node(dev, of_node_to_nid(np));
+	pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
+	if (!pci)
+		return -ENOMEM;
+
 	pci->cfg.ops = (struct gen_pci_cfg_bus_ops *)of_id->data;
 	pci->host.dev.parent = dev;
 	INIT_LIST_HEAD(&pci->host.windows);
-- 
1.8.1.4

Re: [PATCH v9 5/6] PCI: generic: Make pci-host-generic driver numa aware

[David Daney]

On 01/18/2016 08:36 AM, Ganapatrao Kulkarni wrote:
> update numa_node of device associated with pci bus.
> moved down devm_kzalloc to allocate from node memory.
>
> Signed-off-by: Ganapatrao Kulkarni <gkulkarni@caviumnetworks.com>
> ---
>   drivers/pci/host/pci-host-generic.c | 9 ++++++---
>   1 file changed, 6 insertions(+), 3 deletions(-)
>
> diff --git a/drivers/pci/host/pci-host-generic.c b/drivers/pci/host/pci-host-generic.c
> index 5434c90..0e1ce06 100644
> --- a/drivers/pci/host/pci-host-generic.c
> +++ b/drivers/pci/host/pci-host-generic.c
> @@ -215,11 +215,9 @@ static int gen_pci_probe(struct platform_device *pdev)
>   	const struct of_device_id *of_id;
>   	struct device *dev = &pdev->dev;
>   	struct device_node *np = dev->of_node;
> -	struct gen_pci *pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
> +	struct gen_pci *pci;
>   	struct pci_bus *bus, *child;
>
> -	if (!pci)
> -		return -ENOMEM;
>
>   	type = of_get_property(np, "device_type", NULL);
>   	if (!type || strcmp(type, "pci")) {
> @@ -230,6 +228,11 @@ static int gen_pci_probe(struct platform_device *pdev)
>   	of_pci_check_probe_only();
>
>   	of_id = of_match_node(gen_pci_of_match, np);
> +	set_dev_node(dev, of_node_to_nid(np));

This shouldn't be done in individual platform_drivers, but instead in 
the device probing code.

There is code that does this in drivers/of/platform.c and 
drivers/of/device.c  Is that not being called for the pci-host-gweneric 
devices?  If not, we should figure out why, and perhaps attempt to fix 
it rather than doing it here.


> +	pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
> +	if (!pci)
> +		return -ENOMEM;
> +
>   	pci->cfg.ops = (struct gen_pci_cfg_bus_ops *)of_id->data;
>   	pci->host.dev.parent = dev;
>   	INIT_LIST_HEAD(&pci->host.windows);
>

Re: [PATCH v9 5/6] PCI: generic: Make pci-host-generic driver numa aware

[Ganapatrao Kulkarni]

On Mon, Jan 18, 2016 at 11:11 PM, David Daney <ddaney.cavm@gmail.com> wrote:
> On 01/18/2016 08:36 AM, Ganapatrao Kulkarni wrote:
>>
>> update numa_node of device associated with pci bus.
>> moved down devm_kzalloc to allocate from node memory.
>>
>> Signed-off-by: Ganapatrao Kulkarni <gkulkarni@caviumnetworks.com>
>> ---
>>   drivers/pci/host/pci-host-generic.c | 9 ++++++---
>>   1 file changed, 6 insertions(+), 3 deletions(-)
>>
>> diff --git a/drivers/pci/host/pci-host-generic.c
>> b/drivers/pci/host/pci-host-generic.c
>> index 5434c90..0e1ce06 100644
>> --- a/drivers/pci/host/pci-host-generic.c
>> +++ b/drivers/pci/host/pci-host-generic.c
>> @@ -215,11 +215,9 @@ static int gen_pci_probe(struct platform_device
>> *pdev)
>>         const struct of_device_id *of_id;
>>         struct device *dev = &pdev->dev;
>>         struct device_node *np = dev->of_node;
>> -       struct gen_pci *pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
>> +       struct gen_pci *pci;
>>         struct pci_bus *bus, *child;
>>
>> -       if (!pci)
>> -               return -ENOMEM;
>>
>>         type = of_get_property(np, "device_type", NULL);
>>         if (!type || strcmp(type, "pci")) {
>> @@ -230,6 +228,11 @@ static int gen_pci_probe(struct platform_device
>> *pdev)
>>         of_pci_check_probe_only();
>>
>>         of_id = of_match_node(gen_pci_of_match, np);
>> +       set_dev_node(dev, of_node_to_nid(np));
>
>
> This shouldn't be done in individual platform_drivers, but instead in the
> device probing code.
>
> There is code that does this in drivers/of/platform.c and
> drivers/of/device.c  Is that not being called for the pci-host-gweneric
> devices?  If not, we should figure out why, and perhaps attempt to fix it
> rather than doing it here.
is it more appropriate to call of_platform_device_create ?

--- a/drivers/pci/host/pci-host-generic.c
+++ b/drivers/pci/host/pci-host-generic.c
@@ -24,6 +24,7 @@
 #include <linux/of_address.h>
 #include <linux/of_pci.h>
 #include <linux/platform_device.h>
+#include <linux/of_platform.h>

 struct gen_pci;

@@ -466,7 +467,7 @@ static int gen_pci_probe(struct platform_device *pdev)
        of_pci_check_probe_only();

        of_id = of_match_node(gen_pci_of_match, np);
-       set_dev_node(dev, of_node_to_nid(np));
+       of_platform_device_create(np, NULL, NULL);
        pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
        if (!pci)
                return -ENOMEM;

>
>
>
>> +       pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
>> +       if (!pci)
>> +               return -ENOMEM;
>> +
>>         pci->cfg.ops = (struct gen_pci_cfg_bus_ops *)of_id->data;
>>         pci->host.dev.parent = dev;
>>         INIT_LIST_HEAD(&pci->host.windows);
>>
>
thanks
Ganapat

Re: [PATCH v9 5/6] PCI: generic: Make pci-host-generic driver numa aware

[Lorenzo Pieralisi]

On Tue, Jan 19, 2016 at 11:28:56AM +0530, Ganapatrao Kulkarni wrote:
> On Mon, Jan 18, 2016 at 11:11 PM, David Daney <ddaney.cavm-Re5JQEeQqe8AvxtiuMwx3w@public.gmane.org> wrote:
> > On 01/18/2016 08:36 AM, Ganapatrao Kulkarni wrote:
> >>
> >> update numa_node of device associated with pci bus.
> >> moved down devm_kzalloc to allocate from node memory.
> >>
> >> Signed-off-by: Ganapatrao Kulkarni <gkulkarni-M3mlKVOIwJVv6pq1l3V1OdBPR1lH4CV8@public.gmane.org>
> >> ---
> >>   drivers/pci/host/pci-host-generic.c | 9 ++++++---
> >>   1 file changed, 6 insertions(+), 3 deletions(-)
> >>
> >> diff --git a/drivers/pci/host/pci-host-generic.c
> >> b/drivers/pci/host/pci-host-generic.c
> >> index 5434c90..0e1ce06 100644
> >> --- a/drivers/pci/host/pci-host-generic.c
> >> +++ b/drivers/pci/host/pci-host-generic.c
> >> @@ -215,11 +215,9 @@ static int gen_pci_probe(struct platform_device
> >> *pdev)
> >>         const struct of_device_id *of_id;
> >>         struct device *dev = &pdev->dev;
> >>         struct device_node *np = dev->of_node;
> >> -       struct gen_pci *pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
> >> +       struct gen_pci *pci;
> >>         struct pci_bus *bus, *child;
> >>
> >> -       if (!pci)
> >> -               return -ENOMEM;
> >>
> >>         type = of_get_property(np, "device_type", NULL);
> >>         if (!type || strcmp(type, "pci")) {
> >> @@ -230,6 +228,11 @@ static int gen_pci_probe(struct platform_device
> >> *pdev)
> >>         of_pci_check_probe_only();
> >>
> >>         of_id = of_match_node(gen_pci_of_match, np);
> >> +       set_dev_node(dev, of_node_to_nid(np));
> >
> >
> > This shouldn't be done in individual platform_drivers, but instead in the
> > device probing code.
> >
> > There is code that does this in drivers/of/platform.c and
> > drivers/of/device.c  Is that not being called for the pci-host-gweneric
> > devices?  If not, we should figure out why, and perhaps attempt to fix it
> > rather than doing it here.
> is it more appropriate to call of_platform_device_create ?

That's already done to create the platform device by OF core when
populating devices from DT, what David suggested is that you should
add set_dev_node() to core OF code instead of adding it specifically
to the PCI host generic code.

Lorenzo
--
To unsubscribe from this list: send the line "unsubscribe devicetree" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v9 5/6] PCI: generic: Make pci-host-generic driver numa aware

[Ganapatrao Kulkarni]

On Tue, Jan 19, 2016 at 3:11 PM, Lorenzo Pieralisi
<lorenzo.pieralisi@arm.com> wrote:
> On Tue, Jan 19, 2016 at 11:28:56AM +0530, Ganapatrao Kulkarni wrote:
>> On Mon, Jan 18, 2016 at 11:11 PM, David Daney <ddaney.cavm@gmail.com> wrote:
>> > On 01/18/2016 08:36 AM, Ganapatrao Kulkarni wrote:
>> >>
>> >> update numa_node of device associated with pci bus.
>> >> moved down devm_kzalloc to allocate from node memory.
>> >>
>> >> Signed-off-by: Ganapatrao Kulkarni <gkulkarni@caviumnetworks.com>
>> >> ---
>> >>   drivers/pci/host/pci-host-generic.c | 9 ++++++---
>> >>   1 file changed, 6 insertions(+), 3 deletions(-)
>> >>
>> >> diff --git a/drivers/pci/host/pci-host-generic.c
>> >> b/drivers/pci/host/pci-host-generic.c
>> >> index 5434c90..0e1ce06 100644
>> >> --- a/drivers/pci/host/pci-host-generic.c
>> >> +++ b/drivers/pci/host/pci-host-generic.c
>> >> @@ -215,11 +215,9 @@ static int gen_pci_probe(struct platform_device
>> >> *pdev)
>> >>         const struct of_device_id *of_id;
>> >>         struct device *dev = &pdev->dev;
>> >>         struct device_node *np = dev->of_node;
>> >> -       struct gen_pci *pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
>> >> +       struct gen_pci *pci;
>> >>         struct pci_bus *bus, *child;
>> >>
>> >> -       if (!pci)
>> >> -               return -ENOMEM;
>> >>
>> >>         type = of_get_property(np, "device_type", NULL);
>> >>         if (!type || strcmp(type, "pci")) {
>> >> @@ -230,6 +228,11 @@ static int gen_pci_probe(struct platform_device
>> >> *pdev)
>> >>         of_pci_check_probe_only();
>> >>
>> >>         of_id = of_match_node(gen_pci_of_match, np);
>> >> +       set_dev_node(dev, of_node_to_nid(np));
>> >
>> >
>> > This shouldn't be done in individual platform_drivers, but instead in the
>> > device probing code.
>> >
>> > There is code that does this in drivers/of/platform.c and
>> > drivers/of/device.c  Is that not being called for the pci-host-gweneric
>> > devices?  If not, we should figure out why, and perhaps attempt to fix it
>> > rather than doing it here.
>> is it more appropriate to call of_platform_device_create ?
>
> That's already done to create the platform device by OF core when
> populating devices from DT, what David suggested is that you should
> add set_dev_node() to core OF code instead of adding it specifically
> to the PCI host generic code.
This patch was worked on 4.2(inherited to 4.4) to have correct
numa_node populated on sysfs.
however, on 4.4, OF  is fixed to set numa_node(fixed by
56f2de81e020c537f7e35550d13840143cb765cd)

This patch (patch #5) is not required and will be dropped.
thanks for your comments.


>
> Lorenzo
thanks
Ganapat

Re: [PATCH v9 5/6] PCI: generic: Make pci-host-generic driver numa aware

[Bjorn Helgaas]

On Mon, Jan 18, 2016 at 10:06:04PM +0530, Ganapatrao Kulkarni wrote:
> update numa_node of device associated with pci bus.
> moved down devm_kzalloc to allocate from node memory.
> 
> Signed-off-by: Ganapatrao Kulkarni <gkulkarni-M3mlKVOIwJVv6pq1l3V1OdBPR1lH4CV8@public.gmane.org>
> ---
>  drivers/pci/host/pci-host-generic.c | 9 ++++++---
>  1 file changed, 6 insertions(+), 3 deletions(-)
> 
> diff --git a/drivers/pci/host/pci-host-generic.c b/drivers/pci/host/pci-host-generic.c
> index 5434c90..0e1ce06 100644
> --- a/drivers/pci/host/pci-host-generic.c
> +++ b/drivers/pci/host/pci-host-generic.c
> @@ -215,11 +215,9 @@ static int gen_pci_probe(struct platform_device *pdev)
>  	const struct of_device_id *of_id;
>  	struct device *dev = &pdev->dev;
>  	struct device_node *np = dev->of_node;
> -	struct gen_pci *pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
> +	struct gen_pci *pci;
>  	struct pci_bus *bus, *child;
>  
> -	if (!pci)
> -		return -ENOMEM;
>  
>  	type = of_get_property(np, "device_type", NULL);
>  	if (!type || strcmp(type, "pci")) {
> @@ -230,6 +228,11 @@ static int gen_pci_probe(struct platform_device *pdev)
>  	of_pci_check_probe_only();
>  
>  	of_id = of_match_node(gen_pci_of_match, np);
> +	set_dev_node(dev, of_node_to_nid(np));

I like the idea of this patch, but why do we need to call
set_dev_node() here in driver code?  Shouldn't that be done by the
OF/platform device infrastructure so that all drivers for platform devices
get this benefit?

> +	pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
> +	if (!pci)
> +		return -ENOMEM;
> +
>  	pci->cfg.ops = (struct gen_pci_cfg_bus_ops *)of_id->data;
>  	pci->host.dev.parent = dev;
>  	INIT_LIST_HEAD(&pci->host.windows);
> -- 
> 1.8.1.4
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-arch" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH v9 5/6] PCI: generic: Make pci-host-generic driver numa aware

[Bjorn Helgaas]

On Tue, Jan 26, 2016 at 02:50:05PM -0600, Bjorn Helgaas wrote:
> On Mon, Jan 18, 2016 at 10:06:04PM +0530, Ganapatrao Kulkarni wrote:
> > update numa_node of device associated with pci bus.
> > moved down devm_kzalloc to allocate from node memory.
> > 
> > Signed-off-by: Ganapatrao Kulkarni <gkulkarni@caviumnetworks.com>
> > ---
> >  drivers/pci/host/pci-host-generic.c | 9 ++++++---
> >  1 file changed, 6 insertions(+), 3 deletions(-)
> > 
> > diff --git a/drivers/pci/host/pci-host-generic.c b/drivers/pci/host/pci-host-generic.c
> > index 5434c90..0e1ce06 100644
> > --- a/drivers/pci/host/pci-host-generic.c
> > +++ b/drivers/pci/host/pci-host-generic.c
> > @@ -215,11 +215,9 @@ static int gen_pci_probe(struct platform_device *pdev)
> >  	const struct of_device_id *of_id;
> >  	struct device *dev = &pdev->dev;
> >  	struct device_node *np = dev->of_node;
> > -	struct gen_pci *pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
> > +	struct gen_pci *pci;
> >  	struct pci_bus *bus, *child;
> >  
> > -	if (!pci)
> > -		return -ENOMEM;
> >  
> >  	type = of_get_property(np, "device_type", NULL);
> >  	if (!type || strcmp(type, "pci")) {
> > @@ -230,6 +228,11 @@ static int gen_pci_probe(struct platform_device *pdev)
> >  	of_pci_check_probe_only();
> >  
> >  	of_id = of_match_node(gen_pci_of_match, np);
> > +	set_dev_node(dev, of_node_to_nid(np));
> 
> I like the idea of this patch, but why do we need to call
> set_dev_node() here in driver code?  Shouldn't that be done by the
> OF/platform device infrastructure so that all drivers for platform devices
> get this benefit?

Sorry, I'm just a day late and a dollar short.  I see David and Lorenzo
have already responded about this.

> > +	pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
> > +	if (!pci)
> > +		return -ENOMEM;
> > +
> >  	pci->cfg.ops = (struct gen_pci_cfg_bus_ops *)of_id->data;
> >  	pci->host.dev.parent = dev;
> >  	INIT_LIST_HEAD(&pci->host.windows);
> > -- 
> > 1.8.1.4
> > 
> > --
> > To unsubscribe from this list: send the line "unsubscribe linux-arch" in
> > the body of a message to majordomo@vger.kernel.org
> > More majordomo info at  http://vger.kernel.org/majordomo-info.html

[RFC PATCH v9 6/6] topology, cleanup: Avoid redefinition of cpumask_of_pcibus in asm header files.

[Ganapatrao Kulkarni]

At present cpumask_of_pcibus is defined for !CONFIG_NUMA and moving out
to common will allow to use for numa too. This also avoids
redefinition of this macro in respective architecture header files.

Signed-off-by: Ganapatrao Kulkarni <gkulkarni@caviumnetworks.com>
---
 arch/arm64/include/asm/topology.h   | 3 ---
 arch/ia64/include/asm/topology.h    | 4 ----
 arch/metag/include/asm/topology.h   | 3 ---
 arch/powerpc/include/asm/topology.h | 4 ----
 arch/s390/include/asm/pci.h         | 2 +-
 arch/s390/include/asm/topology.h    | 1 +
 arch/sh/include/asm/topology.h      | 3 ---
 arch/x86/include/asm/pci.h          | 2 +-
 arch/x86/include/asm/topology.h     | 1 +
 include/asm-generic/topology.h      | 4 ++--
 10 files changed, 6 insertions(+), 21 deletions(-)

diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h
index 8b57339..6e1f62c 100644
--- a/arch/arm64/include/asm/topology.h
+++ b/arch/arm64/include/asm/topology.h
@@ -26,9 +26,6 @@ const struct cpumask *cpu_coregroup_mask(int cpu);
 
 struct pci_bus;
 int pcibus_to_node(struct pci_bus *bus);
-#define cpumask_of_pcibus(bus)	(pcibus_to_node(bus) == -1 ?		\
-				 cpu_all_mask :				\
-				 cpumask_of_node(pcibus_to_node(bus)))
 
 #endif /* CONFIG_NUMA */
 
diff --git a/arch/ia64/include/asm/topology.h b/arch/ia64/include/asm/topology.h
index 3ad8f69..2778eb6 100644
--- a/arch/ia64/include/asm/topology.h
+++ b/arch/ia64/include/asm/topology.h
@@ -58,10 +58,6 @@ void build_cpu_to_node_map(void);
 
 extern void arch_fix_phys_package_id(int num, u32 slot);
 
-#define cpumask_of_pcibus(bus)	(pcibus_to_node(bus) == -1 ?		\
-				 cpu_all_mask :				\
-				 cpumask_of_node(pcibus_to_node(bus)))
-
 #include <asm-generic/topology.h>
 
 #endif /* _ASM_IA64_TOPOLOGY_H */
diff --git a/arch/metag/include/asm/topology.h b/arch/metag/include/asm/topology.h
index e95f874..b285196 100644
--- a/arch/metag/include/asm/topology.h
+++ b/arch/metag/include/asm/topology.h
@@ -9,9 +9,6 @@
 #define cpumask_of_node(node)	((void)node, cpu_online_mask)
 
 #define pcibus_to_node(bus)	((void)(bus), -1)
-#define cpumask_of_pcibus(bus)	(pcibus_to_node(bus) == -1 ? \
-					cpu_all_mask : \
-					cpumask_of_node(pcibus_to_node(bus)))
 
 #endif
 
diff --git a/arch/powerpc/include/asm/topology.h b/arch/powerpc/include/asm/topology.h
index 8b3b46b..eee025d 100644
--- a/arch/powerpc/include/asm/topology.h
+++ b/arch/powerpc/include/asm/topology.h
@@ -32,10 +32,6 @@ static inline int pcibus_to_node(struct pci_bus *bus)
 }
 #endif
 
-#define cpumask_of_pcibus(bus)	(pcibus_to_node(bus) == -1 ?		\
-				 cpu_all_mask :				\
-				 cpumask_of_node(pcibus_to_node(bus)))
-
 extern int __node_distance(int, int);
 #define node_distance(a, b) __node_distance(a, b)
 
diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h
index c873e68..539fb2d 100644
--- a/arch/s390/include/asm/pci.h
+++ b/arch/s390/include/asm/pci.h
@@ -205,7 +205,7 @@ static inline int __pcibus_to_node(const struct pci_bus *bus)
 }
 
 static inline const struct cpumask *
-cpumask_of_pcibus(const struct pci_bus *bus)
+__cpumask_of_pcibus(const struct pci_bus *bus)
 {
 	return cpu_online_mask;
 }
diff --git a/arch/s390/include/asm/topology.h b/arch/s390/include/asm/topology.h
index 94fc55f..280eafc 100644
--- a/arch/s390/include/asm/topology.h
+++ b/arch/s390/include/asm/topology.h
@@ -78,6 +78,7 @@ static inline const struct cpumask *cpumask_of_node(int node)
 #define parent_node(node) (node)
 
 #define pcibus_to_node(bus) __pcibus_to_node(bus)
+#define cpumask_of_pcibus(bus) __cpumask_of_pcibus(bus)
 
 #define node_distance(a, b) __node_distance(a, b)
 
diff --git a/arch/sh/include/asm/topology.h b/arch/sh/include/asm/topology.h
index b0a282d..357983d 100644
--- a/arch/sh/include/asm/topology.h
+++ b/arch/sh/include/asm/topology.h
@@ -9,9 +9,6 @@
 #define cpumask_of_node(node)	((void)node, cpu_online_mask)
 
 #define pcibus_to_node(bus)	((void)(bus), -1)
-#define cpumask_of_pcibus(bus)	(pcibus_to_node(bus) == -1 ? \
-					cpu_all_mask : \
-					cpumask_of_node(pcibus_to_node(bus)))
 
 #endif
 
diff --git a/arch/x86/include/asm/pci.h b/arch/x86/include/asm/pci.h
index 4625943..4a6f358 100644
--- a/arch/x86/include/asm/pci.h
+++ b/arch/x86/include/asm/pci.h
@@ -121,7 +121,7 @@ static inline int __pcibus_to_node(const struct pci_bus *bus)
 }
 
 static inline const struct cpumask *
-cpumask_of_pcibus(const struct pci_bus *bus)
+__cpumask_of_pcibus(const struct pci_bus *bus)
 {
 	int node;
 
diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h
index 0fb4648..980f6ee 100644
--- a/arch/x86/include/asm/topology.h
+++ b/arch/x86/include/asm/topology.h
@@ -91,6 +91,7 @@ extern void setup_node_to_cpumask_map(void);
 #define parent_node(node) (node)
 
 #define pcibus_to_node(bus) __pcibus_to_node(bus)
+#define cpumask_of_pcibus(bus) __cpumask_of_pcibus(bus)
 
 extern int __node_distance(int, int);
 #define node_distance(a, b) __node_distance(a, b)
diff --git a/include/asm-generic/topology.h b/include/asm-generic/topology.h
index fc824e2..ee305f1 100644
--- a/include/asm-generic/topology.h
+++ b/include/asm-generic/topology.h
@@ -54,14 +54,14 @@
 #define pcibus_to_node(bus)	((void)(bus), -1)
 #endif
 
+#endif	/* !CONFIG_NUMA */
+
 #ifndef cpumask_of_pcibus
 #define cpumask_of_pcibus(bus)	(pcibus_to_node(bus) == -1 ?		\
 				 cpu_all_mask :				\
 				 cpumask_of_node(pcibus_to_node(bus)))
 #endif
 
-#endif	/* CONFIG_NUMA */
-
 #if !defined(CONFIG_NUMA) || !defined(CONFIG_HAVE_MEMORYLESS_NODES)
 
 #ifndef set_numa_mem
-- 
1.8.1.4