[RFC 0/2] Representing interrupt affinity in devicetree

[RFC 0/2] Representing interrupt affinity in devicetree

[Mark Rutland]

Current devicetree bindings for devices which use cpu-affine shared interrupts
assume that interrupts are listed in ascending order of physical cpu id
(MPIDR.Aff{2,1,0}). This is problematic for drivers because:

(1) The driver must convert each physical id to a logical id for the purpose of
    managing interrupts.

(2) In multi-cluster systems the physical ids are not necessarily contiguous,
    and drivers cannot simply iterate over ids from 0 to NR_CPUS.

(3) It is not possible to specify sets of interrupts which are wired to a
    subset of cpus (i.e. clusters) not starting at physical id 0, as we can't
    specify which cpu to start from, and can't skip cpus. This makes it
    impossible to represent some devices (e.g. cpu PMUs) which may not exist
    in the first cluster.

(4) Some devices may either be wired with PPIs or SPIs. It is not possible to
    differentiate the two cases in general from the interrupts list (e.g. when
    a device has multiple PPIs wired to all cpus).

To represent the general case, we require a mechanism to describe the cpu
affinity of a device, and a consistent way to map each interrupt to a cpu (or
set of cpus). So far, the only usable mechanism I've been able to come up with
is the following:

In addition to the interrupts list, we add an optional 'affinity' list - a list
of phandles, where each phandle points to a cpu node or a cluster node (from
Lorenzo Pieralisi's cpu topology bindings [1]). Each element in the interrupts
list has a corresponding element in the affinity list, at the same index.
Interrupts defined as cluster affine are PPIs, and those defined cpu-affine are
SPIs. This would look something like:

> device@0 {
> 	compatible = "example,example-affine-device"
> 	interrupts = <0 130 4>,
> 	             <0 131 4>,
> 	             <1 12 4>;
> 	affinity   = <&cpu0>,
>                    <&cpu1>,
> 	             <&cluster1>;
> };

To request/free interrupts, drivers would walk over all interrupts, and request
the affine cpu(s), which can be automatically mapped to logical ids by common
code. This fixes issues 1, 2, and 3. While 4 is partially solved, it doesn't
provide any additional information when you may have multiple interrupts for a
cpu.

The following patches implement common code that could be used by drivers,
assuming the devicetree uses the cluster topology bindings. To not break
existing bindings, when an affinity parameter is not present, interrupts are
assumed to be in increasing order of MPIDR.Aff0, starting at 0. Logical mapping
is always applied to cpu ids.

The patches are based on v3.7-rc2, with Lorenzo Pieralisi's logical mappings
look-up patch [2] applied.

Using this code, an interrupt registration loop would look something like:

> void request_irqs(...)
> {
> 	int i, irqs, irq, c, cpus, cpu;
>
> 	irqs = pdev->num_resources;
> 	for (i = 0; i < irqs; i++) {
> 		irq = platform_get_irq(pdev, i);
> 		cpus = count_irq_affine_cpus(pdev, i);
>
>		/* SPI if affine to 1 cpu */
>		if (cpus == 1) {
>			cpu = get_irq_affine_cpu(pdev, i, 1);
>			if (cpu < 0)
>				continue;
>			irq_set_affinity(irq, cpumask_of(cpu));
>			request_irq(...);
>
>			continue;
>		}
>
>		/* PPI, build cpumask and register */
> 		for (c = 0; c < cpus; c++) {
> 			cpu = get_irq_affine_cpu(pdev, i, c);
>
> 			/* test, add to mask */
> 			...
> 		}
>		
>		request_percpu_irq(...);
> 	}
> }

Does anyone have any better ideas?

Any and all feedback welcome.

Thanks,
Mark.

[1] http://lists.infradead.org/pipermail/linux-arm-kernel/2012-January/080869.html
[2] http://lists.infradead.org/pipermail/linux-arm-kernel/2012-October/125882.html

Mark Rutland (2):
  of: add of_property_count_phandles
  ARM: Add functions to parse dt irq affinity

 arch/arm/include/asm/dt_irq.h |   64 ++++++++++++++++
 arch/arm/kernel/devtree.c     |  165 +++++++++++++++++++++++++++++++++++++++++
 drivers/of/base.c             |   39 ++++++++++
 include/linux/of.h            |    8 ++
 4 files changed, 276 insertions(+), 0 deletions(-)
 create mode 100644 arch/arm/include/asm/dt_irq.h

[PATCH 1/2] of: add of_property_count_phandles

[Mark Rutland]

Though of_parse_phandle handles lists of phandles, and takes an index
parameter, there is no standard way of discovering how many phandles are
present on a node. This patch adds a function to count how many phandles
are in a phandle list.

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
---
 drivers/of/base.c  |   39 +++++++++++++++++++++++++++++++++++++++
 include/linux/of.h |    8 ++++++++
 2 files changed, 47 insertions(+), 0 deletions(-)

diff --git a/drivers/of/base.c b/drivers/of/base.c
index af3b22a..30ae562 100644
--- a/drivers/of/base.c
+++ b/drivers/of/base.c
@@ -884,6 +884,45 @@ int of_property_count_strings(struct device_node *np, const char *propname)
 EXPORT_SYMBOL_GPL(of_property_count_strings);
 
 /**
+ * of_property_count_phandles - find and return the number of phandles from a
+ * multiple phandles property.
+ * @np:		device node from which the property value is to be read.
+ * @propname:	name of the property to be searched.
+ *
+ * Search for a property in a device tree node and retrieve the number of
+ * phandles contained in it. Returns the number of phandles on success, -ENOENT
+ * if the property does not exist, -ENODATA if there are no phandles, and
+ * -EINVAL if the property is not correctly sized for an array of phandles.
+ */
+int of_property_count_phandles(struct device_node *np, const char *propname)
+{
+	int size, count, i;
+	const __be32 *phandle;
+
+	phandle = of_get_property(np, propname, &size);
+
+	if (!phandle)
+		return -ENOENT;
+	if (size == 0)
+		return -ENODATA;
+	if (size % (sizeof *phandle))
+		return -EINVAL;
+
+	count = size / (sizeof(*phandle));
+
+	/* Check the phandle targets actually exist */
+	for (i = 0; i < count; i++) {
+		struct device_node *node = of_parse_phandle(np, propname, i);
+		if (!node)
+			return -EINVAL;
+		of_node_put(node);
+	}
+
+	return count;
+}
+EXPORT_SYMBOL_GPL(of_property_count_phandles);
+
+/**
  * of_parse_phandle - Resolve a phandle property to a device_node pointer
  * @np: Pointer to device node holding phandle property
  * @phandle_name: Name of property holding a phandle value
diff --git a/include/linux/of.h b/include/linux/of.h
index b4e50d5..1f03f46 100644
--- a/include/linux/of.h
+++ b/include/linux/of.h
@@ -258,6 +258,8 @@ extern int of_modalias_node(struct device_node *node, char *modalias, int len);
 extern struct device_node *of_parse_phandle(struct device_node *np,
 					    const char *phandle_name,
 					    int index);
+extern int of_property_count_phandles(struct device_node *np,
+				      const char *propname);
 extern int of_parse_phandle_with_args(struct device_node *np,
 	const char *list_name, const char *cells_name, int index,
 	struct of_phandle_args *out_args);
@@ -411,6 +413,12 @@ static inline int of_property_match_string(struct device_node *np,
 	return -ENOSYS;
 }
 
+static inline int of_property_count_phandles(struct device_node *np,
+					     const char *propname)
+{
+	return -ENOSYS;
+}
+
 static inline struct device_node *of_parse_phandle(struct device_node *np,
 						   const char *phandle_name,
 						   int index)
-- 
1.7.0.4

[PATCH 2/2] ARM: Add functions to parse dt irq affinity

[Mark Rutland]

This patch adds functions to handle parsing cpu affinity of interrupts
from devicetree. Devices using these functions are dealt with as
follows:

* If the device was not initialised from devicetree, it is considered to
  be affine to all CPUs, and interrupts are assumed to be in order of
  physical CPU id (MPIDR.Aff0).

* If the device was initialised from devicetree, but does not have an
  "affinity" value, the device is considered to be affine to all CPUs,
  and interupts are assumed to be in order of physical CPU id
  (MPIDR.Aff0).

* If the device was initialised from devicetree, and has an "affinity"
  value, then the interrupt at index i in the interrupts list is affine
  to the cpu or cluster pointed to by the entry in the affinity list at
  index i. Interrupts affine to a cluster are PPIs, and interrupts
  affine to a single cpu are SPIs.

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
---
 arch/arm/include/asm/dt_irq.h |   64 ++++++++++++++++
 arch/arm/kernel/devtree.c     |  164 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 228 insertions(+), 0 deletions(-)
 create mode 100644 arch/arm/include/asm/dt_irq.h

diff --git a/arch/arm/include/asm/dt_irq.h b/arch/arm/include/asm/dt_irq.h
new file mode 100644
index 0000000..ed86ca8
--- /dev/null
+++ b/arch/arm/include/asm/dt_irq.h
@@ -0,0 +1,64 @@
+/*
+ *  arch/arm/include/asm/dt_irq.h
+ *
+ *  Copyright (C) 2009 ARM Ltd. <mark.rutland@arm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ */
+#ifndef __ASMARM_DT_IRQ_H
+#define __ASMARM_DT_IRQ_H
+
+#include <linux/cpumask.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+
+#ifdef CONFIG_OF
+
+extern int count_cpus_in_cluster(struct device_node *cluster);
+extern struct device_node *of_get_cluster_cpu(struct device_node *cluster, int idx);
+extern int count_irq_affine_cpus(struct platform_device *pdev, int irq_idx);
+extern int of_get_logical_cpu_id(struct device_node *cpu);
+extern int get_irq_affine_cpu(struct platform_device *pdev, int irq_idx, int cpu_idx);
+extern int get_device_cpu_affinity(struct platform_device *pdev, cpumask_t *mask);
+
+#else /* CONFIG_OF */
+
+static int count_cpus_in_cluster(struct device_node *cluster)
+{
+	return -EINVAL;
+}
+
+static struct device_node *of_get_cluster_cpu(struct device_node *cluster, int idx)
+{
+	return NULL;
+}
+
+static int count_irq_affine_cpus(struct platform_device *pdev, int irq_idx)
+{
+	return 1;
+}
+
+static int of_get_logical_cpu_id(struct device_node *cpu)
+{
+	return -EINVAL;
+}
+
+static int get_irq_affine_cpu(struct platform_device *pdev, int irq_idx, int cpu_idx);
+{
+	if (cpu_idx != 0)
+		return -EINVAL;
+
+	return get_logical_index(irq_idx);
+}
+
+static int get_device_cpu_affinity(struct platform_device *pdev, cpumask_t *mask)
+{
+	cpumask_setall(mask);
+	return 0;
+}
+
+#endif /* CONFIG_OF */
+#endif /* ASMARM_DT_IRQ_H */
diff --git a/arch/arm/kernel/devtree.c b/arch/arm/kernel/devtree.c
index bee7f9d..4026827 100644
--- a/arch/arm/kernel/devtree.c
+++ b/arch/arm/kernel/devtree.c
@@ -132,3 +132,167 @@ struct machine_desc * __init setup_machine_fdt(unsigned int dt_phys)
 
 	return mdesc_best;
 }
+
+int count_cpus_in_cluster(struct device_node *cluster)
+{
+	struct device_node *cpus, *cpu, *cpu_cluster;
+	int count = 0;
+
+	cpus = of_find_node_by_path("/cpus");
+	if (!cpus)
+		return -EINVAL;
+
+	for_each_child_of_node(cpus, cpu) {
+		cpu_cluster = of_parse_phandle(cpu, "cluster", 0);
+		if (cpu_cluster == cluster)
+			count++;
+		of_node_put(cpu_cluster);
+	}
+
+	of_node_put(cpus);
+	return count;
+}
+
+struct device_node *of_get_cluster_cpu(struct device_node *cluster, int idx)
+{
+	struct device_node *cpus, *cpu, *cpu_cluster;
+
+	cpus = of_find_node_by_path("/cpus");
+	if (!cpus)
+		return NULL;
+
+	for_each_child_of_node(cpus, cpu) {
+		cpu_cluster = of_parse_phandle(cpu, "cluster", 0);
+		if (cpu_cluster == cluster && idx == 0) {
+			of_node_put(cpu_cluster);
+			break;
+		}
+
+		idx--;
+		of_node_put(cpu_cluster);
+	}
+
+	of_node_put(cpus);
+	return cpu;
+}
+
+int count_irq_affine_cpus(struct platform_device *pdev, int irq_idx)
+{
+	struct device_node *node, *affine;
+	int ret = -EINVAL;
+
+	if (!pdev)
+		return -EINVAL;
+
+	node = pdev->dev.of_node;
+
+	/*
+	 * For devices not initialised from devicetree, we only support sets of
+	 * SPIs.
+	 */
+	if (!node)
+		return 1;
+
+	affine = of_parse_phandle(node, "affinity", irq_idx);
+
+	/*
+	 * A device initialised from devicetree without an affinity parameter
+	 * is assumed to have a set of cpu-affine SPIs (so 1 cpu per irq).
+	 */
+	if (!affine)
+		return 1;
+
+	if (!of_node_cmp(affine->name, "cpu"))
+		ret = 1;
+	else if (!of_node_cmp(affine->name, "cluster"))
+		ret = count_cpus_in_cluster(affine);
+
+	of_node_put(affine);
+	return ret;
+}
+
+int of_get_logical_cpu_id(struct device_node *cpu)
+{
+	u32 mpidr;
+	if (of_property_read_u32(cpu, "reg", &mpidr))
+		return -EINVAL;
+
+	return get_logical_index(mpidr);
+}
+
+int get_irq_affine_cpu(struct platform_device *pdev, int irq_idx, int cpu_idx)
+{
+	struct device_node *node, *affine, *cpu;
+	int ret = -EINVAL;
+
+	if (!pdev)
+		return -EINVAL;
+
+	node = pdev->dev.of_node;
+
+	/*
+	 * Without devicetree, we only support single cluster systems.
+	 * We assume 1 SPI per CPU.
+	 * CPUs are assumed to be in increasing order of MPIDR.Aff{0},
+	 * starting at 0. MPIDR.Aff{2,1} are assumed to be 0.
+	 */
+	if (!node) {
+		if (cpu_idx != 0)
+			return -EINVAL;
+		return get_logical_index(irq_idx);
+	}
+
+	affine = of_parse_phandle(node, "affinity", irq_idx);
+	if (!affine) {
+		if (cpu_idx != 0)
+			return -EINVAL;
+		return get_logical_index(irq_idx);
+	}
+
+	if (!of_node_cmp(affine->name, "cluster")) {
+		cpu = of_get_cluster_cpu(affine, cpu_idx);
+		if (cpu) {
+			ret = of_get_logical_cpu_id(cpu);
+			of_node_put(cpu);
+		}
+	}
+	else if (!of_node_cmp(affine->name, "cpu")) {
+		ret = of_get_logical_cpu_id(affine);
+	}
+
+	of_node_put(affine);
+	return ret;
+}
+
+int get_device_cpu_affinity(struct platform_device *pdev, cpumask_t *mask)
+{
+	struct device_node *node;
+	int affines, a, cpus, c, cpu;
+
+	node = pdev->dev.of_node;
+
+	if (!node) {
+		cpumask_setall(mask);
+		return 0;
+	}
+
+	affines = of_property_count_phandles(node, "affinity");
+	if (affines == -ENOENT) {
+		cpumask_setall(mask);
+		return 0;
+	}
+
+	if (affines < 0)
+		return -EINVAL;
+
+	for (a = 0; a < affines; a++) {
+		cpus = count_irq_affine_cpus(pdev, a);
+		for (c = 0; c < cpus; c++) {
+			cpu = get_irq_affine_cpu(pdev, a, c);
+			if (cpu >= 0)
+				cpumask_set_cpu(cpu, mask);
+		}
+	}
+
+	return 0;
+}
-- 
1.7.0.4