[PATCH v6 0/8] Introduce automatic DMA configuration for IOMMU masters

[PATCH v6 0/8] Introduce automatic DMA configuration for IOMMU masters

[Will Deacon]

Hello again,

This is version 6 of the patches previously posted here:

  RFCv1: http://lists.infradead.org/pipermail/linux-arm-kernel/2014-August/283023.html
  RFCv2: http://lists.infradead.org/pipermail/linux-arm-kernel/2014-September/283752.html
  RFCv3: http://lists.infradead.org/pipermail/linux-arm-kernel/2014-September/287031.html
  RFCv4: http://lists.infradead.org/pipermail/linux-arm-kernel/2014-November/302711.html
     v5: http://lists.infradead.org/pipermail/linux-arm-kernel/2014-November/307213.html

The only change since v5 is the addition of acks from various maintainers.

Now that the ARM bits have rmk's ack and the IOMMU bits have joro's ack,
I think this is good for merging via the arm-soc tree.

Please let me know if a pull request would be preferable.

Cheers,

Will

--->8

Marek Szyprowski (1):
  iommu: fix initialization without 'add_device' callback

Will Deacon (7):
  iommu: provide early initialisation hook for IOMMU drivers
  dma-mapping: replace set_arch_dma_coherent_ops with arch_setup_dma_ops
  iommu: add new iommu_ops callback for adding an OF device
  iommu: provide helper function to configure an IOMMU for an of master
  dma-mapping: detect and configure IOMMU in of_dma_configure
  arm: call iommu_init before of_platform_populate
  arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

 arch/arm/include/asm/dma-mapping.h | 12 +++---
 arch/arm/kernel/setup.c            |  2 +
 arch/arm/mm/dma-mapping.c          | 83 ++++++++++++++++++++++++++++++++++----
 drivers/iommu/Kconfig              |  2 +-
 drivers/iommu/iommu.c              |  2 +-
 drivers/iommu/of_iommu.c           | 50 +++++++++++++++++++++++
 drivers/of/platform.c              | 50 ++++++++++++-----------
 include/asm-generic/vmlinux.lds.h  |  2 +
 include/linux/dma-mapping.h        | 13 +++---
 include/linux/iommu.h              |  8 ++++
 include/linux/of_iommu.h           | 31 ++++++++++++++
 11 files changed, 211 insertions(+), 44 deletions(-)

-- 
2.1.1

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Will Deacon]

IOMMU drivers must be initialised before any of their upstream devices,
otherwise the relevant iommu_ops won't be configured for the bus in
question. To solve this, a number of IOMMU drivers use initcalls to
initialise the driver before anything has a chance to be probed.

Whilst this solves the immediate problem, it leaves the job of probing
the IOMMU completely separate from the iommu_ops to configure the IOMMU,
which are called on a per-bus basis and require the driver to figure out
exactly which instance of the IOMMU is being requested. In particular,
the add_device callback simply passes a struct device to the driver,
which then has to parse firmware tables or probe buses to identify the
relevant IOMMU instance.

This patch takes the first step in addressing this problem by adding an
early initialisation pass for IOMMU drivers, giving them the ability to
store some per-instance data in their iommu_ops structure and store that
in their of_node. This can later be used when parsing OF masters to
identify the IOMMU instance in question.

Acked-by: Arnd Bergmann <arnd@arndb.de>
Acked-by: Joerg Roedel <jroedel@suse.de>
Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
Tested-by: Robin Murphy <robin.murphy@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
 drivers/iommu/of_iommu.c          | 17 +++++++++++++++++
 include/asm-generic/vmlinux.lds.h |  2 ++
 include/linux/iommu.h             |  2 ++
 include/linux/of_iommu.h          | 25 +++++++++++++++++++++++++
 4 files changed, 46 insertions(+)

diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
index e550ccb7634e..89b903406968 100644
--- a/drivers/iommu/of_iommu.c
+++ b/drivers/iommu/of_iommu.c
@@ -22,6 +22,9 @@
 #include <linux/of.h>
 #include <linux/of_iommu.h>
 
+static const struct of_device_id __iommu_of_table_sentinel
+	__used __section(__iommu_of_table_end);
+
 /**
  * of_get_dma_window - Parse *dma-window property and returns 0 if found.
  *
@@ -89,3 +92,17 @@ int of_get_dma_window(struct device_node *dn, const char *prefix, int index,
 	return 0;
 }
 EXPORT_SYMBOL_GPL(of_get_dma_window);
+
+void __init of_iommu_init(void)
+{
+	struct device_node *np;
+	const struct of_device_id *match, *matches = &__iommu_of_table;
+
+	for_each_matching_node_and_match(np, matches, &match) {
+		const of_iommu_init_fn init_fn = match->data;
+
+		if (init_fn(np))
+			pr_err("Failed to initialise IOMMU %s\n",
+				of_node_full_name(np));
+	}
+}
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index aa70cbda327c..bee5d683074d 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -164,6 +164,7 @@
 #define CLKSRC_OF_TABLES()	OF_TABLE(CONFIG_CLKSRC_OF, clksrc)
 #define IRQCHIP_OF_MATCH_TABLE() OF_TABLE(CONFIG_IRQCHIP, irqchip)
 #define CLK_OF_TABLES()		OF_TABLE(CONFIG_COMMON_CLK, clk)
+#define IOMMU_OF_TABLES()	OF_TABLE(CONFIG_OF_IOMMU, iommu)
 #define RESERVEDMEM_OF_TABLES()	OF_TABLE(CONFIG_OF_RESERVED_MEM, reservedmem)
 #define CPU_METHOD_OF_TABLES()	OF_TABLE(CONFIG_SMP, cpu_method)
 #define EARLYCON_OF_TABLES()	OF_TABLE(CONFIG_SERIAL_EARLYCON, earlycon)
@@ -497,6 +498,7 @@
 	CLK_OF_TABLES()							\
 	RESERVEDMEM_OF_TABLES()						\
 	CLKSRC_OF_TABLES()						\
+	IOMMU_OF_TABLES()						\
 	CPU_METHOD_OF_TABLES()						\
 	KERNEL_DTB()							\
 	IRQCHIP_OF_MATCH_TABLE()					\
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index e6a7c9ff72f2..7b83f9f8e11d 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -103,6 +103,7 @@ enum iommu_attr {
  * @domain_get_attr: Query domain attributes
  * @domain_set_attr: Change domain attributes
  * @pgsize_bitmap: bitmap of supported page sizes
+ * @priv: per-instance data private to the iommu driver
  */
 struct iommu_ops {
 	bool (*capable)(enum iommu_cap);
@@ -133,6 +134,7 @@ struct iommu_ops {
 	u32 (*domain_get_windows)(struct iommu_domain *domain);
 
 	unsigned long pgsize_bitmap;
+	void *priv;
 };
 
 #define IOMMU_GROUP_NOTIFY_ADD_DEVICE		1 /* Device added */
diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h
index 51a560f34bca..5762cdc8effe 100644
--- a/include/linux/of_iommu.h
+++ b/include/linux/of_iommu.h
@@ -1,12 +1,17 @@
 #ifndef __OF_IOMMU_H
 #define __OF_IOMMU_H
 
+#include <linux/iommu.h>
+#include <linux/of.h>
+
 #ifdef CONFIG_OF_IOMMU
 
 extern int of_get_dma_window(struct device_node *dn, const char *prefix,
 			     int index, unsigned long *busno, dma_addr_t *addr,
 			     size_t *size);
 
+extern void of_iommu_init(void);
+
 #else
 
 static inline int of_get_dma_window(struct device_node *dn, const char *prefix,
@@ -16,6 +21,26 @@ static inline int of_get_dma_window(struct device_node *dn, const char *prefix,
 	return -EINVAL;
 }
 
+static inline void of_iommu_init(void) { }
+
 #endif	/* CONFIG_OF_IOMMU */
 
+static inline void of_iommu_set_ops(struct device_node *np,
+				    const struct iommu_ops *ops)
+{
+	np->data = (struct iommu_ops *)ops;
+}
+
+static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
+{
+	return np->data;
+}
+
+extern struct of_device_id __iommu_of_table;
+
+typedef int (*of_iommu_init_fn)(struct device_node *);
+
+#define IOMMU_OF_DECLARE(name, compat, fn) \
+	_OF_DECLARE(iommu, name, compat, fn, of_iommu_init_fn)
+
 #endif /* __OF_IOMMU_H */
-- 
2.1.1

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Rob Herring]

Adding Grant and Pantelis...

On Mon, Dec 1, 2014 at 10:57 AM, Will Deacon <will.deacon@arm.com> wrote:
> IOMMU drivers must be initialised before any of their upstream devices,
> otherwise the relevant iommu_ops won't be configured for the bus in
> question. To solve this, a number of IOMMU drivers use initcalls to
> initialise the driver before anything has a chance to be probed.
>
> Whilst this solves the immediate problem, it leaves the job of probing
> the IOMMU completely separate from the iommu_ops to configure the IOMMU,
> which are called on a per-bus basis and require the driver to figure out
> exactly which instance of the IOMMU is being requested. In particular,
> the add_device callback simply passes a struct device to the driver,
> which then has to parse firmware tables or probe buses to identify the
> relevant IOMMU instance.
>
> This patch takes the first step in addressing this problem by adding an
> early initialisation pass for IOMMU drivers, giving them the ability to
> store some per-instance data in their iommu_ops structure and store that
> in their of_node. This can later be used when parsing OF masters to
> identify the IOMMU instance in question.
>
> Acked-by: Arnd Bergmann <arnd@arndb.de>
> Acked-by: Joerg Roedel <jroedel@suse.de>
> Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
> Tested-by: Robin Murphy <robin.murphy@arm.com>
> Signed-off-by: Will Deacon <will.deacon@arm.com>
> ---
>  drivers/iommu/of_iommu.c          | 17 +++++++++++++++++
>  include/asm-generic/vmlinux.lds.h |  2 ++
>  include/linux/iommu.h             |  2 ++
>  include/linux/of_iommu.h          | 25 +++++++++++++++++++++++++
>  4 files changed, 46 insertions(+)
>
> diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
> index e550ccb7634e..89b903406968 100644
> --- a/drivers/iommu/of_iommu.c
> +++ b/drivers/iommu/of_iommu.c
> @@ -22,6 +22,9 @@
>  #include <linux/of.h>
>  #include <linux/of_iommu.h>
>
> +static const struct of_device_id __iommu_of_table_sentinel
> +       __used __section(__iommu_of_table_end);
> +
>  /**
>   * of_get_dma_window - Parse *dma-window property and returns 0 if found.
>   *
> @@ -89,3 +92,17 @@ int of_get_dma_window(struct device_node *dn, const char *prefix, int index,
>         return 0;
>  }
>  EXPORT_SYMBOL_GPL(of_get_dma_window);
> +
> +void __init of_iommu_init(void)
> +{
> +       struct device_node *np;
> +       const struct of_device_id *match, *matches = &__iommu_of_table;
> +
> +       for_each_matching_node_and_match(np, matches, &match) {
> +               const of_iommu_init_fn init_fn = match->data;
> +
> +               if (init_fn(np))
> +                       pr_err("Failed to initialise IOMMU %s\n",
> +                               of_node_full_name(np));
> +       }
> +}
> diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
> index aa70cbda327c..bee5d683074d 100644
> --- a/include/asm-generic/vmlinux.lds.h
> +++ b/include/asm-generic/vmlinux.lds.h
> @@ -164,6 +164,7 @@
>  #define CLKSRC_OF_TABLES()     OF_TABLE(CONFIG_CLKSRC_OF, clksrc)
>  #define IRQCHIP_OF_MATCH_TABLE() OF_TABLE(CONFIG_IRQCHIP, irqchip)
>  #define CLK_OF_TABLES()                OF_TABLE(CONFIG_COMMON_CLK, clk)
> +#define IOMMU_OF_TABLES()      OF_TABLE(CONFIG_OF_IOMMU, iommu)
>  #define RESERVEDMEM_OF_TABLES()        OF_TABLE(CONFIG_OF_RESERVED_MEM, reservedmem)
>  #define CPU_METHOD_OF_TABLES() OF_TABLE(CONFIG_SMP, cpu_method)
>  #define EARLYCON_OF_TABLES()   OF_TABLE(CONFIG_SERIAL_EARLYCON, earlycon)
> @@ -497,6 +498,7 @@
>         CLK_OF_TABLES()                                                 \
>         RESERVEDMEM_OF_TABLES()                                         \
>         CLKSRC_OF_TABLES()                                              \
> +       IOMMU_OF_TABLES()                                               \
>         CPU_METHOD_OF_TABLES()                                          \
>         KERNEL_DTB()                                                    \
>         IRQCHIP_OF_MATCH_TABLE()                                        \
> diff --git a/include/linux/iommu.h b/include/linux/iommu.h
> index e6a7c9ff72f2..7b83f9f8e11d 100644
> --- a/include/linux/iommu.h
> +++ b/include/linux/iommu.h
> @@ -103,6 +103,7 @@ enum iommu_attr {
>   * @domain_get_attr: Query domain attributes
>   * @domain_set_attr: Change domain attributes
>   * @pgsize_bitmap: bitmap of supported page sizes
> + * @priv: per-instance data private to the iommu driver
>   */
>  struct iommu_ops {
>         bool (*capable)(enum iommu_cap);
> @@ -133,6 +134,7 @@ struct iommu_ops {
>         u32 (*domain_get_windows)(struct iommu_domain *domain);
>
>         unsigned long pgsize_bitmap;
> +       void *priv;
>  };
>
>  #define IOMMU_GROUP_NOTIFY_ADD_DEVICE          1 /* Device added */
> diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h
> index 51a560f34bca..5762cdc8effe 100644
> --- a/include/linux/of_iommu.h
> +++ b/include/linux/of_iommu.h
> @@ -1,12 +1,17 @@
>  #ifndef __OF_IOMMU_H
>  #define __OF_IOMMU_H
>
> +#include <linux/iommu.h>
> +#include <linux/of.h>
> +
>  #ifdef CONFIG_OF_IOMMU
>
>  extern int of_get_dma_window(struct device_node *dn, const char *prefix,
>                              int index, unsigned long *busno, dma_addr_t *addr,
>                              size_t *size);
>
> +extern void of_iommu_init(void);
> +
>  #else
>
>  static inline int of_get_dma_window(struct device_node *dn, const char *prefix,
> @@ -16,6 +21,26 @@ static inline int of_get_dma_window(struct device_node *dn, const char *prefix,
>         return -EINVAL;
>  }
>
> +static inline void of_iommu_init(void) { }
> +
>  #endif /* CONFIG_OF_IOMMU */
>
> +static inline void of_iommu_set_ops(struct device_node *np,
> +                                   const struct iommu_ops *ops)
> +{
> +       np->data = (struct iommu_ops *)ops;
> +}
> +
> +static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
> +{
> +       return np->data;
> +}

This may collide with other users. While use of it is rare, PPC uses
it in its PCI code. The OF_DYNAMIC code frees it but never actually
sets it. There may be some coming usage with the DT overlay code or
that's just a bug. Pantelis or Grant can comment. If not, I think we
really should try to get rid of this pointer rather than expand it's
usage.

I didn't see a user of this. I'm guessing that is coming in a SMMU patch?

Rob

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Marek Szyprowski]

Hello,

On 2014-12-02 00:54, Rob Herring wrote:
> Adding Grant and Pantelis...
>
> On Mon, Dec 1, 2014 at 10:57 AM, Will Deacon <will.deacon@arm.com> wrote:
>> IOMMU drivers must be initialised before any of their upstream devices,
>> otherwise the relevant iommu_ops won't be configured for the bus in
>> question. To solve this, a number of IOMMU drivers use initcalls to
>> initialise the driver before anything has a chance to be probed.
>>
>> Whilst this solves the immediate problem, it leaves the job of probing
>> the IOMMU completely separate from the iommu_ops to configure the IOMMU,
>> which are called on a per-bus basis and require the driver to figure out
>> exactly which instance of the IOMMU is being requested. In particular,
>> the add_device callback simply passes a struct device to the driver,
>> which then has to parse firmware tables or probe buses to identify the
>> relevant IOMMU instance.
>>
>> This patch takes the first step in addressing this problem by adding an
>> early initialisation pass for IOMMU drivers, giving them the ability to
>> store some per-instance data in their iommu_ops structure and store that
>> in their of_node. This can later be used when parsing OF masters to
>> identify the IOMMU instance in question.
>>
>> Acked-by: Arnd Bergmann <arnd@arndb.de>
>> Acked-by: Joerg Roedel <jroedel@suse.de>
>> Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
>> Tested-by: Robin Murphy <robin.murphy@arm.com>
>> Signed-off-by: Will Deacon <will.deacon@arm.com>
>> ---
>>   drivers/iommu/of_iommu.c          | 17 +++++++++++++++++
>>   include/asm-generic/vmlinux.lds.h |  2 ++
>>   include/linux/iommu.h             |  2 ++
>>   include/linux/of_iommu.h          | 25 +++++++++++++++++++++++++
>>   4 files changed, 46 insertions(+)
>>
>> diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
>> index e550ccb7634e..89b903406968 100644
>> --- a/drivers/iommu/of_iommu.c
>> +++ b/drivers/iommu/of_iommu.c
>> @@ -22,6 +22,9 @@
>>   #include <linux/of.h>
>>   #include <linux/of_iommu.h>
>>
>> +static const struct of_device_id __iommu_of_table_sentinel
>> +       __used __section(__iommu_of_table_end);
>> +
>>   /**
>>    * of_get_dma_window - Parse *dma-window property and returns 0 if found.
>>    *
>> @@ -89,3 +92,17 @@ int of_get_dma_window(struct device_node *dn, const char *prefix, int index,
>>          return 0;
>>   }
>>   EXPORT_SYMBOL_GPL(of_get_dma_window);
>> +
>> +void __init of_iommu_init(void)
>> +{
>> +       struct device_node *np;
>> +       const struct of_device_id *match, *matches = &__iommu_of_table;
>> +
>> +       for_each_matching_node_and_match(np, matches, &match) {
>> +               const of_iommu_init_fn init_fn = match->data;
>> +
>> +               if (init_fn(np))
>> +                       pr_err("Failed to initialise IOMMU %s\n",
>> +                               of_node_full_name(np));
>> +       }
>> +}
>> diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
>> index aa70cbda327c..bee5d683074d 100644
>> --- a/include/asm-generic/vmlinux.lds.h
>> +++ b/include/asm-generic/vmlinux.lds.h
>> @@ -164,6 +164,7 @@
>>   #define CLKSRC_OF_TABLES()     OF_TABLE(CONFIG_CLKSRC_OF, clksrc)
>>   #define IRQCHIP_OF_MATCH_TABLE() OF_TABLE(CONFIG_IRQCHIP, irqchip)
>>   #define CLK_OF_TABLES()                OF_TABLE(CONFIG_COMMON_CLK, clk)
>> +#define IOMMU_OF_TABLES()      OF_TABLE(CONFIG_OF_IOMMU, iommu)
>>   #define RESERVEDMEM_OF_TABLES()        OF_TABLE(CONFIG_OF_RESERVED_MEM, reservedmem)
>>   #define CPU_METHOD_OF_TABLES() OF_TABLE(CONFIG_SMP, cpu_method)
>>   #define EARLYCON_OF_TABLES()   OF_TABLE(CONFIG_SERIAL_EARLYCON, earlycon)
>> @@ -497,6 +498,7 @@
>>          CLK_OF_TABLES()                                                 \
>>          RESERVEDMEM_OF_TABLES()                                         \
>>          CLKSRC_OF_TABLES()                                              \
>> +       IOMMU_OF_TABLES()                                               \
>>          CPU_METHOD_OF_TABLES()                                          \
>>          KERNEL_DTB()                                                    \
>>          IRQCHIP_OF_MATCH_TABLE()                                        \
>> diff --git a/include/linux/iommu.h b/include/linux/iommu.h
>> index e6a7c9ff72f2..7b83f9f8e11d 100644
>> --- a/include/linux/iommu.h
>> +++ b/include/linux/iommu.h
>> @@ -103,6 +103,7 @@ enum iommu_attr {
>>    * @domain_get_attr: Query domain attributes
>>    * @domain_set_attr: Change domain attributes
>>    * @pgsize_bitmap: bitmap of supported page sizes
>> + * @priv: per-instance data private to the iommu driver
>>    */
>>   struct iommu_ops {
>>          bool (*capable)(enum iommu_cap);
>> @@ -133,6 +134,7 @@ struct iommu_ops {
>>          u32 (*domain_get_windows)(struct iommu_domain *domain);
>>
>>          unsigned long pgsize_bitmap;
>> +       void *priv;
>>   };
>>
>>   #define IOMMU_GROUP_NOTIFY_ADD_DEVICE          1 /* Device added */
>> diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h
>> index 51a560f34bca..5762cdc8effe 100644
>> --- a/include/linux/of_iommu.h
>> +++ b/include/linux/of_iommu.h
>> @@ -1,12 +1,17 @@
>>   #ifndef __OF_IOMMU_H
>>   #define __OF_IOMMU_H
>>
>> +#include <linux/iommu.h>
>> +#include <linux/of.h>
>> +
>>   #ifdef CONFIG_OF_IOMMU
>>
>>   extern int of_get_dma_window(struct device_node *dn, const char *prefix,
>>                               int index, unsigned long *busno, dma_addr_t *addr,
>>                               size_t *size);
>>
>> +extern void of_iommu_init(void);
>> +
>>   #else
>>
>>   static inline int of_get_dma_window(struct device_node *dn, const char *prefix,
>> @@ -16,6 +21,26 @@ static inline int of_get_dma_window(struct device_node *dn, const char *prefix,
>>          return -EINVAL;
>>   }
>>
>> +static inline void of_iommu_init(void) { }
>> +
>>   #endif /* CONFIG_OF_IOMMU */
>>
>> +static inline void of_iommu_set_ops(struct device_node *np,
>> +                                   const struct iommu_ops *ops)
>> +{
>> +       np->data = (struct iommu_ops *)ops;
>> +}
>> +
>> +static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
>> +{
>> +       return np->data;
>> +}
> This may collide with other users. While use of it is rare, PPC uses
> it in its PCI code. The OF_DYNAMIC code frees it but never actually
> sets it. There may be some coming usage with the DT overlay code or
> that's just a bug. Pantelis or Grant can comment. If not, I think we
> really should try to get rid of this pointer rather than expand it's
> usage.

I think that for the initial version it is ok to use np->data. When 
per-iommu
controller structure is introduced later, it can be reused also for 
performing
of_node to iommu_ops lookup, because IOMMU framework will need to keep track
on all such iommu controllers anyway.

> I didn't see a user of this. I'm guessing that is coming in a SMMU patch?

It is used in of_iommu_configure() function from "[PATCH v6 4/8] iommu:
provide helper function to configure an IOMMU for an of master". Example
driver converted to this framework is available here:
http://www.spinics.net/lists/linux-samsung-soc/msg39168.html

Best regards
-- 
Marek Szyprowski, PhD
Samsung R&D Institute Poland

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Arnd Bergmann]

On Tuesday 02 December 2014 10:23:00 Marek Szyprowski wrote:
> >> +static inline void of_iommu_set_ops(struct device_node *np,
> >> +                                   const struct iommu_ops *ops)
> >> +{
> >> +       np->data = (struct iommu_ops *)ops;
> >> +}
> >> +
> >> +static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
> >> +{
> >> +       return np->data;
> >> +}
> > This may collide with other users. While use of it is rare, PPC uses
> > it in its PCI code. The OF_DYNAMIC code frees it but never actually
> > sets it. There may be some coming usage with the DT overlay code or
> > that's just a bug. Pantelis or Grant can comment. If not, I think we
> > really should try to get rid of this pointer rather than expand it's
> > usage.
> 
> I think that for the initial version it is ok to use np->data. When 
> per-iommu
> controller structure is introduced later, it can be reused also for 
> performing
> of_node to iommu_ops lookup, because IOMMU framework will need to keep track
> on all such iommu controllers anyway.

Agreed. I think in the long run, we will have a 'struct iommu' that is
added into a global linked list and that contains (among other things)
an iommu_ops pointer and a device_node pointer. The of_iommu_get_ops
function then walks the list to find the right iommu instance.

	Arnd

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Will Deacon]

On Tue, Dec 02, 2014 at 09:36:59AM +0000, Arnd Bergmann wrote:
> On Tuesday 02 December 2014 10:23:00 Marek Szyprowski wrote:
> > >> +static inline void of_iommu_set_ops(struct device_node *np,
> > >> +                                   const struct iommu_ops *ops)
> > >> +{
> > >> +       np->data = (struct iommu_ops *)ops;
> > >> +}
> > >> +
> > >> +static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
> > >> +{
> > >> +       return np->data;
> > >> +}
> > > This may collide with other users. While use of it is rare, PPC uses
> > > it in its PCI code. The OF_DYNAMIC code frees it but never actually
> > > sets it. There may be some coming usage with the DT overlay code or
> > > that's just a bug. Pantelis or Grant can comment. If not, I think we
> > > really should try to get rid of this pointer rather than expand it's
> > > usage.
> > 
> > I think that for the initial version it is ok to use np->data. When 
> > per-iommu
> > controller structure is introduced later, it can be reused also for 
> > performing
> > of_node to iommu_ops lookup, because IOMMU framework will need to keep track
> > on all such iommu controllers anyway.
> 
> Agreed. I think in the long run, we will have a 'struct iommu' that is
> added into a global linked list and that contains (among other things)
> an iommu_ops pointer and a device_node pointer. The of_iommu_get_ops
> function then walks the list to find the right iommu instance.

Yup, I plan to look at that after Christmas because I need it for the
per-IOMMU instance pgsize_bitmap anyway.

Will

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Thierry Reding]

On Tue, Dec 02, 2014 at 10:36:59AM +0100, Arnd Bergmann wrote:
> On Tuesday 02 December 2014 10:23:00 Marek Szyprowski wrote:
> > >> +static inline void of_iommu_set_ops(struct device_node *np,
> > >> +                                   const struct iommu_ops *ops)
> > >> +{
> > >> +       np->data = (struct iommu_ops *)ops;
> > >> +}
> > >> +
> > >> +static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
> > >> +{
> > >> +       return np->data;
> > >> +}
> > > This may collide with other users. While use of it is rare, PPC uses
> > > it in its PCI code. The OF_DYNAMIC code frees it but never actually
> > > sets it. There may be some coming usage with the DT overlay code or
> > > that's just a bug. Pantelis or Grant can comment. If not, I think we
> > > really should try to get rid of this pointer rather than expand it's
> > > usage.
> > 
> > I think that for the initial version it is ok to use np->data. When 
> > per-iommu
> > controller structure is introduced later, it can be reused also for 
> > performing
> > of_node to iommu_ops lookup, because IOMMU framework will need to keep track
> > on all such iommu controllers anyway.
> 
> Agreed. I think in the long run, we will have a 'struct iommu' that is
> added into a global linked list and that contains (among other things)
> an iommu_ops pointer and a device_node pointer. The of_iommu_get_ops
> function then walks the list to find the right iommu instance.

If we end up doing that, how is this any different from what Hiroshi and
I initially proposed over a year ago (and then again earlier this year)?

The only remaining difference would be that this current proposal needs
IOMMUs to be registered even before other devices are instantiated for
no apparent reason other than ordering. The original proposal allowed
the IOMMU driver to register just like any other driver and ordering to
be handled via deferred probing.

Thierry
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 819 bytes
Desc: not available
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20141202/124488df/attachment.sig>

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Pantelis Antoniou]

Hi Rob,

> On Dec 2, 2014, at 01:54 , Rob Herring <robherring2@gmail.com> wrote:
> 
> Adding Grant and Pantelis...
> 
> On Mon, Dec 1, 2014 at 10:57 AM, Will Deacon <will.deacon@arm.com> wrote:
>> IOMMU drivers must be initialised before any of their upstream devices,
>> otherwise the relevant iommu_ops won't be configured for the bus in
>> question. To solve this, a number of IOMMU drivers use initcalls to
>> initialise the driver before anything has a chance to be probed.
>> 
>> Whilst this solves the immediate problem, it leaves the job of probing
>> the IOMMU completely separate from the iommu_ops to configure the IOMMU,
>> which are called on a per-bus basis and require the driver to figure out
>> exactly which instance of the IOMMU is being requested. In particular,
>> the add_device callback simply passes a struct device to the driver,
>> which then has to parse firmware tables or probe buses to identify the
>> relevant IOMMU instance.
>> 
>> This patch takes the first step in addressing this problem by adding an
>> early initialisation pass for IOMMU drivers, giving them the ability to
>> store some per-instance data in their iommu_ops structure and store that
>> in their of_node. This can later be used when parsing OF masters to
>> identify the IOMMU instance in question.
>> 
>> Acked-by: Arnd Bergmann <arnd@arndb.de>
>> Acked-by: Joerg Roedel <jroedel@suse.de>
>> Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
>> Tested-by: Robin Murphy <robin.murphy@arm.com>
>> Signed-off-by: Will Deacon <will.deacon@arm.com>
>> ---
>> drivers/iommu/of_iommu.c          | 17 +++++++++++++++++
>> include/asm-generic/vmlinux.lds.h |  2 ++
>> include/linux/iommu.h             |  2 ++
>> include/linux/of_iommu.h          | 25 +++++++++++++++++++++++++
>> 4 files changed, 46 insertions(+)
>> 
>> diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
>> index e550ccb7634e..89b903406968 100644
>> --- a/drivers/iommu/of_iommu.c
>> +++ b/drivers/iommu/of_iommu.c
>> @@ -22,6 +22,9 @@
>> #include <linux/of.h>
>> #include <linux/of_iommu.h>
>> 
>> +static const struct of_device_id __iommu_of_table_sentinel
>> +       __used __section(__iommu_of_table_end);
>> +
>> /**
>>  * of_get_dma_window - Parse *dma-window property and returns 0 if found.
>>  *
>> @@ -89,3 +92,17 @@ int of_get_dma_window(struct device_node *dn, const char *prefix, int index,
>>        return 0;
>> }
>> EXPORT_SYMBOL_GPL(of_get_dma_window);
>> +
>> +void __init of_iommu_init(void)
>> +{
>> +       struct device_node *np;
>> +       const struct of_device_id *match, *matches = &__iommu_of_table;
>> +
>> +       for_each_matching_node_and_match(np, matches, &match) {
>> +               const of_iommu_init_fn init_fn = match->data;
>> +
>> +               if (init_fn(np))
>> +                       pr_err("Failed to initialise IOMMU %s\n",
>> +                               of_node_full_name(np));
>> +       }
>> +}
>> diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
>> index aa70cbda327c..bee5d683074d 100644
>> --- a/include/asm-generic/vmlinux.lds.h
>> +++ b/include/asm-generic/vmlinux.lds.h
>> @@ -164,6 +164,7 @@
>> #define CLKSRC_OF_TABLES()     OF_TABLE(CONFIG_CLKSRC_OF, clksrc)
>> #define IRQCHIP_OF_MATCH_TABLE() OF_TABLE(CONFIG_IRQCHIP, irqchip)
>> #define CLK_OF_TABLES()                OF_TABLE(CONFIG_COMMON_CLK, clk)
>> +#define IOMMU_OF_TABLES()      OF_TABLE(CONFIG_OF_IOMMU, iommu)
>> #define RESERVEDMEM_OF_TABLES()        OF_TABLE(CONFIG_OF_RESERVED_MEM, reservedmem)
>> #define CPU_METHOD_OF_TABLES() OF_TABLE(CONFIG_SMP, cpu_method)
>> #define EARLYCON_OF_TABLES()   OF_TABLE(CONFIG_SERIAL_EARLYCON, earlycon)
>> @@ -497,6 +498,7 @@
>>        CLK_OF_TABLES()                                                 \
>>        RESERVEDMEM_OF_TABLES()                                         \
>>        CLKSRC_OF_TABLES()                                              \
>> +       IOMMU_OF_TABLES()                                               \
>>        CPU_METHOD_OF_TABLES()                                          \
>>        KERNEL_DTB()                                                    \
>>        IRQCHIP_OF_MATCH_TABLE()                                        \
>> diff --git a/include/linux/iommu.h b/include/linux/iommu.h
>> index e6a7c9ff72f2..7b83f9f8e11d 100644
>> --- a/include/linux/iommu.h
>> +++ b/include/linux/iommu.h
>> @@ -103,6 +103,7 @@ enum iommu_attr {
>>  * @domain_get_attr: Query domain attributes
>>  * @domain_set_attr: Change domain attributes
>>  * @pgsize_bitmap: bitmap of supported page sizes
>> + * @priv: per-instance data private to the iommu driver
>>  */
>> struct iommu_ops {
>>        bool (*capable)(enum iommu_cap);
>> @@ -133,6 +134,7 @@ struct iommu_ops {
>>        u32 (*domain_get_windows)(struct iommu_domain *domain);
>> 
>>        unsigned long pgsize_bitmap;
>> +       void *priv;
>> };
>> 
>> #define IOMMU_GROUP_NOTIFY_ADD_DEVICE          1 /* Device added */
>> diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h
>> index 51a560f34bca..5762cdc8effe 100644
>> --- a/include/linux/of_iommu.h
>> +++ b/include/linux/of_iommu.h
>> @@ -1,12 +1,17 @@
>> #ifndef __OF_IOMMU_H
>> #define __OF_IOMMU_H
>> 
>> +#include <linux/iommu.h>
>> +#include <linux/of.h>
>> +
>> #ifdef CONFIG_OF_IOMMU
>> 
>> extern int of_get_dma_window(struct device_node *dn, const char *prefix,
>>                             int index, unsigned long *busno, dma_addr_t *addr,
>>                             size_t *size);
>> 
>> +extern void of_iommu_init(void);
>> +
>> #else
>> 
>> static inline int of_get_dma_window(struct device_node *dn, const char *prefix,
>> @@ -16,6 +21,26 @@ static inline int of_get_dma_window(struct device_node *dn, const char *prefix,
>>        return -EINVAL;
>> }
>> 
>> +static inline void of_iommu_init(void) { }
>> +
>> #endif /* CONFIG_OF_IOMMU */
>> 
>> +static inline void of_iommu_set_ops(struct device_node *np,
>> +                                   const struct iommu_ops *ops)
>> +{
>> +       np->data = (struct iommu_ops *)ops;
>> +}
>> +
>> +static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
>> +{
>> +       return np->data;
>> +}
> 
> This may collide with other users. While use of it is rare, PPC uses
> it in its PCI code. The OF_DYNAMIC code frees it but never actually
> sets it. There may be some coming usage with the DT overlay code or
> that's just a bug. Pantelis or Grant can comment. If not, I think we
> really should try to get rid of this pointer rather than expand it's
> usage.
> 

That assignment is not good as it is. Under OF_DYNAMIC on of_node_release it will attempt
to kfree it.

For now kdup?ing the ops structure should be enough, but I?d like to see this removed eventually.
  
This data pointer just has to go. As far as I can tell it is used in CELL PPC and in the 
freescale timers but IMHO it?s just a bad idea to hang per-driver data in the device node.

It is simple enough to use an IDR or something when you need to grab data related to the
device node instead of stashing stuff in the data pointer.  

> I didn't see a user of this. I'm guessing that is coming in a SMMU patch?
> 
> Rob

Regards

? Pantelis

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Grant Likely]

On Mon, Dec 1, 2014 at 11:54 PM, Rob Herring <robherring2@gmail.com> wrote:
> Adding Grant and Pantelis...
>
> On Mon, Dec 1, 2014 at 10:57 AM, Will Deacon <will.deacon@arm.com> wrote:
>> IOMMU drivers must be initialised before any of their upstream devices,
>> otherwise the relevant iommu_ops won't be configured for the bus in
>> question. To solve this, a number of IOMMU drivers use initcalls to
>> initialise the driver before anything has a chance to be probed.
>>
>> Whilst this solves the immediate problem, it leaves the job of probing
>> the IOMMU completely separate from the iommu_ops to configure the IOMMU,
>> which are called on a per-bus basis and require the driver to figure out
>> exactly which instance of the IOMMU is being requested. In particular,
>> the add_device callback simply passes a struct device to the driver,
>> which then has to parse firmware tables or probe buses to identify the
>> relevant IOMMU instance.
>>
>> This patch takes the first step in addressing this problem by adding an
>> early initialisation pass for IOMMU drivers, giving them the ability to
>> store some per-instance data in their iommu_ops structure and store that
>> in their of_node. This can later be used when parsing OF masters to
>> identify the IOMMU instance in question.
>>
>> Acked-by: Arnd Bergmann <arnd@arndb.de>
>> Acked-by: Joerg Roedel <jroedel@suse.de>
>> Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
>> Tested-by: Robin Murphy <robin.murphy@arm.com>
>> Signed-off-by: Will Deacon <will.deacon@arm.com>
>> ---
>>  drivers/iommu/of_iommu.c          | 17 +++++++++++++++++
>>  include/asm-generic/vmlinux.lds.h |  2 ++
>>  include/linux/iommu.h             |  2 ++
>>  include/linux/of_iommu.h          | 25 +++++++++++++++++++++++++
>>  4 files changed, 46 insertions(+)
>>
>> diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
>> index e550ccb7634e..89b903406968 100644
>> --- a/drivers/iommu/of_iommu.c
>> +++ b/drivers/iommu/of_iommu.c
>> @@ -22,6 +22,9 @@
>>  #include <linux/of.h>
>>  #include <linux/of_iommu.h>
>>
>> +static const struct of_device_id __iommu_of_table_sentinel
>> +       __used __section(__iommu_of_table_end);
>> +
>>  /**
>>   * of_get_dma_window - Parse *dma-window property and returns 0 if found.
>>   *
>> @@ -89,3 +92,17 @@ int of_get_dma_window(struct device_node *dn, const char *prefix, int index,
>>         return 0;
>>  }
>>  EXPORT_SYMBOL_GPL(of_get_dma_window);
>> +
>> +void __init of_iommu_init(void)
>> +{
>> +       struct device_node *np;
>> +       const struct of_device_id *match, *matches = &__iommu_of_table;
>> +
>> +       for_each_matching_node_and_match(np, matches, &match) {
>> +               const of_iommu_init_fn init_fn = match->data;
>> +
>> +               if (init_fn(np))
>> +                       pr_err("Failed to initialise IOMMU %s\n",
>> +                               of_node_full_name(np));
>> +       }
>> +}
>> diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
>> index aa70cbda327c..bee5d683074d 100644
>> --- a/include/asm-generic/vmlinux.lds.h
>> +++ b/include/asm-generic/vmlinux.lds.h
>> @@ -164,6 +164,7 @@
>>  #define CLKSRC_OF_TABLES()     OF_TABLE(CONFIG_CLKSRC_OF, clksrc)
>>  #define IRQCHIP_OF_MATCH_TABLE() OF_TABLE(CONFIG_IRQCHIP, irqchip)
>>  #define CLK_OF_TABLES()                OF_TABLE(CONFIG_COMMON_CLK, clk)
>> +#define IOMMU_OF_TABLES()      OF_TABLE(CONFIG_OF_IOMMU, iommu)
>>  #define RESERVEDMEM_OF_TABLES()        OF_TABLE(CONFIG_OF_RESERVED_MEM, reservedmem)
>>  #define CPU_METHOD_OF_TABLES() OF_TABLE(CONFIG_SMP, cpu_method)
>>  #define EARLYCON_OF_TABLES()   OF_TABLE(CONFIG_SERIAL_EARLYCON, earlycon)
>> @@ -497,6 +498,7 @@
>>         CLK_OF_TABLES()                                                 \
>>         RESERVEDMEM_OF_TABLES()                                         \
>>         CLKSRC_OF_TABLES()                                              \
>> +       IOMMU_OF_TABLES()                                               \
>>         CPU_METHOD_OF_TABLES()                                          \
>>         KERNEL_DTB()                                                    \
>>         IRQCHIP_OF_MATCH_TABLE()                                        \
>> diff --git a/include/linux/iommu.h b/include/linux/iommu.h
>> index e6a7c9ff72f2..7b83f9f8e11d 100644
>> --- a/include/linux/iommu.h
>> +++ b/include/linux/iommu.h
>> @@ -103,6 +103,7 @@ enum iommu_attr {
>>   * @domain_get_attr: Query domain attributes
>>   * @domain_set_attr: Change domain attributes
>>   * @pgsize_bitmap: bitmap of supported page sizes
>> + * @priv: per-instance data private to the iommu driver
>>   */
>>  struct iommu_ops {
>>         bool (*capable)(enum iommu_cap);
>> @@ -133,6 +134,7 @@ struct iommu_ops {
>>         u32 (*domain_get_windows)(struct iommu_domain *domain);
>>
>>         unsigned long pgsize_bitmap;
>> +       void *priv;
>>  };
>>
>>  #define IOMMU_GROUP_NOTIFY_ADD_DEVICE          1 /* Device added */
>> diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h
>> index 51a560f34bca..5762cdc8effe 100644
>> --- a/include/linux/of_iommu.h
>> +++ b/include/linux/of_iommu.h
>> @@ -1,12 +1,17 @@
>>  #ifndef __OF_IOMMU_H
>>  #define __OF_IOMMU_H
>>
>> +#include <linux/iommu.h>
>> +#include <linux/of.h>
>> +
>>  #ifdef CONFIG_OF_IOMMU
>>
>>  extern int of_get_dma_window(struct device_node *dn, const char *prefix,
>>                              int index, unsigned long *busno, dma_addr_t *addr,
>>                              size_t *size);
>>
>> +extern void of_iommu_init(void);
>> +
>>  #else
>>
>>  static inline int of_get_dma_window(struct device_node *dn, const char *prefix,
>> @@ -16,6 +21,26 @@ static inline int of_get_dma_window(struct device_node *dn, const char *prefix,
>>         return -EINVAL;
>>  }
>>
>> +static inline void of_iommu_init(void) { }
>> +
>>  #endif /* CONFIG_OF_IOMMU */
>>
>> +static inline void of_iommu_set_ops(struct device_node *np,
>> +                                   const struct iommu_ops *ops)
>> +{
>> +       np->data = (struct iommu_ops *)ops;
>> +}
>> +
>> +static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
>> +{
>> +       return np->data;
>> +}
>
> This may collide with other users. While use of it is rare, PPC uses
> it in its PCI code. The OF_DYNAMIC code frees it but never actually
> sets it. There may be some coming usage with the DT overlay code or
> that's just a bug. Pantelis or Grant can comment. If not, I think we
> really should try to get rid of this pointer rather than expand it's
> usage.
>
> I didn't see a user of this. I'm guessing that is coming in a SMMU patch?

Good catch. This is not good. The data pointer should be avoided since
there are no controls over its use. Until a better solution can be
implemented, probably the safest thing to do is add a struct iommu_ops
pointer to struct device_node. However, assuming that only a small
portion of nodes will actually have iommu_ops set, I'd rather see a
separate registry that matches device_nodes to iommu_ops.

g.

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Arnd Bergmann]

On Tuesday 02 December 2014 14:16:57 Grant Likely wrote:
> On Mon, Dec 1, 2014 at 11:54 PM, Rob Herring <robherring2@gmail.com> wrote:
> > On Mon, Dec 1, 2014 at 10:57 AM, Will Deacon <will.deacon@arm.com> wrote:
> >>
> >> +static inline void of_iommu_set_ops(struct device_node *np,
> >> +                                   const struct iommu_ops *ops)
> >> +{
> >> +       np->data = (struct iommu_ops *)ops;
> >> +}
> >> +
> >> +static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
> >> +{
> >> +       return np->data;
> >> +}
> >
> > This may collide with other users. While use of it is rare, PPC uses
> > it in its PCI code. The OF_DYNAMIC code frees it but never actually
> > sets it. There may be some coming usage with the DT overlay code or
> > that's just a bug. Pantelis or Grant can comment. If not, I think we
> > really should try to get rid of this pointer rather than expand it's
> > usage.
> >
> > I didn't see a user of this. I'm guessing that is coming in a SMMU patch?
> 
> Good catch. This is not good. The data pointer should be avoided since
> there are no controls over its use. Until a better solution can be
> implemented, probably the safest thing to do is add a struct iommu_ops
> pointer to struct device_node. However, assuming that only a small
> portion of nodes will actually have iommu_ops set, I'd rather see a
> separate registry that matches device_nodes to iommu_ops.

Fair enough. Will, can you take a copy of drivers/dma/of-dma.c and
adapt it as needed? It should be exactly what we need to start
out and can be extended and generalized later.

	ARnd

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Will Deacon]

On Wed, Dec 03, 2014 at 07:57:50PM +0000, Arnd Bergmann wrote:
> On Tuesday 02 December 2014 14:16:57 Grant Likely wrote:
> > On Mon, Dec 1, 2014 at 11:54 PM, Rob Herring <robherring2@gmail.com> wrote:
> > > On Mon, Dec 1, 2014 at 10:57 AM, Will Deacon <will.deacon@arm.com> wrote:
> > >>
> > >> +static inline void of_iommu_set_ops(struct device_node *np,
> > >> +                                   const struct iommu_ops *ops)
> > >> +{
> > >> +       np->data = (struct iommu_ops *)ops;
> > >> +}
> > >> +
> > >> +static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
> > >> +{
> > >> +       return np->data;
> > >> +}
> > >
> > > This may collide with other users. While use of it is rare, PPC uses
> > > it in its PCI code. The OF_DYNAMIC code frees it but never actually
> > > sets it. There may be some coming usage with the DT overlay code or
> > > that's just a bug. Pantelis or Grant can comment. If not, I think we
> > > really should try to get rid of this pointer rather than expand it's
> > > usage.
> > >
> > > I didn't see a user of this. I'm guessing that is coming in a SMMU patch?
> > 
> > Good catch. This is not good. The data pointer should be avoided since
> > there are no controls over its use. Until a better solution can be
> > implemented, probably the safest thing to do is add a struct iommu_ops
> > pointer to struct device_node. However, assuming that only a small
> > portion of nodes will actually have iommu_ops set, I'd rather see a
> > separate registry that matches device_nodes to iommu_ops.
> 
> Fair enough. Will, can you take a copy of drivers/dma/of-dma.c and
> adapt it as needed? It should be exactly what we need to start
> out and can be extended and generalized later.

Sure, I'll add this to my list of stuff to do for 3.20.

Will

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Arnd Bergmann]

On Thursday 04 December 2014 09:49:53 Will Deacon wrote:
> On Wed, Dec 03, 2014 at 07:57:50PM +0000, Arnd Bergmann wrote:
> > On Tuesday 02 December 2014 14:16:57 Grant Likely wrote:
> > > On Mon, Dec 1, 2014 at 11:54 PM, Rob Herring <robherring2@gmail.com> wrote:
> > > > On Mon, Dec 1, 2014 at 10:57 AM, Will Deacon <will.deacon@arm.com> wrote:
> > > >>
> > > >> +static inline void of_iommu_set_ops(struct device_node *np,
> > > >> +                                   const struct iommu_ops *ops)
> > > >> +{
> > > >> +       np->data = (struct iommu_ops *)ops;
> > > >> +}
> > > >> +
> > > >> +static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
> > > >> +{
> > > >> +       return np->data;
> > > >> +}
> > > >
> > > > This may collide with other users. While use of it is rare, PPC uses
> > > > it in its PCI code. The OF_DYNAMIC code frees it but never actually
> > > > sets it. There may be some coming usage with the DT overlay code or
> > > > that's just a bug. Pantelis or Grant can comment. If not, I think we
> > > > really should try to get rid of this pointer rather than expand it's
> > > > usage.
> > > >
> > > > I didn't see a user of this. I'm guessing that is coming in a SMMU patch?
> > > 
> > > Good catch. This is not good. The data pointer should be avoided since
> > > there are no controls over its use. Until a better solution can be
> > > implemented, probably the safest thing to do is add a struct iommu_ops
> > > pointer to struct device_node. However, assuming that only a small
> > > portion of nodes will actually have iommu_ops set, I'd rather see a
> > > separate registry that matches device_nodes to iommu_ops.
> > 
> > Fair enough. Will, can you take a copy of drivers/dma/of-dma.c and
> > adapt it as needed? It should be exactly what we need to start
> > out and can be extended and generalized later.
> 
> Sure, I'll add this to my list of stuff to do for 3.20.

Does that mean the we don't get any of the patches for 3.19 despite the
Acks?

	Arnd

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Will Deacon]

On Thu, Dec 04, 2014 at 10:10:17AM +0000, Arnd Bergmann wrote:
> On Thursday 04 December 2014 09:49:53 Will Deacon wrote:
> > On Wed, Dec 03, 2014 at 07:57:50PM +0000, Arnd Bergmann wrote:
> > > On Tuesday 02 December 2014 14:16:57 Grant Likely wrote:
> > > > On Mon, Dec 1, 2014 at 11:54 PM, Rob Herring <robherring2@gmail.com> wrote:
> > > > > On Mon, Dec 1, 2014 at 10:57 AM, Will Deacon <will.deacon@arm.com> wrote:
> > > > >>
> > > > >> +static inline void of_iommu_set_ops(struct device_node *np,
> > > > >> +                                   const struct iommu_ops *ops)
> > > > >> +{
> > > > >> +       np->data = (struct iommu_ops *)ops;
> > > > >> +}
> > > > >> +
> > > > >> +static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
> > > > >> +{
> > > > >> +       return np->data;
> > > > >> +}
> > > > >
> > > > > This may collide with other users. While use of it is rare, PPC uses
> > > > > it in its PCI code. The OF_DYNAMIC code frees it but never actually
> > > > > sets it. There may be some coming usage with the DT overlay code or
> > > > > that's just a bug. Pantelis or Grant can comment. If not, I think we
> > > > > really should try to get rid of this pointer rather than expand it's
> > > > > usage.
> > > > >
> > > > > I didn't see a user of this. I'm guessing that is coming in a SMMU patch?
> > > > 
> > > > Good catch. This is not good. The data pointer should be avoided since
> > > > there are no controls over its use. Until a better solution can be
> > > > implemented, probably the safest thing to do is add a struct iommu_ops
> > > > pointer to struct device_node. However, assuming that only a small
> > > > portion of nodes will actually have iommu_ops set, I'd rather see a
> > > > separate registry that matches device_nodes to iommu_ops.
> > > 
> > > Fair enough. Will, can you take a copy of drivers/dma/of-dma.c and
> > > adapt it as needed? It should be exactly what we need to start
> > > out and can be extended and generalized later.
> > 
> > Sure, I'll add this to my list of stuff to do for 3.20.
> 
> Does that mean the we don't get any of the patches for 3.19 despite the
> Acks?

Hmm, I don't know how useful they are without the get/set ops and I don't
think I can get those ready for 3.19 given where we currently are.

Grant's suggestion of adding an iommu_ops pointer to device_node would work
as a temporary hack, but anything more advanced is going to need proper
review.

Will

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Arnd Bergmann]

On Thursday 04 December 2014 10:21:27 Will Deacon wrote:
> On Thu, Dec 04, 2014 at 10:10:17AM +0000, Arnd Bergmann wrote:
> > On Thursday 04 December 2014 09:49:53 Will Deacon wrote:
> > > On Wed, Dec 03, 2014 at 07:57:50PM +0000, Arnd Bergmann wrote:
> > > > On Tuesday 02 December 2014 14:16:57 Grant Likely wrote:
> > > > > On Mon, Dec 1, 2014 at 11:54 PM, Rob Herring <robherring2@gmail.com> wrote:
> > > > > > On Mon, Dec 1, 2014 at 10:57 AM, Will Deacon <will.deacon@arm.com> wrote:
> > > > > >>
> > > > > >> +static inline void of_iommu_set_ops(struct device_node *np,
> > > > > >> +                                   const struct iommu_ops *ops)
> > > > > >> +{
> > > > > >> +       np->data = (struct iommu_ops *)ops;
> > > > > >> +}
> > > > > >> +
> > > > > >> +static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
> > > > > >> +{
> > > > > >> +       return np->data;
> > > > > >> +}
> > > > > >
> > > > > > This may collide with other users. While use of it is rare, PPC uses
> > > > > > it in its PCI code. The OF_DYNAMIC code frees it but never actually
> > > > > > sets it. There may be some coming usage with the DT overlay code or
> > > > > > that's just a bug. Pantelis or Grant can comment. If not, I think we
> > > > > > really should try to get rid of this pointer rather than expand it's
> > > > > > usage.
> > > > > >
> > > > > > I didn't see a user of this. I'm guessing that is coming in a SMMU patch?
> > > > > 
> > > > > Good catch. This is not good. The data pointer should be avoided since
> > > > > there are no controls over its use. Until a better solution can be
> > > > > implemented, probably the safest thing to do is add a struct iommu_ops
> > > > > pointer to struct device_node. However, assuming that only a small
> > > > > portion of nodes will actually have iommu_ops set, I'd rather see a
> > > > > separate registry that matches device_nodes to iommu_ops.
> > > > 
> > > > Fair enough. Will, can you take a copy of drivers/dma/of-dma.c and
> > > > adapt it as needed? It should be exactly what we need to start
> > > > out and can be extended and generalized later.
> > > 
> > > Sure, I'll add this to my list of stuff to do for 3.20.
> > 
> > Does that mean the we don't get any of the patches for 3.19 despite the
> > Acks?
> 
> Hmm, I don't know how useful they are without the get/set ops and I don't
> think I can get those ready for 3.19 given where we currently are.
> 
> Grant's suggestion of adding an iommu_ops pointer to device_node would work
> as a temporary hack, but anything more advanced is going to need proper
> review.

Right. I guess it doesn't hurt much if we put the new pointer inside
#ifdef CONFIG_OF_IOMMU, then at least there is no significant size
increase in most DT based platforms.

	Arnd

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Grant Likely]

On Thu, Dec 4, 2014 at 11:19 AM, Arnd Bergmann <arnd@arndb.de> wrote:
> On Thursday 04 December 2014 10:21:27 Will Deacon wrote:
>> On Thu, Dec 04, 2014 at 10:10:17AM +0000, Arnd Bergmann wrote:
>> > On Thursday 04 December 2014 09:49:53 Will Deacon wrote:
>> > > On Wed, Dec 03, 2014 at 07:57:50PM +0000, Arnd Bergmann wrote:
>> > > > On Tuesday 02 December 2014 14:16:57 Grant Likely wrote:
>> > > > > On Mon, Dec 1, 2014 at 11:54 PM, Rob Herring <robherring2@gmail.com> wrote:
>> > > > > > On Mon, Dec 1, 2014 at 10:57 AM, Will Deacon <will.deacon@arm.com> wrote:
>> > > > > >>
>> > > > > >> +static inline void of_iommu_set_ops(struct device_node *np,
>> > > > > >> +                                   const struct iommu_ops *ops)
>> > > > > >> +{
>> > > > > >> +       np->data = (struct iommu_ops *)ops;
>> > > > > >> +}
>> > > > > >> +
>> > > > > >> +static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
>> > > > > >> +{
>> > > > > >> +       return np->data;
>> > > > > >> +}
>> > > > > >
>> > > > > > This may collide with other users. While use of it is rare, PPC uses
>> > > > > > it in its PCI code. The OF_DYNAMIC code frees it but never actually
>> > > > > > sets it. There may be some coming usage with the DT overlay code or
>> > > > > > that's just a bug. Pantelis or Grant can comment. If not, I think we
>> > > > > > really should try to get rid of this pointer rather than expand it's
>> > > > > > usage.
>> > > > > >
>> > > > > > I didn't see a user of this. I'm guessing that is coming in a SMMU patch?
>> > > > >
>> > > > > Good catch. This is not good. The data pointer should be avoided since
>> > > > > there are no controls over its use. Until a better solution can be
>> > > > > implemented, probably the safest thing to do is add a struct iommu_ops
>> > > > > pointer to struct device_node. However, assuming that only a small
>> > > > > portion of nodes will actually have iommu_ops set, I'd rather see a
>> > > > > separate registry that matches device_nodes to iommu_ops.
>> > > >
>> > > > Fair enough. Will, can you take a copy of drivers/dma/of-dma.c and
>> > > > adapt it as needed? It should be exactly what we need to start
>> > > > out and can be extended and generalized later.
>> > >
>> > > Sure, I'll add this to my list of stuff to do for 3.20.
>> >
>> > Does that mean the we don't get any of the patches for 3.19 despite the
>> > Acks?
>>
>> Hmm, I don't know how useful they are without the get/set ops and I don't
>> think I can get those ready for 3.19 given where we currently are.
>>
>> Grant's suggestion of adding an iommu_ops pointer to device_node would work
>> as a temporary hack, but anything more advanced is going to need proper
>> review.
>
> Right. I guess it doesn't hurt much if we put the new pointer inside
> #ifdef CONFIG_OF_IOMMU, then at least there is no significant size
> increase in most DT based platforms.

Yes, I can live with that hack on the proviso that it will be removed by v3.20

Oh, and please put an ugly /* */ comment block in the #ifdef
CONFIG_OF_IOMMU section that makes it really clear that it is an ugly
hack and will be removed in the next release. I don't want anyone
getting ideas that adding pointers to struct device_node is a good
idea.

g.

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Will Deacon]

On Thu, Dec 04, 2014 at 11:25:35AM +0000, Grant Likely wrote:
> On Thu, Dec 4, 2014 at 11:19 AM, Arnd Bergmann <arnd@arndb.de> wrote:
> > On Thursday 04 December 2014 10:21:27 Will Deacon wrote:
> >> > > Sure, I'll add this to my list of stuff to do for 3.20.
> >> >
> >> > Does that mean the we don't get any of the patches for 3.19 despite the
> >> > Acks?
> >>
> >> Hmm, I don't know how useful they are without the get/set ops and I don't
> >> think I can get those ready for 3.19 given where we currently are.
> >>
> >> Grant's suggestion of adding an iommu_ops pointer to device_node would work
> >> as a temporary hack, but anything more advanced is going to need proper
> >> review.
> >
> > Right. I guess it doesn't hurt much if we put the new pointer inside
> > #ifdef CONFIG_OF_IOMMU, then at least there is no significant size
> > increase in most DT based platforms.
> 
> Yes, I can live with that hack on the proviso that it will be removed by v3.20
> 
> Oh, and please put an ugly /* */ comment block in the #ifdef
> CONFIG_OF_IOMMU section that makes it really clear that it is an ugly
> hack and will be removed in the next release. I don't want anyone
> getting ideas that adding pointers to struct device_node is a good
> idea.

Something like the mess below?

Will

--->8

diff --git a/include/linux/of.h b/include/linux/of.h
index 29f0adc5f3e4..6f85c02bc1a6 100644
--- a/include/linux/of.h
+++ b/include/linux/of.h
@@ -43,6 +43,9 @@ struct property {
 #if defined(CONFIG_SPARC)
 struct of_irq_controller;
 #endif
+#ifdef CONFIG_OF_IOMMU
+struct iommu_ops;
+#endif
 
 struct device_node {
 	const char *name;
@@ -65,6 +68,19 @@ struct device_node {
 	unsigned int unique_id;
 	struct of_irq_controller *irq_trans;
 #endif
+#ifdef CONFIG_OF_IOMMU
+/*
+ * HACK! HACK! HACK!
+ *
+ * This is a temporary hack to associate a device_node for an
+ * IOMMU with its set of iommu_ops so that we can probe its upstream DMA
+ * masters on the platform bus by parsing the "iommus" property directly.
+ *
+ * This is going away in 3.20. Please use the of_iommu_{get,set}_ops
+ * functions to get hold of this data.
+ */
+	struct iommu_ops *__iommu_ops_use_accessors;
+#endif
 };
 
 #define MAX_PHANDLE_ARGS 16
diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h
index d03abbb11c34..392ec5f212db 100644
--- a/include/linux/of_iommu.h
+++ b/include/linux/of_iommu.h
@@ -14,6 +14,17 @@ extern int of_get_dma_window(struct device_node *dn, const char *prefix,
 extern void of_iommu_init(void);
 extern struct iommu_ops *of_iommu_configure(struct device *dev);
 
+static inline void of_iommu_set_ops(struct device_node *np,
+				    const struct iommu_ops *ops)
+{
+	np->__iommu_ops_use_accessors = ops;
+}
+
+static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
+{
+	return np->__iommu_ops_use_accessors;
+}
+
 #else
 
 static inline int of_get_dma_window(struct device_node *dn, const char *prefix,
@@ -29,19 +40,15 @@ static inline struct iommu_ops *of_iommu_configure(struct device *dev)
 	return NULL;
 }
 
-#endif	/* CONFIG_OF_IOMMU */
-
 static inline void of_iommu_set_ops(struct device_node *np,
-				    const struct iommu_ops *ops)
-{
-	np->data = (struct iommu_ops *)ops;
-}
-
+				    const struct iommu_ops *ops) { }
 static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
 {
-	return np->data;
+	return NULL;
 }
 
+#endif	/* CONFIG_OF_IOMMU */
+
 extern struct of_device_id __iommu_of_table;
 
 typedef int (*of_iommu_init_fn)(struct device_node *);

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Grant Likely]

On Thu, Dec 4, 2014 at 11:52 AM, Will Deacon <will.deacon@arm.com> wrote:
> On Thu, Dec 04, 2014 at 11:25:35AM +0000, Grant Likely wrote:
>> On Thu, Dec 4, 2014 at 11:19 AM, Arnd Bergmann <arnd@arndb.de> wrote:
>> > On Thursday 04 December 2014 10:21:27 Will Deacon wrote:
>> >> > > Sure, I'll add this to my list of stuff to do for 3.20.
>> >> >
>> >> > Does that mean the we don't get any of the patches for 3.19 despite the
>> >> > Acks?
>> >>
>> >> Hmm, I don't know how useful they are without the get/set ops and I don't
>> >> think I can get those ready for 3.19 given where we currently are.
>> >>
>> >> Grant's suggestion of adding an iommu_ops pointer to device_node would work
>> >> as a temporary hack, but anything more advanced is going to need proper
>> >> review.
>> >
>> > Right. I guess it doesn't hurt much if we put the new pointer inside
>> > #ifdef CONFIG_OF_IOMMU, then at least there is no significant size
>> > increase in most DT based platforms.
>>
>> Yes, I can live with that hack on the proviso that it will be removed by v3.20
>>
>> Oh, and please put an ugly /* */ comment block in the #ifdef
>> CONFIG_OF_IOMMU section that makes it really clear that it is an ugly
>> hack and will be removed in the next release. I don't want anyone
>> getting ideas that adding pointers to struct device_node is a good
>> idea.
>
> Something like the mess below?

Yes... Although it looks like Robin's patch does what is needed
without the hack.

g.

>
> Will
>
> --->8
>
> diff --git a/include/linux/of.h b/include/linux/of.h
> index 29f0adc5f3e4..6f85c02bc1a6 100644
> --- a/include/linux/of.h
> +++ b/include/linux/of.h
> @@ -43,6 +43,9 @@ struct property {
>  #if defined(CONFIG_SPARC)
>  struct of_irq_controller;
>  #endif
> +#ifdef CONFIG_OF_IOMMU
> +struct iommu_ops;
> +#endif
>
>  struct device_node {
>         const char *name;
> @@ -65,6 +68,19 @@ struct device_node {
>         unsigned int unique_id;
>         struct of_irq_controller *irq_trans;
>  #endif
> +#ifdef CONFIG_OF_IOMMU
> +/*
> + * HACK! HACK! HACK!
> + *
> + * This is a temporary hack to associate a device_node for an
> + * IOMMU with its set of iommu_ops so that we can probe its upstream DMA
> + * masters on the platform bus by parsing the "iommus" property directly.
> + *
> + * This is going away in 3.20. Please use the of_iommu_{get,set}_ops
> + * functions to get hold of this data.
> + */
> +       struct iommu_ops *__iommu_ops_use_accessors;
> +#endif
>  };
>
>  #define MAX_PHANDLE_ARGS 16
> diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h
> index d03abbb11c34..392ec5f212db 100644
> --- a/include/linux/of_iommu.h
> +++ b/include/linux/of_iommu.h
> @@ -14,6 +14,17 @@ extern int of_get_dma_window(struct device_node *dn, const char *prefix,
>  extern void of_iommu_init(void);
>  extern struct iommu_ops *of_iommu_configure(struct device *dev);
>
> +static inline void of_iommu_set_ops(struct device_node *np,
> +                                   const struct iommu_ops *ops)
> +{
> +       np->__iommu_ops_use_accessors = ops;
> +}
> +
> +static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
> +{
> +       return np->__iommu_ops_use_accessors;
> +}
> +
>  #else
>
>  static inline int of_get_dma_window(struct device_node *dn, const char *prefix,
> @@ -29,19 +40,15 @@ static inline struct iommu_ops *of_iommu_configure(struct device *dev)
>         return NULL;
>  }
>
> -#endif /* CONFIG_OF_IOMMU */
> -
>  static inline void of_iommu_set_ops(struct device_node *np,
> -                                   const struct iommu_ops *ops)
> -{
> -       np->data = (struct iommu_ops *)ops;
> -}
> -
> +                                   const struct iommu_ops *ops) { }
>  static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
>  {
> -       return np->data;
> +       return NULL;
>  }
>
> +#endif /* CONFIG_OF_IOMMU */
> +
>  extern struct of_device_id __iommu_of_table;
>
>  typedef int (*of_iommu_init_fn)(struct device_node *);

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Robin Murphy]

Hi Arnd,

On 03/12/14 19:57, Arnd Bergmann wrote:
[...]
>> Good catch. This is not good. The data pointer should be avoided since
>> there are no controls over its use. Until a better solution can be
>> implemented, probably the safest thing to do is add a struct iommu_ops
>> pointer to struct device_node. However, assuming that only a small
>> portion of nodes will actually have iommu_ops set, I'd rather see a
>> separate registry that matches device_nodes to iommu_ops.
>
> Fair enough. Will, can you take a copy of drivers/dma/of-dma.c and
> adapt it as needed? It should be exactly what we need to start
> out and can be extended and generalized later.

I'm quite keen to see this series go in, since I'm depending on it to 
make arm64 IOMMU DMA ops "just work". Will and I came to the conclusion 
the other day that we pretty much need to build up some kind of bus 
abstraction based on the probe data in order to be able to assign IOMMU 
groups correctly, which can also subsume this particular problem in the 
long run. Since I've made a start on that already, I've hacked the 
following short-term fix out of it. Tested on my Juno - admittedly with 
only two SMMUs and one master (EHCI) being probed, but it didn't blow up 
or regress anything.

Regards,
Robin.

--->8---
 From 1f3d2612682c239e53f2c20e2ac5d19ef3f5387c Mon Sep 17 00:00:00 2001
Message-Id: 
<1f3d2612682c239e53f2c20e2ac5d19ef3f5387c.1417695078.git.robin.murphy@arm.com>
From: Robin Murphy <robin.murphy@arm.com>
Date: Thu, 4 Dec 2014 11:53:13 +0000
Subject: [PATCH] iommu: store DT-probed IOMMU data privately

Since the data pointer in the DT node is public and may be overwritten
by conflicting code, move the DT-probed IOMMU ops to a private list
where they will be safe.

Signed-off-by: Robin Murphy <robin.murphy@arm.com>
---
  drivers/iommu/of_iommu.c | 38 ++++++++++++++++++++++++++++++++++++++
  include/linux/of_iommu.h | 12 ++----------
  2 files changed, 40 insertions(+), 10 deletions(-)

diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
index 73236d3..5cd451c 100644
--- a/drivers/iommu/of_iommu.c
+++ b/drivers/iommu/of_iommu.c
@@ -94,6 +94,44 @@ int of_get_dma_window(struct device_node *dn, const 
char *prefix, int index,
  }
  EXPORT_SYMBOL_GPL(of_get_dma_window);

+struct of_iommu_node {
+	struct hlist_node list;
+	struct device_node *np;
+	const struct iommu_ops *ops;
+};
+static HLIST_HEAD(of_iommu_list);
+static DEFINE_SPINLOCK(of_iommu_lock);
+
+void of_iommu_set_ops(struct device_node *np, const struct iommu_ops *ops)
+{
+	struct of_iommu_node *iommu = kmalloc(sizeof(*iommu), GFP_KERNEL);
+
+	if (!iommu)
+		return;
+
+	INIT_HLIST_NODE(&iommu->list);
+	iommu->np = np;
+	iommu->ops = ops;
+	spin_lock(&of_iommu_lock);
+	hlist_add_head(&iommu->list, &of_iommu_list);
+	spin_unlock(&of_iommu_lock);
+}
+
+struct iommu_ops *of_iommu_get_ops(struct device_node *np)
+{
+	struct of_iommu_node *node;
+	const struct iommu_ops *ops = NULL;
+
+	spin_lock(&of_iommu_lock);
+	hlist_for_each_entry(node, &of_iommu_list, list)
+		if (node->np == np) {
+			ops = node->ops;
+			break;
+		}
+	spin_unlock(&of_iommu_lock);
+	return (struct iommu_ops *)ops;
+}
+
  struct iommu_ops *of_iommu_configure(struct device *dev)
  {
  	struct of_phandle_args iommu_spec;
diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h
index d03abbb..e27c53a 100644
--- a/include/linux/of_iommu.h
+++ b/include/linux/of_iommu.h
@@ -31,16 +31,8 @@ static inline struct iommu_ops 
*of_iommu_configure(struct device *dev)

  #endif	/* CONFIG_OF_IOMMU */

-static inline void of_iommu_set_ops(struct device_node *np,
-				    const struct iommu_ops *ops)
-{
-	np->data = (struct iommu_ops *)ops;
-}
-
-static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
-{
-	return np->data;
-}
+void of_iommu_set_ops(struct device_node *np, const struct iommu_ops *ops);
+struct iommu_ops *of_iommu_get_ops(struct device_node *np);

  extern struct of_device_id __iommu_of_table;

-- 
1.9.1

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Grant Likely]

On Thu, Dec 4, 2014 at 12:26 PM, Robin Murphy <robin.murphy@arm.com> wrote:
> Hi Arnd,
>
> On 03/12/14 19:57, Arnd Bergmann wrote:
> [...]
>>>
>>> Good catch. This is not good. The data pointer should be avoided since
>>> there are no controls over its use. Until a better solution can be
>>> implemented, probably the safest thing to do is add a struct iommu_ops
>>> pointer to struct device_node. However, assuming that only a small
>>> portion of nodes will actually have iommu_ops set, I'd rather see a
>>> separate registry that matches device_nodes to iommu_ops.
>>
>>
>> Fair enough. Will, can you take a copy of drivers/dma/of-dma.c and
>> adapt it as needed? It should be exactly what we need to start
>> out and can be extended and generalized later.
>
>
> I'm quite keen to see this series go in, since I'm depending on it to make
> arm64 IOMMU DMA ops "just work". Will and I came to the conclusion the other
> day that we pretty much need to build up some kind of bus abstraction based
> on the probe data in order to be able to assign IOMMU groups correctly,
> which can also subsume this particular problem in the long run. Since I've
> made a start on that already, I've hacked the following short-term fix out
> of it. Tested on my Juno - admittedly with only two SMMUs and one master
> (EHCI) being probed, but it didn't blow up or regress anything.
>
> Regards,
> Robin.
>
> --->8---
> From 1f3d2612682c239e53f2c20e2ac5d19ef3f5387c Mon Sep 17 00:00:00 2001
> Message-Id:
> <1f3d2612682c239e53f2c20e2ac5d19ef3f5387c.1417695078.git.robin.murphy@arm.com>
> From: Robin Murphy <robin.murphy@arm.com>
> Date: Thu, 4 Dec 2014 11:53:13 +0000
> Subject: [PATCH] iommu: store DT-probed IOMMU data privately
>
> Since the data pointer in the DT node is public and may be overwritten
> by conflicting code, move the DT-probed IOMMU ops to a private list
> where they will be safe.
>
> Signed-off-by: Robin Murphy <robin.murphy@arm.com>

Looks reasonable to me. Comments below...

> ---
>  drivers/iommu/of_iommu.c | 38 ++++++++++++++++++++++++++++++++++++++
>  include/linux/of_iommu.h | 12 ++----------
>  2 files changed, 40 insertions(+), 10 deletions(-)
>
> diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
> index 73236d3..5cd451c 100644
> --- a/drivers/iommu/of_iommu.c
> +++ b/drivers/iommu/of_iommu.c
> @@ -94,6 +94,44 @@ int of_get_dma_window(struct device_node *dn, const char
> *prefix, int index,
>  }
>  EXPORT_SYMBOL_GPL(of_get_dma_window);
>
> +struct of_iommu_node {
> +       struct hlist_node list;
> +       struct device_node *np;
> +       const struct iommu_ops *ops;
> +};
> +static HLIST_HEAD(of_iommu_list);

Just use a list_head. hlist_head merely saves one pointer in this case.

> +static DEFINE_SPINLOCK(of_iommu_lock);
> +
> +void of_iommu_set_ops(struct device_node *np, const struct iommu_ops *ops)
> +{
> +       struct of_iommu_node *iommu = kmalloc(sizeof(*iommu), GFP_KERNEL);

kzalloc()

> +
> +       if (!iommu)
> +               return;

Shouldn't there be a WARN() here on failure? I don't think failing
silently is desired.

> +
> +       INIT_HLIST_NODE(&iommu->list);
> +       iommu->np = np;
> +       iommu->ops = ops;
> +       spin_lock(&of_iommu_lock);
> +       hlist_add_head(&iommu->list, &of_iommu_list);
> +       spin_unlock(&of_iommu_lock);
> +}
> +
> +struct iommu_ops *of_iommu_get_ops(struct device_node *np)
> +{
> +       struct of_iommu_node *node;
> +       const struct iommu_ops *ops = NULL;
> +
> +       spin_lock(&of_iommu_lock);
> +       hlist_for_each_entry(node, &of_iommu_list, list)
> +               if (node->np == np) {
> +                       ops = node->ops;
> +                       break;
> +               }
> +       spin_unlock(&of_iommu_lock);
> +       return (struct iommu_ops *)ops;

The cast looks fishy. If you need to use a cast, then the data types
are probably wrong. If you drop the const from *ops here and in the
structure then it should probably work fine. Due to the way it is
being used, there isn't any advantage to using const (unless you
changes of_iommu_get_ops() to return a const pointer, then const would
make sense).

> +}
> +
>  struct iommu_ops *of_iommu_configure(struct device *dev)
>  {
>         struct of_phandle_args iommu_spec;
> diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h
> index d03abbb..e27c53a 100644
> --- a/include/linux/of_iommu.h
> +++ b/include/linux/of_iommu.h
> @@ -31,16 +31,8 @@ static inline struct iommu_ops *of_iommu_configure(struct
> device *dev)
>
>  #endif /* CONFIG_OF_IOMMU */
>
> -static inline void of_iommu_set_ops(struct device_node *np,
> -                                   const struct iommu_ops *ops)
> -{
> -       np->data = (struct iommu_ops *)ops;
> -}
> -
> -static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
> -{
> -       return np->data;
> -}
> +void of_iommu_set_ops(struct device_node *np, const struct iommu_ops *ops);
> +struct iommu_ops *of_iommu_get_ops(struct device_node *np);
>
>  extern struct of_device_id __iommu_of_table;
>
> --
> 1.9.1
>
>

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Robin Murphy]

Hi Grant, thanks for the advice - silly micro-optimisations removed, and 
I'll make a note to do so from my in-development code, too ;)

I didn't much like the casting either, so rather than push it elsewhere 
or out to the caller I've just changed the prototype to obviate it 
completely. Since we're also expecting to churn this again to use 
something more suitable than iommu_ops as the private data, I think 
keeping things simple wins over const-correctness for now.

Thanks,
Robin

--->8---
 From b2e8c91ac49bef4008661e4628cd6b7249d84af5 Mon Sep 17 00:00:00 2001
Message-Id: 
<b2e8c91ac49bef4008661e4628cd6b7249d84af5.1417698001.git.robin.murphy@arm.com>
From: Robin Murphy <robin.murphy@arm.com>
Date: Thu, 4 Dec 2014 11:53:13 +0000
Subject: [PATCH v2] iommu: store DT-probed IOMMU data privately

Since the data pointer in the DT node is public and may be overwritten
by conflicting code, move the DT-probed IOMMU ops to a private list
where they will be safe.

Signed-off-by: Robin Murphy <robin.murphy@arm.com>
---
  drivers/iommu/of_iommu.c | 40 ++++++++++++++++++++++++++++++++++++++++
  include/linux/of_iommu.h | 12 ++----------
  2 files changed, 42 insertions(+), 10 deletions(-)

diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
index 73236d3..c7078f6 100644
--- a/drivers/iommu/of_iommu.c
+++ b/drivers/iommu/of_iommu.c
@@ -94,6 +94,46 @@ int of_get_dma_window(struct device_node *dn, const 
char *prefix, int index,
  }
  EXPORT_SYMBOL_GPL(of_get_dma_window);

+struct of_iommu_node {
+	struct list_head list;
+	struct device_node *np;
+	struct iommu_ops *ops;
+};
+static LIST_HEAD(of_iommu_list);
+static DEFINE_SPINLOCK(of_iommu_lock);
+
+void of_iommu_set_ops(struct device_node *np, struct iommu_ops *ops)
+{
+	struct of_iommu_node *iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
+
+	if (!iommu) {
+		__WARN();
+		return;
+	}
+
+	INIT_LIST_HEAD(&iommu->list);
+	iommu->np = np;
+	iommu->ops = ops;
+	spin_lock(&of_iommu_lock);
+	list_add_tail(&iommu->list, &of_iommu_list);
+	spin_unlock(&of_iommu_lock);
+}
+
+struct iommu_ops *of_iommu_get_ops(struct device_node *np)
+{
+	struct of_iommu_node *node;
+	struct iommu_ops *ops = NULL;
+
+	spin_lock(&of_iommu_lock);
+	list_for_each_entry(node, &of_iommu_list, list)
+		if (node->np == np) {
+			ops = node->ops;
+			break;
+		}
+	spin_unlock(&of_iommu_lock);
+	return ops;
+}
+
  struct iommu_ops *of_iommu_configure(struct device *dev)
  {
  	struct of_phandle_args iommu_spec;
diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h
index d03abbb..16c7554 100644
--- a/include/linux/of_iommu.h
+++ b/include/linux/of_iommu.h
@@ -31,16 +31,8 @@ static inline struct iommu_ops 
*of_iommu_configure(struct device *dev)

  #endif	/* CONFIG_OF_IOMMU */

-static inline void of_iommu_set_ops(struct device_node *np,
-				    const struct iommu_ops *ops)
-{
-	np->data = (struct iommu_ops *)ops;
-}
-
-static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
-{
-	return np->data;
-}
+void of_iommu_set_ops(struct device_node *np, struct iommu_ops *ops);
+struct iommu_ops *of_iommu_get_ops(struct device_node *np);

  extern struct of_device_id __iommu_of_table;

-- 
1.9.1

On 04/12/14 12:42, Grant Likely wrote:
> On Thu, Dec 4, 2014 at 12:26 PM, Robin Murphy <robin.murphy@arm.com> wrote:
>> Hi Arnd,
>>
>> On 03/12/14 19:57, Arnd Bergmann wrote:
>> [...]
>>>>
>>>> Good catch. This is not good. The data pointer should be avoided since
>>>> there are no controls over its use. Until a better solution can be
>>>> implemented, probably the safest thing to do is add a struct iommu_ops
>>>> pointer to struct device_node. However, assuming that only a small
>>>> portion of nodes will actually have iommu_ops set, I'd rather see a
>>>> separate registry that matches device_nodes to iommu_ops.
>>>
>>>
>>> Fair enough. Will, can you take a copy of drivers/dma/of-dma.c and
>>> adapt it as needed? It should be exactly what we need to start
>>> out and can be extended and generalized later.
>>
>>
>> I'm quite keen to see this series go in, since I'm depending on it to make
>> arm64 IOMMU DMA ops "just work". Will and I came to the conclusion the other
>> day that we pretty much need to build up some kind of bus abstraction based
>> on the probe data in order to be able to assign IOMMU groups correctly,
>> which can also subsume this particular problem in the long run. Since I've
>> made a start on that already, I've hacked the following short-term fix out
>> of it. Tested on my Juno - admittedly with only two SMMUs and one master
>> (EHCI) being probed, but it didn't blow up or regress anything.
>>
>> Regards,
>> Robin.
>>
>> --->8---
>>  From 1f3d2612682c239e53f2c20e2ac5d19ef3f5387c Mon Sep 17 00:00:00 2001
>> Message-Id:
>> <1f3d2612682c239e53f2c20e2ac5d19ef3f5387c.1417695078.git.robin.murphy@arm.com>
>> From: Robin Murphy <robin.murphy@arm.com>
>> Date: Thu, 4 Dec 2014 11:53:13 +0000
>> Subject: [PATCH] iommu: store DT-probed IOMMU data privately
>>
>> Since the data pointer in the DT node is public and may be overwritten
>> by conflicting code, move the DT-probed IOMMU ops to a private list
>> where they will be safe.
>>
>> Signed-off-by: Robin Murphy <robin.murphy@arm.com>
>
> Looks reasonable to me. Comments below...
>
>> ---
>>   drivers/iommu/of_iommu.c | 38 ++++++++++++++++++++++++++++++++++++++
>>   include/linux/of_iommu.h | 12 ++----------
>>   2 files changed, 40 insertions(+), 10 deletions(-)
>>
>> diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
>> index 73236d3..5cd451c 100644
>> --- a/drivers/iommu/of_iommu.c
>> +++ b/drivers/iommu/of_iommu.c
>> @@ -94,6 +94,44 @@ int of_get_dma_window(struct device_node *dn, const char
>> *prefix, int index,
>>   }
>>   EXPORT_SYMBOL_GPL(of_get_dma_window);
>>
>> +struct of_iommu_node {
>> +       struct hlist_node list;
>> +       struct device_node *np;
>> +       const struct iommu_ops *ops;
>> +};
>> +static HLIST_HEAD(of_iommu_list);
>
> Just use a list_head. hlist_head merely saves one pointer in this case.
>
>> +static DEFINE_SPINLOCK(of_iommu_lock);
>> +
>> +void of_iommu_set_ops(struct device_node *np, const struct iommu_ops *ops)
>> +{
>> +       struct of_iommu_node *iommu = kmalloc(sizeof(*iommu), GFP_KERNEL);
>
> kzalloc()
>
>> +
>> +       if (!iommu)
>> +               return;
>
> Shouldn't there be a WARN() here on failure? I don't think failing
> silently is desired.
>
>> +
>> +       INIT_HLIST_NODE(&iommu->list);
>> +       iommu->np = np;
>> +       iommu->ops = ops;
>> +       spin_lock(&of_iommu_lock);
>> +       hlist_add_head(&iommu->list, &of_iommu_list);
>> +       spin_unlock(&of_iommu_lock);
>> +}
>> +
>> +struct iommu_ops *of_iommu_get_ops(struct device_node *np)
>> +{
>> +       struct of_iommu_node *node;
>> +       const struct iommu_ops *ops = NULL;
>> +
>> +       spin_lock(&of_iommu_lock);
>> +       hlist_for_each_entry(node, &of_iommu_list, list)
>> +               if (node->np == np) {
>> +                       ops = node->ops;
>> +                       break;
>> +               }
>> +       spin_unlock(&of_iommu_lock);
>> +       return (struct iommu_ops *)ops;
>
> The cast looks fishy. If you need to use a cast, then the data types
> are probably wrong. If you drop the const from *ops here and in the
> structure then it should probably work fine. Due to the way it is
> being used, there isn't any advantage to using const (unless you
> changes of_iommu_get_ops() to return a const pointer, then const would
> make sense).
>
>> +}
>> +
>>   struct iommu_ops *of_iommu_configure(struct device *dev)
>>   {
>>          struct of_phandle_args iommu_spec;
>> diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h
>> index d03abbb..e27c53a 100644
>> --- a/include/linux/of_iommu.h
>> +++ b/include/linux/of_iommu.h
>> @@ -31,16 +31,8 @@ static inline struct iommu_ops *of_iommu_configure(struct
>> device *dev)
>>
>>   #endif /* CONFIG_OF_IOMMU */
>>
>> -static inline void of_iommu_set_ops(struct device_node *np,
>> -                                   const struct iommu_ops *ops)
>> -{
>> -       np->data = (struct iommu_ops *)ops;
>> -}
>> -
>> -static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
>> -{
>> -       return np->data;
>> -}
>> +void of_iommu_set_ops(struct device_node *np, const struct iommu_ops *ops);
>> +struct iommu_ops *of_iommu_get_ops(struct device_node *np);
>>
>>   extern struct of_device_id __iommu_of_table;
>>
>> --
>> 1.9.1
>>
>>
>

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Grant Likely]

On Thu, Dec 4, 2014 at 1:43 PM, Robin Murphy <robin.murphy@arm.com> wrote:
> Hi Grant, thanks for the advice - silly micro-optimisations removed, and
> I'll make a note to do so from my in-development code, too ;)
>
> I didn't much like the casting either, so rather than push it elsewhere or
> out to the caller I've just changed the prototype to obviate it completely.
> Since we're also expecting to churn this again to use something more
> suitable than iommu_ops as the private data, I think keeping things simple
> wins over const-correctness for now.
>
> Thanks,
> Robin
>
> --->8---
> From b2e8c91ac49bef4008661e4628cd6b7249d84af5 Mon Sep 17 00:00:00 2001
> Message-Id:
> <b2e8c91ac49bef4008661e4628cd6b7249d84af5.1417698001.git.robin.murphy@arm.com>
> From: Robin Murphy <robin.murphy@arm.com>
> Date: Thu, 4 Dec 2014 11:53:13 +0000
> Subject: [PATCH v2] iommu: store DT-probed IOMMU data privately
>
> Since the data pointer in the DT node is public and may be overwritten
> by conflicting code, move the DT-probed IOMMU ops to a private list
> where they will be safe.
>
> Signed-off-by: Robin Murphy <robin.murphy@arm.com>

Acked-by: Grant Likely <grant.likely@linaro.org>

> ---
>  drivers/iommu/of_iommu.c | 40 ++++++++++++++++++++++++++++++++++++++++
>  include/linux/of_iommu.h | 12 ++----------
>  2 files changed, 42 insertions(+), 10 deletions(-)
>
> diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
> index 73236d3..c7078f6 100644
> --- a/drivers/iommu/of_iommu.c
> +++ b/drivers/iommu/of_iommu.c
> @@ -94,6 +94,46 @@ int of_get_dma_window(struct device_node *dn, const char
> *prefix, int index,
>  }
>  EXPORT_SYMBOL_GPL(of_get_dma_window);
>
> +struct of_iommu_node {
> +       struct list_head list;
> +       struct device_node *np;
> +       struct iommu_ops *ops;
> +};
> +static LIST_HEAD(of_iommu_list);
> +static DEFINE_SPINLOCK(of_iommu_lock);
> +
> +void of_iommu_set_ops(struct device_node *np, struct iommu_ops *ops)
> +{
> +       struct of_iommu_node *iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
> +
> +       if (!iommu) {
> +               __WARN();
> +               return;
> +       }
> +
> +       INIT_LIST_HEAD(&iommu->list);
> +       iommu->np = np;
> +       iommu->ops = ops;
> +       spin_lock(&of_iommu_lock);
> +       list_add_tail(&iommu->list, &of_iommu_list);
> +       spin_unlock(&of_iommu_lock);
> +}
> +
> +struct iommu_ops *of_iommu_get_ops(struct device_node *np)
> +{
> +       struct of_iommu_node *node;
> +       struct iommu_ops *ops = NULL;
> +
> +       spin_lock(&of_iommu_lock);
> +       list_for_each_entry(node, &of_iommu_list, list)
> +               if (node->np == np) {
> +                       ops = node->ops;
> +                       break;
> +               }
> +       spin_unlock(&of_iommu_lock);
> +       return ops;
> +}
> +
>  struct iommu_ops *of_iommu_configure(struct device *dev)
>  {
>         struct of_phandle_args iommu_spec;
> diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h
> index d03abbb..16c7554 100644
> --- a/include/linux/of_iommu.h
> +++ b/include/linux/of_iommu.h
> @@ -31,16 +31,8 @@ static inline struct iommu_ops *of_iommu_configure(struct
> device *dev)
>
>  #endif /* CONFIG_OF_IOMMU */
>
> -static inline void of_iommu_set_ops(struct device_node *np,
> -                                   const struct iommu_ops *ops)
> -{
> -       np->data = (struct iommu_ops *)ops;
> -}
> -
> -static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
> -{
> -       return np->data;
> -}
> +void of_iommu_set_ops(struct device_node *np, struct iommu_ops *ops);
> +struct iommu_ops *of_iommu_get_ops(struct device_node *np);
>
>  extern struct of_device_id __iommu_of_table;
>
> --
> 1.9.1
>
> On 04/12/14 12:42, Grant Likely wrote:
>>
>> On Thu, Dec 4, 2014 at 12:26 PM, Robin Murphy <robin.murphy@arm.com>
>> wrote:
>>>
>>> Hi Arnd,
>>>
>>> On 03/12/14 19:57, Arnd Bergmann wrote:
>>> [...]
>>>>>
>>>>>
>>>>> Good catch. This is not good. The data pointer should be avoided since
>>>>> there are no controls over its use. Until a better solution can be
>>>>> implemented, probably the safest thing to do is add a struct iommu_ops
>>>>> pointer to struct device_node. However, assuming that only a small
>>>>> portion of nodes will actually have iommu_ops set, I'd rather see a
>>>>> separate registry that matches device_nodes to iommu_ops.
>>>>
>>>>
>>>>
>>>> Fair enough. Will, can you take a copy of drivers/dma/of-dma.c and
>>>> adapt it as needed? It should be exactly what we need to start
>>>> out and can be extended and generalized later.
>>>
>>>
>>>
>>> I'm quite keen to see this series go in, since I'm depending on it to
>>> make
>>> arm64 IOMMU DMA ops "just work". Will and I came to the conclusion the
>>> other
>>> day that we pretty much need to build up some kind of bus abstraction
>>> based
>>> on the probe data in order to be able to assign IOMMU groups correctly,
>>> which can also subsume this particular problem in the long run. Since
>>> I've
>>> made a start on that already, I've hacked the following short-term fix
>>> out
>>> of it. Tested on my Juno - admittedly with only two SMMUs and one master
>>> (EHCI) being probed, but it didn't blow up or regress anything.
>>>
>>> Regards,
>>> Robin.
>>>
>>> --->8---
>>>  From 1f3d2612682c239e53f2c20e2ac5d19ef3f5387c Mon Sep 17 00:00:00 2001
>>> Message-Id:
>>>
>>> <1f3d2612682c239e53f2c20e2ac5d19ef3f5387c.1417695078.git.robin.murphy@arm.com>
>>> From: Robin Murphy <robin.murphy@arm.com>
>>> Date: Thu, 4 Dec 2014 11:53:13 +0000
>>> Subject: [PATCH] iommu: store DT-probed IOMMU data privately
>>>
>>> Since the data pointer in the DT node is public and may be overwritten
>>> by conflicting code, move the DT-probed IOMMU ops to a private list
>>> where they will be safe.
>>>
>>> Signed-off-by: Robin Murphy <robin.murphy@arm.com>
>>
>>
>> Looks reasonable to me. Comments below...
>>
>>> ---
>>>   drivers/iommu/of_iommu.c | 38 ++++++++++++++++++++++++++++++++++++++
>>>   include/linux/of_iommu.h | 12 ++----------
>>>   2 files changed, 40 insertions(+), 10 deletions(-)
>>>
>>> diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
>>> index 73236d3..5cd451c 100644
>>> --- a/drivers/iommu/of_iommu.c
>>> +++ b/drivers/iommu/of_iommu.c
>>> @@ -94,6 +94,44 @@ int of_get_dma_window(struct device_node *dn, const
>>> char
>>> *prefix, int index,
>>>   }
>>>   EXPORT_SYMBOL_GPL(of_get_dma_window);
>>>
>>> +struct of_iommu_node {
>>> +       struct hlist_node list;
>>> +       struct device_node *np;
>>> +       const struct iommu_ops *ops;
>>> +};
>>> +static HLIST_HEAD(of_iommu_list);
>>
>>
>> Just use a list_head. hlist_head merely saves one pointer in this case.
>>
>>> +static DEFINE_SPINLOCK(of_iommu_lock);
>>> +
>>> +void of_iommu_set_ops(struct device_node *np, const struct iommu_ops
>>> *ops)
>>> +{
>>> +       struct of_iommu_node *iommu = kmalloc(sizeof(*iommu),
>>> GFP_KERNEL);
>>
>>
>> kzalloc()
>>
>>> +
>>> +       if (!iommu)
>>> +               return;
>>
>>
>> Shouldn't there be a WARN() here on failure? I don't think failing
>> silently is desired.
>>
>>> +
>>> +       INIT_HLIST_NODE(&iommu->list);
>>> +       iommu->np = np;
>>> +       iommu->ops = ops;
>>> +       spin_lock(&of_iommu_lock);
>>> +       hlist_add_head(&iommu->list, &of_iommu_list);
>>> +       spin_unlock(&of_iommu_lock);
>>> +}
>>> +
>>> +struct iommu_ops *of_iommu_get_ops(struct device_node *np)
>>> +{
>>> +       struct of_iommu_node *node;
>>> +       const struct iommu_ops *ops = NULL;
>>> +
>>> +       spin_lock(&of_iommu_lock);
>>> +       hlist_for_each_entry(node, &of_iommu_list, list)
>>> +               if (node->np == np) {
>>> +                       ops = node->ops;
>>> +                       break;
>>> +               }
>>> +       spin_unlock(&of_iommu_lock);
>>> +       return (struct iommu_ops *)ops;
>>
>>
>> The cast looks fishy. If you need to use a cast, then the data types
>> are probably wrong. If you drop the const from *ops here and in the
>> structure then it should probably work fine. Due to the way it is
>> being used, there isn't any advantage to using const (unless you
>> changes of_iommu_get_ops() to return a const pointer, then const would
>> make sense).
>>
>>> +}
>>> +
>>>   struct iommu_ops *of_iommu_configure(struct device *dev)
>>>   {
>>>          struct of_phandle_args iommu_spec;
>>> diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h
>>> index d03abbb..e27c53a 100644
>>> --- a/include/linux/of_iommu.h
>>> +++ b/include/linux/of_iommu.h
>>> @@ -31,16 +31,8 @@ static inline struct iommu_ops
>>> *of_iommu_configure(struct
>>> device *dev)
>>>
>>>   #endif /* CONFIG_OF_IOMMU */
>>>
>>> -static inline void of_iommu_set_ops(struct device_node *np,
>>> -                                   const struct iommu_ops *ops)
>>> -{
>>> -       np->data = (struct iommu_ops *)ops;
>>> -}
>>> -
>>> -static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
>>> -{
>>> -       return np->data;
>>> -}
>>> +void of_iommu_set_ops(struct device_node *np, const struct iommu_ops
>>> *ops);
>>> +struct iommu_ops *of_iommu_get_ops(struct device_node *np);
>>>
>>>   extern struct of_device_id __iommu_of_table;
>>>
>>> --
>>> 1.9.1
>>>
>>>
>>
>
>

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Thierry Reding]

On Thu, Dec 04, 2014 at 01:43:32PM +0000, Robin Murphy wrote:
> Hi Grant, thanks for the advice - silly micro-optimisations removed, and
> I'll make a note to do so from my in-development code, too ;)
> 
> I didn't much like the casting either, so rather than push it elsewhere or
> out to the caller I've just changed the prototype to obviate it completely.
> Since we're also expecting to churn this again to use something more
> suitable than iommu_ops as the private data, I think keeping things simple
> wins over const-correctness for now.
> 
> Thanks,
> Robin
> 
> --->8---
> From b2e8c91ac49bef4008661e4628cd6b7249d84af5 Mon Sep 17 00:00:00 2001
> Message-Id: <b2e8c91ac49bef4008661e4628cd6b7249d84af5.1417698001.git.robin.murphy@arm.com>
> From: Robin Murphy <robin.murphy@arm.com>
> Date: Thu, 4 Dec 2014 11:53:13 +0000
> Subject: [PATCH v2] iommu: store DT-probed IOMMU data privately
> 
> Since the data pointer in the DT node is public and may be overwritten
> by conflicting code, move the DT-probed IOMMU ops to a private list
> where they will be safe.
> 
> Signed-off-by: Robin Murphy <robin.murphy@arm.com>
> ---
>  drivers/iommu/of_iommu.c | 40 ++++++++++++++++++++++++++++++++++++++++
>  include/linux/of_iommu.h | 12 ++----------
>  2 files changed, 42 insertions(+), 10 deletions(-)
> 
> diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
> index 73236d3..c7078f6 100644
> --- a/drivers/iommu/of_iommu.c
> +++ b/drivers/iommu/of_iommu.c
> @@ -94,6 +94,46 @@ int of_get_dma_window(struct device_node *dn, const char
> *prefix, int index,
>  }
>  EXPORT_SYMBOL_GPL(of_get_dma_window);
> 
> +struct of_iommu_node {
> +	struct list_head list;
> +	struct device_node *np;
> +	struct iommu_ops *ops;

Why can't this be const? Why would anyone ever need to modify it? Also
drivers do define their iommu_ops structures const these days, so you'll
either still have to cast away at some point or the compiler will
complain once you start calling this from drivers.

Thierry
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 819 bytes
Desc: not available
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20141204/88e4256b/attachment-0001.sig>

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Robin Murphy]

Hi Thierry,

On 04/12/14 14:49, Thierry Reding wrote:
> On Thu, Dec 04, 2014 at 01:43:32PM +0000, Robin Murphy wrote:
>> Hi Grant, thanks for the advice - silly micro-optimisations removed, and
>> I'll make a note to do so from my in-development code, too ;)
>>
>> I didn't much like the casting either, so rather than push it elsewhere or
>> out to the caller I've just changed the prototype to obviate it completely.
>> Since we're also expecting to churn this again to use something more
>> suitable than iommu_ops as the private data, I think keeping things simple
>> wins over const-correctness for now.
>>
>> Thanks,
>> Robin
>>
>> --->8---
>>  From b2e8c91ac49bef4008661e4628cd6b7249d84af5 Mon Sep 17 00:00:00 2001
>> Message-Id: <b2e8c91ac49bef4008661e4628cd6b7249d84af5.1417698001.git.robin.murphy@arm.com>
>> From: Robin Murphy <robin.murphy@arm.com>
>> Date: Thu, 4 Dec 2014 11:53:13 +0000
>> Subject: [PATCH v2] iommu: store DT-probed IOMMU data privately
>>
>> Since the data pointer in the DT node is public and may be overwritten
>> by conflicting code, move the DT-probed IOMMU ops to a private list
>> where they will be safe.
>>
>> Signed-off-by: Robin Murphy <robin.murphy@arm.com>
>> ---
>>   drivers/iommu/of_iommu.c | 40 ++++++++++++++++++++++++++++++++++++++++
>>   include/linux/of_iommu.h | 12 ++----------
>>   2 files changed, 42 insertions(+), 10 deletions(-)
>>
>> diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
>> index 73236d3..c7078f6 100644
>> --- a/drivers/iommu/of_iommu.c
>> +++ b/drivers/iommu/of_iommu.c
>> @@ -94,6 +94,46 @@ int of_get_dma_window(struct device_node *dn, const char
>> *prefix, int index,
>>   }
>>   EXPORT_SYMBOL_GPL(of_get_dma_window);
>>
>> +struct of_iommu_node {
>> +	struct list_head list;
>> +	struct device_node *np;
>> +	struct iommu_ops *ops;
>
> Why can't this be const? Why would anyone ever need to modify it? Also
> drivers do define their iommu_ops structures const these days, so you'll
> either still have to cast away at some point or the compiler will
> complain once you start calling this from drivers.
>

...whereas if we make everything const then the drivers that _do_ want 
to use the private data introduced by this series and thus pass mutable 
per-instance iommu_ops will be the ones having to cast. We have no users 
either way until this series is merged, and the need to stash the 
per-instance data somewhere other than np->data is in the way of getting 
it merged - this is just a quick hack to address that. I think we've 
already agreed that mutable per-instance iommu_ops holding private data 
aren't particularly pleasant and will (hopefully) go away in the next 
iteration[1], at which point this all changes anyway.

Regards,
Robin.

[1]:http://article.gmane.org/gmane.linux.kernel.iommu/7554

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Grant Likely]

On Thu, Dec 4, 2014 at 5:42 PM, Robin Murphy <robin.murphy@arm.com> wrote:
> Hi Thierry,
>
>
> On 04/12/14 14:49, Thierry Reding wrote:
>>
>> On Thu, Dec 04, 2014 at 01:43:32PM +0000, Robin Murphy wrote:
>>>
>>> Hi Grant, thanks for the advice - silly micro-optimisations removed, and
>>> I'll make a note to do so from my in-development code, too ;)
>>>
>>> I didn't much like the casting either, so rather than push it elsewhere
>>> or
>>> out to the caller I've just changed the prototype to obviate it
>>> completely.
>>> Since we're also expecting to churn this again to use something more
>>> suitable than iommu_ops as the private data, I think keeping things
>>> simple
>>> wins over const-correctness for now.
>>>
>>> Thanks,
>>> Robin
>>>
>>> --->8---
>>>  From b2e8c91ac49bef4008661e4628cd6b7249d84af5 Mon Sep 17 00:00:00 2001
>>> Message-Id:
>>> <b2e8c91ac49bef4008661e4628cd6b7249d84af5.1417698001.git.robin.murphy@arm.com>
>>> From: Robin Murphy <robin.murphy@arm.com>
>>> Date: Thu, 4 Dec 2014 11:53:13 +0000
>>> Subject: [PATCH v2] iommu: store DT-probed IOMMU data privately
>>>
>>> Since the data pointer in the DT node is public and may be overwritten
>>> by conflicting code, move the DT-probed IOMMU ops to a private list
>>> where they will be safe.
>>>
>>> Signed-off-by: Robin Murphy <robin.murphy@arm.com>
>>> ---
>>>   drivers/iommu/of_iommu.c | 40 ++++++++++++++++++++++++++++++++++++++++
>>>   include/linux/of_iommu.h | 12 ++----------
>>>   2 files changed, 42 insertions(+), 10 deletions(-)
>>>
>>> diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
>>> index 73236d3..c7078f6 100644
>>> --- a/drivers/iommu/of_iommu.c
>>> +++ b/drivers/iommu/of_iommu.c
>>> @@ -94,6 +94,46 @@ int of_get_dma_window(struct device_node *dn, const
>>> char
>>> *prefix, int index,
>>>   }
>>>   EXPORT_SYMBOL_GPL(of_get_dma_window);
>>>
>>> +struct of_iommu_node {
>>> +       struct list_head list;
>>> +       struct device_node *np;
>>> +       struct iommu_ops *ops;
>>
>>
>> Why can't this be const? Why would anyone ever need to modify it? Also
>> drivers do define their iommu_ops structures const these days, so you'll
>> either still have to cast away at some point or the compiler will
>> complain once you start calling this from drivers.
>>
>
> ...whereas if we make everything const then the drivers that _do_ want to
> use the private data introduced by this series and thus pass mutable
> per-instance iommu_ops will be the ones having to cast. We have no users
> either way until this series is merged, and the need to stash the
> per-instance data somewhere other than np->data is in the way of getting it
> merged - this is just a quick hack to address that. I think we've already
> agreed that mutable per-instance iommu_ops holding private data aren't
> particularly pleasant and will (hopefully) go away in the next iteration[1],
> at which point this all changes anyway.

Do you expect drivers to modify that *priv pointer after the ops
structure is registered? I'd be very surprised if that was the use
case. It's fine for the driver to register a non-const version, but
once it is registered, the infrastructure can treat it as const from
then on.

g.

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Robin Murphy]

Hi Grant,

On 04/12/14 17:58, Grant Likely wrote:
[...]
>>>> +struct of_iommu_node {
>>>> +       struct list_head list;
>>>> +       struct device_node *np;
>>>> +       struct iommu_ops *ops;
>>>
>>>
>>> Why can't this be const? Why would anyone ever need to modify it? Also
>>> drivers do define their iommu_ops structures const these days, so you'll
>>> either still have to cast away at some point or the compiler will
>>> complain once you start calling this from drivers.
>>>
>>
>> ...whereas if we make everything const then the drivers that _do_ want to
>> use the private data introduced by this series and thus pass mutable
>> per-instance iommu_ops will be the ones having to cast. We have no users
>> either way until this series is merged, and the need to stash the
>> per-instance data somewhere other than np->data is in the way of getting it
>> merged - this is just a quick hack to address that. I think we've already
>> agreed that mutable per-instance iommu_ops holding private data aren't
>> particularly pleasant and will (hopefully) go away in the next iteration[1],
>> at which point this all changes anyway.
>
> Do you expect drivers to modify that *priv pointer after the ops
> structure is registered? I'd be very surprised if that was the use
> case. It's fine for the driver to register a non-const version, but
> once it is registered, the infrastructure can treat it as const from
> then on.

Possibly not - certainly my current port of the ARM SMMU which makes use 
of *priv is only ever reading it - although we did also wave around 
reasons for mutable ops like dynamically changing the pgsize_bitmap and 
possibly even swizzling individual ops for runtime reconfiguration. On 
consideration though, I'd agree that things like that are mad enough to 
stay well within individual drivers if they did ever happen, and 
certainly shouldn't apply to this bit of the infrastructure at any rate.

Here's a respin the other way - making the get/set infrastructure fully 
const and fixing up the callsite in of_iommu_configure instead to avoid 
the warning. Trying to chase const-correctness beyond that quickly got 
too invasive and out of scope for this fix - I'm just keen to get the 
merge-blocker out of the way.

Regards,
Robin.

--->8---
 From 9eba5081aaf4fa8ed5158675a6e622be11a64ae2 Mon Sep 17 00:00:00 2001
Message-Id: 
<9eba5081aaf4fa8ed5158675a6e622be11a64ae2.1417719305.git.robin.murphy@arm.com>
From: Robin Murphy <robin.murphy@arm.com>
Date: Thu, 4 Dec 2014 11:53:13 +0000
Subject: [PATCH v3] iommu: store DT-probed IOMMU data privately

Since the data pointer in the DT node is public and may be overwritten
by conflicting code, move the DT-probed IOMMU ops to a private list
where they will be safe.

Signed-off-by: Robin Murphy <robin.murphy@arm.com>
---
  drivers/iommu/of_iommu.c | 40 +++++++++++++++++++++++++++++++++++++++-
  include/linux/of_iommu.h | 12 ++----------
  2 files changed, 41 insertions(+), 11 deletions(-)

diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
index 73236d3..39f581f 100644
--- a/drivers/iommu/of_iommu.c
+++ b/drivers/iommu/of_iommu.c
@@ -94,6 +94,44 @@ int of_get_dma_window(struct device_node *dn, const 
char *prefix, int index,
  }
  EXPORT_SYMBOL_GPL(of_get_dma_window);

+struct of_iommu_node {
+	struct list_head list;
+	struct device_node *np;
+	const struct iommu_ops *ops;
+};
+static LIST_HEAD(of_iommu_list);
+static DEFINE_SPINLOCK(of_iommu_lock);
+
+void of_iommu_set_ops(struct device_node *np, const struct iommu_ops *ops)
+{
+	struct of_iommu_node *iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
+
+	if (WARN_ON(!iommu))
+		return;
+
+	INIT_LIST_HEAD(&iommu->list);
+	iommu->np = np;
+	iommu->ops = ops;
+	spin_lock(&of_iommu_lock);
+	list_add_tail(&iommu->list, &of_iommu_list);
+	spin_unlock(&of_iommu_lock);
+}
+
+const struct iommu_ops *of_iommu_get_ops(struct device_node *np)
+{
+	struct of_iommu_node *node;
+	const struct iommu_ops *ops = NULL;
+
+	spin_lock(&of_iommu_lock);
+	list_for_each_entry(node, &of_iommu_list, list)
+		if (node->np == np) {
+			ops = node->ops;
+			break;
+		}
+	spin_unlock(&of_iommu_lock);
+	return ops;
+}
+
  struct iommu_ops *of_iommu_configure(struct device *dev)
  {
  	struct of_phandle_args iommu_spec;
@@ -110,7 +148,7 @@ struct iommu_ops *of_iommu_configure(struct device *dev)
  					   "#iommu-cells", idx,
  					   &iommu_spec)) {
  		np = iommu_spec.np;
-		ops = of_iommu_get_ops(np);
+		ops = (struct iommu_ops *)of_iommu_get_ops(np);

  		if (!ops || !ops->of_xlate || ops->of_xlate(dev, &iommu_spec))
  			goto err_put_node;
diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h
index d03abbb..83f6d0b 100644
--- a/include/linux/of_iommu.h
+++ b/include/linux/of_iommu.h
@@ -31,16 +31,8 @@ static inline struct iommu_ops 
*of_iommu_configure(struct device *dev)

  #endif	/* CONFIG_OF_IOMMU */

-static inline void of_iommu_set_ops(struct device_node *np,
-				    const struct iommu_ops *ops)
-{
-	np->data = (struct iommu_ops *)ops;
-}
-
-static inline struct iommu_ops *of_iommu_get_ops(struct device_node *np)
-{
-	return np->data;
-}
+void of_iommu_set_ops(struct device_node *np, const struct iommu_ops *ops);
+const struct iommu_ops *of_iommu_get_ops(struct device_node *np);

  extern struct of_device_id __iommu_of_table;

-- 
1.9.1

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Will Deacon]

On Thu, Dec 04, 2014 at 07:42:28PM +0000, Robin Murphy wrote:
> On 04/12/14 17:58, Grant Likely wrote:
> [...]
> >>>> +struct of_iommu_node {
> >>>> +       struct list_head list;
> >>>> +       struct device_node *np;
> >>>> +       struct iommu_ops *ops;
> >>>
> >>>
> >>> Why can't this be const? Why would anyone ever need to modify it? Also
> >>> drivers do define their iommu_ops structures const these days, so you'll
> >>> either still have to cast away at some point or the compiler will
> >>> complain once you start calling this from drivers.
> >>>
> >>
> >> ...whereas if we make everything const then the drivers that _do_ want to
> >> use the private data introduced by this series and thus pass mutable
> >> per-instance iommu_ops will be the ones having to cast. We have no users
> >> either way until this series is merged, and the need to stash the
> >> per-instance data somewhere other than np->data is in the way of getting it
> >> merged - this is just a quick hack to address that. I think we've already
> >> agreed that mutable per-instance iommu_ops holding private data aren't
> >> particularly pleasant and will (hopefully) go away in the next iteration[1],
> >> at which point this all changes anyway.
> >
> > Do you expect drivers to modify that *priv pointer after the ops
> > structure is registered? I'd be very surprised if that was the use
> > case. It's fine for the driver to register a non-const version, but
> > once it is registered, the infrastructure can treat it as const from
> > then on.
> 
> Possibly not - certainly my current port of the ARM SMMU which makes use 
> of *priv is only ever reading it - although we did also wave around 
> reasons for mutable ops like dynamically changing the pgsize_bitmap and 
> possibly even swizzling individual ops for runtime reconfiguration. On 
> consideration though, I'd agree that things like that are mad enough to 
> stay well within individual drivers if they did ever happen, and 
> certainly shouldn't apply to this bit of the infrastructure at any rate.

I certainly need to update the pgsize_bitmap at runtime because I don't
know the supported page sizes until I've both (a) probed the hardware
and (b) allocated page tables for a domain. We've already discussed
moving the pgsize_bitmap out of the ops, but moving it somewhere where
it remains const doesn't really help.

Can I just take the patch that Grant acked, in the interest of getting
something merged? As you say, there's plenty of planned changes in this
area anyway. I plan to send Olof a pull request this afternoon.

Will

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Arnd Bergmann]

On Friday 05 December 2014 12:10:37 Will Deacon wrote:
> On Thu, Dec 04, 2014 at 07:42:28PM +0000, Robin Murphy wrote:
> > On 04/12/14 17:58, Grant Likely wrote:
> > [...]
> > >>>> +struct of_iommu_node {
> > >>>> +       struct list_head list;
> > >>>> +       struct device_node *np;
> > >>>> +       struct iommu_ops *ops;
> > >>>
> > >>>
> > >>> Why can't this be const? Why would anyone ever need to modify it? Also
> > >>> drivers do define their iommu_ops structures const these days, so you'll
> > >>> either still have to cast away at some point or the compiler will
> > >>> complain once you start calling this from drivers.
> > >>>
> > >>
> > >> ...whereas if we make everything const then the drivers that _do_ want to
> > >> use the private data introduced by this series and thus pass mutable
> > >> per-instance iommu_ops will be the ones having to cast. We have no users
> > >> either way until this series is merged, and the need to stash the
> > >> per-instance data somewhere other than np->data is in the way of getting it
> > >> merged - this is just a quick hack to address that. I think we've already
> > >> agreed that mutable per-instance iommu_ops holding private data aren't
> > >> particularly pleasant and will (hopefully) go away in the next iteration[1],
> > >> at which point this all changes anyway.
> > >
> > > Do you expect drivers to modify that *priv pointer after the ops
> > > structure is registered? I'd be very surprised if that was the use
> > > case. It's fine for the driver to register a non-const version, but
> > > once it is registered, the infrastructure can treat it as const from
> > > then on.
> > 
> > Possibly not - certainly my current port of the ARM SMMU which makes use 
> > of *priv is only ever reading it - although we did also wave around 
> > reasons for mutable ops like dynamically changing the pgsize_bitmap and 
> > possibly even swizzling individual ops for runtime reconfiguration. On 
> > consideration though, I'd agree that things like that are mad enough to 
> > stay well within individual drivers if they did ever happen, and 
> > certainly shouldn't apply to this bit of the infrastructure at any rate.
> 
> I certainly need to update the pgsize_bitmap at runtime because I don't
> know the supported page sizes until I've both (a) probed the hardware
> and (b) allocated page tables for a domain. We've already discussed
> moving the pgsize_bitmap out of the ops, but moving it somewhere where
> it remains const doesn't really help.
> 
> Can I just take the patch that Grant acked, in the interest of getting
> something merged? As you say, there's plenty of planned changes in this
> area anyway. I plan to send Olof a pull request this afternoon.
> 

I think that would be ok. The fix later should be to move the
private data pointer into of_iommu_node, but I think that will
require a larger set of changes, so let's defer that.

	Arnd

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Robin Murphy]

Hi Will,

On 05/12/14 12:10, Will Deacon wrote:
[...]
>>> Do you expect drivers to modify that *priv pointer after the ops
>>> structure is registered? I'd be very surprised if that was the use
>>> case. It's fine for the driver to register a non-const version, but
>>> once it is registered, the infrastructure can treat it as const from
>>> then on.
>>
>> Possibly not - certainly my current port of the ARM SMMU which makes use
>> of *priv is only ever reading it - although we did also wave around
>> reasons for mutable ops like dynamically changing the pgsize_bitmap and
>> possibly even swizzling individual ops for runtime reconfiguration. On
>> consideration though, I'd agree that things like that are mad enough to
>> stay well within individual drivers if they did ever happen, and
>> certainly shouldn't apply to this bit of the infrastructure at any rate.
>
> I certainly need to update the pgsize_bitmap at runtime because I don't
> know the supported page sizes until I've both (a) probed the hardware
> and (b) allocated page tables for a domain. We've already discussed
> moving the pgsize_bitmap out of the ops, but moving it somewhere where
> it remains const doesn't really help.

We can safely cast the call to get_ops in the SMMU driver though, since 
we'll know that we put a mutable per-instance ops in there in the first 
place. At least that way drivers that aren't taking advantage and just 
pass their static const ops around shouldn't provoke warnings. I 
deliberately didn't touch anything beyond get_ops as that would be too 
disruptive.

> Can I just take the patch that Grant acked, in the interest of getting
> something merged? As you say, there's plenty of planned changes in this
> area anyway. I plan to send Olof a pull request this afternoon.

Grant, Thierry? Personally I'm not fussed either way - the sooner 
something goes in, the sooner I can carry on working at replacing it :D

Robin.

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Grant Likely]

On Fri, Dec 5, 2014 at 12:35 PM, Robin Murphy <robin.murphy@arm.com> wrote:
> Hi Will,
>
> On 05/12/14 12:10, Will Deacon wrote:
> [...]
>>>>
>>>> Do you expect drivers to modify that *priv pointer after the ops
>>>> structure is registered? I'd be very surprised if that was the use
>>>> case. It's fine for the driver to register a non-const version, but
>>>> once it is registered, the infrastructure can treat it as const from
>>>> then on.
>>>
>>>
>>> Possibly not - certainly my current port of the ARM SMMU which makes use
>>> of *priv is only ever reading it - although we did also wave around
>>> reasons for mutable ops like dynamically changing the pgsize_bitmap and
>>> possibly even swizzling individual ops for runtime reconfiguration. On
>>> consideration though, I'd agree that things like that are mad enough to
>>> stay well within individual drivers if they did ever happen, and
>>> certainly shouldn't apply to this bit of the infrastructure at any rate.
>>
>>
>> I certainly need to update the pgsize_bitmap at runtime because I don't
>> know the supported page sizes until I've both (a) probed the hardware
>> and (b) allocated page tables for a domain. We've already discussed
>> moving the pgsize_bitmap out of the ops, but moving it somewhere where
>> it remains const doesn't really help.
>
>
> We can safely cast the call to get_ops in the SMMU driver though, since
> we'll know that we put a mutable per-instance ops in there in the first
> place. At least that way drivers that aren't taking advantage and just pass
> their static const ops around shouldn't provoke warnings. I deliberately
> didn't touch anything beyond get_ops as that would be too disruptive.
>
>> Can I just take the patch that Grant acked, in the interest of getting
>> something merged? As you say, there's plenty of planned changes in this
>> area anyway. I plan to send Olof a pull request this afternoon.
>
>
> Grant, Thierry? Personally I'm not fussed either way - the sooner something
> goes in, the sooner I can carry on working at replacing it :D

I've already acked it. Why are we still talking about it?  :-D

g.

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Thierry Reding]

On Fri, Dec 05, 2014 at 01:06:52PM +0000, Grant Likely wrote:
> On Fri, Dec 5, 2014 at 12:35 PM, Robin Murphy <robin.murphy@arm.com> wrote:
> > Hi Will,
> >
> > On 05/12/14 12:10, Will Deacon wrote:
> > [...]
> >>>>
> >>>> Do you expect drivers to modify that *priv pointer after the ops
> >>>> structure is registered? I'd be very surprised if that was the use
> >>>> case. It's fine for the driver to register a non-const version, but
> >>>> once it is registered, the infrastructure can treat it as const from
> >>>> then on.
> >>>
> >>>
> >>> Possibly not - certainly my current port of the ARM SMMU which makes use
> >>> of *priv is only ever reading it - although we did also wave around
> >>> reasons for mutable ops like dynamically changing the pgsize_bitmap and
> >>> possibly even swizzling individual ops for runtime reconfiguration. On
> >>> consideration though, I'd agree that things like that are mad enough to
> >>> stay well within individual drivers if they did ever happen, and
> >>> certainly shouldn't apply to this bit of the infrastructure at any rate.
> >>
> >>
> >> I certainly need to update the pgsize_bitmap at runtime because I don't
> >> know the supported page sizes until I've both (a) probed the hardware
> >> and (b) allocated page tables for a domain. We've already discussed
> >> moving the pgsize_bitmap out of the ops, but moving it somewhere where
> >> it remains const doesn't really help.
> >
> >
> > We can safely cast the call to get_ops in the SMMU driver though, since
> > we'll know that we put a mutable per-instance ops in there in the first
> > place. At least that way drivers that aren't taking advantage and just pass
> > their static const ops around shouldn't provoke warnings. I deliberately
> > didn't touch anything beyond get_ops as that would be too disruptive.
> >
> >> Can I just take the patch that Grant acked, in the interest of getting
> >> something merged? As you say, there's plenty of planned changes in this
> >> area anyway. I plan to send Olof a pull request this afternoon.
> >
> >
> > Grant, Thierry? Personally I'm not fussed either way - the sooner something
> > goes in, the sooner I can carry on working at replacing it :D
> 
> I've already acked it. Why are we still talking about it?  :-D

Am I missing something? Why is there a need to rush things? Are there
actually drivers that depend on this that will be merged during the 3.19
merge window? It seems like that'd be cutting it really close given
where we are in the release cycle.

If that's not the case, why even bother getting this hack into 3.19 if
nobody uses it and we're going to change it in 3.20 anyway?

Thierry
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 819 bytes
Desc: not available
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20141205/cdcf4793/attachment.sig>

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Grant Likely]

On Fri, Dec 5, 2014 at 1:18 PM, Thierry Reding <thierry.reding@gmail.com> wrote:
> On Fri, Dec 05, 2014 at 01:06:52PM +0000, Grant Likely wrote:
>> On Fri, Dec 5, 2014 at 12:35 PM, Robin Murphy <robin.murphy@arm.com> wrote:
>> > Hi Will,
>> >
>> > On 05/12/14 12:10, Will Deacon wrote:
>> > [...]
>> >>>>
>> >>>> Do you expect drivers to modify that *priv pointer after the ops
>> >>>> structure is registered? I'd be very surprised if that was the use
>> >>>> case. It's fine for the driver to register a non-const version, but
>> >>>> once it is registered, the infrastructure can treat it as const from
>> >>>> then on.
>> >>>
>> >>>
>> >>> Possibly not - certainly my current port of the ARM SMMU which makes use
>> >>> of *priv is only ever reading it - although we did also wave around
>> >>> reasons for mutable ops like dynamically changing the pgsize_bitmap and
>> >>> possibly even swizzling individual ops for runtime reconfiguration. On
>> >>> consideration though, I'd agree that things like that are mad enough to
>> >>> stay well within individual drivers if they did ever happen, and
>> >>> certainly shouldn't apply to this bit of the infrastructure at any rate.
>> >>
>> >>
>> >> I certainly need to update the pgsize_bitmap at runtime because I don't
>> >> know the supported page sizes until I've both (a) probed the hardware
>> >> and (b) allocated page tables for a domain. We've already discussed
>> >> moving the pgsize_bitmap out of the ops, but moving it somewhere where
>> >> it remains const doesn't really help.
>> >
>> >
>> > We can safely cast the call to get_ops in the SMMU driver though, since
>> > we'll know that we put a mutable per-instance ops in there in the first
>> > place. At least that way drivers that aren't taking advantage and just pass
>> > their static const ops around shouldn't provoke warnings. I deliberately
>> > didn't touch anything beyond get_ops as that would be too disruptive.
>> >
>> >> Can I just take the patch that Grant acked, in the interest of getting
>> >> something merged? As you say, there's plenty of planned changes in this
>> >> area anyway. I plan to send Olof a pull request this afternoon.
>> >
>> >
>> > Grant, Thierry? Personally I'm not fussed either way - the sooner something
>> > goes in, the sooner I can carry on working at replacing it :D
>>
>> I've already acked it. Why are we still talking about it?  :-D
>
> Am I missing something? Why is there a need to rush things? Are there
> actually drivers that depend on this that will be merged during the 3.19
> merge window? It seems like that'd be cutting it really close given
> where we are in the release cycle.
>
> If that's not the case, why even bother getting this hack into 3.19 if
> nobody uses it and we're going to change it in 3.20 anyway?

I also acked the non-hack version, the patch that doesn't try to make
everything const. I assumed that was the one that we are talking about
merging.

g.

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Thierry Reding]

On Fri, Dec 05, 2014 at 01:21:31PM +0000, Grant Likely wrote:
> On Fri, Dec 5, 2014 at 1:18 PM, Thierry Reding <thierry.reding@gmail.com> wrote:
> > On Fri, Dec 05, 2014 at 01:06:52PM +0000, Grant Likely wrote:
> >> On Fri, Dec 5, 2014 at 12:35 PM, Robin Murphy <robin.murphy@arm.com> wrote:
> >> > Hi Will,
> >> >
> >> > On 05/12/14 12:10, Will Deacon wrote:
> >> > [...]
> >> >>>>
> >> >>>> Do you expect drivers to modify that *priv pointer after the ops
> >> >>>> structure is registered? I'd be very surprised if that was the use
> >> >>>> case. It's fine for the driver to register a non-const version, but
> >> >>>> once it is registered, the infrastructure can treat it as const from
> >> >>>> then on.
> >> >>>
> >> >>>
> >> >>> Possibly not - certainly my current port of the ARM SMMU which makes use
> >> >>> of *priv is only ever reading it - although we did also wave around
> >> >>> reasons for mutable ops like dynamically changing the pgsize_bitmap and
> >> >>> possibly even swizzling individual ops for runtime reconfiguration. On
> >> >>> consideration though, I'd agree that things like that are mad enough to
> >> >>> stay well within individual drivers if they did ever happen, and
> >> >>> certainly shouldn't apply to this bit of the infrastructure at any rate.
> >> >>
> >> >>
> >> >> I certainly need to update the pgsize_bitmap at runtime because I don't
> >> >> know the supported page sizes until I've both (a) probed the hardware
> >> >> and (b) allocated page tables for a domain. We've already discussed
> >> >> moving the pgsize_bitmap out of the ops, but moving it somewhere where
> >> >> it remains const doesn't really help.
> >> >
> >> >
> >> > We can safely cast the call to get_ops in the SMMU driver though, since
> >> > we'll know that we put a mutable per-instance ops in there in the first
> >> > place. At least that way drivers that aren't taking advantage and just pass
> >> > their static const ops around shouldn't provoke warnings. I deliberately
> >> > didn't touch anything beyond get_ops as that would be too disruptive.
> >> >
> >> >> Can I just take the patch that Grant acked, in the interest of getting
> >> >> something merged? As you say, there's plenty of planned changes in this
> >> >> area anyway. I plan to send Olof a pull request this afternoon.
> >> >
> >> >
> >> > Grant, Thierry? Personally I'm not fussed either way - the sooner something
> >> > goes in, the sooner I can carry on working at replacing it :D
> >>
> >> I've already acked it. Why are we still talking about it?  :-D
> >
> > Am I missing something? Why is there a need to rush things? Are there
> > actually drivers that depend on this that will be merged during the 3.19
> > merge window? It seems like that'd be cutting it really close given
> > where we are in the release cycle.
> >
> > If that's not the case, why even bother getting this hack into 3.19 if
> > nobody uses it and we're going to change it in 3.20 anyway?
> 
> I also acked the non-hack version, the patch that doesn't try to make
> everything const. I assumed that was the one that we are talking about
> merging.

Actually not making everything const would be a hack. Drivers already
mark their struct iommu_ops as const.

But I'm more referring to the series as a whole. It seems like there are
various issues that still need to be ironed out, and there's committment
to do that before 3.20, so unless there are drivers that need any of the
unfinished patches for 3.19 I don't see why we should be merging them in
the first place.

If getting them into 3.19 is merely to resolve dependencies then it's
not going to work well anyway. Since this is all going to change in 3.20
anyway we'd likely have new dependencies that need to be handled, so
might just as well do it properly at that time.

Thierry
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 819 bytes
Desc: not available
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20141205/f1fe5879/attachment.sig>

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Marek Szyprowski]

Hello,

On 2014-12-05 14:18, Thierry Reding wrote:
> On Fri, Dec 05, 2014 at 01:06:52PM +0000, Grant Likely wrote:
>> On Fri, Dec 5, 2014 at 12:35 PM, Robin Murphy <robin.murphy@arm.com> wrote:
>>> Hi Will,
>>>
>>> On 05/12/14 12:10, Will Deacon wrote:
>>> [...]
>>>>>> Do you expect drivers to modify that *priv pointer after the ops
>>>>>> structure is registered? I'd be very surprised if that was the use
>>>>>> case. It's fine for the driver to register a non-const version, but
>>>>>> once it is registered, the infrastructure can treat it as const from
>>>>>> then on.
>>>>>
>>>>> Possibly not - certainly my current port of the ARM SMMU which makes use
>>>>> of *priv is only ever reading it - although we did also wave around
>>>>> reasons for mutable ops like dynamically changing the pgsize_bitmap and
>>>>> possibly even swizzling individual ops for runtime reconfiguration. On
>>>>> consideration though, I'd agree that things like that are mad enough to
>>>>> stay well within individual drivers if they did ever happen, and
>>>>> certainly shouldn't apply to this bit of the infrastructure at any rate.
>>>>
>>>> I certainly need to update the pgsize_bitmap at runtime because I don't
>>>> know the supported page sizes until I've both (a) probed the hardware
>>>> and (b) allocated page tables for a domain. We've already discussed
>>>> moving the pgsize_bitmap out of the ops, but moving it somewhere where
>>>> it remains const doesn't really help.
>>>
>>> We can safely cast the call to get_ops in the SMMU driver though, since
>>> we'll know that we put a mutable per-instance ops in there in the first
>>> place. At least that way drivers that aren't taking advantage and just pass
>>> their static const ops around shouldn't provoke warnings. I deliberately
>>> didn't touch anything beyond get_ops as that would be too disruptive.
>>>
>>>> Can I just take the patch that Grant acked, in the interest of getting
>>>> something merged? As you say, there's plenty of planned changes in this
>>>> area anyway. I plan to send Olof a pull request this afternoon.
>>>
>>> Grant, Thierry? Personally I'm not fussed either way - the sooner something
>>> goes in, the sooner I can carry on working at replacing it :D
>> I've already acked it. Why are we still talking about it?  :-D
> Am I missing something? Why is there a need to rush things? Are there
> actually drivers that depend on this that will be merged during the 3.19
> merge window? It seems like that'd be cutting it really close given
> where we are in the release cycle.
>
> If that's not the case, why even bother getting this hack into 3.19 if
> nobody uses it and we're going to change it in 3.20 anyway?

There are Exynos SYSMMU patches ready & waiting for this gets merged...

Best regards
-- 
Marek Szyprowski, PhD
Samsung R&D Institute Poland

[PATCH v6 1/8] iommu: provide early initialisation hook for IOMMU drivers

[Arnd Bergmann]

On Thursday 04 December 2014 12:26:58 Robin Murphy wrote:
> 
> +struct of_iommu_node {
> +       struct hlist_node list;
> +       struct device_node *np;
> +       const struct iommu_ops *ops;
> +};
> +static HLIST_HEAD(of_iommu_list);
> +static DEFINE_SPINLOCK(of_iommu_lock);

Looks good to me. For 3.20, I would hope to get consensus on renaming this
structure to 'struct iommmu' and adding additional members into it as needed,
but let's do that another day. Please address Grant's feedback and send
a new version.

	Arnd

[PATCH v6 2/8] dma-mapping: replace set_arch_dma_coherent_ops with arch_setup_dma_ops

[Will Deacon]

set_arch_dma_coherent_ops is called from of_dma_configure in order to
swizzle the architectural dma-mapping functions over to a cache-coherent
implementation. This is currently implemented only for ARM.

In anticipation of re-using this mechanism for IOMMU-backed dma-mapping
ops too, this patch replaces the function with a broader
arch_setup_dma_ops callback which will be extended in future.

Acked-by: Arnd Bergmann <arnd@arndb.de>
Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
Tested-by: Robin Murphy <robin.murphy@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
 arch/arm/include/asm/dma-mapping.h |  8 ++++----
 drivers/of/platform.c              | 31 +++++++++++++------------------
 include/linux/dma-mapping.h        |  7 ++-----
 3 files changed, 19 insertions(+), 27 deletions(-)

diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
index 85738b200023..dc3420e77758 100644
--- a/arch/arm/include/asm/dma-mapping.h
+++ b/arch/arm/include/asm/dma-mapping.h
@@ -121,12 +121,12 @@ static inline unsigned long dma_max_pfn(struct device *dev)
 }
 #define dma_max_pfn(dev) dma_max_pfn(dev)
 
-static inline int set_arch_dma_coherent_ops(struct device *dev)
+static inline void arch_setup_dma_ops(struct device *dev, bool coherent)
 {
-	set_dma_ops(dev, &arm_coherent_dma_ops);
-	return 0;
+	if (coherent)
+		set_dma_ops(dev, &arm_coherent_dma_ops);
 }
-#define set_arch_dma_coherent_ops(dev)	set_arch_dma_coherent_ops(dev)
+#define arch_setup_dma_ops arch_setup_dma_ops
 
 static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
 {
diff --git a/drivers/of/platform.c b/drivers/of/platform.c
index 3b64d0bf5bba..ff1f4e9afccb 100644
--- a/drivers/of/platform.c
+++ b/drivers/of/platform.c
@@ -164,6 +164,8 @@ static void of_dma_configure(struct device *dev)
 {
 	u64 dma_addr, paddr, size;
 	int ret;
+	bool coherent;
+	unsigned long offset;
 
 	/*
 	 * Set default dma-mask to 32 bit. Drivers are expected to setup
@@ -178,28 +180,21 @@ static void of_dma_configure(struct device *dev)
 	if (!dev->dma_mask)
 		dev->dma_mask = &dev->coherent_dma_mask;
 
-	/*
-	 * if dma-coherent property exist, call arch hook to setup
-	 * dma coherent operations.
-	 */
-	if (of_dma_is_coherent(dev->of_node)) {
-		set_arch_dma_coherent_ops(dev);
-		dev_dbg(dev, "device is dma coherent\n");
-	}
-
-	/*
-	 * if dma-ranges property doesn't exist - just return else
-	 * setup the dma offset
-	 */
 	ret = of_dma_get_range(dev->of_node, &dma_addr, &paddr, &size);
 	if (ret < 0) {
-		dev_dbg(dev, "no dma range information to setup\n");
-		return;
+		dma_addr = offset = 0;
+		size = dev->coherent_dma_mask;
+	} else {
+		offset = PFN_DOWN(paddr - dma_addr);
+		dev_dbg(dev, "dma_pfn_offset(%#08lx)\n", dev->dma_pfn_offset);
 	}
+	dev->dma_pfn_offset = offset;
+
+	coherent = of_dma_is_coherent(dev->of_node);
+	dev_dbg(dev, "device is%sdma coherent\n",
+		coherent ? " " : " not ");
 
-	/* DMA ranges found. Calculate and set dma_pfn_offset */
-	dev->dma_pfn_offset = PFN_DOWN(paddr - dma_addr);
-	dev_dbg(dev, "dma_pfn_offset(%#08lx)\n", dev->dma_pfn_offset);
+	arch_setup_dma_ops(dev, coherent);
 }
 
 /**
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index d5d388160f42..8a1560f95d4a 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -129,11 +129,8 @@ static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask)
 
 extern u64 dma_get_required_mask(struct device *dev);
 
-#ifndef set_arch_dma_coherent_ops
-static inline int set_arch_dma_coherent_ops(struct device *dev)
-{
-	return 0;
-}
+#ifndef arch_setup_dma_ops
+static inline void arch_setup_dma_ops(struct device *dev, bool coherent) { }
 #endif
 
 static inline unsigned int dma_get_max_seg_size(struct device *dev)
-- 
2.1.1

[PATCH v6 2/8] dma-mapping: replace set_arch_dma_coherent_ops with arch_setup_dma_ops

[Rob Herring]

On Mon, Dec 1, 2014 at 10:57 AM, Will Deacon <will.deacon@arm.com> wrote:
> set_arch_dma_coherent_ops is called from of_dma_configure in order to
> swizzle the architectural dma-mapping functions over to a cache-coherent
> implementation. This is currently implemented only for ARM.
>
> In anticipation of re-using this mechanism for IOMMU-backed dma-mapping
> ops too, this patch replaces the function with a broader
> arch_setup_dma_ops callback which will be extended in future.
>
> Acked-by: Arnd Bergmann <arnd@arndb.de>
> Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
> Tested-by: Robin Murphy <robin.murphy@arm.com>
> Signed-off-by: Will Deacon <will.deacon@arm.com>

One comment below, but for the DT parts:

Acked-by: Rob Herring <robh@kernel.org>

> ---
>  arch/arm/include/asm/dma-mapping.h |  8 ++++----
>  drivers/of/platform.c              | 31 +++++++++++++------------------
>  include/linux/dma-mapping.h        |  7 ++-----
>  3 files changed, 19 insertions(+), 27 deletions(-)
>
> diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
> index 85738b200023..dc3420e77758 100644
> --- a/arch/arm/include/asm/dma-mapping.h
> +++ b/arch/arm/include/asm/dma-mapping.h
> @@ -121,12 +121,12 @@ static inline unsigned long dma_max_pfn(struct device *dev)
>  }
>  #define dma_max_pfn(dev) dma_max_pfn(dev)
>
> -static inline int set_arch_dma_coherent_ops(struct device *dev)
> +static inline void arch_setup_dma_ops(struct device *dev, bool coherent)
>  {
> -       set_dma_ops(dev, &arm_coherent_dma_ops);
> -       return 0;
> +       if (coherent)
> +               set_dma_ops(dev, &arm_coherent_dma_ops);
>  }
> -#define set_arch_dma_coherent_ops(dev) set_arch_dma_coherent_ops(dev)
> +#define arch_setup_dma_ops arch_setup_dma_ops
>
>  static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
>  {
> diff --git a/drivers/of/platform.c b/drivers/of/platform.c
> index 3b64d0bf5bba..ff1f4e9afccb 100644
> --- a/drivers/of/platform.c
> +++ b/drivers/of/platform.c
> @@ -164,6 +164,8 @@ static void of_dma_configure(struct device *dev)
>  {
>         u64 dma_addr, paddr, size;
>         int ret;
> +       bool coherent;
> +       unsigned long offset;
>
>         /*
>          * Set default dma-mask to 32 bit. Drivers are expected to setup
> @@ -178,28 +180,21 @@ static void of_dma_configure(struct device *dev)
>         if (!dev->dma_mask)
>                 dev->dma_mask = &dev->coherent_dma_mask;
>
> -       /*
> -        * if dma-coherent property exist, call arch hook to setup
> -        * dma coherent operations.
> -        */
> -       if (of_dma_is_coherent(dev->of_node)) {
> -               set_arch_dma_coherent_ops(dev);
> -               dev_dbg(dev, "device is dma coherent\n");
> -       }
> -
> -       /*
> -        * if dma-ranges property doesn't exist - just return else
> -        * setup the dma offset
> -        */
>         ret = of_dma_get_range(dev->of_node, &dma_addr, &paddr, &size);
>         if (ret < 0) {
> -               dev_dbg(dev, "no dma range information to setup\n");
> -               return;
> +               dma_addr = offset = 0;
> +               size = dev->coherent_dma_mask;

It looks like size is not used.

> +       } else {
> +               offset = PFN_DOWN(paddr - dma_addr);
> +               dev_dbg(dev, "dma_pfn_offset(%#08lx)\n", dev->dma_pfn_offset);
>         }
> +       dev->dma_pfn_offset = offset;
> +
> +       coherent = of_dma_is_coherent(dev->of_node);
> +       dev_dbg(dev, "device is%sdma coherent\n",
> +               coherent ? " " : " not ");
>
> -       /* DMA ranges found. Calculate and set dma_pfn_offset */
> -       dev->dma_pfn_offset = PFN_DOWN(paddr - dma_addr);
> -       dev_dbg(dev, "dma_pfn_offset(%#08lx)\n", dev->dma_pfn_offset);
> +       arch_setup_dma_ops(dev, coherent);
>  }
>
>  /**
> diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
> index d5d388160f42..8a1560f95d4a 100644
> --- a/include/linux/dma-mapping.h
> +++ b/include/linux/dma-mapping.h
> @@ -129,11 +129,8 @@ static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask)
>
>  extern u64 dma_get_required_mask(struct device *dev);
>
> -#ifndef set_arch_dma_coherent_ops
> -static inline int set_arch_dma_coherent_ops(struct device *dev)
> -{
> -       return 0;
> -}
> +#ifndef arch_setup_dma_ops
> +static inline void arch_setup_dma_ops(struct device *dev, bool coherent) { }
>  #endif
>
>  static inline unsigned int dma_get_max_seg_size(struct device *dev)
> --
> 2.1.1
>
>
> _______________________________________________
> linux-arm-kernel mailing list
> linux-arm-kernel at lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[PATCH v6 2/8] dma-mapping: replace set_arch_dma_coherent_ops with arch_setup_dma_ops

[Arnd Bergmann]

On Monday 01 December 2014 16:58:09 Rob Herring wrote:
> > @@ -178,28 +180,21 @@ static void of_dma_configure(struct device *dev)
> >         if (!dev->dma_mask)
> >                 dev->dma_mask = &dev->coherent_dma_mask;
> >
> > -       /*
> > -        * if dma-coherent property exist, call arch hook to setup
> > -        * dma coherent operations.
> > -        */
> > -       if (of_dma_is_coherent(dev->of_node)) {
> > -               set_arch_dma_coherent_ops(dev);
> > -               dev_dbg(dev, "device is dma coherent\n");
> > -       }
> > -
> > -       /*
> > -        * if dma-ranges property doesn't exist - just return else
> > -        * setup the dma offset
> > -        */
> >         ret = of_dma_get_range(dev->of_node, &dma_addr, &paddr, &size);
> >         if (ret < 0) {
> > -               dev_dbg(dev, "no dma range information to setup\n");
> > -               return;
> > +               dma_addr = offset = 0;
> > +               size = dev->coherent_dma_mask;
> 
> It looks like size is not used.

Catalin has proposed a patch for handling the size right, but it's
still under discussion.

	Arnd

[PATCH v6 3/8] iommu: add new iommu_ops callback for adding an OF device

[Will Deacon]

This patch adds a new function to the iommu_ops structure to allow an
OF device to be added to a specific IOMMU instance using the recently
merged generic devicetree binding for IOMMUs. The callback (of_xlate)
takes a struct device representing the master and an of_phandle_args
representing the IOMMU and the correspondong IDs for the new master.

Acked-by: Arnd Bergmann <arnd@arndb.de>
Acked-by: Joerg Roedel <jroedel@suse.de>
Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
Tested-by: Robin Murphy <robin.murphy@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
 include/linux/iommu.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index 7b83f9f8e11d..415c7613d02c 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -21,6 +21,7 @@
 
 #include <linux/errno.h>
 #include <linux/err.h>
+#include <linux/of.h>
 #include <linux/types.h>
 #include <trace/events/iommu.h>
 
@@ -102,6 +103,7 @@ enum iommu_attr {
  * @remove_device: remove device from iommu grouping
  * @domain_get_attr: Query domain attributes
  * @domain_set_attr: Change domain attributes
+ * @of_xlate: add OF master IDs to iommu grouping
  * @pgsize_bitmap: bitmap of supported page sizes
  * @priv: per-instance data private to the iommu driver
  */
@@ -133,6 +135,10 @@ struct iommu_ops {
 	/* Get the numer of window per domain */
 	u32 (*domain_get_windows)(struct iommu_domain *domain);
 
+#ifdef CONFIG_OF_IOMMU
+	int (*of_xlate)(struct device *dev, struct of_phandle_args *args);
+#endif
+
 	unsigned long pgsize_bitmap;
 	void *priv;
 };
-- 
2.1.1

[PATCH v6 4/8] iommu: provide helper function to configure an IOMMU for an of master

[Will Deacon]

The generic IOMMU device-tree bindings can be used to add arbitrary OF
masters to an IOMMU with a compliant binding.

This patch introduces of_iommu_configure, which does exactly that.

Acked-by: Arnd Bergmann <arnd@arndb.de>
Acked-by: Joerg Roedel <jroedel@suse.de>
Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
Tested-by: Robin Murphy <robin.murphy@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
 drivers/iommu/Kconfig    |  2 +-
 drivers/iommu/of_iommu.c | 33 +++++++++++++++++++++++++++++++++
 include/linux/of_iommu.h |  6 ++++++
 3 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index dd5112265cc9..6d13f962f156 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -15,7 +15,7 @@ if IOMMU_SUPPORT
 
 config OF_IOMMU
        def_bool y
-       depends on OF
+       depends on OF && IOMMU_API
 
 config FSL_PAMU
 	bool "Freescale IOMMU support"
diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c
index 89b903406968..73236d3cc955 100644
--- a/drivers/iommu/of_iommu.c
+++ b/drivers/iommu/of_iommu.c
@@ -18,6 +18,7 @@
  */
 
 #include <linux/export.h>
+#include <linux/iommu.h>
 #include <linux/limits.h>
 #include <linux/of.h>
 #include <linux/of_iommu.h>
@@ -93,6 +94,38 @@ int of_get_dma_window(struct device_node *dn, const char *prefix, int index,
 }
 EXPORT_SYMBOL_GPL(of_get_dma_window);
 
+struct iommu_ops *of_iommu_configure(struct device *dev)
+{
+	struct of_phandle_args iommu_spec;
+	struct device_node *np;
+	struct iommu_ops *ops = NULL;
+	int idx = 0;
+
+	/*
+	 * We don't currently walk up the tree looking for a parent IOMMU.
+	 * See the `Notes:' section of
+	 * Documentation/devicetree/bindings/iommu/iommu.txt
+	 */
+	while (!of_parse_phandle_with_args(dev->of_node, "iommus",
+					   "#iommu-cells", idx,
+					   &iommu_spec)) {
+		np = iommu_spec.np;
+		ops = of_iommu_get_ops(np);
+
+		if (!ops || !ops->of_xlate || ops->of_xlate(dev, &iommu_spec))
+			goto err_put_node;
+
+		of_node_put(np);
+		idx++;
+	}
+
+	return ops;
+
+err_put_node:
+	of_node_put(np);
+	return NULL;
+}
+
 void __init of_iommu_init(void)
 {
 	struct device_node *np;
diff --git a/include/linux/of_iommu.h b/include/linux/of_iommu.h
index 5762cdc8effe..d03abbb11c34 100644
--- a/include/linux/of_iommu.h
+++ b/include/linux/of_iommu.h
@@ -1,6 +1,7 @@
 #ifndef __OF_IOMMU_H
 #define __OF_IOMMU_H
 
+#include <linux/device.h>
 #include <linux/iommu.h>
 #include <linux/of.h>
 
@@ -11,6 +12,7 @@ extern int of_get_dma_window(struct device_node *dn, const char *prefix,
 			     size_t *size);
 
 extern void of_iommu_init(void);
+extern struct iommu_ops *of_iommu_configure(struct device *dev);
 
 #else
 
@@ -22,6 +24,10 @@ static inline int of_get_dma_window(struct device_node *dn, const char *prefix,
 }
 
 static inline void of_iommu_init(void) { }
+static inline struct iommu_ops *of_iommu_configure(struct device *dev)
+{
+	return NULL;
+}
 
 #endif	/* CONFIG_OF_IOMMU */
 
-- 
2.1.1

[PATCH v6 5/8] iommu: fix initialization without 'add_device' callback

[Will Deacon]

From: Marek Szyprowski <m.szyprowski@samsung.com>

IOMMU drivers can be initialized from of_iommu helpers. Such drivers don't
need to provide device_add callbacks to operate properly, so there is no
need to fail initialization if the callback is missing.

Acked-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Marek Szyprowski <m.szyprowski@samsung.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
 drivers/iommu/iommu.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index ed8b04867b1f..02f798b7e295 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -737,7 +737,7 @@ static int add_iommu_group(struct device *dev, void *data)
 	const struct iommu_ops *ops = cb->ops;
 
 	if (!ops->add_device)
-		return -ENODEV;
+		return 0;
 
 	WARN_ON(dev->iommu_group);
 
-- 
2.1.1

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Will Deacon]

This patch extends of_dma_configure so that it sets up the IOMMU for a
device, as well as the coherent/non-coherent DMA mapping ops.

Acked-by: Arnd Bergmann <arnd@arndb.de>
Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
Tested-by: Robin Murphy <robin.murphy@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
 arch/arm/include/asm/dma-mapping.h |  4 +++-
 drivers/of/platform.c              | 21 ++++++++++++++-------
 include/linux/dma-mapping.h        |  8 +++++++-
 3 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
index dc3420e77758..f3c0d953f6a2 100644
--- a/arch/arm/include/asm/dma-mapping.h
+++ b/arch/arm/include/asm/dma-mapping.h
@@ -121,7 +121,9 @@ static inline unsigned long dma_max_pfn(struct device *dev)
 }
 #define dma_max_pfn(dev) dma_max_pfn(dev)
 
-static inline void arch_setup_dma_ops(struct device *dev, bool coherent)
+static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
+				      u64 size, struct iommu_ops *iommu,
+				      bool coherent)
 {
 	if (coherent)
 		set_dma_ops(dev, &arm_coherent_dma_ops);
diff --git a/drivers/of/platform.c b/drivers/of/platform.c
index ff1f4e9afccb..b89caf8c7586 100644
--- a/drivers/of/platform.c
+++ b/drivers/of/platform.c
@@ -19,6 +19,7 @@
 #include <linux/slab.h>
 #include <linux/of_address.h>
 #include <linux/of_device.h>
+#include <linux/of_iommu.h>
 #include <linux/of_irq.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
@@ -166,6 +167,7 @@ static void of_dma_configure(struct device *dev)
 	int ret;
 	bool coherent;
 	unsigned long offset;
+	struct iommu_ops *iommu;
 
 	/*
 	 * Set default dma-mask to 32 bit. Drivers are expected to setup
@@ -194,7 +196,16 @@ static void of_dma_configure(struct device *dev)
 	dev_dbg(dev, "device is%sdma coherent\n",
 		coherent ? " " : " not ");
 
-	arch_setup_dma_ops(dev, coherent);
+	iommu = of_iommu_configure(dev);
+	dev_dbg(dev, "device is%sbehind an iommu\n",
+		iommu ? " " : " not ");
+
+	arch_setup_dma_ops(dev, dma_addr, size, iommu, coherent);
+}
+
+static void of_dma_deconfigure(struct device *dev)
+{
+	arch_teardown_dma_ops(dev);
 }
 
 /**
@@ -223,16 +234,12 @@ static struct platform_device *of_platform_device_create_pdata(
 	if (!dev)
 		goto err_clear_flag;
 
-	of_dma_configure(&dev->dev);
 	dev->dev.bus = &platform_bus_type;
 	dev->dev.platform_data = platform_data;
-
-	/* We do not fill the DMA ops for platform devices by default.
-	 * This is currently the responsibility of the platform code
-	 * to do such, possibly using a device notifier
-	 */
+	of_dma_configure(&dev->dev);
 
 	if (of_device_add(dev) != 0) {
+		of_dma_deconfigure(&dev->dev);
 		platform_device_put(dev);
 		goto err_clear_flag;
 	}
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index 8a1560f95d4a..c3007cb4bfa6 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -130,7 +130,13 @@ static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask)
 extern u64 dma_get_required_mask(struct device *dev);
 
 #ifndef arch_setup_dma_ops
-static inline void arch_setup_dma_ops(struct device *dev, bool coherent) { }
+static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
+				      u64 size, struct iommu_ops *iommu,
+				      bool coherent) { }
+#endif
+
+#ifndef arch_teardown_dma_ops
+static inline void arch_teardown_dma_ops(struct device *dev) { }
 #endif
 
 static inline unsigned int dma_get_max_seg_size(struct device *dev)
-- 
2.1.1

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Rob Herring]

On Mon, Dec 1, 2014 at 10:57 AM, Will Deacon <will.deacon@arm.com> wrote:
> This patch extends of_dma_configure so that it sets up the IOMMU for a
> device, as well as the coherent/non-coherent DMA mapping ops.
>
> Acked-by: Arnd Bergmann <arnd@arndb.de>
> Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
> Tested-by: Robin Murphy <robin.murphy@arm.com>
> Signed-off-by: Will Deacon <will.deacon@arm.com>

Acked-by: Rob Herring <robh@kernel.org>

> ---
>  arch/arm/include/asm/dma-mapping.h |  4 +++-
>  drivers/of/platform.c              | 21 ++++++++++++++-------
>  include/linux/dma-mapping.h        |  8 +++++++-
>  3 files changed, 24 insertions(+), 9 deletions(-)
>
> diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
> index dc3420e77758..f3c0d953f6a2 100644
> --- a/arch/arm/include/asm/dma-mapping.h
> +++ b/arch/arm/include/asm/dma-mapping.h
> @@ -121,7 +121,9 @@ static inline unsigned long dma_max_pfn(struct device *dev)
>  }
>  #define dma_max_pfn(dev) dma_max_pfn(dev)
>
> -static inline void arch_setup_dma_ops(struct device *dev, bool coherent)
> +static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
> +                                     u64 size, struct iommu_ops *iommu,
> +                                     bool coherent)
>  {
>         if (coherent)
>                 set_dma_ops(dev, &arm_coherent_dma_ops);
> diff --git a/drivers/of/platform.c b/drivers/of/platform.c
> index ff1f4e9afccb..b89caf8c7586 100644
> --- a/drivers/of/platform.c
> +++ b/drivers/of/platform.c
> @@ -19,6 +19,7 @@
>  #include <linux/slab.h>
>  #include <linux/of_address.h>
>  #include <linux/of_device.h>
> +#include <linux/of_iommu.h>
>  #include <linux/of_irq.h>
>  #include <linux/of_platform.h>
>  #include <linux/platform_device.h>
> @@ -166,6 +167,7 @@ static void of_dma_configure(struct device *dev)
>         int ret;
>         bool coherent;
>         unsigned long offset;
> +       struct iommu_ops *iommu;
>
>         /*
>          * Set default dma-mask to 32 bit. Drivers are expected to setup
> @@ -194,7 +196,16 @@ static void of_dma_configure(struct device *dev)
>         dev_dbg(dev, "device is%sdma coherent\n",
>                 coherent ? " " : " not ");
>
> -       arch_setup_dma_ops(dev, coherent);
> +       iommu = of_iommu_configure(dev);
> +       dev_dbg(dev, "device is%sbehind an iommu\n",
> +               iommu ? " " : " not ");
> +
> +       arch_setup_dma_ops(dev, dma_addr, size, iommu, coherent);
> +}
> +
> +static void of_dma_deconfigure(struct device *dev)
> +{
> +       arch_teardown_dma_ops(dev);
>  }
>
>  /**
> @@ -223,16 +234,12 @@ static struct platform_device *of_platform_device_create_pdata(
>         if (!dev)
>                 goto err_clear_flag;
>
> -       of_dma_configure(&dev->dev);
>         dev->dev.bus = &platform_bus_type;
>         dev->dev.platform_data = platform_data;
> -
> -       /* We do not fill the DMA ops for platform devices by default.
> -        * This is currently the responsibility of the platform code
> -        * to do such, possibly using a device notifier
> -        */
> +       of_dma_configure(&dev->dev);
>
>         if (of_device_add(dev) != 0) {
> +               of_dma_deconfigure(&dev->dev);
>                 platform_device_put(dev);
>                 goto err_clear_flag;
>         }
> diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
> index 8a1560f95d4a..c3007cb4bfa6 100644
> --- a/include/linux/dma-mapping.h
> +++ b/include/linux/dma-mapping.h
> @@ -130,7 +130,13 @@ static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask)
>  extern u64 dma_get_required_mask(struct device *dev);
>
>  #ifndef arch_setup_dma_ops
> -static inline void arch_setup_dma_ops(struct device *dev, bool coherent) { }
> +static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
> +                                     u64 size, struct iommu_ops *iommu,
> +                                     bool coherent) { }
> +#endif
> +
> +#ifndef arch_teardown_dma_ops
> +static inline void arch_teardown_dma_ops(struct device *dev) { }
>  #endif
>
>  static inline unsigned int dma_get_max_seg_size(struct device *dev)
> --
> 2.1.1
>
>
> _______________________________________________
> linux-arm-kernel mailing list
> linux-arm-kernel at lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Rob Clark]

On Mon, Dec 1, 2014 at 11:57 AM, Will Deacon <will.deacon@arm.com> wrote:
> This patch extends of_dma_configure so that it sets up the IOMMU for a
> device, as well as the coherent/non-coherent DMA mapping ops.
>
> Acked-by: Arnd Bergmann <arnd@arndb.de>
> Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
> Tested-by: Robin Murphy <robin.murphy@arm.com>
> Signed-off-by: Will Deacon <will.deacon@arm.com>
> ---
>  arch/arm/include/asm/dma-mapping.h |  4 +++-
>  drivers/of/platform.c              | 21 ++++++++++++++-------
>  include/linux/dma-mapping.h        |  8 +++++++-
>  3 files changed, 24 insertions(+), 9 deletions(-)
>
> diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
> index dc3420e77758..f3c0d953f6a2 100644
> --- a/arch/arm/include/asm/dma-mapping.h
> +++ b/arch/arm/include/asm/dma-mapping.h
> @@ -121,7 +121,9 @@ static inline unsigned long dma_max_pfn(struct device *dev)
>  }
>  #define dma_max_pfn(dev) dma_max_pfn(dev)
>
> -static inline void arch_setup_dma_ops(struct device *dev, bool coherent)
> +static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
> +                                     u64 size, struct iommu_ops *iommu,
> +                                     bool coherent)
>  {
>         if (coherent)
>                 set_dma_ops(dev, &arm_coherent_dma_ops);
> diff --git a/drivers/of/platform.c b/drivers/of/platform.c
> index ff1f4e9afccb..b89caf8c7586 100644
> --- a/drivers/of/platform.c
> +++ b/drivers/of/platform.c
> @@ -19,6 +19,7 @@
>  #include <linux/slab.h>
>  #include <linux/of_address.h>
>  #include <linux/of_device.h>
> +#include <linux/of_iommu.h>
>  #include <linux/of_irq.h>
>  #include <linux/of_platform.h>
>  #include <linux/platform_device.h>
> @@ -166,6 +167,7 @@ static void of_dma_configure(struct device *dev)
>         int ret;
>         bool coherent;
>         unsigned long offset;
> +       struct iommu_ops *iommu;
>
>         /*
>          * Set default dma-mask to 32 bit. Drivers are expected to setup
> @@ -194,7 +196,16 @@ static void of_dma_configure(struct device *dev)
>         dev_dbg(dev, "device is%sdma coherent\n",
>                 coherent ? " " : " not ");
>
> -       arch_setup_dma_ops(dev, coherent);
> +       iommu = of_iommu_configure(dev);
> +       dev_dbg(dev, "device is%sbehind an iommu\n",
> +               iommu ? " " : " not ");
> +
> +       arch_setup_dma_ops(dev, dma_addr, size, iommu, coherent);


so, what is the way for a driver that explicitly wants to manage it's
own device virtual address space to opt out of this?  I suspect that
won't be the common case, but for a gpu, if dma layer all of a sudden
thinks it is in control of the gpu's virtual address space, things are
going to end in tears..

BR,
-R


> +}
> +
> +static void of_dma_deconfigure(struct device *dev)
> +{
> +       arch_teardown_dma_ops(dev);
>  }
>
>  /**
> @@ -223,16 +234,12 @@ static struct platform_device *of_platform_device_create_pdata(
>         if (!dev)
>                 goto err_clear_flag;
>
> -       of_dma_configure(&dev->dev);
>         dev->dev.bus = &platform_bus_type;
>         dev->dev.platform_data = platform_data;
> -
> -       /* We do not fill the DMA ops for platform devices by default.
> -        * This is currently the responsibility of the platform code
> -        * to do such, possibly using a device notifier
> -        */
> +       of_dma_configure(&dev->dev);
>
>         if (of_device_add(dev) != 0) {
> +               of_dma_deconfigure(&dev->dev);
>                 platform_device_put(dev);
>                 goto err_clear_flag;
>         }
> diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
> index 8a1560f95d4a..c3007cb4bfa6 100644
> --- a/include/linux/dma-mapping.h
> +++ b/include/linux/dma-mapping.h
> @@ -130,7 +130,13 @@ static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask)
>  extern u64 dma_get_required_mask(struct device *dev);
>
>  #ifndef arch_setup_dma_ops
> -static inline void arch_setup_dma_ops(struct device *dev, bool coherent) { }
> +static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
> +                                     u64 size, struct iommu_ops *iommu,
> +                                     bool coherent) { }
> +#endif
> +
> +#ifndef arch_teardown_dma_ops
> +static inline void arch_teardown_dma_ops(struct device *dev) { }
>  #endif
>
>  static inline unsigned int dma_get_max_seg_size(struct device *dev)
> --
> 2.1.1
>
> _______________________________________________
> iommu mailing list
> iommu at lists.linux-foundation.org
> https://lists.linuxfoundation.org/mailman/listinfo/iommu

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Will Deacon]

On Wed, Dec 10, 2014 at 02:52:56PM +0000, Rob Clark wrote:
> On Mon, Dec 1, 2014 at 11:57 AM, Will Deacon <will.deacon@arm.com> wrote:
> > This patch extends of_dma_configure so that it sets up the IOMMU for a
> > device, as well as the coherent/non-coherent DMA mapping ops.
> >
> > Acked-by: Arnd Bergmann <arnd@arndb.de>
> > Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
> > Tested-by: Robin Murphy <robin.murphy@arm.com>
> > Signed-off-by: Will Deacon <will.deacon@arm.com>
> > ---
> >  arch/arm/include/asm/dma-mapping.h |  4 +++-
> >  drivers/of/platform.c              | 21 ++++++++++++++-------
> >  include/linux/dma-mapping.h        |  8 +++++++-
> >  3 files changed, 24 insertions(+), 9 deletions(-)
> >
> > diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
> > index dc3420e77758..f3c0d953f6a2 100644
> > --- a/arch/arm/include/asm/dma-mapping.h
> > +++ b/arch/arm/include/asm/dma-mapping.h
> > @@ -121,7 +121,9 @@ static inline unsigned long dma_max_pfn(struct device *dev)
> >  }
> >  #define dma_max_pfn(dev) dma_max_pfn(dev)
> >
> > -static inline void arch_setup_dma_ops(struct device *dev, bool coherent)
> > +static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
> > +                                     u64 size, struct iommu_ops *iommu,
> > +                                     bool coherent)
> >  {
> >         if (coherent)
> >                 set_dma_ops(dev, &arm_coherent_dma_ops);
> > diff --git a/drivers/of/platform.c b/drivers/of/platform.c
> > index ff1f4e9afccb..b89caf8c7586 100644
> > --- a/drivers/of/platform.c
> > +++ b/drivers/of/platform.c
> > @@ -19,6 +19,7 @@
> >  #include <linux/slab.h>
> >  #include <linux/of_address.h>
> >  #include <linux/of_device.h>
> > +#include <linux/of_iommu.h>
> >  #include <linux/of_irq.h>
> >  #include <linux/of_platform.h>
> >  #include <linux/platform_device.h>
> > @@ -166,6 +167,7 @@ static void of_dma_configure(struct device *dev)
> >         int ret;
> >         bool coherent;
> >         unsigned long offset;
> > +       struct iommu_ops *iommu;
> >
> >         /*
> >          * Set default dma-mask to 32 bit. Drivers are expected to setup
> > @@ -194,7 +196,16 @@ static void of_dma_configure(struct device *dev)
> >         dev_dbg(dev, "device is%sdma coherent\n",
> >                 coherent ? " " : " not ");
> >
> > -       arch_setup_dma_ops(dev, coherent);
> > +       iommu = of_iommu_configure(dev);
> > +       dev_dbg(dev, "device is%sbehind an iommu\n",
> > +               iommu ? " " : " not ");
> > +
> > +       arch_setup_dma_ops(dev, dma_addr, size, iommu, coherent);
> 
> 
> so, what is the way for a driver that explicitly wants to manage it's
> own device virtual address space to opt out of this?  I suspect that
> won't be the common case, but for a gpu, if dma layer all of a sudden
> thinks it is in control of the gpu's virtual address space, things are
> going to end in tears..

I think you'll need to detach from the DMA domain, then have the driver
manage everything itself. As you say, it's not the common case, so you
may need to add some hooks for detaching from the default domain and
swizzling your DMA ops.

Will

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Robin Murphy]

On 10/12/14 15:08, Will Deacon wrote:
> On Wed, Dec 10, 2014 at 02:52:56PM +0000, Rob Clark wrote:
>> On Mon, Dec 1, 2014 at 11:57 AM, Will Deacon <will.deacon@arm.com> wrote:
>>> This patch extends of_dma_configure so that it sets up the IOMMU for a
>>> device, as well as the coherent/non-coherent DMA mapping ops.
>>>
>>> Acked-by: Arnd Bergmann <arnd@arndb.de>
>>> Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
>>> Tested-by: Robin Murphy <robin.murphy@arm.com>
>>> Signed-off-by: Will Deacon <will.deacon@arm.com>
>>> ---
>>>   arch/arm/include/asm/dma-mapping.h |  4 +++-
>>>   drivers/of/platform.c              | 21 ++++++++++++++-------
>>>   include/linux/dma-mapping.h        |  8 +++++++-
>>>   3 files changed, 24 insertions(+), 9 deletions(-)
>>>
>>> diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
>>> index dc3420e77758..f3c0d953f6a2 100644
>>> --- a/arch/arm/include/asm/dma-mapping.h
>>> +++ b/arch/arm/include/asm/dma-mapping.h
>>> @@ -121,7 +121,9 @@ static inline unsigned long dma_max_pfn(struct device *dev)
>>>   }
>>>   #define dma_max_pfn(dev) dma_max_pfn(dev)
>>>
>>> -static inline void arch_setup_dma_ops(struct device *dev, bool coherent)
>>> +static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
>>> +                                     u64 size, struct iommu_ops *iommu,
>>> +                                     bool coherent)
>>>   {
>>>          if (coherent)
>>>                  set_dma_ops(dev, &arm_coherent_dma_ops);
>>> diff --git a/drivers/of/platform.c b/drivers/of/platform.c
>>> index ff1f4e9afccb..b89caf8c7586 100644
>>> --- a/drivers/of/platform.c
>>> +++ b/drivers/of/platform.c
>>> @@ -19,6 +19,7 @@
>>>   #include <linux/slab.h>
>>>   #include <linux/of_address.h>
>>>   #include <linux/of_device.h>
>>> +#include <linux/of_iommu.h>
>>>   #include <linux/of_irq.h>
>>>   #include <linux/of_platform.h>
>>>   #include <linux/platform_device.h>
>>> @@ -166,6 +167,7 @@ static void of_dma_configure(struct device *dev)
>>>          int ret;
>>>          bool coherent;
>>>          unsigned long offset;
>>> +       struct iommu_ops *iommu;
>>>
>>>          /*
>>>           * Set default dma-mask to 32 bit. Drivers are expected to setup
>>> @@ -194,7 +196,16 @@ static void of_dma_configure(struct device *dev)
>>>          dev_dbg(dev, "device is%sdma coherent\n",
>>>                  coherent ? " " : " not ");
>>>
>>> -       arch_setup_dma_ops(dev, coherent);
>>> +       iommu = of_iommu_configure(dev);
>>> +       dev_dbg(dev, "device is%sbehind an iommu\n",
>>> +               iommu ? " " : " not ");
>>> +
>>> +       arch_setup_dma_ops(dev, dma_addr, size, iommu, coherent);
>>
>>
>> so, what is the way for a driver that explicitly wants to manage it's
>> own device virtual address space to opt out of this?  I suspect that
>> won't be the common case, but for a gpu, if dma layer all of a sudden
>> thinks it is in control of the gpu's virtual address space, things are
>> going to end in tears..
>
> I think you'll need to detach from the DMA domain, then have the driver
> manage everything itself. As you say, it's not the common case, so you
> may need to add some hooks for detaching from the default domain and
> swizzling your DMA ops.
>

Surely if that's a problem then it's an existing one anyway? If the 
IOMMU driver wants to do its own thing here all it has to do is return 
-ENOTHANKSIDRATHERNOT from its of_xlate call (or be even more boring and 
simply not implement it), and the device gets whatever default DMA ops 
the old of_dma_configure would have given it, with zero impact from this 
series. I only see a potential issue if you have one driver running 
multiple IOMMUs, some of which are happy to pass off control to DMA 
mapping and some not, in which case either you make the driver clever 
enough to identify its clients and handle of_xlate correctly, or work 
around it with the DT representation.

Robin.

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Laurent Pinchart]

Hi Robin,

On Wednesday 10 December 2014 15:54:10 Robin Murphy wrote:
> On 10/12/14 15:08, Will Deacon wrote:
> > On Wed, Dec 10, 2014 at 02:52:56PM +0000, Rob Clark wrote:
> >> On Mon, Dec 1, 2014 at 11:57 AM, Will Deacon <will.deacon@arm.com> wrote:
> >>> This patch extends of_dma_configure so that it sets up the IOMMU for a
> >>> device, as well as the coherent/non-coherent DMA mapping ops.
> >>> 
> >>> Acked-by: Arnd Bergmann <arnd@arndb.de>
> >>> Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
> >>> Tested-by: Robin Murphy <robin.murphy@arm.com>
> >>> Signed-off-by: Will Deacon <will.deacon@arm.com>
> >>> ---
> >>> 
> >>>   arch/arm/include/asm/dma-mapping.h |  4 +++-
> >>>   drivers/of/platform.c              | 21 ++++++++++++++-------
> >>>   include/linux/dma-mapping.h        |  8 +++++++-
> >>>   3 files changed, 24 insertions(+), 9 deletions(-)
> >>> 
> >>> diff --git a/arch/arm/include/asm/dma-mapping.h
> >>> b/arch/arm/include/asm/dma-mapping.h index dc3420e77758..f3c0d953f6a2
> >>> 100644
> >>> --- a/arch/arm/include/asm/dma-mapping.h
> >>> +++ b/arch/arm/include/asm/dma-mapping.h
> >>> @@ -121,7 +121,9 @@ static inline unsigned long dma_max_pfn(struct
> >>> device *dev)>>> 
> >>>   }
> >>>   #define dma_max_pfn(dev) dma_max_pfn(dev)
> >>> 
> >>> -static inline void arch_setup_dma_ops(struct device *dev, bool
> >>> coherent)
> >>> +static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
> >>> +                                     u64 size, struct iommu_ops *iommu,
> >>> +                                     bool coherent)
> >>> 
> >>>   {
> >>>   
> >>>          if (coherent)
> >>>          
> >>>                  set_dma_ops(dev, &arm_coherent_dma_ops);
> >>> 
> >>> diff --git a/drivers/of/platform.c b/drivers/of/platform.c
> >>> index ff1f4e9afccb..b89caf8c7586 100644
> >>> --- a/drivers/of/platform.c
> >>> +++ b/drivers/of/platform.c
> >>> @@ -19,6 +19,7 @@
> >>> 
> >>>   #include <linux/slab.h>
> >>>   #include <linux/of_address.h>
> >>>   #include <linux/of_device.h>
> >>> 
> >>> +#include <linux/of_iommu.h>
> >>> 
> >>>   #include <linux/of_irq.h>
> >>>   #include <linux/of_platform.h>
> >>>   #include <linux/platform_device.h>
> >>> 
> >>> @@ -166,6 +167,7 @@ static void of_dma_configure(struct device *dev)
> >>> 
> >>>          int ret;
> >>>          bool coherent;
> >>>          unsigned long offset;
> >>> 
> >>> +       struct iommu_ops *iommu;
> >>> 
> >>>          /*
> >>>          
> >>>           * Set default dma-mask to 32 bit. Drivers are expected to
> >>>           setup
> >>> 
> >>> @@ -194,7 +196,16 @@ static void of_dma_configure(struct device *dev)
> >>> 
> >>>          dev_dbg(dev, "device is%sdma coherent\n",
> >>>          
> >>>                  coherent ? " " : " not ");
> >>> 
> >>> -       arch_setup_dma_ops(dev, coherent);
> >>> +       iommu = of_iommu_configure(dev);
> >>> +       dev_dbg(dev, "device is%sbehind an iommu\n",
> >>> +               iommu ? " " : " not ");
> >>> +
> >>> +       arch_setup_dma_ops(dev, dma_addr, size, iommu, coherent);
> >> 
> >> so, what is the way for a driver that explicitly wants to manage it's
> >> own device virtual address space to opt out of this?  I suspect that
> >> won't be the common case, but for a gpu, if dma layer all of a sudden
> >> thinks it is in control of the gpu's virtual address space, things are
> >> going to end in tears..
> > 
> > I think you'll need to detach from the DMA domain, then have the driver
> > manage everything itself. As you say, it's not the common case, so you
> > may need to add some hooks for detaching from the default domain and
> > swizzling your DMA ops.
> 
> Surely if that's a problem then it's an existing one anyway? If the
> IOMMU driver wants to do its own thing here all it has to do is return
> -ENOTHANKSIDRATHERNOT from its of_xlate call (or be even more boring and
> simply not implement it), and the device gets whatever default DMA ops
> the old of_dma_configure would have given it, with zero impact from this
> series. I only see a potential issue if you have one driver running
> multiple IOMMUs, some of which are happy to pass off control to DMA
> mapping and some not, in which case either you make the driver clever
> enough to identify its clients and handle of_xlate correctly, or work
> around it with the DT representation.

Exactly for that reason, wouldn't it be better to let the client device 
whether it wants to use the DMA mapping API abstraction or manually control 
the IOMMU mappings ?

-- 
Regards,

Laurent Pinchart

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Laurent Pinchart]

Hi Will,

On Wednesday 10 December 2014 15:08:53 Will Deacon wrote:
> On Wed, Dec 10, 2014 at 02:52:56PM +0000, Rob Clark wrote:
> > On Mon, Dec 1, 2014 at 11:57 AM, Will Deacon <will.deacon@arm.com> wrote:
> > > This patch extends of_dma_configure so that it sets up the IOMMU for a
> > > device, as well as the coherent/non-coherent DMA mapping ops.
> > > 
> > > Acked-by: Arnd Bergmann <arnd@arndb.de>
> > > Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
> > > Tested-by: Robin Murphy <robin.murphy@arm.com>
> > > Signed-off-by: Will Deacon <will.deacon@arm.com>
> > > ---
> > > 
> > >  arch/arm/include/asm/dma-mapping.h |  4 +++-
> > >  drivers/of/platform.c              | 21 ++++++++++++++-------
> > >  include/linux/dma-mapping.h        |  8 +++++++-
> > >  3 files changed, 24 insertions(+), 9 deletions(-)
> > > 
> > > diff --git a/arch/arm/include/asm/dma-mapping.h
> > > b/arch/arm/include/asm/dma-mapping.h index dc3420e77758..f3c0d953f6a2
> > > 100644
> > > --- a/arch/arm/include/asm/dma-mapping.h
> > > +++ b/arch/arm/include/asm/dma-mapping.h
> > > @@ -121,7 +121,9 @@ static inline unsigned long dma_max_pfn(struct
> > > device *dev)> > 
> > >  }
> > >  #define dma_max_pfn(dev) dma_max_pfn(dev)
> > > 
> > > -static inline void arch_setup_dma_ops(struct device *dev, bool
> > > coherent)
> > > +static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
> > > +                                     u64 size, struct iommu_ops *iommu,
> > > +                                     bool coherent)
> > >  {
> > >         if (coherent)
> > >                 set_dma_ops(dev, &arm_coherent_dma_ops);
> > > diff --git a/drivers/of/platform.c b/drivers/of/platform.c
> > > index ff1f4e9afccb..b89caf8c7586 100644
> > > --- a/drivers/of/platform.c
> > > +++ b/drivers/of/platform.c
> > > @@ -19,6 +19,7 @@
> > >  #include <linux/slab.h>
> > >  #include <linux/of_address.h>
> > >  #include <linux/of_device.h>
> > > +#include <linux/of_iommu.h>
> > >  #include <linux/of_irq.h>
> > >  #include <linux/of_platform.h>
> > >  #include <linux/platform_device.h>
> > > @@ -166,6 +167,7 @@ static void of_dma_configure(struct device *dev)
> > >         int ret;
> > >         bool coherent;
> > >         unsigned long offset;
> > > +       struct iommu_ops *iommu;
> > > 
> > >         /*
> > >          * Set default dma-mask to 32 bit. Drivers are expected to setup
> > > @@ -194,7 +196,16 @@ static void of_dma_configure(struct device *dev)
> > >         dev_dbg(dev, "device is%sdma coherent\n",
> > >                 coherent ? " " : " not ");
> > > 
> > > -       arch_setup_dma_ops(dev, coherent);
> > > +       iommu = of_iommu_configure(dev);
> > > +       dev_dbg(dev, "device is%sbehind an iommu\n",
> > > +               iommu ? " " : " not ");
> > > +
> > > +       arch_setup_dma_ops(dev, dma_addr, size, iommu, coherent);
> > 
> > so, what is the way for a driver that explicitly wants to manage it's
> > own device virtual address space to opt out of this?  I suspect that
> > won't be the common case, but for a gpu, if dma layer all of a sudden
> > thinks it is in control of the gpu's virtual address space, things are
> > going to end in tears..
> 
> I think you'll need to detach from the DMA domain, then have the driver
> manage everything itself. As you say, it's not the common case, so you
> may need to add some hooks for detaching from the default domain and
> swizzling your DMA ops.

I'm wondering if it's such an exotic case after all. I can see two reasons not 
to use the default domain. In addition to special requirements coming from the 
bus master side, the IOMMU itself might not support one domain per bus master 
(I'm of course raising the issue from a very selfish Renesas IPMMU point of 
view).

In that case the IOMMU driver can still detach the device from the default 
domain in the .of_xlate() callback and attach it to its private domain. I'm 
however wondering whether this case isn't common enough to avoid creating 
default domains in common code in the first place. IOMMU drivers that support 
one domain per master would then create device domain in their .of_xlate() 
callback.

-- 
Regards,

Laurent Pinchart

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Laurent Pinchart]

Hi Will,

On Sunday 14 December 2014 17:49:34 Laurent Pinchart wrote:
> On Wednesday 10 December 2014 15:08:53 Will Deacon wrote:
> > On Wed, Dec 10, 2014 at 02:52:56PM +0000, Rob Clark wrote:
> > > On Mon, Dec 1, 2014 at 11:57 AM, Will Deacon wrote:
> > > > This patch extends of_dma_configure so that it sets up the IOMMU for a
> > > > device, as well as the coherent/non-coherent DMA mapping ops.
> > > > 
> > > > Acked-by: Arnd Bergmann <arnd@arndb.de>
> > > > Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
> > > > Tested-by: Robin Murphy <robin.murphy@arm.com>
> > > > Signed-off-by: Will Deacon <will.deacon@arm.com>
> > > > ---
> > > > 
> > > >  arch/arm/include/asm/dma-mapping.h |  4 +++-
> > > >  drivers/of/platform.c              | 21 ++++++++++++++-------
> > > >  include/linux/dma-mapping.h        |  8 +++++++-
> > > >  3 files changed, 24 insertions(+), 9 deletions(-)
> > > > 
> > > > diff --git a/arch/arm/include/asm/dma-mapping.h
> > > > b/arch/arm/include/asm/dma-mapping.h index dc3420e77758..f3c0d953f6a2
> > > > 100644
> > > > --- a/arch/arm/include/asm/dma-mapping.h
> > > > +++ b/arch/arm/include/asm/dma-mapping.h
> > > > @@ -121,7 +121,9 @@ static inline unsigned long dma_max_pfn(struct
> > > > device *dev)
> > > >  }
> > > >  #define dma_max_pfn(dev) dma_max_pfn(dev)
> > > > 
> > > > -static inline void arch_setup_dma_ops(struct device *dev, bool
> > > > coherent)
> > > > +static inline void arch_setup_dma_ops(struct device *dev, u64
> > > > dma_base,
> > > > +                                     u64 size, struct iommu_ops
> > > > *iommu,
> > > > +                                     bool coherent)
> > > >  {
> > > >         if (coherent)
> > > >                 set_dma_ops(dev, &arm_coherent_dma_ops);
> > > > diff --git a/drivers/of/platform.c b/drivers/of/platform.c
> > > > index ff1f4e9afccb..b89caf8c7586 100644
> > > > --- a/drivers/of/platform.c
> > > > +++ b/drivers/of/platform.c
> > > > @@ -19,6 +19,7 @@
> > > >  #include <linux/slab.h>
> > > >  #include <linux/of_address.h>
> > > >  #include <linux/of_device.h>
> > > > +#include <linux/of_iommu.h>
> > > >  #include <linux/of_irq.h>
> > > >  #include <linux/of_platform.h>
> > > >  #include <linux/platform_device.h>
> > > > @@ -166,6 +167,7 @@ static void of_dma_configure(struct device *dev)
> > > >         int ret;
> > > >         bool coherent;
> > > >         unsigned long offset;
> > > > +       struct iommu_ops *iommu;
> > > >         /*
> > > >          * Set default dma-mask to 32 bit. Drivers are expected to
> > > >          setup
> > > > @@ -194,7 +196,16 @@ static void of_dma_configure(struct device *dev)
> > > >         dev_dbg(dev, "device is%sdma coherent\n",
> > > >                 coherent ? " " : " not ");
> > > > -       arch_setup_dma_ops(dev, coherent);
> > > > +       iommu = of_iommu_configure(dev);
> > > > +       dev_dbg(dev, "device is%sbehind an iommu\n",
> > > > +               iommu ? " " : " not ");
> > > > +
> > > > +       arch_setup_dma_ops(dev, dma_addr, size, iommu, coherent);
> > > 
> > > so, what is the way for a driver that explicitly wants to manage it's
> > > own device virtual address space to opt out of this?  I suspect that
> > > won't be the common case, but for a gpu, if dma layer all of a sudden
> > > thinks it is in control of the gpu's virtual address space, things are
> > > going to end in tears..
> > 
> > I think you'll need to detach from the DMA domain, then have the driver
> > manage everything itself. As you say, it's not the common case, so you
> > may need to add some hooks for detaching from the default domain and
> > swizzling your DMA ops.
> 
> I'm wondering if it's such an exotic case after all. I can see two reasons
> not to use the default domain. In addition to special requirements coming
> from the bus master side, the IOMMU itself might not support one domain per
> bus master (I'm of course raising the issue from a very selfish Renesas
> IPMMU point of view).
> 
> In that case the IOMMU driver can still detach the device from the default
> domain in the .of_xlate() callback and attach it to its private domain.

I actually spoke too fast, the .of_xlate() callback is called before 
arch_setup_dma_ops(). The operation could be performed in the .add_device() 
callback. I'm having trouble understanding how you envision .add_device, 
.remove_device and .of_xlate working together in the DT case. Could you please 
detail what you think each callback should be responsible of exactly ?

> I'm however wondering whether this case isn't common enough to avoid
> creating default domains in common code in the first place. IOMMU drivers
> that support one domain per master would then create device domain in their
> .of_xlate() callback.

-- 
Regards,

Laurent Pinchart

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Will Deacon]

On Sun, Dec 14, 2014 at 03:59:38PM +0000, Laurent Pinchart wrote:
> On Sunday 14 December 2014 17:49:34 Laurent Pinchart wrote:
> > On Wednesday 10 December 2014 15:08:53 Will Deacon wrote:
> > > On Wed, Dec 10, 2014 at 02:52:56PM +0000, Rob Clark wrote:
> > > > so, what is the way for a driver that explicitly wants to manage it's
> > > > own device virtual address space to opt out of this?  I suspect that
> > > > won't be the common case, but for a gpu, if dma layer all of a sudden
> > > > thinks it is in control of the gpu's virtual address space, things are
> > > > going to end in tears..
> > > 
> > > I think you'll need to detach from the DMA domain, then have the driver
> > > manage everything itself. As you say, it's not the common case, so you
> > > may need to add some hooks for detaching from the default domain and
> > > swizzling your DMA ops.
> > 
> > I'm wondering if it's such an exotic case after all. I can see two reasons
> > not to use the default domain. In addition to special requirements coming
> > from the bus master side, the IOMMU itself might not support one domain per
> > bus master (I'm of course raising the issue from a very selfish Renesas
> > IPMMU point of view).
> > 
> > In that case the IOMMU driver can still detach the device from the default
> > domain in the .of_xlate() callback and attach it to its private domain.
> 
> I actually spoke too fast, the .of_xlate() callback is called before 
> arch_setup_dma_ops(). The operation could be performed in the .add_device() 
> callback. I'm having trouble understanding how you envision .add_device, 
> .remove_device and .of_xlate working together in the DT case. Could you please 
> detail what you think each callback should be responsible of exactly ?

I think the sequence goes something like:


1: of_iommu_init: early initialisiation of IOMMU driver, stashes its private
                  data via of_iommu_set_ops

2: ->of_xlate: called for each master upstream of the IOMMU before the master
               has probed. Passes back the DT info, which can be used to
	       get hold of the private data from earlier. Per-device data
	       can be stored in archdata.iommu (but I'd like to change this,
	       see below).

3. ->domain_init: called from dma-mapping to create default domain (this
                  should be replaced by ->get_default_domain in future).

3. ->attach_device: called by dma-mapping to attach device to the default
                    domain

4. ->add_device: called just before the master is probed


Now, the thing missing here is the instantiation of iommu_groups. They
require a fully initialised struct device because of some sysfs magic
that goes on in iommu_group_add_device. The questions I have are:

  - How should we describe/create iommu_groups for platform devices?

  - When do the groups actually need to be initialised (currently likely
    to be in add_device).

Will

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Will Deacon]

On Sun, Dec 14, 2014 at 03:49:34PM +0000, Laurent Pinchart wrote:
> Hi Will,

Hi Laurent,

> On Wednesday 10 December 2014 15:08:53 Will Deacon wrote:
> > On Wed, Dec 10, 2014 at 02:52:56PM +0000, Rob Clark wrote:
> > > so, what is the way for a driver that explicitly wants to manage it's
> > > own device virtual address space to opt out of this?  I suspect that
> > > won't be the common case, but for a gpu, if dma layer all of a sudden
> > > thinks it is in control of the gpu's virtual address space, things are
> > > going to end in tears..
> > 
> > I think you'll need to detach from the DMA domain, then have the driver
> > manage everything itself. As you say, it's not the common case, so you
> > may need to add some hooks for detaching from the default domain and
> > swizzling your DMA ops.
> 
> I'm wondering if it's such an exotic case after all. I can see two reasons not 
> to use the default domain. In addition to special requirements coming from the 
> bus master side, the IOMMU itself might not support one domain per bus master 
> (I'm of course raising the issue from a very selfish Renesas IPMMU point of 
> view).

Do you mean that certain masters must be grouped into the same domain, or
that the IOMMU can fail with -ENOSPC?

For the former, we need a way to represent IOMMU groups for the platform
bus. For the latter, we should have a per-IOMMU default domain instead of
creating one per master as we currently do for ARM.

Joerg has talked about adding a ->get_default_domain callback to the IOMMU
layer, but I've not seen any code and my attempt at using it also got
pretty complicated:

  http://lists.infradead.org/pipermail/linux-arm-kernel/2014-November/304076.html

Marek also said he might be taking a look.

Will

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Laurent Pinchart]

Hi Will,

On Monday 15 December 2014 16:40:41 Will Deacon wrote:
> On Sun, Dec 14, 2014 at 03:49:34PM +0000, Laurent Pinchart wrote:
> > On Wednesday 10 December 2014 15:08:53 Will Deacon wrote:
> > > On Wed, Dec 10, 2014 at 02:52:56PM +0000, Rob Clark wrote:
> > > > so, what is the way for a driver that explicitly wants to manage it's
> > > > own device virtual address space to opt out of this?  I suspect that
> > > > won't be the common case, but for a gpu, if dma layer all of a sudden
> > > > thinks it is in control of the gpu's virtual address space, things are
> > > > going to end in tears..
> > > 
> > > I think you'll need to detach from the DMA domain, then have the driver
> > > manage everything itself. As you say, it's not the common case, so you
> > > may need to add some hooks for detaching from the default domain and
> > > swizzling your DMA ops.
> > 
> > I'm wondering if it's such an exotic case after all. I can see two reasons
> > not to use the default domain. In addition to special requirements coming
> > from the bus master side, the IOMMU itself might not support one domain
> > per bus master (I'm of course raising the issue from a very selfish
> > Renesas IPMMU point of view).
> 
> Do you mean that certain masters must be grouped into the same domain, or
> that the IOMMU can fail with -ENOSPC?

My IOMMU has hardware supports for 4 domains, and serves N masters (where N is 
dependent on the SoC but is > 4). In its current form the driver supports a 
single domain and thus detaches devices from the default domain in the 
add_device callback:

http://git.linuxtv.org/cgit.cgi/pinchartl/fbdev.git/tree/drivers/iommu/ipmmu-vmsa.c?h=iommu/devel/arm-lpae

	/*
	 * Detach the device from the default ARM VA mapping and attach it to
	 * our private mapping.
	 */
	arm_iommu_detach_device(dev);
	ret = arm_iommu_attach_device(dev, mmu->mapping);
	if (ret < 0) {
		dev_err(dev, "Failed to attach device to VA mapping\n");
		return ret;
	}

I would have implemented that in the of_xlate callback, but that's too early 
as the ARM default domain isn't created yet@that point.

Using a single domain is a bit of a waste of resources in my case, so an 
evolution would be to create four domains and assign devices to them based on 
a policy. The policy could be fixed (round-robin for instance), or 
configurable (possibly through DT, although it's really a policy, not a 
hardware description).

> For the former, we need a way to represent IOMMU groups for the platform
> bus.

To be honest I'm not entirely sure how IOMMU groups are supposed to be used. I 
understand they can be used by VFIO to group several masters that will be able 
to see each other's memory through the same page table, and also that a page 
table could be shared between multiple groups. When it comes to group 
creation, though, things get fuzzy. I started with creating one group per 
master in my driver, which is probably not the thing to do. The Exynos IOMMU 
driver used to do the same, until Marek's patch series converting it to DT-
based instantiation (on top of your patch set) has removed groups altogether. 
Groups seem to be more or less optional, except in a couple of places (for 
instance the remove_device callback will not be called by the 
BUS_NOTIFY_DEL_DEVICE notifier if the device isn't part of an iommu group).

I'd appreciate if someone could clarify this to help me make an informed 
opinion on the topic.

> For the latter, we should have a per-IOMMU default domain instead of
> creating one per master as we currently do for ARM.
> 
> Joerg has talked about adding a ->get_default_domain callback to the IOMMU
> layer, but I've not seen any code and my attempt at using it also got
> pretty complicated:
> 
> http://lists.infradead.org/pipermail/linux-arm-kernel/2014-November/304076.
> html

Thank you for the pointer. I'll reply to the patch.

> Marek also said he might be taking a look.

-- 
Regards,

Laurent Pinchart

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Will Deacon]

On Mon, Dec 15, 2014 at 05:16:50PM +0000, Laurent Pinchart wrote:
> On Monday 15 December 2014 16:40:41 Will Deacon wrote:
> > On Sun, Dec 14, 2014 at 03:49:34PM +0000, Laurent Pinchart wrote:
> > > On Wednesday 10 December 2014 15:08:53 Will Deacon wrote:
> > > > On Wed, Dec 10, 2014 at 02:52:56PM +0000, Rob Clark wrote:
> > > > > so, what is the way for a driver that explicitly wants to manage it's
> > > > > own device virtual address space to opt out of this?  I suspect that
> > > > > won't be the common case, but for a gpu, if dma layer all of a sudden
> > > > > thinks it is in control of the gpu's virtual address space, things are
> > > > > going to end in tears..
> > > > 
> > > > I think you'll need to detach from the DMA domain, then have the driver
> > > > manage everything itself. As you say, it's not the common case, so you
> > > > may need to add some hooks for detaching from the default domain and
> > > > swizzling your DMA ops.
> > > 
> > > I'm wondering if it's such an exotic case after all. I can see two reasons
> > > not to use the default domain. In addition to special requirements coming
> > > from the bus master side, the IOMMU itself might not support one domain
> > > per bus master (I'm of course raising the issue from a very selfish
> > > Renesas IPMMU point of view).
> > 
> > Do you mean that certain masters must be grouped into the same domain, or
> > that the IOMMU can fail with -ENOSPC?
> 
> My IOMMU has hardware supports for 4 domains, and serves N masters (where N is 
> dependent on the SoC but is > 4). In its current form the driver supports a 
> single domain and thus detaches devices from the default domain in the 
> add_device callback:

Hmm, ok. Ideally, you wouldn't need to do any of that in the driver, but I
can understand why you decided to go down that route.

> 	/*
> 	 * Detach the device from the default ARM VA mapping and attach it to
> 	 * our private mapping.
> 	 */
> 	arm_iommu_detach_device(dev);
> 	ret = arm_iommu_attach_device(dev, mmu->mapping);
> 	if (ret < 0) {
> 		dev_err(dev, "Failed to attach device to VA mapping\n");
> 		return ret;
> 	}
> 
> I would have implemented that in the of_xlate callback, but that's too early 
> as the ARM default domain isn't created yet at that point.

Yup, the mythical ->get_default_domain might be the right place instead.

> Using a single domain is a bit of a waste of resources in my case, so an 
> evolution would be to create four domains and assign devices to them based on 
> a policy. The policy could be fixed (round-robin for instance), or 
> configurable (possibly through DT, although it's really a policy, not a 
> hardware description).

I think having one default domain, which is home to all of the masters that
don't have any DMA restrictions is a good use of the hardware. That then
leaves you with three domains to cover VFIO, devices with DMA limitations
and potentially device isolation (if we had a way to describe that).

> > For the former, we need a way to represent IOMMU groups for the platform
> > bus.
> 
> To be honest I'm not entirely sure how IOMMU groups are supposed to be used. I 
> understand they can be used by VFIO to group several masters that will be able 
> to see each other's memory through the same page table, and also that a page 
> table could be shared between multiple groups. When it comes to group 
> creation, though, things get fuzzy. I started with creating one group per 
> master in my driver, which is probably not the thing to do. The Exynos IOMMU 
> driver used to do the same, until Marek's patch series converting it to DT-
> based instantiation (on top of your patch set) has removed groups altogether. 
> Groups seem to be more or less optional, except in a couple of places (for 
> instance the remove_device callback will not be called by the 
> BUS_NOTIFY_DEL_DEVICE notifier if the device isn't part of an iommu group).
> 
> I'd appreciate if someone could clarify this to help me make an informed 
> opinion on the topic.

Ok, an iommu_group is the minimum granularity for which a specific IOMMU
can offer address translation. So, if your IPMMU can isolate an arbitrary
device (assuming there is a domain available), then each device is in its
own iommu_group. This isn't always the case, for example if two masters are
behind some sort of bridge that makes them indistinguishable to the IOMMU
(perhaps they appear to have the same master ID), then they would have to
be in the same iommu_group. Essentially, iommu_groups are a property of
the hardware and should be instantiated by the bus. PCI does this, but
we don't yet have anything for the platform bus.

VFIO puts multiple groups (now called vfio_groups) into a container. The
container is synoymous to an iommu_domain (i.e. a shared address space).

Will

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Arnd Bergmann]

On Monday 15 December 2014 18:09:33 Will Deacon wrote:
> > Using a single domain is a bit of a waste of resources in my case, so an 
> > evolution would be to create four domains and assign devices to them based on 
> > a policy. The policy could be fixed (round-robin for instance), or 
> > configurable (possibly through DT, although it's really a policy, not a 
> > hardware description).

I think in case of the ARM SMMU, we concluded that the grouping is indeed
best done in DT, because of there is no good algorithmic way to come
up with a set of bitmasks that make up a proper grouping into domains.
 
> I think having one default domain, which is home to all of the masters that
> don't have any DMA restrictions is a good use of the hardware. That then
> leaves you with three domains to cover VFIO, devices with DMA limitations
> and potentially device isolation (if we had a way to describe that).

Yes, I agree. There are also a number of degrees to which one might want
to enable IOMMUs at boot time:

- force-disable: use swiotlb only and turn off all IOMMUs or program them
  with a static linear mapping if they cannot be disabled in hardware
- soft-enable: use IOMMUs only for devices whose dma-mask does not cover
  all the physical memory. This would provide the highest performance
- force-enable: use the IOMMU for any device that has an 'iommus' property,
  to catch any wild DMA pointer accesses.
- secure-enable: like force-enable, but use as many separate domains as
  possible to provide isolation between devices as well.

	Arnd

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Will Deacon]

On Tue, Dec 16, 2014 at 12:08:15PM +0000, Arnd Bergmann wrote:
> On Monday 15 December 2014 18:09:33 Will Deacon wrote:
> > > Using a single domain is a bit of a waste of resources in my case, so an 
> > > evolution would be to create four domains and assign devices to them based on 
> > > a policy. The policy could be fixed (round-robin for instance), or 
> > > configurable (possibly through DT, although it's really a policy, not a 
> > > hardware description).
> 
> I think in case of the ARM SMMU, we concluded that the grouping is indeed
> best done in DT, because of there is no good algorithmic way to come
> up with a set of bitmasks that make up a proper grouping into domains.

I think that's a slightly different case. The `grouping' in the DT, is on a
per-master basis where a master may have a set of StreamIDs, which can be
expressed in a more efficient (per-IOMMU) manner that cannot easily be
determined at runtime.

For iommu_group creation, that could be done in the of code by treating the
master IDs as bus IDs on a per-IOMMU bus; when a new device is probed, we
can look at the set of devices with intersecting IDs and create a group
containing those. This is similar to walking a PCI topology to establish DMA
aliases.

The problem with all of this is how we distinguish the different ID formats
in the `iommus' device-tree property. For the ARM SMMU, we could have:

  (1) [easy case] A device has a list of StreamIDs

  (2) A device has a list of SMR mask/value pairs

  (3) A (bus) device has a range translation for a downstream bus (e.g.
      a PCI host controller which needs to convert RequesterID to StreamID).

>From the SMMU driver's perspective, they will all look like of_xlate calls
unless we augment the generic IOMMU bindings with extra properties to
identify the format. It also makes it difficult to pick a sensible value for
#iommu-cells, as it depends on the format.

Will

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Arnd Bergmann]

On Wednesday 17 December 2014 12:09:49 Will Deacon wrote:
> On Tue, Dec 16, 2014 at 12:08:15PM +0000, Arnd Bergmann wrote:
> > On Monday 15 December 2014 18:09:33 Will Deacon wrote:
> > > > Using a single domain is a bit of a waste of resources in my case, so an 
> > > > evolution would be to create four domains and assign devices to them based on 
> > > > a policy. The policy could be fixed (round-robin for instance), or 
> > > > configurable (possibly through DT, although it's really a policy, not a 
> > > > hardware description).
> > 
> > I think in case of the ARM SMMU, we concluded that the grouping is indeed
> > best done in DT, because of there is no good algorithmic way to come
> > up with a set of bitmasks that make up a proper grouping into domains.
> 
> I think that's a slightly different case. The `grouping' in the DT, is on a
> per-master basis where a master may have a set of StreamIDs, which can be
> expressed in a more efficient (per-IOMMU) manner that cannot easily be
> determined at runtime.
> 
> For iommu_group creation, that could be done in the of code by treating the
> master IDs as bus IDs on a per-IOMMU bus; when a new device is probed, we
> can look at the set of devices with intersecting IDs and create a group
> containing those. This is similar to walking a PCI topology to establish DMA
> aliases.
> 
> The problem with all of this is how we distinguish the different ID formats
> in the `iommus' device-tree property. For the ARM SMMU, we could have:
> 
>   (1) [easy case] A device has a list of StreamIDs
> 
>   (2) A device has a list of SMR mask/value pairs

I was under the impression that using the format from (2), we could
describe all devices that fall into (1). In the worst case, we would
create an iommu group that is somewhat larger than one using discrete
StreamID values, but I would hope that this does not cause actual
troubles.

If all devices on each iommu fall into either 1 or 2, but you never mix
the two on one iommu, this could be handled by supporting either
#iommu-cells=<1> or <2> in the smmu driver. That way, the xlate function
will know which method to apply by looking at the iommu's #iommu-cells
property.

>   (3) A (bus) device has a range translation for a downstream bus (e.g.
>       a PCI host controller which needs to convert RequesterID to StreamID).
> 
> From the SMMU driver's perspective, they will all look like of_xlate calls
> unless we augment the generic IOMMU bindings with extra properties to
> identify the format. It also makes it difficult to pick a sensible value for
> #iommu-cells, as it depends on the format.

I would hope that PCI is the only case we need to worry about for a while.
This means we just need to come up with another property or a set of properties
that we can put into a PCI host controller device node in order to describe
the mapping. These properties could be iommu-specific, so we add something
to the PCI core that calls a new iommu callback function that takes the
device node of the PCI host and the bus/device/function number as inputs.

In arm_setup_iommu_dma_ops(), we can then do something like

	if (dev_is_pci(dev)) {
		struct pci_dev *pdev = to_pci_dev(dev);
		struct device_node *node;
		unsigned int bdf;

		node = find_pci_host_bridge(pdev->bus)->parent->of_node;
		bdf = PCI_DEVID(pdev->bus->number, dev->devfn);

		iommu_setup_pci_dev(pdev, node, bdf);
	}

	Arnd

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Will Deacon]

On Wed, Dec 17, 2014 at 02:15:12PM +0000, Arnd Bergmann wrote:
> On Wednesday 17 December 2014 12:09:49 Will Deacon wrote:
> > On Tue, Dec 16, 2014 at 12:08:15PM +0000, Arnd Bergmann wrote:
> > > On Monday 15 December 2014 18:09:33 Will Deacon wrote:
> > > > > Using a single domain is a bit of a waste of resources in my case, so an 
> > > > > evolution would be to create four domains and assign devices to them based on 
> > > > > a policy. The policy could be fixed (round-robin for instance), or 
> > > > > configurable (possibly through DT, although it's really a policy, not a 
> > > > > hardware description).
> > > 
> > > I think in case of the ARM SMMU, we concluded that the grouping is indeed
> > > best done in DT, because of there is no good algorithmic way to come
> > > up with a set of bitmasks that make up a proper grouping into domains.
> > 
> > I think that's a slightly different case. The `grouping' in the DT, is on a
> > per-master basis where a master may have a set of StreamIDs, which can be
> > expressed in a more efficient (per-IOMMU) manner that cannot easily be
> > determined at runtime.
> > 
> > For iommu_group creation, that could be done in the of code by treating the
> > master IDs as bus IDs on a per-IOMMU bus; when a new device is probed, we
> > can look at the set of devices with intersecting IDs and create a group
> > containing those. This is similar to walking a PCI topology to establish DMA
> > aliases.
> > 
> > The problem with all of this is how we distinguish the different ID formats
> > in the `iommus' device-tree property. For the ARM SMMU, we could have:
> > 
> >   (1) [easy case] A device has a list of StreamIDs
> > 
> >   (2) A device has a list of SMR mask/value pairs
> 
> I was under the impression that using the format from (2), we could
> describe all devices that fall into (1). In the worst case, we would
> create an iommu group that is somewhat larger than one using discrete
> StreamID values, but I would hope that this does not cause actual
> troubles.

The icky part is that an ARM SMMU can have one of two indexing schemes in
hardware:

  (1) The StreamID is used as a linear index into an (MMIO) table, which
      has pointers to contexts (translation tables)

  (2) The StreamID is matched against a mask/value pair, which generates
      an index into the same table mentioned above

Currently, the driver probes ID registers at runtime to figure out which
indexing scheme is implemented. If we start encoding scheme-specific data in
the device-tree, then the binding will differ based on hardware properties
that aren't otherwise described. Is there precedent for this sort of thing
elsewhere?

> If all devices on each iommu fall into either 1 or 2, but you never mix
> the two on one iommu, this could be handled by supporting either
> #iommu-cells=<1> or <2> in the smmu driver. That way, the xlate function
> will know which method to apply by looking at the iommu's #iommu-cells
> property.

Yes, that would work if you're ok with the above (i.e. we never see a mix
on the same IOMMU instance).

> >   (3) A (bus) device has a range translation for a downstream bus (e.g.
> >       a PCI host controller which needs to convert RequesterID to StreamID).
> > 
> > From the SMMU driver's perspective, they will all look like of_xlate calls
> > unless we augment the generic IOMMU bindings with extra properties to
> > identify the format. It also makes it difficult to pick a sensible value for
> > #iommu-cells, as it depends on the format.
> 
> I would hope that PCI is the only case we need to worry about for a while.
> This means we just need to come up with another property or a set of properties
> that we can put into a PCI host controller device node in order to describe
> the mapping. These properties could be iommu-specific, so we add something
> to the PCI core that calls a new iommu callback function that takes the
> device node of the PCI host and the bus/device/function number as inputs.
> 
> In arm_setup_iommu_dma_ops(), we can then do something like
> 
> 	if (dev_is_pci(dev)) {
> 		struct pci_dev *pdev = to_pci_dev(dev);
> 		struct device_node *node;
> 		unsigned int bdf;
> 
> 		node = find_pci_host_bridge(pdev->bus)->parent->of_node;
> 		bdf = PCI_DEVID(pdev->bus->number, dev->devfn);
> 
> 		iommu_setup_pci_dev(pdev, node, bdf);
> 	}

The other way to do this is have the IOMMU driver check dev_is_pci(dev)
in add_device, then call an of_xlate_pci_bdf library function which could
do the translation on-demand.

Will

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Arnd Bergmann]

On Wednesday 17 December 2014 14:45:18 Will Deacon wrote:
> On Wed, Dec 17, 2014 at 02:15:12PM +0000, Arnd Bergmann wrote:
> > On Wednesday 17 December 2014 12:09:49 Will Deacon wrote:
> > > On Tue, Dec 16, 2014 at 12:08:15PM +0000, Arnd Bergmann wrote:
> > > > On Monday 15 December 2014 18:09:33 Will Deacon wrote:
> > > For iommu_group creation, that could be done in the of code by treating the
> > > master IDs as bus IDs on a per-IOMMU bus; when a new device is probed, we
> > > can look at the set of devices with intersecting IDs and create a group
> > > containing those. This is similar to walking a PCI topology to establish DMA
> > > aliases.
> > > 
> > > The problem with all of this is how we distinguish the different ID formats
> > > in the `iommus' device-tree property. For the ARM SMMU, we could have:
> > > 
> > >   (1) [easy case] A device has a list of StreamIDs
> > > 
> > >   (2) A device has a list of SMR mask/value pairs
> > 
> > I was under the impression that using the format from (2), we could
> > describe all devices that fall into (1). In the worst case, we would
> > create an iommu group that is somewhat larger than one using discrete
> > StreamID values, but I would hope that this does not cause actual
> > troubles.
> 
> The icky part is that an ARM SMMU can have one of two indexing schemes in
> hardware:
> 
>   (1) The StreamID is used as a linear index into an (MMIO) table, which
>       has pointers to contexts (translation tables)
> 
>   (2) The StreamID is matched against a mask/value pair, which generates
>       an index into the same table mentioned above
> 
> Currently, the driver probes ID registers at runtime to figure out which
> indexing scheme is implemented. If we start encoding scheme-specific data in
> the device-tree, then the binding will differ based on hardware properties
> that aren't otherwise described. Is there precedent for this sort of thing
> elsewhere?

We should probably have different compatible strings in that case.
Is it always hardwired which scheme gets used, or can the SMMU
be reconfigured between the two?

> > If all devices on each iommu fall into either 1 or 2, but you never mix
> > the two on one iommu, this could be handled by supporting either
> > #iommu-cells=<1> or <2> in the smmu driver. That way, the xlate function
> > will know which method to apply by looking at the iommu's #iommu-cells
> > property.
> 
> Yes, that would work if you're ok with the above (i.e. we never see a mix
> on the same IOMMU instance).

Ok

> > >   (3) A (bus) device has a range translation for a downstream bus (e.g.
> > >       a PCI host controller which needs to convert RequesterID to StreamID).
> > > 
> > > From the SMMU driver's perspective, they will all look like of_xlate calls
> > > unless we augment the generic IOMMU bindings with extra properties to
> > > identify the format. It also makes it difficult to pick a sensible value for
> > > #iommu-cells, as it depends on the format.
> > 
> > I would hope that PCI is the only case we need to worry about for a while.
> > This means we just need to come up with another property or a set of properties
> > that we can put into a PCI host controller device node in order to describe
> > the mapping. These properties could be iommu-specific, so we add something
> > to the PCI core that calls a new iommu callback function that takes the
> > device node of the PCI host and the bus/device/function number as inputs.
> > 
> > In arm_setup_iommu_dma_ops(), we can then do something like
> > 
> > 	if (dev_is_pci(dev)) {
> > 		struct pci_dev *pdev = to_pci_dev(dev);
> > 		struct device_node *node;
> > 		unsigned int bdf;
> > 
> > 		node = find_pci_host_bridge(pdev->bus)->parent->of_node;
> > 		bdf = PCI_DEVID(pdev->bus->number, dev->devfn);
> > 
> > 		iommu_setup_pci_dev(pdev, node, bdf);
> > 	}
> 
> The other way to do this is have the IOMMU driver check dev_is_pci(dev)
> in add_device, then call an of_xlate_pci_bdf library function which could
> do the translation on-demand.

We'd still need to find the device node for the host controller in
common code, otherwise we don't have an of_xlate function to call.

	Arnd

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Will Deacon]

On Wed, Dec 17, 2014 at 03:35:29PM +0000, Arnd Bergmann wrote:
> On Wednesday 17 December 2014 14:45:18 Will Deacon wrote:
> > On Wed, Dec 17, 2014 at 02:15:12PM +0000, Arnd Bergmann wrote:
> > > On Wednesday 17 December 2014 12:09:49 Will Deacon wrote:
> > The icky part is that an ARM SMMU can have one of two indexing schemes in
> > hardware:
> > 
> >   (1) The StreamID is used as a linear index into an (MMIO) table, which
> >       has pointers to contexts (translation tables)
> > 
> >   (2) The StreamID is matched against a mask/value pair, which generates
> >       an index into the same table mentioned above
> > 
> > Currently, the driver probes ID registers at runtime to figure out which
> > indexing scheme is implemented. If we start encoding scheme-specific data in
> > the device-tree, then the binding will differ based on hardware properties
> > that aren't otherwise described. Is there precedent for this sort of thing
> > elsewhere?
> 
> We should probably have different compatible strings in that case.
> Is it always hardwired which scheme gets used, or can the SMMU
> be reconfigured between the two?

It's a hard-wired thing that is advertised in a read-only ID register. I'm
not too keen on a different compatible string for that, because there are
all sorts of features like this that could quickly get messy. How about
using #iommu-cells to distinguish between them, as Robin suggested? If
iommu-cells is #2, then we're going to be using stream-matching and fail
the probe if it's not supported by the IOMMU.

> > > I would hope that PCI is the only case we need to worry about for a while.
> > > This means we just need to come up with another property or a set of properties
> > > that we can put into a PCI host controller device node in order to describe
> > > the mapping. These properties could be iommu-specific, so we add something
> > > to the PCI core that calls a new iommu callback function that takes the
> > > device node of the PCI host and the bus/device/function number as inputs.
> > > 
> > > In arm_setup_iommu_dma_ops(), we can then do something like
> > > 
> > > 	if (dev_is_pci(dev)) {
> > > 		struct pci_dev *pdev = to_pci_dev(dev);
> > > 		struct device_node *node;
> > > 		unsigned int bdf;
> > > 
> > > 		node = find_pci_host_bridge(pdev->bus)->parent->of_node;
> > > 		bdf = PCI_DEVID(pdev->bus->number, dev->devfn);
> > > 
> > > 		iommu_setup_pci_dev(pdev, node, bdf);
> > > 	}
> > 
> > The other way to do this is have the IOMMU driver check dev_is_pci(dev)
> > in add_device, then call an of_xlate_pci_bdf library function which could
> > do the translation on-demand.
> 
> We'd still need to find the device node for the host controller in
> common code, otherwise we don't have an of_xlate function to call.

I guess I was hoping that the translation code could be generic. I don't
really see anything special about adding a constant to a magic number to
obtain another magic number.

Will

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Arnd Bergmann]

On Wednesday 17 December 2014 17:17:52 Will Deacon wrote:
> > > > I would hope that PCI is the only case we need to worry about for a while.
> > > > This means we just need to come up with another property or a set of properties
> > > > that we can put into a PCI host controller device node in order to describe
> > > > the mapping. These properties could be iommu-specific, so we add something
> > > > to the PCI core that calls a new iommu callback function that takes the
> > > > device node of the PCI host and the bus/device/function number as inputs.
> > > > 
> > > > In arm_setup_iommu_dma_ops(), we can then do something like
> > > > 
> > > >   if (dev_is_pci(dev)) {
> > > >           struct pci_dev *pdev = to_pci_dev(dev);
> > > >           struct device_node *node;
> > > >           unsigned int bdf;
> > > > 
> > > >           node = find_pci_host_bridge(pdev->bus)->parent->of_node;
> > > >           bdf = PCI_DEVID(pdev->bus->number, dev->devfn);
> > > > 
> > > >           iommu_setup_pci_dev(pdev, node, bdf);
> > > >   }
> > > 
> > > The other way to do this is have the IOMMU driver check dev_is_pci(dev)
> > > in add_device, then call an of_xlate_pci_bdf library function which could
> > > do the translation on-demand.
> > 
> > We'd still need to find the device node for the host controller in
> > common code, otherwise we don't have an of_xlate function to call.
> 
> I guess I was hoping that the translation code could be generic. I don't
> really see anything special about adding a constant to a magic number to
> obtain another magic number.
> 

If that's all we need, that's fine.

I was fearing that we'd get different host controllers using different
parts of the bdf number. E.g. one might pass down just bus/device
while another uses bus/device/function, so we'd need a shift and
an offset.

As part of the AMD review I found out that their SMMU implementation
only has 15 bits of address space, while bdf is 16 bits, so they
cut off the top bit and get aliasing between bus 0+n and bus 128+n.

Another SoC might have more aliasing if they have multiple domains.


	Arnd

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Will Deacon]

On Wed, Dec 17, 2014 at 07:48:29PM +0000, Arnd Bergmann wrote:
> On Wednesday 17 December 2014 17:17:52 Will Deacon wrote:
> > > > > I would hope that PCI is the only case we need to worry about for a while.
> > > > > This means we just need to come up with another property or a set of properties
> > > > > that we can put into a PCI host controller device node in order to describe
> > > > > the mapping. These properties could be iommu-specific, so we add something
> > > > > to the PCI core that calls a new iommu callback function that takes the
> > > > > device node of the PCI host and the bus/device/function number as inputs.
> > > > > 
> > > > > In arm_setup_iommu_dma_ops(), we can then do something like
> > > > > 
> > > > >   if (dev_is_pci(dev)) {
> > > > >           struct pci_dev *pdev = to_pci_dev(dev);
> > > > >           struct device_node *node;
> > > > >           unsigned int bdf;
> > > > > 
> > > > >           node = find_pci_host_bridge(pdev->bus)->parent->of_node;
> > > > >           bdf = PCI_DEVID(pdev->bus->number, dev->devfn);
> > > > > 
> > > > >           iommu_setup_pci_dev(pdev, node, bdf);
> > > > >   }
> > > > 
> > > > The other way to do this is have the IOMMU driver check dev_is_pci(dev)
> > > > in add_device, then call an of_xlate_pci_bdf library function which could
> > > > do the translation on-demand.
> > > 
> > > We'd still need to find the device node for the host controller in
> > > common code, otherwise we don't have an of_xlate function to call.
> > 
> > I guess I was hoping that the translation code could be generic. I don't
> > really see anything special about adding a constant to a magic number to
> > obtain another magic number.
> > 
> 
> If that's all we need, that's fine.
> 
> I was fearing that we'd get different host controllers using different
> parts of the bdf number. E.g. one might pass down just bus/device
> while another uses bus/device/function, so we'd need a shift and
> an offset.

We could still encode that as adding a constant, modulo the streamid
width though, right? I agree that it would be nasty, because we'd have to
list a whole bunch of offsets for each function rather than group things
into ranges. Still, it gives us something to start with.

FWIW, this (adding an offset) is also the direction that the ACPI IORT
description is going in, so at least we have parity there.

> As part of the AMD review I found out that their SMMU implementation
> only has 15 bits of address space, while bdf is 16 bits, so they
> cut off the top bit and get aliasing between bus 0+n and bus 128+n.
> 
> Another SoC might have more aliasing if they have multiple domains.

This particular aliasing is probably going to be pretty common; the ARM
SMMU typically only supports 15-bit StreamIDs (there is an extension to
16-bit, but I haven't seen one built yet and the driver doesn't even support
it).

Will

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Arnd Bergmann]

On Sunday 21 December 2014 10:04:51 Will Deacon wrote:
> On Wed, Dec 17, 2014 at 07:48:29PM +0000, Arnd Bergmann wrote:
> > On Wednesday 17 December 2014 17:17:52 Will Deacon wrote:
> > > > > > I would hope that PCI is the only case we need to worry about for a while.
> > > > > > This means we just need to come up with another property or a set of properties
> > > > > > that we can put into a PCI host controller device node in order to describe
> > > > > > the mapping. These properties could be iommu-specific, so we add something
> > > > > > to the PCI core that calls a new iommu callback function that takes the
> > > > > > device node of the PCI host and the bus/device/function number as inputs.
> > > > > > 
> > > > > > In arm_setup_iommu_dma_ops(), we can then do something like
> > > > > > 
> > > > > >   if (dev_is_pci(dev)) {
> > > > > >           struct pci_dev *pdev = to_pci_dev(dev);
> > > > > >           struct device_node *node;
> > > > > >           unsigned int bdf;
> > > > > > 
> > > > > >           node = find_pci_host_bridge(pdev->bus)->parent->of_node;
> > > > > >           bdf = PCI_DEVID(pdev->bus->number, dev->devfn);
> > > > > > 
> > > > > >           iommu_setup_pci_dev(pdev, node, bdf);
> > > > > >   }
> > > > > 
> > > > > The other way to do this is have the IOMMU driver check dev_is_pci(dev)
> > > > > in add_device, then call an of_xlate_pci_bdf library function which could
> > > > > do the translation on-demand.
> > > > 
> > > > We'd still need to find the device node for the host controller in
> > > > common code, otherwise we don't have an of_xlate function to call.
> > > 
> > > I guess I was hoping that the translation code could be generic. I don't
> > > really see anything special about adding a constant to a magic number to
> > > obtain another magic number.
> > > 
> > 
> > If that's all we need, that's fine.
> > 
> > I was fearing that we'd get different host controllers using different
> > parts of the bdf number. E.g. one might pass down just bus/device
> > while another uses bus/device/function, so we'd need a shift and
> > an offset.
> 
> We could still encode that as adding a constant, modulo the streamid
> width though, right? I agree that it would be nasty, because we'd have to
> list a whole bunch of offsets for each function rather than group things
> into ranges. Still, it gives us something to start with.
> 
> FWIW, this (adding an offset) is also the direction that the ACPI IORT
> description is going in, so at least we have parity there.

Ok, in that case, I think just using the 'iommus' property would work,
with yet another format to cover the SMMU case of passing a range
of StreamIDs. With #iommu-cells=<3>, we can have a tuple of
mask/value/length that would be able to cover all cases.

> > As part of the AMD review I found out that their SMMU implementation
> > only has 15 bits of address space, while bdf is 16 bits, so they
> > cut off the top bit and get aliasing between bus 0+n and bus 128+n.
> > 
> > Another SoC might have more aliasing if they have multiple domains.
> 
> This particular aliasing is probably going to be pretty common; the ARM
> SMMU typically only supports 15-bit StreamIDs (there is an extension to
> 16-bit, but I haven't seen one built yet and the driver doesn't even support
> it).

In case of AMD, I asked the maintainers to put a bus-range = <0 127> into
the PCI host node to address that. I've seen something similar while
discussing another SoC that had multiple PCI hosts connected to the
same SMMU and my comment back then was a recommendation to not have distinct
bus-range properties in each one (e.g. <0 31>, <32 63>, <64 95>, <96 127>
for four hosts), but I suppose that is actually the cleanest way to deal
with it.

Do you think it's possible that we might have to deal with a single PCI host
that is connected two different SMMU instances for the purposes of extending
the StreamID space from 15 to 16 bits? I think we would have trouble
expressing this with the current syntax.

	Arnd

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Will Deacon]

Hi Arnd,

Sorry for the delay on this, I had to do a bit of digging.

On Mon, Dec 22, 2014 at 01:36:01PM +0000, Arnd Bergmann wrote:
> Do you think it's possible that we might have to deal with a single PCI host
> that is connected two different SMMU instances for the purposes of extending
> the StreamID space from 15 to 16 bits? I think we would have trouble
> expressing this with the current syntax.

Unfortunately, this sounds like something we may well have to support.
Whilst the SMMUv2 architecture did grow a late extension to support 16-bit
StreamIDs, that may have been too late for all silicon vendors and, as such,
I'm not confident that such systems will present a single software interface
for their SMMU.

Will

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Arnd Bergmann]

On Wednesday 07 January 2015 18:57:05 Will Deacon wrote:
> 
> Sorry for the delay on this, I had to do a bit of digging.
> 
> On Mon, Dec 22, 2014 at 01:36:01PM +0000, Arnd Bergmann wrote:
> > Do you think it's possible that we might have to deal with a single PCI host
> > that is connected two different SMMU instances for the purposes of extending
> > the StreamID space from 15 to 16 bits? I think we would have trouble
> > expressing this with the current syntax.
> 
> Unfortunately, this sounds like something we may well have to support.
> Whilst the SMMUv2 architecture did grow a late extension to support 16-bit
> StreamIDs, that may have been too late for all silicon vendors and, as such,
> I'm not confident that such systems will present a single software interface
> for their SMMU.

So it's technically possible to connect two SMMU instances to a single
PCIe root complex? I knew that there is at least one vendor (AMD) that
can only do 128 bus numbers on PCIe, which seems a much simpler workaround.

	Arnd

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Will Deacon]

On Wed, Jan 07, 2015 at 07:29:15PM +0000, Arnd Bergmann wrote:
> On Wednesday 07 January 2015 18:57:05 Will Deacon wrote:
> > Sorry for the delay on this, I had to do a bit of digging.
> > 
> > On Mon, Dec 22, 2014 at 01:36:01PM +0000, Arnd Bergmann wrote:
> > > Do you think it's possible that we might have to deal with a single PCI host
> > > that is connected two different SMMU instances for the purposes of extending
> > > the StreamID space from 15 to 16 bits? I think we would have trouble
> > > expressing this with the current syntax.
> > 
> > Unfortunately, this sounds like something we may well have to support.
> > Whilst the SMMUv2 architecture did grow a late extension to support 16-bit
> > StreamIDs, that may have been too late for all silicon vendors and, as such,
> > I'm not confident that such systems will present a single software interface
> > for their SMMU.
> 
> So it's technically possible to connect two SMMU instances to a single
> PCIe root complex? I knew that there is at least one vendor (AMD) that
> can only do 128 bus numbers on PCIe, which seems a much simpler workaround.

I think you'd need to add some additional logic around the host controller,
but I don't see why it wouldn't be possible to send the first 32k SIDs off
to one SMMU and the other 32k off somewhere else -- those IDs are carried on
the bus, after all.

That said, something being `technically possible' isn't always a good reason
for us to support it :)

Will

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Robin Murphy]

Hi Will,

On 17/12/14 12:09, Will Deacon wrote:
> On Tue, Dec 16, 2014 at 12:08:15PM +0000, Arnd Bergmann wrote:
>> On Monday 15 December 2014 18:09:33 Will Deacon wrote:
>>>> Using a single domain is a bit of a waste of resources in my case, so an
>>>> evolution would be to create four domains and assign devices to them based on
>>>> a policy. The policy could be fixed (round-robin for instance), or
>>>> configurable (possibly through DT, although it's really a policy, not a
>>>> hardware description).
>>
>> I think in case of the ARM SMMU, we concluded that the grouping is indeed
>> best done in DT, because of there is no good algorithmic way to come
>> up with a set of bitmasks that make up a proper grouping into domains.
>
> I think that's a slightly different case. The `grouping' in the DT, is on a
> per-master basis where a master may have a set of StreamIDs, which can be
> expressed in a more efficient (per-IOMMU) manner that cannot easily be
> determined at runtime.

I'm pretty convinced that automatic SMR allocation is solvable; I was 
planning to take a look at it soon, once I've finished warming up the 
dormant mathematician in me on implementing the OF group allocator. So 
far I'm lucky that stream indexing is sufficient for the bits of our 
hardware that actually work.

> For iommu_group creation, that could be done in the of code by treating the
> master IDs as bus IDs on a per-IOMMU bus; when a new device is probed, we
> can look at the set of devices with intersecting IDs and create a group
> containing those. This is similar to walking a PCI topology to establish DMA
> aliases.

Thanks to the USB EHCI/OHCI device model I have this prototyped already, 
via having the IOMMU driver register an arbitrary ID of its own choosing 
per of_xlate call, such that any devices claiming the same ID must be in 
the same group (i.e. the driver can perform its own 
translation/abstraction between the DT master ID and the "bus ID" for 
grouping). The tricky missing bit for the SMMU driver itself is 
reprogramming the S2CRs dynamically as devices are added to/removed from 
a group - with that solved I don't see much need left for any explicit 
grouping in DT.

> The problem with all of this is how we distinguish the different ID formats
> in the `iommus' device-tree property. For the ARM SMMU, we could have:
>
>    (1) [easy case] A device has a list of StreamIDs
>
>    (2) A device has a list of SMR mask/value pairs
>
>    (3) A (bus) device has a range translation for a downstream bus (e.g.
>        a PCI host controller which needs to convert RequesterID to StreamID).
>
>  From the SMMU driver's perspective, they will all look like of_xlate calls
> unless we augment the generic IOMMU bindings with extra properties to
> identify the format. It also makes it difficult to pick a sensible value for
> #iommu-cells, as it depends on the format.

I thoroughly dislike the idea, but one /could/ simply abuse the generic 
bindings well within the current framework, e.g.

#1: iommus = <&smmu 0 streamID 0 0>, <&smmu 0 streamID 0 0>, ...;
#2: iommus = <&smmu 1 mask value 0>;
#3: iommus = <&smmu 2 start end offset>;

and have a big complicated of_xlate that knows the secret. I don't think 
it would be unreasonable to constrain all masters on the same SMMU to 
use the same format either - that way you can lose the type indicator 
and the padding and the driver simply inspects #iommu-cells in 
of_iommu_init and sets up that instance to take the right path in its 
of_xlate calls later. Having a handful of single-stream-ID masters 
behind the same SMMU as a whacking great bus controller with a massive 
range seems like the kind of system-designer-insanity we should 
emphatically NOT be looking to accommodate if we can possibly help it.

That said, for the time being I see definite use-cases for #1, which 
already works, and #3, for which I really think we should be looking to 
add a parallel generic "iommu-ranges" or "iommu-bus-range" binding for 
bus controllers, rather than trying to overload "iommus" beyond 
describing actual bus masters.

Robin.

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Will Deacon]

On Wed, Dec 17, 2014 at 02:27:30PM +0000, Robin Murphy wrote:
> On 17/12/14 12:09, Will Deacon wrote:
> > On Tue, Dec 16, 2014 at 12:08:15PM +0000, Arnd Bergmann wrote:
> >> On Monday 15 December 2014 18:09:33 Will Deacon wrote:
> >>>> Using a single domain is a bit of a waste of resources in my case, so an
> >>>> evolution would be to create four domains and assign devices to them based on
> >>>> a policy. The policy could be fixed (round-robin for instance), or
> >>>> configurable (possibly through DT, although it's really a policy, not a
> >>>> hardware description).
> >>
> >> I think in case of the ARM SMMU, we concluded that the grouping is indeed
> >> best done in DT, because of there is no good algorithmic way to come
> >> up with a set of bitmasks that make up a proper grouping into domains.
> >
> > I think that's a slightly different case. The `grouping' in the DT, is on a
> > per-master basis where a master may have a set of StreamIDs, which can be
> > expressed in a more efficient (per-IOMMU) manner that cannot easily be
> > determined at runtime.
> 
> I'm pretty convinced that automatic SMR allocation is solvable; I was 
> planning to take a look at it soon, once I've finished warming up the 
> dormant mathematician in me on implementing the OF group allocator. So 
> far I'm lucky that stream indexing is sufficient for the bits of our 
> hardware that actually work.

Hmm, a few of us have looked at that and failed, so I'm all ears if you
think you can do it. The main issue is that you need *complete* knowledge
of the allocated StreamID space before you can safely create any SMR entry
unless you restrict yourself to consecutive, power-of-2 sized ID regions
for each master (which doesn't match reality, unfortunately).

> > For iommu_group creation, that could be done in the of code by treating the
> > master IDs as bus IDs on a per-IOMMU bus; when a new device is probed, we
> > can look at the set of devices with intersecting IDs and create a group
> > containing those. This is similar to walking a PCI topology to establish DMA
> > aliases.
> 
> Thanks to the USB EHCI/OHCI device model I have this prototyped already, 
> via having the IOMMU driver register an arbitrary ID of its own choosing 
> per of_xlate call, such that any devices claiming the same ID must be in 
> the same group (i.e. the driver can perform its own 
> translation/abstraction between the DT master ID and the "bus ID" for 
> grouping). The tricky missing bit for the SMMU driver itself is 
> reprogramming the S2CRs dynamically as devices are added to/removed from 
> a group - with that solved I don't see much need left for any explicit 
> grouping in DT.

Hmm, is it actually safe to reprogram those on-the-fly? In particular, the
spec says "SMMUv1 does not provide any special support for updating context
bank state." whilst SMMUv2 isn't much better...

> > The problem with all of this is how we distinguish the different ID formats
> > in the `iommus' device-tree property. For the ARM SMMU, we could have:
> >
> >    (1) [easy case] A device has a list of StreamIDs
> >
> >    (2) A device has a list of SMR mask/value pairs
> >
> >    (3) A (bus) device has a range translation for a downstream bus (e.g.
> >        a PCI host controller which needs to convert RequesterID to StreamID).
> >
> >  From the SMMU driver's perspective, they will all look like of_xlate calls
> > unless we augment the generic IOMMU bindings with extra properties to
> > identify the format. It also makes it difficult to pick a sensible value for
> > #iommu-cells, as it depends on the format.
> 
> I thoroughly dislike the idea, but one /could/ simply abuse the generic 
> bindings well within the current framework, e.g.
> 
> #1: iommus = <&smmu 0 streamID 0 0>, <&smmu 0 streamID 0 0>, ...;
> #2: iommus = <&smmu 1 mask value 0>;
> #3: iommus = <&smmu 2 start end offset>;
> 
> and have a big complicated of_xlate that knows the secret. I don't think 
> it would be unreasonable to constrain all masters on the same SMMU to 
> use the same format either - that way you can lose the type indicator 
> and the padding and the driver simply inspects #iommu-cells in 
> of_iommu_init and sets up that instance to take the right path in its 
> of_xlate calls later. Having a handful of single-stream-ID masters 
> behind the same SMMU as a whacking great bus controller with a massive 
> range seems like the kind of system-designer-insanity we should 
> emphatically NOT be looking to accommodate if we can possibly help it.
> 
> That said, for the time being I see definite use-cases for #1, which 
> already works, and #3, for which I really think we should be looking to 
> add a parallel generic "iommu-ranges" or "iommu-bus-range" binding for 
> bus controllers, rather than trying to overload "iommus" beyond 
> describing actual bus masters.

Yeah, let's avoid overloading the current bindings too much. Adding the
ranges property makes perfect sense, I'm just trying to to straighten
out how the "iommus" property should differ depending on the presence of
SMR groups.

Will

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Arnd Bergmann]

On Wednesday 17 December 2014 15:01:58 Will Deacon wrote:
> On Wed, Dec 17, 2014 at 02:27:30PM +0000, Robin Murphy wrote:
> > On 17/12/14 12:09, Will Deacon wrote:
> > > I think that's a slightly different case. The `grouping' in the DT, is on a
> > > per-master basis where a master may have a set of StreamIDs, which can be
> > > expressed in a more efficient (per-IOMMU) manner that cannot easily be
> > > determined at runtime.
> > 
> > I'm pretty convinced that automatic SMR allocation is solvable; I was 
> > planning to take a look at it soon, once I've finished warming up the 
> > dormant mathematician in me on implementing the OF group allocator. So 
> > far I'm lucky that stream indexing is sufficient for the bits of our 
> > hardware that actually work.
> 
> Hmm, a few of us have looked at that and failed, so I'm all ears if you
> think you can do it. The main issue is that you need *complete* knowledge
> of the allocated StreamID space before you can safely create any SMR entry
> unless you restrict yourself to consecutive, power-of-2 sized ID regions
> for each master (which doesn't match reality, unfortunately).

Right, scanning the entire device tree in the SMMU driver is
definitely something to avoid.

> > > The problem with all of this is how we distinguish the different ID formats
> > > in the `iommus' device-tree property. For the ARM SMMU, we could have:
> > >
> > >    (1) [easy case] A device has a list of StreamIDs
> > >
> > >    (2) A device has a list of SMR mask/value pairs
> > >
> > >    (3) A (bus) device has a range translation for a downstream bus (e.g.
> > >        a PCI host controller which needs to convert RequesterID to StreamID).
> > >
> > >  From the SMMU driver's perspective, they will all look like of_xlate calls
> > > unless we augment the generic IOMMU bindings with extra properties to
> > > identify the format. It also makes it difficult to pick a sensible value for
> > > #iommu-cells, as it depends on the format.
> > 
> > I thoroughly dislike the idea, but one /could/ simply abuse the generic 
> > bindings well within the current framework, e.g.
> > 
> > #1: iommus = <&smmu 0 streamID 0 0>, <&smmu 0 streamID 0 0>, ...;
> > #2: iommus = <&smmu 1 mask value 0>;
> > #3: iommus = <&smmu 2 start end offset>;
> > 
> > and have a big complicated of_xlate that knows the secret. I don't think 
> > it would be unreasonable to constrain all masters on the same SMMU to 
> > use the same format either - that way you can lose the type indicator 
> > and the padding and the driver simply inspects #iommu-cells in 
> > of_iommu_init and sets up that instance to take the right path in its 
> > of_xlate calls later. Having a handful of single-stream-ID masters 
> > behind the same SMMU as a whacking great bus controller with a massive 
> > range seems like the kind of system-designer-insanity we should 
> > emphatically NOT be looking to accommodate if we can possibly help it.
> > 
> > That said, for the time being I see definite use-cases for #1, which 
> > already works, and #3, for which I really think we should be looking to 
> > add a parallel generic "iommu-ranges" or "iommu-bus-range" binding for 
> > bus controllers, rather than trying to overload "iommus" beyond 
> > describing actual bus masters.
> 
> Yeah, let's avoid overloading the current bindings too much. Adding the
> ranges property makes perfect sense, I'm just trying to to straighten
> out how the "iommus" property should differ depending on the presence of
> SMR groups.

Is a range something that is defined in the hardware spec, or is that
just what you expect the PCI host controllers to use? 

	Arnd

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Will Deacon]

On Wed, Dec 17, 2014 at 03:38:22PM +0000, Arnd Bergmann wrote:
> On Wednesday 17 December 2014 15:01:58 Will Deacon wrote:
> > On Wed, Dec 17, 2014 at 02:27:30PM +0000, Robin Murphy wrote:
> > > I thoroughly dislike the idea, but one /could/ simply abuse the generic 
> > > bindings well within the current framework, e.g.
> > > 
> > > #1: iommus = <&smmu 0 streamID 0 0>, <&smmu 0 streamID 0 0>, ...;
> > > #2: iommus = <&smmu 1 mask value 0>;
> > > #3: iommus = <&smmu 2 start end offset>;
> > > 
> > > and have a big complicated of_xlate that knows the secret. I don't think 
> > > it would be unreasonable to constrain all masters on the same SMMU to 
> > > use the same format either - that way you can lose the type indicator 
> > > and the padding and the driver simply inspects #iommu-cells in 
> > > of_iommu_init and sets up that instance to take the right path in its 
> > > of_xlate calls later. Having a handful of single-stream-ID masters 
> > > behind the same SMMU as a whacking great bus controller with a massive 
> > > range seems like the kind of system-designer-insanity we should 
> > > emphatically NOT be looking to accommodate if we can possibly help it.
> > > 
> > > That said, for the time being I see definite use-cases for #1, which 
> > > already works, and #3, for which I really think we should be looking to 
> > > add a parallel generic "iommu-ranges" or "iommu-bus-range" binding for 
> > > bus controllers, rather than trying to overload "iommus" beyond 
> > > describing actual bus masters.
> > 
> > Yeah, let's avoid overloading the current bindings too much. Adding the
> > ranges property makes perfect sense, I'm just trying to to straighten
> > out how the "iommus" property should differ depending on the presence of
> > SMR groups.
> 
> Is a range something that is defined in the hardware spec, or is that
> just what you expect the PCI host controllers to use? 

The range would be a Linux-ism for converting a bus ID on bus foo (e.g. a
requester ID on a PCI bus) into a master ID for the IOMMU for that bus
(e.g. a stream ID for an ARM SMMU). However, I don't think the code doing
that mapping really needs knowledge about what the IDs represent.

Will

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Laurent Pinchart]

Hi Will,

On Monday 15 December 2014 18:09:33 Will Deacon wrote:
> On Mon, Dec 15, 2014 at 05:16:50PM +0000, Laurent Pinchart wrote:
> > On Monday 15 December 2014 16:40:41 Will Deacon wrote:
> >> On Sun, Dec 14, 2014 at 03:49:34PM +0000, Laurent Pinchart wrote:
> >>> On Wednesday 10 December 2014 15:08:53 Will Deacon wrote:
> >>>> On Wed, Dec 10, 2014 at 02:52:56PM +0000, Rob Clark wrote:
> >>>>> so, what is the way for a driver that explicitly wants to manage
> >>>>> it's own device virtual address space to opt out of this?  I
> >>>>> suspect that won't be the common case, but for a gpu, if dma layer
> >>>>> all of a sudden thinks it is in control of the gpu's virtual
> >>>>> address space, things are going to end in tears..
> >>>> 
> >>>> I think you'll need to detach from the DMA domain, then have the
> >>>> driver manage everything itself. As you say, it's not the common
> >>>> case, so you may need to add some hooks for detaching from the
> >>>> default domain and swizzling your DMA ops.
> >>> 
> >>> I'm wondering if it's such an exotic case after all. I can see two
> >>> reasons not to use the default domain. In addition to special
> >>> requirements coming from the bus master side, the IOMMU itself might
> >>> not support one domain per bus master (I'm of course raising the issue
> >>> from a very selfish Renesas IPMMU point of view).
> >> 
> >> Do you mean that certain masters must be grouped into the same domain,
> >> or that the IOMMU can fail with -ENOSPC?
> > 
> > My IOMMU has hardware supports for 4 domains, and serves N masters (where
> > N is dependent on the SoC but is > 4). In its current form the driver
> > supports a single domain and thus detaches devices from the default
> > domain in the add_device callback:
>
> Hmm, ok. Ideally, you wouldn't need to do any of that in the driver, but I
> can understand why you decided to go down that route.

I'm of course open to alternative suggestions :-)

> > 	/*
> > 	 * Detach the device from the default ARM VA mapping and attach it to
> > 	 * our private mapping.
> > 	 */
> > 	arm_iommu_detach_device(dev);
> > 	ret = arm_iommu_attach_device(dev, mmu->mapping);
> > 	if (ret < 0) {
> > 	
> > 		dev_err(dev, "Failed to attach device to VA mapping\n");
> > 		return ret;
> > 	
> > 	}
> > 
> > I would have implemented that in the of_xlate callback, but that's too
> > early as the ARM default domain isn't created yet at that point.
> 
> Yup, the mythical ->get_default_domain might be the right place instead.
> 
> > Using a single domain is a bit of a waste of resources in my case, so an
> > evolution would be to create four domains and assign devices to them based
> > on a policy. The policy could be fixed (round-robin for instance), or
> > configurable (possibly through DT, although it's really a policy, not a
> > hardware description).
> 
> I think having one default domain, which is home to all of the masters that
> don't have any DMA restrictions is a good use of the hardware. That then
> leaves you with three domains to cover VFIO, devices with DMA limitations
> and potentially device isolation (if we had a way to describe that).
> 
> >> For the former, we need a way to represent IOMMU groups for the platform
> >> bus.
> > 
> > To be honest I'm not entirely sure how IOMMU groups are supposed to be
> > used. I understand they can be used by VFIO to group several masters that
> > will be able to see each other's memory through the same page table, and
> > also that a page table could be shared between multiple groups. When it
> > comes to group creation, though, things get fuzzy. I started with
> > creating one group per master in my driver, which is probably not the
> > thing to do. The Exynos IOMMU driver used to do the same, until Marek's
> > patch series converting it to DT- based instantiation (on top of your
> > patch set) has removed groups altogether. Groups seem to be more or less
> > optional, except in a couple of places (for instance the remove_device
> > callback will not be called by the BUS_NOTIFY_DEL_DEVICE notifier if the
> > device isn't part of an iommu group).
> > 
> > I'd appreciate if someone could clarify this to help me make an informed
> > opinion on the topic.
> 
> Ok, an iommu_group is the minimum granularity for which a specific IOMMU
> can offer address translation. So, if your IPMMU can isolate an arbitrary
> device (assuming there is a domain available), then each device is in its
> own iommu_group. This isn't always the case, for example if two masters are
> behind some sort of bridge that makes them indistinguishable to the IOMMU
> (perhaps they appear to have the same master ID), then they would have to
> be in the same iommu_group. Essentially, iommu_groups are a property of
> the hardware and should be instantiated by the bus. PCI does this, but
> we don't yet have anything for the platform bus.
> 
> VFIO puts multiple groups (now called vfio_groups) into a container. The
> container is synoymous to an iommu_domain (i.e. a shared address space).

Thank you for the explanation, that was very informative. It would be nice to 
capture that somewhere in Documentation/ ;-)

I'll thus create one group per bus master in the of_xlate function, as my 
platforms don't have masters indistinguishable to the IOMMUs (at least not to 
my knowledge).

-- 
Regards,

Laurent Pinchart

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Laurent Pinchart]

Hi Will,

On Monday 01 December 2014 16:57:12 Will Deacon wrote:
> This patch extends of_dma_configure so that it sets up the IOMMU for a
> device, as well as the coherent/non-coherent DMA mapping ops.
> 
> Acked-by: Arnd Bergmann <arnd@arndb.de>
> Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
> Tested-by: Robin Murphy <robin.murphy@arm.com>
> Signed-off-by: Will Deacon <will.deacon@arm.com>
> ---
>  arch/arm/include/asm/dma-mapping.h |  4 +++-
>  drivers/of/platform.c              | 21 ++++++++++++++-------
>  include/linux/dma-mapping.h        |  8 +++++++-
>  3 files changed, 24 insertions(+), 9 deletions(-)
> 
> diff --git a/arch/arm/include/asm/dma-mapping.h
> b/arch/arm/include/asm/dma-mapping.h index dc3420e77758..f3c0d953f6a2
> 100644
> --- a/arch/arm/include/asm/dma-mapping.h
> +++ b/arch/arm/include/asm/dma-mapping.h
> @@ -121,7 +121,9 @@ static inline unsigned long dma_max_pfn(struct device
> *dev) }
>  #define dma_max_pfn(dev) dma_max_pfn(dev)
> 
> -static inline void arch_setup_dma_ops(struct device *dev, bool coherent)
> +static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
> +				      u64 size, struct iommu_ops *iommu,
> +				      bool coherent)
>  {
>  	if (coherent)
>  		set_dma_ops(dev, &arm_coherent_dma_ops);
> diff --git a/drivers/of/platform.c b/drivers/of/platform.c
> index ff1f4e9afccb..b89caf8c7586 100644
> --- a/drivers/of/platform.c
> +++ b/drivers/of/platform.c
> @@ -19,6 +19,7 @@
>  #include <linux/slab.h>
>  #include <linux/of_address.h>
>  #include <linux/of_device.h>
> +#include <linux/of_iommu.h>
>  #include <linux/of_irq.h>
>  #include <linux/of_platform.h>
>  #include <linux/platform_device.h>
> @@ -166,6 +167,7 @@ static void of_dma_configure(struct device *dev)
>  	int ret;
>  	bool coherent;
>  	unsigned long offset;
> +	struct iommu_ops *iommu;
> 
>  	/*
>  	 * Set default dma-mask to 32 bit. Drivers are expected to setup
> @@ -194,7 +196,16 @@ static void of_dma_configure(struct device *dev)
>  	dev_dbg(dev, "device is%sdma coherent\n",
>  		coherent ? " " : " not ");
> 
> -	arch_setup_dma_ops(dev, coherent);
> +	iommu = of_iommu_configure(dev);
> +	dev_dbg(dev, "device is%sbehind an iommu\n",
> +		iommu ? " " : " not ");
> +
> +	arch_setup_dma_ops(dev, dma_addr, size, iommu, coherent);
> +}
> +
> +static void of_dma_deconfigure(struct device *dev)
> +{
> +	arch_teardown_dma_ops(dev);
>  }
> 
>  /**
> @@ -223,16 +234,12 @@ static struct platform_device
> *of_platform_device_create_pdata( if (!dev)
>  		goto err_clear_flag;
> 
> -	of_dma_configure(&dev->dev);
>  	dev->dev.bus = &platform_bus_type;
>  	dev->dev.platform_data = platform_data;
> -
> -	/* We do not fill the DMA ops for platform devices by default.
> -	 * This is currently the responsibility of the platform code
> -	 * to do such, possibly using a device notifier
> -	 */
> +	of_dma_configure(&dev->dev);
> 
>  	if (of_device_add(dev) != 0) {
> +		of_dma_deconfigure(&dev->dev);

Don't you also need to call of_dma_deconfigure() when the device is destroyed 
? Otherwise the default domain created by arch_setup_dma_ops() will be leaked.

>  		platform_device_put(dev);
>  		goto err_clear_flag;
>  	}
> diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
> index 8a1560f95d4a..c3007cb4bfa6 100644
> --- a/include/linux/dma-mapping.h
> +++ b/include/linux/dma-mapping.h
> @@ -130,7 +130,13 @@ static inline int dma_coerce_mask_and_coherent(struct
> device *dev, u64 mask) extern u64 dma_get_required_mask(struct device
> *dev);
> 
>  #ifndef arch_setup_dma_ops
> -static inline void arch_setup_dma_ops(struct device *dev, bool coherent) {
> } +static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
> +				      u64 size, struct iommu_ops *iommu,
> +				      bool coherent) { }
> +#endif
> +
> +#ifndef arch_teardown_dma_ops
> +static inline void arch_teardown_dma_ops(struct device *dev) { }
>  #endif
> 
>  static inline unsigned int dma_get_max_seg_size(struct device *dev)

-- 
Regards,

Laurent Pinchart

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Will Deacon]

On Sun, Dec 14, 2014 at 03:51:13PM +0000, Laurent Pinchart wrote:
> On Monday 01 December 2014 16:57:12 Will Deacon wrote:
> > This patch extends of_dma_configure so that it sets up the IOMMU for a
> > device, as well as the coherent/non-coherent DMA mapping ops.
> > 
> > Acked-by: Arnd Bergmann <arnd@arndb.de>
> > Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
> > Tested-by: Robin Murphy <robin.murphy@arm.com>
> > Signed-off-by: Will Deacon <will.deacon@arm.com>

[...]

> > +static void of_dma_deconfigure(struct device *dev)
> > +{
> > +	arch_teardown_dma_ops(dev);
> >  }
> > 
> >  /**
> > @@ -223,16 +234,12 @@ static struct platform_device
> > *of_platform_device_create_pdata( if (!dev)
> >  		goto err_clear_flag;
> > 
> > -	of_dma_configure(&dev->dev);
> >  	dev->dev.bus = &platform_bus_type;
> >  	dev->dev.platform_data = platform_data;
> > -
> > -	/* We do not fill the DMA ops for platform devices by default.
> > -	 * This is currently the responsibility of the platform code
> > -	 * to do such, possibly using a device notifier
> > -	 */
> > +	of_dma_configure(&dev->dev);
> > 
> >  	if (of_device_add(dev) != 0) {
> > +		of_dma_deconfigure(&dev->dev);
> 
> Don't you also need to call of_dma_deconfigure() when the device is destroyed 
> ? Otherwise the default domain created by arch_setup_dma_ops() will be leaked.

Something like below?

Will

--->8

diff --git a/drivers/of/platform.c b/drivers/of/platform.c
index b89caf8c7586..ec29c25b4fce 100644
--- a/drivers/of/platform.c
+++ b/drivers/of/platform.c
@@ -525,6 +525,7 @@ static int of_platform_device_destroy(struct device *dev, void *data)
                amba_device_unregister(to_amba_device(dev));
 #endif
 
+       of_dma_deconfigure(dev);
        of_node_clear_flag(dev->of_node, OF_POPULATED);
        of_node_clear_flag(dev->of_node, OF_POPULATED_BUS);
        return 0;

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Laurent Pinchart]

Hi Will,

On Monday 15 December 2014 11:32:52 Will Deacon wrote:
> On Sun, Dec 14, 2014 at 03:51:13PM +0000, Laurent Pinchart wrote:
> > On Monday 01 December 2014 16:57:12 Will Deacon wrote:
> > > This patch extends of_dma_configure so that it sets up the IOMMU for a
> > > device, as well as the coherent/non-coherent DMA mapping ops.
> > > 
> > > Acked-by: Arnd Bergmann <arnd@arndb.de>
> > > Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
> > > Tested-by: Robin Murphy <robin.murphy@arm.com>
> > > Signed-off-by: Will Deacon <will.deacon@arm.com>
> 
> [...]
> 
> > > +static void of_dma_deconfigure(struct device *dev)
> > > +{
> > > +	arch_teardown_dma_ops(dev);
> > >  }
> > >  
> > >  /**
> > > @@ -223,16 +234,12 @@ static struct platform_device
> > > *of_platform_device_create_pdata( if (!dev)
> > > 
> > >  		goto err_clear_flag;
> > > 
> > > -	of_dma_configure(&dev->dev);
> > >  	dev->dev.bus = &platform_bus_type;
> > >  	dev->dev.platform_data = platform_data;
 > > -
> > > -	/* We do not fill the DMA ops for platform devices by default.
> > > -	 * This is currently the responsibility of the platform code
> > > -	 * to do such, possibly using a device notifier
> > > -	 */
> > > +	of_dma_configure(&dev->dev);
> > >  	if (of_device_add(dev) != 0) {
> > > +		of_dma_deconfigure(&dev->dev);
> > 
> > Don't you also need to call of_dma_deconfigure() when the device is
> > destroyed ? Otherwise the default domain created by arch_setup_dma_ops()
> > will be leaked.
>
> Something like below?

Yes, something like that. This will however cause a dev_warn("Not attached") 
message to be printed for devices that have no IOMMU, that should be fixed in 
the arch code.

> --->8
> 
> diff --git a/drivers/of/platform.c b/drivers/of/platform.c
> index b89caf8c7586..ec29c25b4fce 100644
> --- a/drivers/of/platform.c
> +++ b/drivers/of/platform.c
> @@ -525,6 +525,7 @@ static int of_platform_device_destroy(struct device
> *dev, void *data) amba_device_unregister(to_amba_device(dev));
>  #endif
> 
> +       of_dma_deconfigure(dev);
>         of_node_clear_flag(dev->of_node, OF_POPULATED);
>         of_node_clear_flag(dev->of_node, OF_POPULATED_BUS);
>         return 0;

-- 
Regards,

Laurent Pinchart

[PATCH v6 6/8] dma-mapping: detect and configure IOMMU in of_dma_configure

[Will Deacon]

On Wed, Dec 17, 2014 at 12:19:45AM +0000, Laurent Pinchart wrote:
> On Monday 15 December 2014 11:32:52 Will Deacon wrote:
> > On Sun, Dec 14, 2014 at 03:51:13PM +0000, Laurent Pinchart wrote:
> > > On Monday 01 December 2014 16:57:12 Will Deacon wrote:
> > > > +	of_dma_configure(&dev->dev);
> > > >  	if (of_device_add(dev) != 0) {
> > > > +		of_dma_deconfigure(&dev->dev);
> > > 
> > > Don't you also need to call of_dma_deconfigure() when the device is
> > > destroyed ? Otherwise the default domain created by arch_setup_dma_ops()
> > > will be leaked.
> >
> > Something like below?
> 
> Yes, something like that. This will however cause a dev_warn("Not attached") 
> message to be printed for devices that have no IOMMU, that should be fixed in 
> the arch code.

Good catch, additional patch below to fix that up.

Will

--->8

diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index 09645f00bd17..c757f3d2d01e 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -2023,7 +2023,10 @@ static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
 
 static void arm_teardown_iommu_dma_ops(struct device *dev)
 {
-       struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+       struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);
+
+       if (!mapping)
+               return;
 
        arm_iommu_detach_device(dev);
        arm_iommu_release_mapping(mapping);

[PATCH v6 7/8] arm: call iommu_init before of_platform_populate

[Will Deacon]

We need to ensure that the IOMMUs in the system have a chance to perform
some basic initialisation before we start adding masters to them.

This patch adds a call to of_iommu_init before of_platform_populate.

Acked-by: Russell King <rmk+kernel@arm.linux.org.uk>
Acked-by: Arnd Bergmann <arnd@arndb.de>
Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
 arch/arm/kernel/setup.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c
index c03106378b49..1cfc94aa7fa2 100644
--- a/arch/arm/kernel/setup.c
+++ b/arch/arm/kernel/setup.c
@@ -18,6 +18,7 @@
 #include <linux/bootmem.h>
 #include <linux/seq_file.h>
 #include <linux/screen_info.h>
+#include <linux/of_iommu.h>
 #include <linux/of_platform.h>
 #include <linux/init.h>
 #include <linux/kexec.h>
@@ -806,6 +807,7 @@ static int __init customize_machine(void)
 	 * machine from the device tree, if no callback is provided,
 	 * otherwise we would always need an init_machine callback.
 	 */
+	of_iommu_init();
 	if (machine_desc->init_machine)
 		machine_desc->init_machine();
 #ifdef CONFIG_OF
-- 
2.1.1

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Will Deacon]

This patch plumbs the existing ARM IOMMU DMA infrastructure (which isn't
actually called outside of a few drivers) into arch_setup_dma_ops, so
that we can use IOMMUs for DMA transfers in a more generic fashion.

Since this significantly complicates the arch_setup_dma_ops function,
it is moved out of line into dma-mapping.c. If CONFIG_ARM_DMA_USE_IOMMU
is not set, the iommu parameter is ignored and the normal ops are used
instead.

Acked-by: Russell King <rmk+kernel@arm.linux.org.uk>
Acked-by: Arnd Bergmann <arnd@arndb.de>
Acked-by: Marek Szyprowski <m.szyprowski@samsung.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
 arch/arm/include/asm/dma-mapping.h | 12 +++---
 arch/arm/mm/dma-mapping.c          | 83 ++++++++++++++++++++++++++++++++++----
 2 files changed, 81 insertions(+), 14 deletions(-)

diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h
index f3c0d953f6a2..9410b7e548fc 100644
--- a/arch/arm/include/asm/dma-mapping.h
+++ b/arch/arm/include/asm/dma-mapping.h
@@ -121,14 +121,12 @@ static inline unsigned long dma_max_pfn(struct device *dev)
 }
 #define dma_max_pfn(dev) dma_max_pfn(dev)
 
-static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base,
-				      u64 size, struct iommu_ops *iommu,
-				      bool coherent)
-{
-	if (coherent)
-		set_dma_ops(dev, &arm_coherent_dma_ops);
-}
 #define arch_setup_dma_ops arch_setup_dma_ops
+extern void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
+			       struct iommu_ops *iommu, bool coherent);
+
+#define arch_teardown_dma_ops arch_teardown_dma_ops
+extern void arch_teardown_dma_ops(struct device *dev);
 
 static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
 {
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index e8907117861e..09645f00bd17 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -1947,9 +1947,8 @@ EXPORT_SYMBOL_GPL(arm_iommu_release_mapping);
  *	arm_iommu_create_mapping)
  *
  * Attaches specified io address space mapping to the provided device,
- * this replaces the dma operations (dma_map_ops pointer) with the
- * IOMMU aware version. More than one client might be attached to
- * the same io address space mapping.
+ * More than one client might be attached to the same io address space
+ * mapping.
  */
 int arm_iommu_attach_device(struct device *dev,
 			    struct dma_iommu_mapping *mapping)
@@ -1962,7 +1961,6 @@ int arm_iommu_attach_device(struct device *dev,
 
 	kref_get(&mapping->kref);
 	dev->archdata.mapping = mapping;
-	set_dma_ops(dev, &iommu_ops);
 
 	pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev));
 	return 0;
@@ -1974,7 +1972,6 @@ EXPORT_SYMBOL_GPL(arm_iommu_attach_device);
  * @dev: valid struct device pointer
  *
  * Detaches the provided device from a previously attached map.
- * This voids the dma operations (dma_map_ops pointer)
  */
 void arm_iommu_detach_device(struct device *dev)
 {
@@ -1989,10 +1986,82 @@ void arm_iommu_detach_device(struct device *dev)
 	iommu_detach_device(mapping->domain, dev);
 	kref_put(&mapping->kref, release_iommu_mapping);
 	dev->archdata.mapping = NULL;
-	set_dma_ops(dev, NULL);
 
 	pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev));
 }
 EXPORT_SYMBOL_GPL(arm_iommu_detach_device);
 
-#endif
+static struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent)
+{
+	return coherent ? &iommu_coherent_ops : &iommu_ops;
+}
+
+static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
+				    struct iommu_ops *iommu)
+{
+	struct dma_iommu_mapping *mapping;
+
+	if (!iommu)
+		return false;
+
+	mapping = arm_iommu_create_mapping(dev->bus, dma_base, size);
+	if (IS_ERR(mapping)) {
+		pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n",
+				size, dev_name(dev));
+		return false;
+	}
+
+	if (arm_iommu_attach_device(dev, mapping)) {
+		pr_warn("Failed to attached device %s to IOMMU_mapping\n",
+				dev_name(dev));
+		arm_iommu_release_mapping(mapping);
+		return false;
+	}
+
+	return true;
+}
+
+static void arm_teardown_iommu_dma_ops(struct device *dev)
+{
+	struct dma_iommu_mapping *mapping = dev->archdata.mapping;
+
+	arm_iommu_detach_device(dev);
+	arm_iommu_release_mapping(mapping);
+}
+
+#else
+
+static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size,
+				    struct iommu_ops *iommu)
+{
+	return false;
+}
+
+static void arm_teardown_iommu_dma_ops(struct device *dev) { }
+
+#define arm_get_iommu_dma_map_ops arm_get_dma_map_ops
+
+#endif	/* CONFIG_ARM_DMA_USE_IOMMU */
+
+static struct dma_map_ops *arm_get_dma_map_ops(bool coherent)
+{
+	return coherent ? &arm_coherent_dma_ops : &arm_dma_ops;
+}
+
+void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
+			struct iommu_ops *iommu, bool coherent)
+{
+	struct dma_map_ops *dma_ops;
+
+	if (arm_setup_iommu_dma_ops(dev, dma_base, size, iommu))
+		dma_ops = arm_get_iommu_dma_map_ops(coherent);
+	else
+		dma_ops = arm_get_dma_map_ops(coherent);
+
+	set_dma_ops(dev, dma_ops);
+}
+
+void arch_teardown_dma_ops(struct device *dev)
+{
+	arm_teardown_iommu_dma_ops(dev);
+}
-- 
2.1.1

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Alexandre Courbot]

On 12/02/2014 01:57 AM, Will Deacon wrote:
> This patch plumbs the existing ARM IOMMU DMA infrastructure (which isn't
> actually called outside of a few drivers) into arch_setup_dma_ops, so
> that we can use IOMMUs for DMA transfers in a more generic fashion.
>
> Since this significantly complicates the arch_setup_dma_ops function,
> it is moved out of line into dma-mapping.c. If CONFIG_ARM_DMA_USE_IOMMU
> is not set, the iommu parameter is ignored and the normal ops are used
> instead.

A series for IOMMU support with Tegra/Nouveau ceased to work after this 
patch. The Tegra IOMMU is not registered by the time the DT is parsed, 
and thus all devices end up without the proper DMA ops set up because 
the phandle to the IOMMU cannot be resolved. Subsequently calling 
arm_iommu_create_mapping() and arm_iommu_attach_device() from the driver 
(as I used to do until 3.18) does not help since the call to 
set_dma_ops() has been moved out of arm_iommu_attach_device(). Therefore 
there seems to be no way for a device to gets its correct DMA ops unless 
the IOMMU is ready by the time the DT is parsed.

Also potentially affected by this are the Rockchip DRM and OMAP3 ISP 
drivers, which follow the same pattern.

This raises the following questions:

1) Why are arm_iommu_create_mapping() and arm_iommu_attach_device() 
still public since they cannot set the DMA ops and thus seem to be 
useless outside of arch_setup_dma_ops()?

2) Say you want to use the IOMMU API in your driver, and have an iommu 
property in your device's DT node. If by chance your IOMMU is registered 
early, you will already have a mapping automatically created even before 
your probe function is called. Can this be avoided? Is it even safe?

The issue is workarounded (at least for me) with the following patch:

--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -1961,6 +1961,7 @@ int arm_iommu_attach_device(struct device *dev,

         kref_get(&mapping->kref);
         dev->archdata.mapping = mapping;
+       set_dma_ops(dev, &iommu_ops);

         pr_debug("Attached IOMMU controller to %s device.\n", 
dev_name(dev));
         return 0;

This allows arm_iommu_create_mapping() and arm_iommu_attach_device() to 
set the correct DMA ops when called from the driver. But it doesn't look 
like mergeable material and I'd like to know whether there is a better 
way to handle this, or whether I should just use the IOMMU API directly. 
Knowing that even this might not be safe if ARM_DMA_USE_IOMMU is enabled 
because of point 2) above.

So basically I'm afraid I might not even be able to use the IOMMU safely 
after this. Or am I missing anything?

Thanks,
Alex.

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Will Deacon]

Hi Alex,

On Wed, Jan 14, 2015 at 09:00:24AM +0000, Alexandre Courbot wrote:
> On 12/02/2014 01:57 AM, Will Deacon wrote:
> > This patch plumbs the existing ARM IOMMU DMA infrastructure (which isn't
> > actually called outside of a few drivers) into arch_setup_dma_ops, so
> > that we can use IOMMUs for DMA transfers in a more generic fashion.
> >
> > Since this significantly complicates the arch_setup_dma_ops function,
> > it is moved out of line into dma-mapping.c. If CONFIG_ARM_DMA_USE_IOMMU
> > is not set, the iommu parameter is ignored and the normal ops are used
> > instead.
> 
> A series for IOMMU support with Tegra/Nouveau ceased to work after this 
> patch.

Which series? This code shouldn't even be executed unless you start using
of_xlate and the generic bindings.

> The Tegra IOMMU is not registered by the time the DT is parsed, 
> and thus all devices end up without the proper DMA ops set up because 
> the phandle to the IOMMU cannot be resolved.

You might want to look at the patches posted for the exynos, renesas and ARM
SMMUs for some hints in how to use the new API.

> Subsequently calling arm_iommu_create_mapping() and
> arm_iommu_attach_device() from the driver (as I used to do until 3.18)
> does not help since the call to set_dma_ops() has been moved out of
> arm_iommu_attach_device(). Therefore there seems to be no way for a device
> to gets its correct DMA ops unless the IOMMU is ready by the time the DT
> is parsed.
> 
> Also potentially affected by this are the Rockchip DRM and OMAP3 ISP 
> drivers, which follow the same pattern.

I don't understand why any code currently in mainline should be affected.
Please can you elaborate on the failure case?

> This raises the following questions:
> 
> 1) Why are arm_iommu_create_mapping() and arm_iommu_attach_device() 
> still public since they cannot set the DMA ops and thus seem to be 
> useless outside of arch_setup_dma_ops()?

It has callers outside of the file. I'd like to make it static, but that
means doing some non-trivial porting of all the callers, which I'm also
unable to test.

> 2) Say you want to use the IOMMU API in your driver, and have an iommu 
> property in your device's DT node. If by chance your IOMMU is registered 
> early, you will already have a mapping automatically created even before 
> your probe function is called. Can this be avoided? Is it even safe?

Currently, I think you have to either teardown the ops manually or return
an error from of_xlate. Thierry was also looking at this sort of thing,
so it might be worth talking to him.

Will

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Heiko Stübner]

Hi Will, Alexandre, Daniel,

Am Mittwoch, 14. Januar 2015, 10:46:10 schrieb Will Deacon:
> Hi Alex,
> 
> On Wed, Jan 14, 2015 at 09:00:24AM +0000, Alexandre Courbot wrote:
> > On 12/02/2014 01:57 AM, Will Deacon wrote:
> > > This patch plumbs the existing ARM IOMMU DMA infrastructure (which isn't
> > > actually called outside of a few drivers) into arch_setup_dma_ops, so
> > > that we can use IOMMUs for DMA transfers in a more generic fashion.
> > > 
> > > Since this significantly complicates the arch_setup_dma_ops function,
> > > it is moved out of line into dma-mapping.c. If CONFIG_ARM_DMA_USE_IOMMU
> > > is not set, the iommu parameter is ignored and the normal ops are used
> > > instead.
> > 
> > A series for IOMMU support with Tegra/Nouveau ceased to work after this
> > patch.
> 
> Which series? This code shouldn't even be executed unless you start using
> of_xlate and the generic bindings.
> 
> > The Tegra IOMMU is not registered by the time the DT is parsed,
> > and thus all devices end up without the proper DMA ops set up because
> > the phandle to the IOMMU cannot be resolved.
> 
> You might want to look at the patches posted for the exynos, renesas and ARM
> SMMUs for some hints in how to use the new API.
> 
> > Subsequently calling arm_iommu_create_mapping() and
> > arm_iommu_attach_device() from the driver (as I used to do until 3.18)
> > does not help since the call to set_dma_ops() has been moved out of
> > arm_iommu_attach_device(). Therefore there seems to be no way for a device
> > to gets its correct DMA ops unless the IOMMU is ready by the time the DT
> > is parsed.
> > 
> > Also potentially affected by this are the Rockchip DRM and OMAP3 ISP
> > drivers, which follow the same pattern.
> 
> I don't understand why any code currently in mainline should be affected.
> Please can you elaborate on the failure case?

As Alexandre suspected the new Rockchip drm code seems to be affected by
this. I hadn't played with the drm code before last weekend and was then
stumbling over different iommu related issues. As I hadn't to much contact
with iommus till now I didn't get very far.

But with Alexandre's bandaid patch of adding
	set_dma_ops(dev, &iommu_ops);
to arm_iommu_attach_device both problems go away.


So to elaborate on the two failure cases:

When attaching either hdmi or vga connectors at runtime, I get

[drm:drm_mode_debug_printmodeline] Modeline 26:"1024x768" 85 94500 1024 1072 1168 1376 768 769 772 808 0x40 0x5
[drm:drm_crtc_helper_set_mode] [CRTC:14]
[drm:vop_crtc_dpms] crtc[14] mode[0]
rockchip-vop ff930000.vop: Attached to iommu domain
[drm] processing encoder TMDS-18
[drm] processing encoder DAC-22
[drm] processing connector HDMI-A-1
[drm] processing connector VGA-1
[drm:vop_win_update] [PLANE:12] [FB:-1->24] update
rk_iommu ff930300.iommu: Page fault at 0x2d400500 of type read
rk_iommu ff930300.iommu: iova = 0x2d400500: dte_index: 0xb5 pte_index: 0x0 page_offset: 0x500
rk_iommu ff930300.iommu: mmu_dte_addr: 0x2e3b3000 dte at 0x2e3b32d4: 0x000000 valid: 0 pte at 0x00000000: 0x000000 valid: 0 page at 0x00000000 flags: 0x0
[drm:drm_crtc_helper_set_mode] [ENCODER:22:DAC-22] set [MODE:26:1024x768]

===============
When my wip vga-connector is plugged in at boot, I get


[drm:drm_target_preferred] found mode 1280x1024
[drm:drm_setup_crtcs] picking CRTCs for 4096x4096 config
[drm:drm_setup_crtcs] desired mode 1280x1024 set on crtc 14 (0,0)
------------[ cut here ]------------
WARNING: CPU: 1 PID: 33 at mm/page_alloc.c:2645 __alloc_pages_nodemask+0x18c/0x6a8()
Modules linked in:
CPU: 1 PID: 33 Comm: kworker/u8:1 Not tainted 3.19.0-rc1+ #1512
Hardware name: Rockchip Cortex-A9 (Device Tree)
Workqueue: deferwq deferred_probe_work_func
[<c00148e4>] (unwind_backtrace) from [<c00111e0>] (show_stack+0x10/0x14)
[<c00111e0>] (show_stack) from [<c0426564>] (dump_stack+0x6c/0x84)
[<c0426564>] (dump_stack) from [<c0021f34>] (warn_slowpath_common+0x80/0xac)
[<c0021f34>] (warn_slowpath_common) from [<c0021f78>] (warn_slowpath_null+0x18/0x1c)
[<c0021f78>] (warn_slowpath_null) from [<c008b114>] (__alloc_pages_nodemask+0x18c/0x6a8)
[<c008b114>] (__alloc_pages_nodemask) from [<c001a274>] (__dma_alloc_buffer.isra.18+0x2c/0x80)
[<c001a274>] (__dma_alloc_buffer.isra.18) from [<c001a2dc>] (__alloc_remap_buffer.isra.22+0x14/0x5c)
[<c001a2dc>] (__alloc_remap_buffer.isra.22) from [<c001a490>] (__dma_alloc+0x16c/0x1d8)
[<c001a490>] (__dma_alloc) from [<c001a614>] (arm_dma_alloc+0x84/0x90)
[<c001a614>] (arm_dma_alloc) from [<c022f610>] (rockchip_gem_create_object+0x8c/0xc4)
[<c022f610>] (rockchip_gem_create_object) from [<c022f158>] (rockchip_drm_fbdev_create+0x6c/0x1ec)
[<c022f158>] (rockchip_drm_fbdev_create) from [<c021499c>] (drm_fb_helper_initial_config+0x230/0x328)
[<c021499c>] (drm_fb_helper_initial_config) from [<c022f388>] (rockchip_drm_fbdev_init+0xa4/0xc0)
[<c022f388>] (rockchip_drm_fbdev_init) from [<c022ea18>] (rockchip_drm_load+0x1b8/0x1f4)
[<c022ea18>] (rockchip_drm_load) from [<c021c218>] (drm_dev_register+0x80/0x100)
[<c021c218>] (drm_dev_register) from [<c022e77c>] (rockchip_drm_bind+0x48/0x74)
[<c022e77c>] (rockchip_drm_bind) from [<c0234a58>] (try_to_bring_up_master.part.2+0xa4/0xf4)
[<c0234a58>] (try_to_bring_up_master.part.2) from [<c0234c58>] (component_add+0x9c/0x104)
[<c0234c58>] (component_add) from [<c023a1e4>] (platform_drv_probe+0x48/0x90)
[<c023a1e4>] (platform_drv_probe) from [<c0238c30>] (driver_probe_device+0x130/0x340)
[<c0238c30>] (driver_probe_device) from [<c0237270>] (bus_for_each_drv+0x70/0x84)
[<c0237270>] (bus_for_each_drv) from [<c0238a8c>] (device_attach+0x64/0x88)
[<c0238a8c>] (device_attach) from [<c0238090>] (bus_probe_device+0x28/0x98)
[<c0238090>] (bus_probe_device) from [<c0238548>] (deferred_probe_work_func+0x78/0xa4)
[<c0238548>] (deferred_probe_work_func) from [<c0034008>] (process_one_work+0x1c8/0x2f4)
[<c0034008>] (process_one_work) from [<c0034448>] (worker_thread+0x2e8/0x450)
[<c0034448>] (worker_thread) from [<c00380a8>] (kthread+0xdc/0xf0)
[<c00380a8>] (kthread) from [<c000e8b8>] (ret_from_fork+0x14/0x3c)
---[ end trace ce23b3730f8c0d32 ]---
[drm:rockchip_gem_create_object] *ERROR* failed to allocate 0x500000 byte dma buffer


where Daniel Kurtz already deducted in private:

"But, more importantly, this call stack has "arm_dma_alloc", which
suggests that are not actually using the iommu dma allocators.
The allocation should have been handled by arm_iommu_alloc_attrs(),
but for some reason, it is not.
The iommu allocator should have been installed as the allocator for
the drm device by the call to arm_iommu_attach_device() in
rockchip_drm_load."


> > This raises the following questions:
> > 
> > 1) Why are arm_iommu_create_mapping() and arm_iommu_attach_device()
> > still public since they cannot set the DMA ops and thus seem to be
> > useless outside of arch_setup_dma_ops()?
> 
> It has callers outside of the file. I'd like to make it static, but that
> means doing some non-trivial porting of all the callers, which I'm also
> unable to test.
> 
> > 2) Say you want to use the IOMMU API in your driver, and have an iommu
> > property in your device's DT node. If by chance your IOMMU is registered
> > early, you will already have a mapping automatically created even before
> > your probe function is called. Can this be avoided? Is it even safe?
> 
> Currently, I think you have to either teardown the ops manually or return
> an error from of_xlate. Thierry was also looking at this sort of thing,
> so it might be worth talking to him.
> 
> Will
-------------- next part --------------
[drm:output_poll_execute] [CONNECTOR:23:VGA-1] status updated from disconnected to connected
[drm:drm_fb_helper_hotplug_event] 
[drm:drm_helper_probe_single_connector_modes_merge_bits] [CONNECTOR:20:HDMI-A-1]
[drm:drm_helper_probe_single_connector_modes_merge_bits] [CONNECTOR:20:HDMI-A-1] disconnected
[drm:drm_helper_probe_single_connector_modes_merge_bits] [CONNECTOR:23:VGA-1]
[drm:drm_mode_debug_printmodeline] Modeline 26:"1280x1024" 60 108000 1280 1328 1440 1688 1024 1025 1028 1066 0x48 0x5
[drm:drm_mode_prune_invalid] Not using 1280x1024 mode 12
[drm:drm_mode_debug_printmodeline] Modeline 27:"1152x864" 0 108000 1152 1216 1344 1600 864 865 868 900 0x40 0x5
[drm:drm_mode_prune_invalid] Not using 1152x864 mode 12
[drm:drm_mode_debug_printmodeline] Modeline 38:"1280x1024" 0 135000 1280 1296 1440 1688 1024 1025 1028 1066 0x40 0x5
[drm:drm_mode_prune_invalid] Not using 1280x1024 mode 12
[drm:drm_helper_probe_single_connector_modes_merge_bits] [CONNECTOR:23:VGA-1] probed modes :
[drm:drm_mode_debug_printmodeline] Modeline 28:"1024x768" 85 94500 1024 1072 1168 1376 768 769 772 808 0x40 0x5
[drm:drm_mode_debug_printmodeline] Modeline 39:"1024x768" 75 78800 1024 1040 1136 1312 768 769 772 800 0x40 0x5
[drm:drm_mode_debug_printmodeline] Modeline 40:"1024x768" 70 75000 1024 1048 1184 1328 768 771 777 806 0x40 0xa
[drm:drm_mode_debug_printmodeline] Modeline 41:"1024x768" 60 65000 1024 1048 1184 1344 768 771 777 806 0x40 0xa
[drm:drm_mode_debug_printmodeline] Modeline 42:"832x624" 75 57284 832 864 928 1152 624 625 628 667 0x40 0xa
[drm:drm_mode_debug_printmodeline] Modeline 29:"800x600" 85 56250 800 832 896 1048 600 601 604 631 0x40 0x5
[drm:drm_mode_debug_printmodeline] Modeline 43:"800x600" 75 49500 800 816 896 1056 600 601 604 625 0x40 0x5
[drm:drm_mode_debug_printmodeline] Modeline 44:"800x600" 72 50000 800 856 976 1040 600 637 643 666 0x40 0x5
[drm:drm_mode_debug_printmodeline] Modeline 31:"800x600" 60 40000 800 840 968 1056 600 601 605 628 0x40 0x5
[drm:drm_mode_debug_printmodeline] Modeline 32:"800x600" 56 36000 800 824 896 1024 600 601 603 625 0x40 0x5
[drm:drm_mode_debug_printmodeline] Modeline 30:"640x480" 85 36000 640 696 752 832 480 481 484 509 0x40 0xa
[drm:drm_mode_debug_printmodeline] Modeline 33:"640x480" 75 31500 640 656 720 840 480 481 484 500 0x40 0xa
[drm:drm_mode_debug_printmodeline] Modeline 34:"640x480" 73 31500 640 664 704 832 480 489 491 520 0x40 0xa
[drm:drm_mode_debug_printmodeline] Modeline 35:"640x480" 67 30240 640 704 768 864 480 483 486 525 0x40 0xa
[drm:drm_mode_debug_printmodeline] Modeline 36:"640x480" 60 25200 640 656 752 800 480 490 492 525 0x40 0xa
[drm:drm_mode_debug_printmodeline] Modeline 37:"720x400" 70 28320 720 738 846 900 400 412 414 449 0x40 0x6
[drm:drm_setup_crtcs] 
[drm:drm_enable_connectors] connector 20 enabled? no
[drm:drm_enable_connectors] connector 23 enabled? yes
[drm:drm_target_preferred] looking for cmdline mode on connector 23
[drm:drm_target_preferred] looking for preferred mode on connector 23 0
[drm:drm_target_preferred] found mode 1024x768
[drm:drm_setup_crtcs] picking CRTCs for 4096x4096 config
[drm:drm_setup_crtcs] desired mode 1024x768 set on crtc 14 (0,0)
[drm:drm_crtc_helper_set_config] 
[drm:drm_crtc_helper_set_config] [CRTC:8] [NOFB]
[drm:vop_crtc_dpms] crtc[8] mode[3]
[drm:vop_crtc_dpms] desired dpms mode is same as previous one.
[drm:vop_crtc_dpms] crtc[14] mode[3]
[drm:vop_crtc_dpms] desired dpms mode is same as previous one.
[drm:drm_crtc_helper_set_config] 
[drm:drm_crtc_helper_set_config] [CRTC:14] [FB:24] #connectors=1 (x y) (0 0)
[drm:drm_crtc_helper_set_config] crtc has no fb, full mode set
[drm:drm_crtc_helper_set_config] modes are different, full mode set
[drm:drm_mode_debug_printmodeline] Modeline 0:"" 0 0 0 0 0 0 0 0 0 0 0x0 0x0
[drm:drm_mode_debug_printmodeline] Modeline 26:"1024x768" 85 94500 1024 1072 1168 1376 768 769 772 808 0x40 0x5
[drm:drm_crtc_helper_set_config] encoder changed, full mode switch
[drm:drm_crtc_helper_set_config] [CONNECTOR:20:HDMI-A-1] to [NOCRTC]
[drm:drm_crtc_helper_set_config] crtc changed, full mode switch
[drm:drm_crtc_helper_set_config] [CONNECTOR:23:VGA-1] to [CRTC:14]
[drm:drm_crtc_helper_set_config] attempting to set mode from userspace
[drm:drm_mode_debug_printmodeline] Modeline 26:"1024x768" 85 94500 1024 1072 1168 1376 768 769 772 808 0x40 0x5
[drm:drm_crtc_helper_set_mode] [CRTC:14]
[drm:vop_crtc_dpms] crtc[14] mode[0]
rockchip-vop ff930000.vop: Attached to iommu domain
[drm] processing encoder TMDS-18
[drm] processing encoder DAC-22
[drm] processing connector HDMI-A-1
[drm] processing connector VGA-1
[drm:vop_win_update] [PLANE:12] [FB:-1->24] update
rk_iommu ff930300.iommu: Page fault at 0x2d400500 of type read
rk_iommu ff930300.iommu: iova = 0x2d400500: dte_index: 0xb5 pte_index: 0x0 page_offset: 0x500
rk_iommu ff930300.iommu: mmu_dte_addr: 0x2e3b3000 dte at 0x2e3b32d4: 0x000000 valid: 0 pte at 0x00000000: 0x000000 valid: 0 page at 0x00000000 flags: 0x0
[drm:drm_crtc_helper_set_mode] [ENCODER:22:DAC-22] set [MODE:26:1024x768]
[drm:drm_crtc_helper_set_config] Setting connector DPMS state to on
[drm:drm_crtc_helper_set_config] 	[CONNECTOR:23:VGA-1] set DPMS on
[drm:vop_crtc_dpms] crtc[8] mode[3]
[drm:vop_crtc_dpms] desired dpms mode is same as previous one.
[drm:drm_do_probe_ddc_edid] drm: skipping non-existent adapter rk3x-i2c
[drm:output_poll_execute] [CONNECTOR:23:VGA-1] status updated from connected to disconnected
[drm:drm_fb_helper_hotplug_event] 
[drm:drm_helper_probe_single_connector_modes_merge_bits] [CONNECTOR:20:HDMI-A-1]
[drm:drm_helper_probe_single_connector_modes_merge_bits] [CONNECTOR:20:HDMI-A-1] disconnected
[drm:drm_helper_probe_single_connector_modes_merge_bits] [CONNECTOR:23:VGA-1]
[drm:drm_do_probe_ddc_edid] drm: skipping non-existent adapter rk3x-i2c
[drm:drm_helper_probe_single_connector_modes_merge_bits] [CONNECTOR:23:VGA-1] disconnected
[drm:drm_setup_crtcs] 
[drm:drm_enable_connectors] connector 20 enabled? no
[drm:drm_enable_connectors] connector 23 enabled? no
[drm:drm_setup_crtcs] picking CRTCs for 4096x4096 config
[drm:drm_crtc_helper_set_config] 
[drm:drm_crtc_helper_set_config] [CRTC:8] [NOFB]
[drm:vop_crtc_dpms] crtc[8] mode[3]
[drm:vop_crtc_dpms] desired dpms mode is same as previous one.
[drm:drm_crtc_helper_set_config] 
[drm:drm_crtc_helper_set_config] [CRTC:14] [NOFB]
[drm:vop_crtc_dpms] crtc[8] mode[3]
[drm:vop_crtc_dpms] desired dpms mode is same as previous one.
[drm:vop_crtc_dpms] crtc[14] mode[3]
rk_iommu ff930300.iommu: Enable stall request timed out, status: 0x00004b
rk_iommu ff930300.iommu: Disable paging request timed out, status: 0x00004b
rockchip-vop ff930000.vop: Detached from iommu domain
-------------- next part --------------
rockchip-drm display-subsystem: bound ff940000.vop (ops vop_component_ops)
rockchip-drm display-subsystem: bound ff930000.vop (ops vop_component_ops)
dwhdmi-rockchip ff980000.hdmi: Detected HDMI controller 0x20:0xa:0xa0:0xc1
rockchip-drm display-subsystem: bound ff980000.hdmi (ops dw_hdmi_rockchip_ops)
rockchip-drm display-subsystem: bound vga-bridge (ops vga_bridge_ops)
[drm] Supports vblank timestamp caching Rev 2 (21.10.2013).
[drm] No driver support for vblank timestamp query.
[drm:vop_crtc_dpms] crtc[8] mode[3]
[drm:vop_crtc_dpms] desired dpms mode is same as previous one.
[drm:vop_crtc_dpms] crtc[14] mode[3]
[drm:vop_crtc_dpms] desired dpms mode is same as previous one.
[drm:drm_helper_probe_single_connector_modes_merge_bits] [CONNECTOR:20:HDMI-A-1]
[drm:drm_helper_probe_single_connector_modes_merge_bits] [CONNECTOR:20:HDMI-A-1] disconnected
[drm:drm_helper_probe_single_connector_modes_merge_bits] [CONNECTOR:23:VGA-1]
[drm:drm_helper_probe_single_connector_modes_merge_bits] [CONNECTOR:23:VGA-1] probed modes :
[drm:drm_mode_debug_printmodeline] Modeline 25:"1280x1024" 60 108000 1280 1328 1440 1688 1024 1025 1028 1066 0x48 0x5
[drm:drm_mode_debug_printmodeline] Modeline 37:"1280x1024" 75 135000 1280 1296 1440 1688 1024 1025 1028 1066 0x40 0x5
[drm:drm_mode_debug_printmodeline] Modeline 26:"1152x864" 75 108000 1152 1216 1344 1600 864 865 868 900 0x40 0x5
[drm:drm_mode_debug_printmodeline] Modeline 27:"1024x768" 85 94500 1024 1072 1168 1376 768 769 772 808 0x40 0x5
[drm:drm_mode_debug_printmodeline] Modeline 38:"1024x768" 75 78800 1024 1040 1136 1312 768 769 772 800 0x40 0x5
[drm:drm_mode_debug_printmodeline] Modeline 39:"1024x768" 70 75000 1024 1048 1184 1328 768 771 777 806 0x40 0xa
[drm:drm_mode_debug_printmodeline] Modeline 40:"1024x768" 60 65000 1024 1048 1184 1344 768 771 777 806 0x40 0xa
[drm:drm_mode_debug_printmodeline] Modeline 41:"832x624" 75 57284 832 864 928 1152 624 625 628 667 0x40 0xa
[drm:drm_mode_debug_printmodeline] Modeline 28:"800x600" 85 56250 800 832 896 1048 600 601 604 631 0x40 0x5
[drm:drm_mode_debug_printmodeline] Modeline 42:"800x600" 75 49500 800 816 896 1056 600 601 604 625 0x40 0x5
[drm:drm_mode_debug_printmodeline] Modeline 43:"800x600" 72 50000 800 856 976 1040 600 637 643 666 0x40 0x5
[drm:drm_mode_debug_printmodeline] Modeline 30:"800x600" 60 40000 800 840 968 1056 600 601 605 628 0x40 0x5
[drm:drm_mode_debug_printmodeline] Modeline 31:"800x600" 56 36000 800 824 896 1024 600 601 603 625 0x40 0x5
[drm:drm_mode_debug_printmodeline] Modeline 29:"640x480" 85 36000 640 696 752 832 480 481 484 509 0x40 0xa
[drm:drm_mode_debug_printmodeline] Modeline 32:"640x480" 75 31500 640 656 720 840 480 481 484 500 0x40 0xa
[drm:drm_mode_debug_printmodeline] Modeline 33:"640x480" 73 31500 640 664 704 832 480 489 491 520 0x40 0xa
[drm:drm_mode_debug_printmodeline] Modeline 34:"640x480" 67 30240 640 704 768 864 480 483 486 525 0x40 0xa
[drm:drm_mode_debug_printmodeline] Modeline 35:"640x480" 60 25200 640 656 752 800 480 490 492 525 0x40 0xa
[drm:drm_mode_debug_printmodeline] Modeline 36:"720x400" 70 28320 720 738 846 900 400 412 414 449 0x40 0x6
[drm:drm_setup_crtcs] 
[drm:drm_enable_connectors] connector 20 enabled? no
[drm:drm_enable_connectors] connector 23 enabled? yes
[drm:drm_target_preferred] looking for cmdline mode on connector 23
[drm:drm_target_preferred] looking for preferred mode on connector 23 0
[drm:drm_target_preferred] found mode 1280x1024
[drm:drm_setup_crtcs] picking CRTCs for 4096x4096 config
[drm:drm_setup_crtcs] desired mode 1280x1024 set on crtc 14 (0,0)
------------[ cut here ]------------
WARNING: CPU: 1 PID: 33 at mm/page_alloc.c:2645 __alloc_pages_nodemask+0x18c/0x6a8()
Modules linked in:
CPU: 1 PID: 33 Comm: kworker/u8:1 Not tainted 3.19.0-rc1+ #1512
Hardware name: Rockchip Cortex-A9 (Device Tree)
Workqueue: deferwq deferred_probe_work_func
[<c00148e4>] (unwind_backtrace) from [<c00111e0>] (show_stack+0x10/0x14)
[<c00111e0>] (show_stack) from [<c0426564>] (dump_stack+0x6c/0x84)
[<c0426564>] (dump_stack) from [<c0021f34>] (warn_slowpath_common+0x80/0xac)
[<c0021f34>] (warn_slowpath_common) from [<c0021f78>] (warn_slowpath_null+0x18/0x1c)
[<c0021f78>] (warn_slowpath_null) from [<c008b114>] (__alloc_pages_nodemask+0x18c/0x6a8)
[<c008b114>] (__alloc_pages_nodemask) from [<c001a274>] (__dma_alloc_buffer.isra.18+0x2c/0x80)
[<c001a274>] (__dma_alloc_buffer.isra.18) from [<c001a2dc>] (__alloc_remap_buffer.isra.22+0x14/0x5c)
[<c001a2dc>] (__alloc_remap_buffer.isra.22) from [<c001a490>] (__dma_alloc+0x16c/0x1d8)
[<c001a490>] (__dma_alloc) from [<c001a614>] (arm_dma_alloc+0x84/0x90)
[<c001a614>] (arm_dma_alloc) from [<c022f610>] (rockchip_gem_create_object+0x8c/0xc4)
[<c022f610>] (rockchip_gem_create_object) from [<c022f158>] (rockchip_drm_fbdev_create+0x6c/0x1ec)
[<c022f158>] (rockchip_drm_fbdev_create) from [<c021499c>] (drm_fb_helper_initial_config+0x230/0x328)
[<c021499c>] (drm_fb_helper_initial_config) from [<c022f388>] (rockchip_drm_fbdev_init+0xa4/0xc0)
[<c022f388>] (rockchip_drm_fbdev_init) from [<c022ea18>] (rockchip_drm_load+0x1b8/0x1f4)
[<c022ea18>] (rockchip_drm_load) from [<c021c218>] (drm_dev_register+0x80/0x100)
[<c021c218>] (drm_dev_register) from [<c022e77c>] (rockchip_drm_bind+0x48/0x74)
[<c022e77c>] (rockchip_drm_bind) from [<c0234a58>] (try_to_bring_up_master.part.2+0xa4/0xf4)
[<c0234a58>] (try_to_bring_up_master.part.2) from [<c0234c58>] (component_add+0x9c/0x104)
[<c0234c58>] (component_add) from [<c023a1e4>] (platform_drv_probe+0x48/0x90)
[<c023a1e4>] (platform_drv_probe) from [<c0238c30>] (driver_probe_device+0x130/0x340)
[<c0238c30>] (driver_probe_device) from [<c0237270>] (bus_for_each_drv+0x70/0x84)
[<c0237270>] (bus_for_each_drv) from [<c0238a8c>] (device_attach+0x64/0x88)
[<c0238a8c>] (device_attach) from [<c0238090>] (bus_probe_device+0x28/0x98)
[<c0238090>] (bus_probe_device) from [<c0238548>] (deferred_probe_work_func+0x78/0xa4)
[<c0238548>] (deferred_probe_work_func) from [<c0034008>] (process_one_work+0x1c8/0x2f4)
[<c0034008>] (process_one_work) from [<c0034448>] (worker_thread+0x2e8/0x450)
[<c0034448>] (worker_thread) from [<c00380a8>] (kthread+0xdc/0xf0)
[<c00380a8>] (kthread) from [<c000e8b8>] (ret_from_fork+0x14/0x3c)
---[ end trace ce23b3730f8c0d32 ]---
[drm:rockchip_gem_create_object] *ERROR* failed to allocate 0x500000 byte dma buffer

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Will Deacon]

On Wed, Jan 14, 2015 at 01:51:36PM +0000, Heiko St?bner wrote:
> Am Mittwoch, 14. Januar 2015, 10:46:10 schrieb Will Deacon:
> > On Wed, Jan 14, 2015 at 09:00:24AM +0000, Alexandre Courbot wrote:
> > > On 12/02/2014 01:57 AM, Will Deacon wrote:
> > > > This patch plumbs the existing ARM IOMMU DMA infrastructure (which isn't
> > > > actually called outside of a few drivers) into arch_setup_dma_ops, so
> > > > that we can use IOMMUs for DMA transfers in a more generic fashion.
> > > > 
> > > > Since this significantly complicates the arch_setup_dma_ops function,
> > > > it is moved out of line into dma-mapping.c. If CONFIG_ARM_DMA_USE_IOMMU
> > > > is not set, the iommu parameter is ignored and the normal ops are used
> > > > instead.
> > > 
> > > A series for IOMMU support with Tegra/Nouveau ceased to work after this
> > > patch.
> > 
> > Which series? This code shouldn't even be executed unless you start using
> > of_xlate and the generic bindings.
> > 
> > > The Tegra IOMMU is not registered by the time the DT is parsed,
> > > and thus all devices end up without the proper DMA ops set up because
> > > the phandle to the IOMMU cannot be resolved.
> > 
> > You might want to look at the patches posted for the exynos, renesas and ARM
> > SMMUs for some hints in how to use the new API.
> > 
> > > Subsequently calling arm_iommu_create_mapping() and
> > > arm_iommu_attach_device() from the driver (as I used to do until 3.18)
> > > does not help since the call to set_dma_ops() has been moved out of
> > > arm_iommu_attach_device(). Therefore there seems to be no way for a device
> > > to gets its correct DMA ops unless the IOMMU is ready by the time the DT
> > > is parsed.
> > > 
> > > Also potentially affected by this are the Rockchip DRM and OMAP3 ISP
> > > drivers, which follow the same pattern.
> > 
> > I don't understand why any code currently in mainline should be affected.
> > Please can you elaborate on the failure case?
> 
> As Alexandre suspected the new Rockchip drm code seems to be affected by
> this. I hadn't played with the drm code before last weekend and was then
> stumbling over different iommu related issues. As I hadn't to much contact
> with iommus till now I didn't get very far.
> 
> But with Alexandre's bandaid patch of adding
> 	set_dma_ops(dev, &iommu_ops);
> to arm_iommu_attach_device both problems go away.
> 
> 
> So to elaborate on the two failure cases:

Aha, I see what you mean now -- the issue is that attaching to an IOMMU
domain no longer swizzles the DMA ops. Furthermore, we also need to take
into account the coherency of the device, which we didn't do before (and
assumedly "worked" because all of the users happened to be non-coherent).

Maybe we just need to add something like arm_iommu_swizzle_dma_ops, so
that direct users of the arm_iommu_* API can manage things manually?

Will

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Thierry Reding]

On Wed, Jan 14, 2015 at 07:17:50PM +0000, Will Deacon wrote:
> On Wed, Jan 14, 2015 at 01:51:36PM +0000, Heiko St?bner wrote:
> > Am Mittwoch, 14. Januar 2015, 10:46:10 schrieb Will Deacon:
> > > On Wed, Jan 14, 2015 at 09:00:24AM +0000, Alexandre Courbot wrote:
> > > > On 12/02/2014 01:57 AM, Will Deacon wrote:
> > > > > This patch plumbs the existing ARM IOMMU DMA infrastructure (which isn't
> > > > > actually called outside of a few drivers) into arch_setup_dma_ops, so
> > > > > that we can use IOMMUs for DMA transfers in a more generic fashion.
> > > > > 
> > > > > Since this significantly complicates the arch_setup_dma_ops function,
> > > > > it is moved out of line into dma-mapping.c. If CONFIG_ARM_DMA_USE_IOMMU
> > > > > is not set, the iommu parameter is ignored and the normal ops are used
> > > > > instead.
> > > > 
> > > > A series for IOMMU support with Tegra/Nouveau ceased to work after this
> > > > patch.
> > > 
> > > Which series? This code shouldn't even be executed unless you start using
> > > of_xlate and the generic bindings.
> > > 
> > > > The Tegra IOMMU is not registered by the time the DT is parsed,
> > > > and thus all devices end up without the proper DMA ops set up because
> > > > the phandle to the IOMMU cannot be resolved.
> > > 
> > > You might want to look at the patches posted for the exynos, renesas and ARM
> > > SMMUs for some hints in how to use the new API.
> > > 
> > > > Subsequently calling arm_iommu_create_mapping() and
> > > > arm_iommu_attach_device() from the driver (as I used to do until 3.18)
> > > > does not help since the call to set_dma_ops() has been moved out of
> > > > arm_iommu_attach_device(). Therefore there seems to be no way for a device
> > > > to gets its correct DMA ops unless the IOMMU is ready by the time the DT
> > > > is parsed.
> > > > 
> > > > Also potentially affected by this are the Rockchip DRM and OMAP3 ISP
> > > > drivers, which follow the same pattern.
> > > 
> > > I don't understand why any code currently in mainline should be affected.
> > > Please can you elaborate on the failure case?
> > 
> > As Alexandre suspected the new Rockchip drm code seems to be affected by
> > this. I hadn't played with the drm code before last weekend and was then
> > stumbling over different iommu related issues. As I hadn't to much contact
> > with iommus till now I didn't get very far.
> > 
> > But with Alexandre's bandaid patch of adding
> > 	set_dma_ops(dev, &iommu_ops);
> > to arm_iommu_attach_device both problems go away.
> > 
> > 
> > So to elaborate on the two failure cases:
> 
> Aha, I see what you mean now -- the issue is that attaching to an IOMMU
> domain no longer swizzles the DMA ops. Furthermore, we also need to take
> into account the coherency of the device, which we didn't do before (and
> assumedly "worked" because all of the users happened to be non-coherent).
> 
> Maybe we just need to add something like arm_iommu_swizzle_dma_ops, so
> that direct users of the arm_iommu_* API can manage things manually?

Why does this even have to be an ARM-specific API? Couldn't this equally
well be generic across all platforms so that we get unified handling of
IOMMU through the DMA API?

Thierry
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 819 bytes
Desc: not available
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20150115/93c23635/attachment.sig>

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Will Deacon]

On Thu, Jan 15, 2015 at 08:30:06AM +0000, Thierry Reding wrote:
> On Wed, Jan 14, 2015 at 07:17:50PM +0000, Will Deacon wrote:
> > On Wed, Jan 14, 2015 at 01:51:36PM +0000, Heiko St?bner wrote:
> > > As Alexandre suspected the new Rockchip drm code seems to be affected by
> > > this. I hadn't played with the drm code before last weekend and was then
> > > stumbling over different iommu related issues. As I hadn't to much contact
> > > with iommus till now I didn't get very far.
> > > 
> > > But with Alexandre's bandaid patch of adding
> > > 	set_dma_ops(dev, &iommu_ops);
> > > to arm_iommu_attach_device both problems go away.
> > > 
> > > 
> > > So to elaborate on the two failure cases:
> > 
> > Aha, I see what you mean now -- the issue is that attaching to an IOMMU
> > domain no longer swizzles the DMA ops. Furthermore, we also need to take
> > into account the coherency of the device, which we didn't do before (and
> > assumedly "worked" because all of the users happened to be non-coherent).
> > 
> > Maybe we just need to add something like arm_iommu_swizzle_dma_ops, so
> > that direct users of the arm_iommu_* API can manage things manually?
> 
> Why does this even have to be an ARM-specific API? Couldn't this equally
> well be generic across all platforms so that we get unified handling of
> IOMMU through the DMA API?

It's already an ARM-specific API. Having something generic would be great,
but I was thinking more about a short-term fix for the current issue rather
than implementing something brand new.

Will

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Alexandre Courbot]

On 01/14/2015 07:46 PM, Will Deacon wrote:
> Hi Alex,
>
> On Wed, Jan 14, 2015 at 09:00:24AM +0000, Alexandre Courbot wrote:
>> On 12/02/2014 01:57 AM, Will Deacon wrote:
>>> This patch plumbs the existing ARM IOMMU DMA infrastructure (which isn't
>>> actually called outside of a few drivers) into arch_setup_dma_ops, so
>>> that we can use IOMMUs for DMA transfers in a more generic fashion.
>>>
>>> Since this significantly complicates the arch_setup_dma_ops function,
>>> it is moved out of line into dma-mapping.c. If CONFIG_ARM_DMA_USE_IOMMU
>>> is not set, the iommu parameter is ignored and the normal ops are used
>>> instead.
>>
>> A series for IOMMU support with Tegra/Nouveau ceased to work after this
>> patch.
>
> Which series? This code shouldn't even be executed unless you start using
> of_xlate and the generic bindings.
>
>> The Tegra IOMMU is not registered by the time the DT is parsed,
>> and thus all devices end up without the proper DMA ops set up because
>> the phandle to the IOMMU cannot be resolved.
>
> You might want to look at the patches posted for the exynos, renesas and ARM
> SMMUs for some hints in how to use the new API.
>
>> Subsequently calling arm_iommu_create_mapping() and
>> arm_iommu_attach_device() from the driver (as I used to do until 3.18)
>> does not help since the call to set_dma_ops() has been moved out of
>> arm_iommu_attach_device(). Therefore there seems to be no way for a device
>> to gets its correct DMA ops unless the IOMMU is ready by the time the DT
>> is parsed.
>>
>> Also potentially affected by this are the Rockchip DRM and OMAP3 ISP
>> drivers, which follow the same pattern.
>
> I don't understand why any code currently in mainline should be affected.
> Please can you elaborate on the failure case?

Here is the sequence of events:

1) DT is populated and of_dma_configure() is called for every device. 
Here is the stack trace:

of_dma_configure
of_platform_device_create_pdata
of_platform_bus_create
of_platform_bus_create
of_platform_populate
customize_machine
do_one_initcall

of_dma_configure() is supposed to set the iommu_ops for every device 
(and this is currently the only place in the kernel where this can 
happen), but since the IOMMU is not ready yet it falls back to the 
default arm_dma_ops.

2) Driver calls arm_iommu_create_mapping() and arm_iommu_attach_device() 
(which were already called during step 1 btw). Both calls succeed, but 
since arm_iommu_attach_device() does not change the ops anymore we are 
still left with arm_dma_ops.

3) Things don't go well. :)

There are several problems with the way things are currently setup IMHO:

1) CONFIG_ARM_DMA_USE_IOMMU forcibly makes the DMA API go through the 
IOMMU. This effectively forbids the simultaneous use of the IOMMU API 
and DMA API because of address-space collisions issue. Being a kernel 
config option, there is no way to turn this behavior off which will 
likely become a problem for multi-configs where some platforms might 
want this and some other not.

2) IIUC arm_iommu_*() are now not supposed to be called directly by 
drivers anymore. At least that's what their current behavior makes me 
think, as well as the fact that they are not defined if 
CONFIG_ARM_DMA_USE_IOMMU is not set. Yet they are still public and 
drivers that use them have not been updated consequently.

3) This is a longer-standing issue, but IIUC the fact that the IOMMU VM 
used by the DMA API is not available effectively prevents anyone from 
using the DMA API behind a IOMMU and the IOMMU API simultaneously (which 
you now have to assume is always the case because of 1)).

These are huge issues - under the current conditions the only safe path 
for me is to eschew the DMA API completely and port my work to the IOMMU 
API, and I suspect this will affect other people as well.

>
>> This raises the following questions:
>>
>> 1) Why are arm_iommu_create_mapping() and arm_iommu_attach_device()
>> still public since they cannot set the DMA ops and thus seem to be
>> useless outside of arch_setup_dma_ops()?
>
> It has callers outside of the file. I'd like to make it static, but that
> means doing some non-trivial porting of all the callers, which I'm also
> unable to test.

Yeah, but all the callers of these functions are broken anyway as of 
3.18-rc4...

>
>> 2) Say you want to use the IOMMU API in your driver, and have an iommu
>> property in your device's DT node. If by chance your IOMMU is registered
>> early, you will already have a mapping automatically created even before
>> your probe function is called. Can this be avoided? Is it even safe?
>
> Currently, I think you have to either teardown the ops manually or return
> an error from of_xlate. Thierry was also looking at this sort of thing,
> so it might be worth talking to him.

Tearing down the ops manually does not sound like something drivers 
should do - how can they even be aware of the private dma_ops anyway?

Thanks,
Alex.

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Thierry Reding]

On Wed, Jan 14, 2015 at 10:46:10AM +0000, Will Deacon wrote:
> On Wed, Jan 14, 2015 at 09:00:24AM +0000, Alexandre Courbot wrote:
[...]
> > 2) Say you want to use the IOMMU API in your driver, and have an iommu 
> > property in your device's DT node. If by chance your IOMMU is registered 
> > early, you will already have a mapping automatically created even before 
> > your probe function is called. Can this be avoided? Is it even safe?
> 
> Currently, I think you have to either teardown the ops manually or return
> an error from of_xlate. Thierry was also looking at this sort of thing,
> so it might be worth talking to him.

I already explained in earlier threads why I think this is a bad idea.
It's completely unnatural for any driver to manually tear down something
that it didn't want set up in the first place. It also means that you
have to carefully audit any users of these IOMMU APIs to make sure that
they do tear down. That doesn't sound like a good incremental approach,
as evidenced by the breakage that Alex and Heiko have encountered.

The solution for me has been to completely side-step the issue and not
register the IOMMU with the new mechanism at all. That is, there's no
.of_xlate() implementation, which means that the ARM DMA API glue won't
try to be smart and use the IOMMU in ways it's not meant to be used.
This has several advantages, such as that I can also use the regular
driver model for suspend/resume of the IOMMU, and I get to enjoy the
benefits of devres in the IOMMU driver. Probe ordering is still a tiny
issue, but we can easily solve that using explicit initcall ordering
(which really isn't any worse than IOMMU_OF_DECLARE()).

Ideally of course I could use deferred probing to solve that. Hiroshi
and I posted patches to implement that over a year ago[0], but it was
mostly ignored after some, in my opinion, fruitful discussion.

That patch requires drivers to explicitly call iommu_attach() to
advertise that they want to use the IOMMU. This was done because a
previous attempt to call iommu_attach() from the driver core was
rejected on the sole argument that "it doesn't belong in the core",
without further explanation.

Note that with that approach things all happen at driver probe time, so
we can simply return -EPROBE_DEFER when the IOMMU isn't ready yet. At
the same time the IOMMU can be a regular driver just like any of the
other resource providers like clocks, regulators, etc.

That all said, I'm fine with the current situation, too. Nobody can
force the Tegra SMMU driver to register via IOMMU_OF_DECLARE() and I'm
actively recommending the use of the IOMMU API directly. That way we
get the control we need for drivers where it matters most (i.e. where
close control of the address space and switching of address spaces per
process are needed). Any other drivers, like USB or SDMMC simply use
the DMA API and get their buffers from a contiguous pool.

Thierry

[0]: https://lkml.org/lkml/2014/6/26/476
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 819 bytes
Desc: not available
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20150115/7220303e/attachment.sig>

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Will Deacon]

On Thu, Jan 15, 2015 at 08:28:44AM +0000, Thierry Reding wrote:
> On Wed, Jan 14, 2015 at 10:46:10AM +0000, Will Deacon wrote:
> > On Wed, Jan 14, 2015 at 09:00:24AM +0000, Alexandre Courbot wrote:
> [...]
> > > 2) Say you want to use the IOMMU API in your driver, and have an iommu 
> > > property in your device's DT node. If by chance your IOMMU is registered 
> > > early, you will already have a mapping automatically created even before 
> > > your probe function is called. Can this be avoided? Is it even safe?
> > 
> > Currently, I think you have to either teardown the ops manually or return
> > an error from of_xlate. Thierry was also looking at this sort of thing,
> > so it might be worth talking to him.
> 
> I already explained in earlier threads why I think this is a bad idea.
> It's completely unnatural for any driver to manually tear down something
> that it didn't want set up in the first place. It also means that you
> have to carefully audit any users of these IOMMU APIs to make sure that
> they do tear down. That doesn't sound like a good incremental approach,
> as evidenced by the breakage that Alex and Heiko have encountered.

Well, perhaps we hide that behind a get_iommu API or something. We *do*
need this manual teardown step to support things like VFIO, so it makes
sense to reuse it for other users too imo.

> The solution for me has been to completely side-step the issue and not
> register the IOMMU with the new mechanism at all. That is, there's no
> .of_xlate() implementation, which means that the ARM DMA API glue won't
> try to be smart and use the IOMMU in ways it's not meant to be used.
> This has several advantages, such as that I can also use the regular
> driver model for suspend/resume of the IOMMU, and I get to enjoy the
> benefits of devres in the IOMMU driver. Probe ordering is still a tiny
> issue, but we can easily solve that using explicit initcall ordering
> (which really isn't any worse than IOMMU_OF_DECLARE()).

That's a pity. I'd much rather extend what we currently have to satisfy
your use-case. Ho-hum.

Will

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Laurent Pinchart]

On Thursday 15 January 2015 11:12:17 Will Deacon wrote:
> On Thu, Jan 15, 2015 at 08:28:44AM +0000, Thierry Reding wrote:
> > On Wed, Jan 14, 2015 at 10:46:10AM +0000, Will Deacon wrote:
> > > On Wed, Jan 14, 2015 at 09:00:24AM +0000, Alexandre Courbot wrote:
> > [...]
> > 
> >>> 2) Say you want to use the IOMMU API in your driver, and have an iommu
> >>> property in your device's DT node. If by chance your IOMMU is
> >>> registered early, you will already have a mapping automatically
> >>> created even before your probe function is called. Can this be
> >>> avoided? Is it even safe?
> >> 
> >> Currently, I think you have to either teardown the ops manually or
> >> return an error from of_xlate. Thierry was also looking at this sort of
> >> thing, so it might be worth talking to him.
> > 
> > I already explained in earlier threads why I think this is a bad idea.
> > It's completely unnatural for any driver to manually tear down something
> > that it didn't want set up in the first place. It also means that you
> > have to carefully audit any users of these IOMMU APIs to make sure that
> > they do tear down. That doesn't sound like a good incremental approach,
> > as evidenced by the breakage that Alex and Heiko have encountered.
> 
> Well, perhaps we hide that behind a get_iommu API or something. We *do*
> need this manual teardown step to support things like VFIO, so it makes
> sense to reuse it for other users too imo.
> 
> > The solution for me has been to completely side-step the issue and not
> > register the IOMMU with the new mechanism at all. That is, there's no
> > .of_xlate() implementation, which means that the ARM DMA API glue won't
> > try to be smart and use the IOMMU in ways it's not meant to be used.

That will break when someone will want to use the same IOMMU type for devices 
that use the DMA mapping API to hide the IOMMU. That might not be the case for 
your IOMMU today, but it's pretty fragile, we need to fix it.

> > This has several advantages, such as that I can also use the regular
> > driver model for suspend/resume of the IOMMU, and I get to enjoy the
> > benefits of devres in the IOMMU driver. Probe ordering is still a tiny
> > issue, but we can easily solve that using explicit initcall ordering
> > (which really isn't any worse than IOMMU_OF_DECLARE()).
> 
> That's a pity. I'd much rather extend what we currently have to satisfy
> your use-case. Ho-hum.

Assuming we want the IOMMU to be handled transparently for the majority of 
devices I only see two ways to fix this,

The first way is to create a default DMA mapping unconditionally and let 
drivers that can't live with it tear it down. That's what is implemented 
today.

The second way is to implement a mechanism to let drivers signal that they 
want to handle DMA mappings themselves. As the mappings need in the general 
case to be created before the probe function is called we can't signal this by 
calling a function in probe(). A new flag field for struct device_driver is a 
possible solution. This would however require delaying the creation of DMA 
mappings until right before probe time. Attaching to the IOMMU could be pushed 
to right before probe() as well, which would have the added benefit of making 
IOMMU driver implementable as real platform drivers.

-- 
Regards,

Laurent Pinchart

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Alexandre Courbot]

On 01/16/2015 08:18 AM, Laurent Pinchart wrote:
> On Thursday 15 January 2015 11:12:17 Will Deacon wrote:
>> On Thu, Jan 15, 2015 at 08:28:44AM +0000, Thierry Reding wrote:
>>> On Wed, Jan 14, 2015 at 10:46:10AM +0000, Will Deacon wrote:
>>>> On Wed, Jan 14, 2015 at 09:00:24AM +0000, Alexandre Courbot wrote:
>>> [...]
>>>
>>>>> 2) Say you want to use the IOMMU API in your driver, and have an iommu
>>>>> property in your device's DT node. If by chance your IOMMU is
>>>>> registered early, you will already have a mapping automatically
>>>>> created even before your probe function is called. Can this be
>>>>> avoided? Is it even safe?
>>>>
>>>> Currently, I think you have to either teardown the ops manually or
>>>> return an error from of_xlate. Thierry was also looking at this sort of
>>>> thing, so it might be worth talking to him.
>>>
>>> I already explained in earlier threads why I think this is a bad idea.
>>> It's completely unnatural for any driver to manually tear down something
>>> that it didn't want set up in the first place. It also means that you
>>> have to carefully audit any users of these IOMMU APIs to make sure that
>>> they do tear down. That doesn't sound like a good incremental approach,
>>> as evidenced by the breakage that Alex and Heiko have encountered.
>>
>> Well, perhaps we hide that behind a get_iommu API or something. We *do*
>> need this manual teardown step to support things like VFIO, so it makes
>> sense to reuse it for other users too imo.
>>
>>> The solution for me has been to completely side-step the issue and not
>>> register the IOMMU with the new mechanism at all. That is, there's no
>>> .of_xlate() implementation, which means that the ARM DMA API glue won't
>>> try to be smart and use the IOMMU in ways it's not meant to be used.
>
> That will break when someone will want to use the same IOMMU type for devices
> that use the DMA mapping API to hide the IOMMU. That might not be the case for
> your IOMMU today, but it's pretty fragile, we need to fix it.
>
>>> This has several advantages, such as that I can also use the regular
>>> driver model for suspend/resume of the IOMMU, and I get to enjoy the
>>> benefits of devres in the IOMMU driver. Probe ordering is still a tiny
>>> issue, but we can easily solve that using explicit initcall ordering
>>> (which really isn't any worse than IOMMU_OF_DECLARE()).
>>
>> That's a pity. I'd much rather extend what we currently have to satisfy
>> your use-case. Ho-hum.
>
> Assuming we want the IOMMU to be handled transparently for the majority of
> devices I only see two ways to fix this,
>
> The first way is to create a default DMA mapping unconditionally and let
> drivers that can't live with it tear it down. That's what is implemented
> today.

I strongly support Thierry's point that drivers should not have to tear 
down things they don't need. The issue we are facing today is a very 
good illustration of why one should not have to do this.

Everybody hates to receive unsollicited email with a link that says "to 
unsubscribe, click here". Let's not import that unpleasant culture into 
the kernel.

I am arriving late in this discussion, but what is wrong with asking 
drivers to explicitly state that they want the DMA API to be backed by 
the IOMMU instead of forcibly making it work that way?

>
> The second way is to implement a mechanism to let drivers signal that they
> want to handle DMA mappings themselves. As the mappings need in the general
> case to be created before the probe function  is called

Sorry for being ignorant here, but why is that?

 > we can't signal this by
> calling a function in probe(). A new flag field for struct device_driver is a
> possible solution. This would however require delaying the creation of DMA
> mappings until right before probe time. Attaching to the IOMMU could be pushed
> to right before probe() as well, which would have the added benefit of making
> IOMMU driver implementable as real platform drivers.

Keeping the ability to write IOMMU drivers as platform drivers would be 
nice indeed.

The problem with the opt-out flag though is that one will need to check 
every single driver out there to see whether it stills behave correctly 
if its hardware is suddently put behind a IOMMU. Doing it the other way 
(a flag that enables IOMMU if available) sounds safer to me.

What we have right now is a mechanism that basically makes it impossible 
to use the DMA API on many ARM platforms if ARM_DMA_USE_IOMMU is set 
(and I suspect it would also make the IOMMU unusable as well, without 
any way to fix things). This is quite concerning.

Even more concerning is that -rc5 is about to be released and we have 
in-tree drivers (Rockchip DRM) that are not working as they should 
because of this patch. Will, what is your plan to fix this? Do we have 
stuff that absolutely depends on this patch? If not, can we just revert 
it until all these issues are solved?

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Laurent Pinchart]

Hi Alexandre,

On Sunday 18 January 2015 15:54:34 Alexandre Courbot wrote:
> On 01/16/2015 08:18 AM, Laurent Pinchart wrote:
> > On Thursday 15 January 2015 11:12:17 Will Deacon wrote:
> >> On Thu, Jan 15, 2015 at 08:28:44AM +0000, Thierry Reding wrote:
> >>> On Wed, Jan 14, 2015 at 10:46:10AM +0000, Will Deacon wrote:
> >>>> On Wed, Jan 14, 2015 at 09:00:24AM +0000, Alexandre Courbot wrote:
> >>> [...]
> >>> 
> >>>>> 2) Say you want to use the IOMMU API in your driver, and have an iommu
> >>>>> property in your device's DT node. If by chance your IOMMU is
> >>>>> registered early, you will already have a mapping automatically
> >>>>> created even before your probe function is called. Can this be
> >>>>> avoided? Is it even safe?
> >>>> 
> >>>> Currently, I think you have to either teardown the ops manually or
> >>>> return an error from of_xlate. Thierry was also looking at this sort of
> >>>> thing, so it might be worth talking to him.
> >>> 
> >>> I already explained in earlier threads why I think this is a bad idea.
> >>> It's completely unnatural for any driver to manually tear down something
> >>> that it didn't want set up in the first place. It also means that you
> >>> have to carefully audit any users of these IOMMU APIs to make sure that
> >>> they do tear down. That doesn't sound like a good incremental approach,
> >>> as evidenced by the breakage that Alex and Heiko have encountered.
> >> 
> >> Well, perhaps we hide that behind a get_iommu API or something. We *do*
> >> need this manual teardown step to support things like VFIO, so it makes
> >> sense to reuse it for other users too imo.
> >> 
> >>> The solution for me has been to completely side-step the issue and not
> >>> register the IOMMU with the new mechanism at all. That is, there's no
> >>> .of_xlate() implementation, which means that the ARM DMA API glue won't
> >>> try to be smart and use the IOMMU in ways it's not meant to be used.
> > 
> > That will break when someone will want to use the same IOMMU type for
> > devices that use the DMA mapping API to hide the IOMMU. That might not be
> > the case for your IOMMU today, but it's pretty fragile, we need to fix
> > it.
> > 
> >>> This has several advantages, such as that I can also use the regular
> >>> driver model for suspend/resume of the IOMMU, and I get to enjoy the
> >>> benefits of devres in the IOMMU driver. Probe ordering is still a tiny
> >>> issue, but we can easily solve that using explicit initcall ordering
> >>> (which really isn't any worse than IOMMU_OF_DECLARE()).
> >> 
> >> That's a pity. I'd much rather extend what we currently have to satisfy
> >> your use-case. Ho-hum.
> > 
> > Assuming we want the IOMMU to be handled transparently for the majority of
> > devices I only see two ways to fix this,
> > 
> > The first way is to create a default DMA mapping unconditionally and let
> > drivers that can't live with it tear it down. That's what is implemented
> > today.
> 
> I strongly support Thierry's point that drivers should not have to tear
> down things they don't need. The issue we are facing today is a very
> good illustration of why one should not have to do this.
> 
> Everybody hates to receive unsollicited email with a link that says "to
> unsubscribe, click here". Let's not import that unpleasant culture into
> the kernel.
> 
> I am arriving late in this discussion, but what is wrong with asking
> drivers to explicitly state that they want the DMA API to be backed by
> the IOMMU instead of forcibly making it work that way?

The vast majority of the drivers are not IOMMU-aware. We would thus need to 
add a call at the beginning of the probe function of nearly every driver that 
can perform DMA to state that the driver doesn't need to handle any IOMMU that 
might be present in the system itself. I don't think that's a better solution.

Explicitly tearing down mappings in drivers that want to manage IOMMUs isn't a 
solution I like either. A possibly better solution would be to call a function 
to state that the DMA mapping API shouldn't not handle IOMMUs. Something like

	dma_mapping_ignore_iommu(dev);

at the beginning of the probe function of such drivers could do. The function 
would perform behind the scene all operations needed to tear down everything 
that shouldn't have been set up.

> > The second way is to implement a mechanism to let drivers signal that they
> > want to handle DMA mappings themselves. As the mappings need in the
> > general case to be created before the probe function  is called
> 
> Sorry for being ignorant here, but why is that?

Because a driver can call the DMA mapping API in its probe function, to 
allocate DMA coherent memory for instance. We need to ensure that the DMA 
mapping IOMMU has set up the required IOMMU ops by that time. As explained 
above I don't like the idea of sprinkling explicit calls to initialize IOMMU 
support in the vast majority of drivers, especially when they shouldn't be 
IOMMU-aware, so we then need to initialize everything that is needed before 
the first call to the DMA mapping API.

> > we can't signal this by calling a function in probe(). A new flag field
> > for struct device_driver is a possible solution. This would however
> > require delaying the creation of DMA mappings until right before probe
> > time. Attaching to the IOMMU could be pushed to right before probe() as
> > well, which would have the added benefit of making IOMMU driver
> > implementable as real platform drivers.
> 
> Keeping the ability to write IOMMU drivers as platform drivers would be
> nice indeed.
> 
> The problem with the opt-out flag though is that one will need to check
> every single driver out there to see whether it stills behave correctly
> if its hardware is suddently put behind a IOMMU.

That's actually my default assumption :-) On ARM platforms at least, for many 
devices, whether an IOMMU is present or not is an integration property, not a 
property of the device. The same way a device shouldn't care about the exact 
bus topology that connects it to the CPU and memory, it shouldn't care about 
whether an IOMMU is present on that bus, except in special cases.

> Doing it the other way (a flag that enables IOMMU if available) sounds safer
> to me.
>
> What we have right now is a mechanism that basically makes it impossible
> to use the DMA API on many ARM platforms if ARM_DMA_USE_IOMMU is set
> (and I suspect it would also make the IOMMU unusable as well, without
> any way to fix things). This is quite concerning.
> 
> Even more concerning is that -rc5 is about to be released and we have
> in-tree drivers (Rockchip DRM) that are not working as they should
> because of this patch. Will, what is your plan to fix this? Do we have
> stuff that absolutely depends on this patch? If not, can we just revert
> it until all these issues are solved?

-- 
Regards,

Laurent Pinchart

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Will Deacon]

On Sun, Jan 18, 2015 at 11:18:51AM +0000, Laurent Pinchart wrote:
> On Sunday 18 January 2015 15:54:34 Alexandre Courbot wrote:
> > On 01/16/2015 08:18 AM, Laurent Pinchart wrote:
> > > On Thursday 15 January 2015 11:12:17 Will Deacon wrote:
> > >> On Thu, Jan 15, 2015 at 08:28:44AM +0000, Thierry Reding wrote:
> > >>> On Wed, Jan 14, 2015 at 10:46:10AM +0000, Will Deacon wrote:
> > >>>> On Wed, Jan 14, 2015 at 09:00:24AM +0000, Alexandre Courbot wrote:
> > >>> [...]
> > >>> 
> > >>>>> 2) Say you want to use the IOMMU API in your driver, and have an iommu
> > >>>>> property in your device's DT node. If by chance your IOMMU is
> > >>>>> registered early, you will already have a mapping automatically
> > >>>>> created even before your probe function is called. Can this be
> > >>>>> avoided? Is it even safe?
> > >>>> 
> > >>>> Currently, I think you have to either teardown the ops manually or
> > >>>> return an error from of_xlate. Thierry was also looking at this sort of
> > >>>> thing, so it might be worth talking to him.
> > >>> 
> > >>> I already explained in earlier threads why I think this is a bad idea.
> > >>> It's completely unnatural for any driver to manually tear down something
> > >>> that it didn't want set up in the first place. It also means that you
> > >>> have to carefully audit any users of these IOMMU APIs to make sure that
> > >>> they do tear down. That doesn't sound like a good incremental approach,
> > >>> as evidenced by the breakage that Alex and Heiko have encountered.
> > >> 
> > >> Well, perhaps we hide that behind a get_iommu API or something. We *do*
> > >> need this manual teardown step to support things like VFIO, so it makes
> > >> sense to reuse it for other users too imo.
> > >> 
> > >>> The solution for me has been to completely side-step the issue and not
> > >>> register the IOMMU with the new mechanism at all. That is, there's no
> > >>> .of_xlate() implementation, which means that the ARM DMA API glue won't
> > >>> try to be smart and use the IOMMU in ways it's not meant to be used.
> > > 
> > > That will break when someone will want to use the same IOMMU type for
> > > devices that use the DMA mapping API to hide the IOMMU. That might not be
> > > the case for your IOMMU today, but it's pretty fragile, we need to fix
> > > it.
> > > 
> > >>> This has several advantages, such as that I can also use the regular
> > >>> driver model for suspend/resume of the IOMMU, and I get to enjoy the
> > >>> benefits of devres in the IOMMU driver. Probe ordering is still a tiny
> > >>> issue, but we can easily solve that using explicit initcall ordering
> > >>> (which really isn't any worse than IOMMU_OF_DECLARE()).
> > >> 
> > >> That's a pity. I'd much rather extend what we currently have to satisfy
> > >> your use-case. Ho-hum.
> > > 
> > > Assuming we want the IOMMU to be handled transparently for the majority of
> > > devices I only see two ways to fix this,
> > > 
> > > The first way is to create a default DMA mapping unconditionally and let
> > > drivers that can't live with it tear it down. That's what is implemented
> > > today.
> > 
> > I strongly support Thierry's point that drivers should not have to tear
> > down things they don't need. The issue we are facing today is a very
> > good illustration of why one should not have to do this.
> > 
> > Everybody hates to receive unsollicited email with a link that says "to
> > unsubscribe, click here". Let's not import that unpleasant culture into
> > the kernel.
> > 
> > I am arriving late in this discussion, but what is wrong with asking
> > drivers to explicitly state that they want the DMA API to be backed by
> > the IOMMU instead of forcibly making it work that way?
> 
> The vast majority of the drivers are not IOMMU-aware. We would thus need to 
> add a call at the beginning of the probe function of nearly every driver that 
> can perform DMA to state that the driver doesn't need to handle any IOMMU that 
> might be present in the system itself. I don't think that's a better solution.
> 
> Explicitly tearing down mappings in drivers that want to manage IOMMUs isn't a 
> solution I like either. A possibly better solution would be to call a function 
> to state that the DMA mapping API shouldn't not handle IOMMUs. Something like
> 
> 	dma_mapping_ignore_iommu(dev);
> 
> at the beginning of the probe function of such drivers could do. The function 
> would perform behind the scene all operations needed to tear down everything 
> that shouldn't have been set up.

An alternative would be to add a flag to platform_driver, like we have for
"prevent_deferred_probe" which is something like "prevent_dma_configure".

For the moment, that would actually teardown the DMA configuration in
platform_drv_probe, but if things are reordering in future then we can avoid
setting up the ops altogether without an API change.

Will

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Laurent Pinchart]

Hi Will,

On Monday 19 January 2015 11:12:02 Will Deacon wrote:
> On Sun, Jan 18, 2015 at 11:18:51AM +0000, Laurent Pinchart wrote:
> > On Sunday 18 January 2015 15:54:34 Alexandre Courbot wrote:
> >> On 01/16/2015 08:18 AM, Laurent Pinchart wrote:
> >>> On Thursday 15 January 2015 11:12:17 Will Deacon wrote:
> >>>> On Thu, Jan 15, 2015 at 08:28:44AM +0000, Thierry Reding wrote:
> >>>>> On Wed, Jan 14, 2015 at 10:46:10AM +0000, Will Deacon wrote:
> >>>>>> On Wed, Jan 14, 2015 at 09:00:24AM +0000, Alexandre Courbot wrote:
> >>>>> [...]
> >>>>> 
> >>>>>>> 2) Say you want to use the IOMMU API in your driver, and have an
> >>>>>>> iommu property in your device's DT node. If by chance your IOMMU
> >>>>>>> is registered early, you will already have a mapping automatically
> >>>>>>> created even before your probe function is called. Can this be
> >>>>>>> avoided? Is it even safe?
> >>>>>> 
> >>>>>> Currently, I think you have to either teardown the ops manually or
> >>>>>> return an error from of_xlate. Thierry was also looking at this
> >>>>>> sort of thing, so it might be worth talking to him.
> >>>>> 
> >>>>> I already explained in earlier threads why I think this is a bad
> >>>>> idea. It's completely unnatural for any driver to manually tear down
> >>>>> something that it didn't want set up in the first place. It also
> >>>>> means that you have to carefully audit any users of these IOMMU APIs
> >>>>> to make sure that they do tear down. That doesn't sound like a good
> >>>>> incremental approach, as evidenced by the breakage that Alex and
> >>>>> Heiko have encountered.
> >>>> 
> >>>> Well, perhaps we hide that behind a get_iommu API or something. We
> >>>> *do* need this manual teardown step to support things like VFIO, so
> >>>> it makes sense to reuse it for other users too imo.
> >>>> 
> >>>>> The solution for me has been to completely side-step the issue and
> >>>>> not register the IOMMU with the new mechanism at all. That is,
> >>>>> there's no .of_xlate() implementation, which means that the ARM DMA
> >>>>> API glue won't try to be smart and use the IOMMU in ways it's not
> >>>>> meant to be used.
> >>> 
> >>> That will break when someone will want to use the same IOMMU type for
> >>> devices that use the DMA mapping API to hide the IOMMU. That might not
> >>> be the case for your IOMMU today, but it's pretty fragile, we need to
> >>> fix it.
> >>> 
> >>>>> This has several advantages, such as that I can also use the regular
> >>>>> driver model for suspend/resume of the IOMMU, and I get to enjoy the
> >>>>> benefits of devres in the IOMMU driver. Probe ordering is still a
> >>>>> tiny issue, but we can easily solve that using explicit initcall
> >>>>> ordering (which really isn't any worse than IOMMU_OF_DECLARE()).
> >>>> 
> >>>> That's a pity. I'd much rather extend what we currently have to
> >>>> satisfy your use-case. Ho-hum.
> >>> 
> >>> Assuming we want the IOMMU to be handled transparently for the
> >>> majority of devices I only see two ways to fix this,
> >>> 
> >>> The first way is to create a default DMA mapping unconditionally and
> >>> let drivers that can't live with it tear it down. That's what is
> >>> implemented today.
> >> 
> >> I strongly support Thierry's point that drivers should not have to tear
> >> down things they don't need. The issue we are facing today is a very
> >> good illustration of why one should not have to do this.
> >> 
> >> Everybody hates to receive unsollicited email with a link that says "to
> >> unsubscribe, click here". Let's not import that unpleasant culture into
> >> the kernel.
> >> 
> >> I am arriving late in this discussion, but what is wrong with asking
> >> drivers to explicitly state that they want the DMA API to be backed by
> >> the IOMMU instead of forcibly making it work that way?
> > 
> > The vast majority of the drivers are not IOMMU-aware. We would thus need
> > to add a call at the beginning of the probe function of nearly every
> > driver that can perform DMA to state that the driver doesn't need to
> > handle any IOMMU that might be present in the system itself. I don't think
> > that's a better solution.
> > 
> > Explicitly tearing down mappings in drivers that want to manage IOMMUs
> > isn't a solution I like either. A possibly better solution would be to
> > call a function to state that the DMA mapping API shouldn't not handle
> > IOMMUs. Something like
> >
> > 	dma_mapping_ignore_iommu(dev);
> > 
> > at the beginning of the probe function of such drivers could do. The
> > function would perform behind the scene all operations needed to tear
> > down everything that shouldn't have been set up.
> 
> An alternative would be to add a flag to platform_driver, like we have for
> "prevent_deferred_probe" which is something like "prevent_dma_configure".

That's a solution I have proposed (albeit as a struct device_driver field, but 
that's a small detail), so I'm fine with it :-)

> For the moment, that would actually teardown the DMA configuration in
> platform_drv_probe, but if things are reordering in future then we can avoid
> setting up the ops altogether without an API change.

-- 
Regards,

Laurent Pinchart

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Thierry Reding]

On Mon, Jan 19, 2015 at 01:34:24PM +0200, Laurent Pinchart wrote:
> Hi Will,
> 
> On Monday 19 January 2015 11:12:02 Will Deacon wrote:
> > On Sun, Jan 18, 2015 at 11:18:51AM +0000, Laurent Pinchart wrote:
> > > On Sunday 18 January 2015 15:54:34 Alexandre Courbot wrote:
> > >> On 01/16/2015 08:18 AM, Laurent Pinchart wrote:
> > >>> On Thursday 15 January 2015 11:12:17 Will Deacon wrote:
> > >>>> On Thu, Jan 15, 2015 at 08:28:44AM +0000, Thierry Reding wrote:
> > >>>>> On Wed, Jan 14, 2015 at 10:46:10AM +0000, Will Deacon wrote:
> > >>>>>> On Wed, Jan 14, 2015 at 09:00:24AM +0000, Alexandre Courbot wrote:
> > >>>>> [...]
> > >>>>> 
> > >>>>>>> 2) Say you want to use the IOMMU API in your driver, and have an
> > >>>>>>> iommu property in your device's DT node. If by chance your IOMMU
> > >>>>>>> is registered early, you will already have a mapping automatically
> > >>>>>>> created even before your probe function is called. Can this be
> > >>>>>>> avoided? Is it even safe?
> > >>>>>> 
> > >>>>>> Currently, I think you have to either teardown the ops manually or
> > >>>>>> return an error from of_xlate. Thierry was also looking at this
> > >>>>>> sort of thing, so it might be worth talking to him.
> > >>>>> 
> > >>>>> I already explained in earlier threads why I think this is a bad
> > >>>>> idea. It's completely unnatural for any driver to manually tear down
> > >>>>> something that it didn't want set up in the first place. It also
> > >>>>> means that you have to carefully audit any users of these IOMMU APIs
> > >>>>> to make sure that they do tear down. That doesn't sound like a good
> > >>>>> incremental approach, as evidenced by the breakage that Alex and
> > >>>>> Heiko have encountered.
> > >>>> 
> > >>>> Well, perhaps we hide that behind a get_iommu API or something. We
> > >>>> *do* need this manual teardown step to support things like VFIO, so
> > >>>> it makes sense to reuse it for other users too imo.
> > >>>> 
> > >>>>> The solution for me has been to completely side-step the issue and
> > >>>>> not register the IOMMU with the new mechanism at all. That is,
> > >>>>> there's no .of_xlate() implementation, which means that the ARM DMA
> > >>>>> API glue won't try to be smart and use the IOMMU in ways it's not
> > >>>>> meant to be used.
> > >>> 
> > >>> That will break when someone will want to use the same IOMMU type for
> > >>> devices that use the DMA mapping API to hide the IOMMU. That might not
> > >>> be the case for your IOMMU today, but it's pretty fragile, we need to
> > >>> fix it.
> > >>> 
> > >>>>> This has several advantages, such as that I can also use the regular
> > >>>>> driver model for suspend/resume of the IOMMU, and I get to enjoy the
> > >>>>> benefits of devres in the IOMMU driver. Probe ordering is still a
> > >>>>> tiny issue, but we can easily solve that using explicit initcall
> > >>>>> ordering (which really isn't any worse than IOMMU_OF_DECLARE()).
> > >>>> 
> > >>>> That's a pity. I'd much rather extend what we currently have to
> > >>>> satisfy your use-case. Ho-hum.
> > >>> 
> > >>> Assuming we want the IOMMU to be handled transparently for the
> > >>> majority of devices I only see two ways to fix this,
> > >>> 
> > >>> The first way is to create a default DMA mapping unconditionally and
> > >>> let drivers that can't live with it tear it down. That's what is
> > >>> implemented today.
> > >> 
> > >> I strongly support Thierry's point that drivers should not have to tear
> > >> down things they don't need. The issue we are facing today is a very
> > >> good illustration of why one should not have to do this.
> > >> 
> > >> Everybody hates to receive unsollicited email with a link that says "to
> > >> unsubscribe, click here". Let's not import that unpleasant culture into
> > >> the kernel.
> > >> 
> > >> I am arriving late in this discussion, but what is wrong with asking
> > >> drivers to explicitly state that they want the DMA API to be backed by
> > >> the IOMMU instead of forcibly making it work that way?
> > > 
> > > The vast majority of the drivers are not IOMMU-aware. We would thus need
> > > to add a call at the beginning of the probe function of nearly every
> > > driver that can perform DMA to state that the driver doesn't need to
> > > handle any IOMMU that might be present in the system itself. I don't think
> > > that's a better solution.
> > > 
> > > Explicitly tearing down mappings in drivers that want to manage IOMMUs
> > > isn't a solution I like either. A possibly better solution would be to
> > > call a function to state that the DMA mapping API shouldn't not handle
> > > IOMMUs. Something like
> > >
> > > 	dma_mapping_ignore_iommu(dev);
> > > 
> > > at the beginning of the probe function of such drivers could do. The
> > > function would perform behind the scene all operations needed to tear
> > > down everything that shouldn't have been set up.
> > 
> > An alternative would be to add a flag to platform_driver, like we have for
> > "prevent_deferred_probe" which is something like "prevent_dma_configure".
> 
> That's a solution I have proposed (albeit as a struct device_driver field, but 
> that's a small detail), so I'm fine with it :-)

I think Marek had proposed something similar initially as well. I don't
see an immediate downside to that solution. It's still somewhat ugly in
that a lot of stuff is set up before it's known to actually be used at
all, but it seems like there's some consensus that this can be improved
later on, so I have no objections to such a patch.

Of course that doesn't solve the current breakage for the Rockchip DRM
and OMAP ISP drivers.

There's an additional issue with this automatic type of setting up
mapping: on Tegra quite a few devices can use the IOMMU. Among those are
SD/MMC controllers, HDA, SATA and things like the AHB bus. Creating a
separate domain (== address space) for each of those devices is wasteful
and there's currently no way to make them use a single virtual address
space. Each of them really has no use for a full 4 GiB of address space.
My earliest attempt to implement that was as a policy decision within
the IOMMU driver, but that wasn't very well received.

I'm now wondering whether the same could be done using this new type of
flag. Perhaps it can be assumed that every driver that doesn't want its
IOMMU master hooked up to the DMA API can (and will) manually manage the
virtual address space. Conversely every driver that wants to use the DMA
API to abstract away the existence of an IOMMU could be considered part
of the group where a separate domain isn't necessary.

Thierry
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 819 bytes
Desc: not available
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20150119/dd111ee4/attachment.sig>

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Laurent Pinchart]

Hi Thierry and Will,

On Monday 19 January 2015 13:31:00 Thierry Reding wrote:
> On Mon, Jan 19, 2015 at 01:34:24PM +0200, Laurent Pinchart wrote:
> > On Monday 19 January 2015 11:12:02 Will Deacon wrote:
> >> On Sun, Jan 18, 2015 at 11:18:51AM +0000, Laurent Pinchart wrote:
> >>> On Sunday 18 January 2015 15:54:34 Alexandre Courbot wrote:
> >>>> On 01/16/2015 08:18 AM, Laurent Pinchart wrote:
> >>>>> On Thursday 15 January 2015 11:12:17 Will Deacon wrote:

[snip]

> >>>> I am arriving late in this discussion, but what is wrong with asking
> >>>> drivers to explicitly state that they want the DMA API to be backed
> >>>> by the IOMMU instead of forcibly making it work that way?
> >>> 
> >>> The vast majority of the drivers are not IOMMU-aware. We would thus
> >>> need to add a call at the beginning of the probe function of nearly
> >>> every driver that can perform DMA to state that the driver doesn't need
> >>> to handle any IOMMU that might be present in the system itself. I don't
> >>> think that's a better solution.
> >>> 
> >>> Explicitly tearing down mappings in drivers that want to manage IOMMUs
> >>> isn't a solution I like either. A possibly better solution would be to
> >>> call a function to state that the DMA mapping API shouldn't not handle
> >>> IOMMUs. Something like
> >>> 
> >>> 	dma_mapping_ignore_iommu(dev);
> >>> 
> >>> at the beginning of the probe function of such drivers could do. The
> >>> function would perform behind the scene all operations needed to tear
> >>> down everything that shouldn't have been set up.
> >> 
> >> An alternative would be to add a flag to platform_driver, like we have
> >> for "prevent_deferred_probe" which is something like
> >> "prevent_dma_configure".
> > 
> > That's a solution I have proposed (albeit as a struct device_driver field,
> > but that's a small detail), so I'm fine with it :-)
> 
> I think Marek had proposed something similar initially as well. I don't
> see an immediate downside to that solution. It's still somewhat ugly in
> that a lot of stuff is set up before it's known to actually be used at
> all, but it seems like there's some consensus that this can be improved
> later on, so I have no objections to such a patch.
> 
> Of course that doesn't solve the current breakage for the Rockchip DRM
> and OMAP ISP drivers.

And, as I came to realize after a long bisect yesternight, the Renesas IPMMU 
driver :-/ Basically any platform that relied on arm_iommu_attach_device() to 
set the IOMMU DMA ops is now broken.

> There's an additional issue with this automatic type of setting up
> mapping: on Tegra quite a few devices can use the IOMMU. Among those are
> SD/MMC controllers, HDA, SATA and things like the AHB bus. Creating a
> separate domain (== address space) for each of those devices is wasteful
> and there's currently no way to make them use a single virtual address
> space. Each of them really has no use for a full 4 GiB of address space.
> My earliest attempt to implement that was as a policy decision within
> the IOMMU driver, but that wasn't very well received.

A similar discussion started at http://www.spinics.net/lists/arm-kernel/msg385805.html but didn't go very far.

In the end it's really a policy decision. The question is how to express that 
policy. As policies in DT are frowned upon we have several subsystems 
currently hacking around similar issues by implementing heuristics or defaults 
that nobody really complained about so far. We might be able to do the same 
for IOMMUs as a first step.

> I'm now wondering whether the same could be done using this new type of
> flag. Perhaps it can be assumed that every driver that doesn't want its
> IOMMU master hooked up to the DMA API can (and will) manually manage the
> virtual address space. Conversely every driver that wants to use the DMA
> API to abstract away the existence of an IOMMU could be considered part
> of the group where a separate domain isn't necessary.

-- 
Regards,

Laurent Pinchart

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Will Deacon]

On Tue, Jan 20, 2015 at 03:14:01PM +0000, Laurent Pinchart wrote:
> On Monday 19 January 2015 13:31:00 Thierry Reding wrote:
> > On Mon, Jan 19, 2015 at 01:34:24PM +0200, Laurent Pinchart wrote:
> > > On Monday 19 January 2015 11:12:02 Will Deacon wrote:
> > >> On Sun, Jan 18, 2015 at 11:18:51AM +0000, Laurent Pinchart wrote:
> > >>> On Sunday 18 January 2015 15:54:34 Alexandre Courbot wrote:
> > >>>> On 01/16/2015 08:18 AM, Laurent Pinchart wrote:
> > >>>>> On Thursday 15 January 2015 11:12:17 Will Deacon wrote:
> 
> [snip]
> 
> > >>>> I am arriving late in this discussion, but what is wrong with asking
> > >>>> drivers to explicitly state that they want the DMA API to be backed
> > >>>> by the IOMMU instead of forcibly making it work that way?
> > >>> 
> > >>> The vast majority of the drivers are not IOMMU-aware. We would thus
> > >>> need to add a call at the beginning of the probe function of nearly
> > >>> every driver that can perform DMA to state that the driver doesn't need
> > >>> to handle any IOMMU that might be present in the system itself. I don't
> > >>> think that's a better solution.
> > >>> 
> > >>> Explicitly tearing down mappings in drivers that want to manage IOMMUs
> > >>> isn't a solution I like either. A possibly better solution would be to
> > >>> call a function to state that the DMA mapping API shouldn't not handle
> > >>> IOMMUs. Something like
> > >>> 
> > >>> 	dma_mapping_ignore_iommu(dev);
> > >>> 
> > >>> at the beginning of the probe function of such drivers could do. The
> > >>> function would perform behind the scene all operations needed to tear
> > >>> down everything that shouldn't have been set up.
> > >> 
> > >> An alternative would be to add a flag to platform_driver, like we have
> > >> for "prevent_deferred_probe" which is something like
> > >> "prevent_dma_configure".
> > > 
> > > That's a solution I have proposed (albeit as a struct device_driver field,
> > > but that's a small detail), so I'm fine with it :-)
> > 
> > I think Marek had proposed something similar initially as well. I don't
> > see an immediate downside to that solution. It's still somewhat ugly in
> > that a lot of stuff is set up before it's known to actually be used at
> > all, but it seems like there's some consensus that this can be improved
> > later on, so I have no objections to such a patch.
> > 
> > Of course that doesn't solve the current breakage for the Rockchip DRM
> > and OMAP ISP drivers.
> 
> And, as I came to realize after a long bisect yesternight, the Renesas IPMMU 
> driver :-/ Basically any platform that relied on arm_iommu_attach_device() to 
> set the IOMMU DMA ops is now broken.

We could restore the set_dma_ops call in arm_iommu_attach_device as a
temporary hack (along with a big fat comment), since arch_setup_dma_ops
actually sets the ops correct *after* the call to
arm_get_iommu_dma_map_ops...

It doesn't provide any motivation for people to consider moving over to the
new framework, but it fixes the current issues affecting mainline.

Will

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Will Deacon]

On Tue, Jan 20, 2015 at 03:19:10PM +0000, Will Deacon wrote:
> We could restore the set_dma_ops call in arm_iommu_attach_device as a
> temporary hack (along with a big fat comment), since arch_setup_dma_ops
> actually sets the ops correct *after* the call to
> arm_get_iommu_dma_map_ops...

s/arm_get_iommu_dma_map_ops/arm_setup_iommu_dma_ops/

Will

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Laurent Pinchart]

Hi Will,

On Tuesday 20 January 2015 15:19:11 Will Deacon wrote:
> On Tue, Jan 20, 2015 at 03:14:01PM +0000, Laurent Pinchart wrote:
> > On Monday 19 January 2015 13:31:00 Thierry Reding wrote:
> >> On Mon, Jan 19, 2015 at 01:34:24PM +0200, Laurent Pinchart wrote:
> >>> On Monday 19 January 2015 11:12:02 Will Deacon wrote:
> >>>> On Sun, Jan 18, 2015 at 11:18:51AM +0000, Laurent Pinchart wrote:
> >>>>> On Sunday 18 January 2015 15:54:34 Alexandre Courbot wrote:
> >>>>>> On 01/16/2015 08:18 AM, Laurent Pinchart wrote:
> >>>>>>> On Thursday 15 January 2015 11:12:17 Will Deacon wrote:
> >
> > [snip]
> > 
> >>>>>> I am arriving late in this discussion, but what is wrong with
> >>>>>> asking drivers to explicitly state that they want the DMA API to be
> >>>>>> backed by the IOMMU instead of forcibly making it work that way?
> >>>>> 
> >>>>> The vast majority of the drivers are not IOMMU-aware. We would thus
> >>>>> need to add a call at the beginning of the probe function of nearly
> >>>>> every driver that can perform DMA to state that the driver doesn't
> >>>>> need to handle any IOMMU that might be present in the system itself.
> >>>>> I don't think that's a better solution.
> >>>>> 
> >>>>> Explicitly tearing down mappings in drivers that want to manage
> >>>>> IOMMUs isn't a solution I like either. A possibly better solution
> >>>>> would be to call a function to state that the DMA mapping API
> >>>>> shouldn't not handle IOMMUs. Something like
> >>>>> 
> >>>>> 	dma_mapping_ignore_iommu(dev);
> >>>>> 
> >>>>> at the beginning of the probe function of such drivers could do. The
> >>>>> function would perform behind the scene all operations needed to
> >>>>> tear down everything that shouldn't have been set up.
> >>>> 
> >>>> An alternative would be to add a flag to platform_driver, like we
> >>>> have for "prevent_deferred_probe" which is something like
> >>>> "prevent_dma_configure".
> >>> 
> >>> That's a solution I have proposed (albeit as a struct device_driver
> >>> field, but that's a small detail), so I'm fine with it :-)
> >> 
> >> I think Marek had proposed something similar initially as well. I don't
> >> see an immediate downside to that solution. It's still somewhat ugly in
> >> that a lot of stuff is set up before it's known to actually be used at
> >> all, but it seems like there's some consensus that this can be improved
> >> later on, so I have no objections to such a patch.
> >> 
> >> Of course that doesn't solve the current breakage for the Rockchip DRM
> >> and OMAP ISP drivers.
> > 
> > And, as I came to realize after a long bisect yesternight, the Renesas
> > IPMMU driver :-/ Basically any platform that relied on
> > arm_iommu_attach_device() to set the IOMMU DMA ops is now broken.
> 
> We could restore the set_dma_ops call in arm_iommu_attach_device as a
> temporary hack (along with a big fat comment), since arch_setup_dma_ops
> actually sets the ops correct *after* the call to
> arm_get_iommu_dma_map_ops...
> 
> It doesn't provide any motivation for people to consider moving over to the
> new framework, but it fixes the current issues affecting mainline.

I'm all for incentives, but I think avoiding a major v3.19 regression would be 
good, too :-) I wanted to test your LPAE page table allocator yesterday with 
the Renesas IPMMU driver, and ended up spending the whole night bisecting the 
regression instead.

-- 
Regards,

Laurent Pinchart

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Thierry Reding]

On Sun, Jan 18, 2015 at 01:18:51PM +0200, Laurent Pinchart wrote:
> Hi Alexandre,
> 
> On Sunday 18 January 2015 15:54:34 Alexandre Courbot wrote:
> > On 01/16/2015 08:18 AM, Laurent Pinchart wrote:
[...]
> > > The second way is to implement a mechanism to let drivers signal that they
> > > want to handle DMA mappings themselves. As the mappings need in the
> > > general case to be created before the probe function  is called
> > 
> > Sorry for being ignorant here, but why is that?
> 
> Because a driver can call the DMA mapping API in its probe function, to 
> allocate DMA coherent memory for instance. We need to ensure that the DMA 
> mapping IOMMU has set up the required IOMMU ops by that time. As explained 
> above I don't like the idea of sprinkling explicit calls to initialize IOMMU 
> support in the vast majority of drivers, especially when they shouldn't be 
> IOMMU-aware, so we then need to initialize everything that is needed before 
> the first call to the DMA mapping API.

The original patch that Hiroshi posted based on my comments was to have
the driver core call iommu_attach(), which would then set up everything
needed right before calling into the driver's ->probe(). This works
quite nicely because by definition the driver can't allocate any DMA
before ->probe(). And, like you said, this allows deferred probe to be
used.

To me it's so obviously the right solution that I remain flabbergasted
with how much resistance it's received (or how much it's being ignored).

Thierry
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 819 bytes
Desc: not available
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20150119/7415b52c/attachment.sig>

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Will Deacon]

On Mon, Jan 19, 2015 at 12:43:06PM +0000, Thierry Reding wrote:
> On Sun, Jan 18, 2015 at 01:18:51PM +0200, Laurent Pinchart wrote:
> > On Sunday 18 January 2015 15:54:34 Alexandre Courbot wrote:
> > > On 01/16/2015 08:18 AM, Laurent Pinchart wrote:
> [...]
> > > > The second way is to implement a mechanism to let drivers signal that they
> > > > want to handle DMA mappings themselves. As the mappings need in the
> > > > general case to be created before the probe function  is called
> > > 
> > > Sorry for being ignorant here, but why is that?
> > 
> > Because a driver can call the DMA mapping API in its probe function, to 
> > allocate DMA coherent memory for instance. We need to ensure that the DMA 
> > mapping IOMMU has set up the required IOMMU ops by that time. As explained 
> > above I don't like the idea of sprinkling explicit calls to initialize IOMMU 
> > support in the vast majority of drivers, especially when they shouldn't be 
> > IOMMU-aware, so we then need to initialize everything that is needed before 
> > the first call to the DMA mapping API.
> 
> The original patch that Hiroshi posted based on my comments was to have
> the driver core call iommu_attach(), which would then set up everything
> needed right before calling into the driver's ->probe(). This works
> quite nicely because by definition the driver can't allocate any DMA
> before ->probe(). And, like you said, this allows deferred probe to be
> used.
> 
> To me it's so obviously the right solution that I remain flabbergasted
> with how much resistance it's received (or how much it's being ignored).

Have you considered reposting the patches based on what we currently have
(which has the advantage of identifying a specific IOMMU instance)?

Will

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Thierry Reding]

On Mon, Jan 19, 2015 at 12:50:52PM +0000, Will Deacon wrote:
> On Mon, Jan 19, 2015 at 12:43:06PM +0000, Thierry Reding wrote:
> > On Sun, Jan 18, 2015 at 01:18:51PM +0200, Laurent Pinchart wrote:
> > > On Sunday 18 January 2015 15:54:34 Alexandre Courbot wrote:
> > > > On 01/16/2015 08:18 AM, Laurent Pinchart wrote:
> > [...]
> > > > > The second way is to implement a mechanism to let drivers signal that they
> > > > > want to handle DMA mappings themselves. As the mappings need in the
> > > > > general case to be created before the probe function  is called
> > > > 
> > > > Sorry for being ignorant here, but why is that?
> > > 
> > > Because a driver can call the DMA mapping API in its probe function, to 
> > > allocate DMA coherent memory for instance. We need to ensure that the DMA 
> > > mapping IOMMU has set up the required IOMMU ops by that time. As explained 
> > > above I don't like the idea of sprinkling explicit calls to initialize IOMMU 
> > > support in the vast majority of drivers, especially when they shouldn't be 
> > > IOMMU-aware, so we then need to initialize everything that is needed before 
> > > the first call to the DMA mapping API.
> > 
> > The original patch that Hiroshi posted based on my comments was to have
> > the driver core call iommu_attach(), which would then set up everything
> > needed right before calling into the driver's ->probe(). This works
> > quite nicely because by definition the driver can't allocate any DMA
> > before ->probe(). And, like you said, this allows deferred probe to be
> > used.
> > 
> > To me it's so obviously the right solution that I remain flabbergasted
> > with how much resistance it's received (or how much it's being ignored).
> 
> Have you considered reposting the patches based on what we currently have
> (which has the advantage of identifying a specific IOMMU instance)?

No, I hadn't. Initially my patches included a solution for identifying
individual IOMMU instances, too. There was a small registry with a list
of struct iommus. That was supposed to get used to store per-instance
data (by being embedded in a driver-specific structure). I'd need to
look in more detail how that could be done with the infrastructure that
your patchset creates. I'm somewhat burried below other tasks right now
so don't expect to have any time to look into this anytime before -rc6
or -rc7 at the earliest.

Thierry
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 819 bytes
Desc: not available
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20150119/aa153d5a/attachment.sig>

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Laurent Pinchart]

Hi Thierry,

On Monday 19 January 2015 14:36:38 Thierry Reding wrote:
> On Mon, Jan 19, 2015 at 12:50:52PM +0000, Will Deacon wrote:
> > On Mon, Jan 19, 2015 at 12:43:06PM +0000, Thierry Reding wrote:
> >> On Sun, Jan 18, 2015 at 01:18:51PM +0200, Laurent Pinchart wrote:
> >> > On Sunday 18 January 2015 15:54:34 Alexandre Courbot wrote:
> >> > > On 01/16/2015 08:18 AM, Laurent Pinchart wrote:
> >> [...]
> >> 
> >>>>> The second way is to implement a mechanism to let drivers signal
> >>>>> that they want to handle DMA mappings themselves. As the mappings
> >>>>> need in the general case to be created before the probe function 
> >>>>> is called
> >>>> 
> >>>> Sorry for being ignorant here, but why is that?
> >>> 
> >>> Because a driver can call the DMA mapping API in its probe function,
> >>> to allocate DMA coherent memory for instance. We need to ensure that
> >>> the DMA mapping IOMMU has set up the required IOMMU ops by that time.
> >>> As explained above I don't like the idea of sprinkling explicit calls
> >>> to initialize IOMMU support in the vast majority of drivers,
> >>> especially when they shouldn't be IOMMU-aware, so we then need to
> >>> initialize everything that is needed before the first call to the DMA
> >>> mapping API.
> >> 
> >> The original patch that Hiroshi posted based on my comments was to have
> >> the driver core call iommu_attach(), which would then set up everything
> >> needed right before calling into the driver's ->probe(). This works
> >> quite nicely because by definition the driver can't allocate any DMA
> >> before ->probe(). And, like you said, this allows deferred probe to be
> >> used.
> >> 
> >> To me it's so obviously the right solution that I remain flabbergasted
> >> with how much resistance it's received (or how much it's being ignored).
> > 
> > Have you considered reposting the patches based on what we currently have
> > (which has the advantage of identifying a specific IOMMU instance)?
> 
> No, I hadn't. Initially my patches included a solution for identifying
> individual IOMMU instances, too. There was a small registry with a list
> of struct iommus. That was supposed to get used to store per-instance
> data (by being embedded in a driver-specific structure). I'd need to
> look in more detail how that could be done with the infrastructure that
> your patchset creates. I'm somewhat burried below other tasks right now
> so don't expect to have any time to look into this anytime before -rc6
> or -rc7 at the earliest.

I'm pretty buried below other tasks as well, but if you find time to work on 
this I'll try to prioritize testing and review.

-- 
Regards,

Laurent Pinchart

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Arnd Bergmann]

On Sunday 18 January 2015 13:18:51 Laurent Pinchart wrote:
> On Sunday 18 January 2015 15:54:34 Alexandre Courbot wrote:
> > On 01/16/2015 08:18 AM, Laurent Pinchart wrote:
> > > On Thursday 15 January 2015 11:12:17 Will Deacon wrote:
> > >> On Thu, Jan 15, 2015 at 08:28:44AM +0000, Thierry Reding wrote:
> > >>> This has several advantages, such as that I can also use the regular
> > >>> driver model for suspend/resume of the IOMMU, and I get to enjoy the
> > >>> benefits of devres in the IOMMU driver. Probe ordering is still a tiny
> > >>> issue, but we can easily solve that using explicit initcall ordering
> > >>> (which really isn't any worse than IOMMU_OF_DECLARE()).
> > >> 
> > >> That's a pity. I'd much rather extend what we currently have to satisfy
> > >> your use-case. Ho-hum.
> > > 
> > > Assuming we want the IOMMU to be handled transparently for the majority of
> > > devices I only see two ways to fix this,
> > > 
> > > The first way is to create a default DMA mapping unconditionally and let
> > > drivers that can't live with it tear it down. That's what is implemented
> > > today.
> > 
> > I strongly support Thierry's point that drivers should not have to tear
> > down things they don't need. The issue we are facing today is a very
> > good illustration of why one should not have to do this.
> > 
> > Everybody hates to receive unsollicited email with a link that says "to
> > unsubscribe, click here". Let's not import that unpleasant culture into
> > the kernel.
> > 
> > I am arriving late in this discussion, but what is wrong with asking
> > drivers to explicitly state that they want the DMA API to be backed by
> > the IOMMU instead of forcibly making it work that way?
> 
> The vast majority of the drivers are not IOMMU-aware. We would thus need to 
> add a call at the beginning of the probe function of nearly every driver that 
> can perform DMA to state that the driver doesn't need to handle any IOMMU that 
> might be present in the system itself. I don't think that's a better solution.

Right, abstracting the presence of an IOMMU (along with things like
cache management) is the whole point for having the dma-mapping API.

The iommu driver should instead be able to make the decision on whether
the device uses the iommu for DMA or not. In some cases, it's not an
option because the iommu is mandatory for all DMA and there is no working
passthrough mode. In other cases, it depends on the dma mask: as long as
the device's dma_mask covers all of RAM, we can avoid using the IOMMU
and get better performance (and also avoid setting up tables that may
need to be removed again), but when the dma mask is too small, we have
to use the iommu or fall back to swiotlb (which is currently not implemeted
on arm32).

> > > we can't signal this by calling a function in probe(). A new flag field
> > > for struct device_driver is a possible solution. This would however
> > > require delaying the creation of DMA mappings until right before probe
> > > time. Attaching to the IOMMU could be pushed to right before probe() as
> > > well, which would have the added benefit of making IOMMU driver
> > > implementable as real platform drivers.
> > 
> > Keeping the ability to write IOMMU drivers as platform drivers would be
> > nice indeed.
> > 
> > The problem with the opt-out flag though is that one will need to check
> > every single driver out there to see whether it stills behave correctly
> > if its hardware is suddently put behind a IOMMU.
> 
> That's actually my default assumption :-) On ARM platforms at least, for many 
> devices, whether an IOMMU is present or not is an integration property, not a 
> property of the device. The same way a device shouldn't care about the exact 
> bus topology that connects it to the CPU and memory, it shouldn't care about 
> whether an IOMMU is present on that bus, except in special cases.

Agreed. This must work by default, or basically all arm64 machines are
broken. At the moment, arm64 does not support IOMMUs properly and uses
uses swiotlb instead, but it's a huge performance bottleneck. On arm32,
very few systems need an IOMMU at the moment, but it's getting more common.

> > Doing it the other way (a flag that enables IOMMU if available) sounds safer
> > to me.
> >
> > What we have right now is a mechanism that basically makes it impossible
> > to use the DMA API on many ARM platforms if ARM_DMA_USE_IOMMU is set
> > (and I suspect it would also make the IOMMU unusable as well, without
> > any way to fix things). This is quite concerning.
> > 
> > Even more concerning is that -rc5 is about to be released and we have
> > in-tree drivers (Rockchip DRM) that are not working as they should
> > because of this patch. Will, what is your plan to fix this? Do we have
> > stuff that absolutely depends on this patch? If not, can we just revert
> > it until all these issues are solved?

keystone, shmobile, mvebu and highbank all have PCI buses that are unable
to access all of RAM, with different kinds of hacks to work around that.

	Arnd

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Laurent Pinchart]

Hi Arnd,

On Monday 19 January 2015 17:13:14 Arnd Bergmann wrote:
> On Sunday 18 January 2015 13:18:51 Laurent Pinchart wrote:
> > On Sunday 18 January 2015 15:54:34 Alexandre Courbot wrote:
> >> On 01/16/2015 08:18 AM, Laurent Pinchart wrote:
> >>> On Thursday 15 January 2015 11:12:17 Will Deacon wrote:
> >>>> On Thu, Jan 15, 2015 at 08:28:44AM +0000, Thierry Reding wrote:
> >>>>> This has several advantages, such as that I can also use the regular
> >>>>> driver model for suspend/resume of the IOMMU, and I get to enjoy the
> >>>>> benefits of devres in the IOMMU driver. Probe ordering is still a
> >>>>> tiny issue, but we can easily solve that using explicit initcall
> >>>>> ordering (which really isn't any worse than IOMMU_OF_DECLARE()).
> >>>> 
> >>>> That's a pity. I'd much rather extend what we currently have to
> >>>> satisfy your use-case. Ho-hum.
> >>> 
> >>> Assuming we want the IOMMU to be handled transparently for the
> >>> majority of devices I only see two ways to fix this,
> >>> 
> >>> The first way is to create a default DMA mapping unconditionally and
> >>> let drivers that can't live with it tear it down. That's what is
> >>> implemented today.
> >> 
> >> I strongly support Thierry's point that drivers should not have to tear
> >> down things they don't need. The issue we are facing today is a very
> >> good illustration of why one should not have to do this.
> >> 
> >> Everybody hates to receive unsollicited email with a link that says "to
> >> unsubscribe, click here". Let's not import that unpleasant culture into
> >> the kernel.
> >> 
> >> I am arriving late in this discussion, but what is wrong with asking
> >> drivers to explicitly state that they want the DMA API to be backed by
> >> the IOMMU instead of forcibly making it work that way?
> > 
> > The vast majority of the drivers are not IOMMU-aware. We would thus need
> > to add a call at the beginning of the probe function of nearly every
> > driver that can perform DMA to state that the driver doesn't need to
> > handle any IOMMU that might be present in the system itself. I don't think
> > that's a better solution.
>
> Right, abstracting the presence of an IOMMU (along with things like
> cache management) is the whole point for having the dma-mapping API.
> 
> The iommu driver should instead be able to make the decision on whether
> the device uses the iommu for DMA or not. In some cases, it's not an
> option because the iommu is mandatory for all DMA and there is no working
> passthrough mode. In other cases, it depends on the dma mask: as long as
> the device's dma_mask covers all of RAM, we can avoid using the IOMMU
> and get better performance (and also avoid setting up tables that may
> need to be removed again), but when the dma mask is too small, we have
> to use the iommu or fall back to swiotlb (which is currently not implemeted
> on arm32).
> 
> >>> we can't signal this by calling a function in probe(). A new flag
> >>> field for struct device_driver is a possible solution. This would
> >>> however require delaying the creation of DMA mappings until right
> >>> before probe time. Attaching to the IOMMU could be pushed to right
> >>> before probe() as well, which would have the added benefit of making
> >>> IOMMU driver implementable as real platform drivers.
> >> 
> >> Keeping the ability to write IOMMU drivers as platform drivers would be
> >> nice indeed.
> >> 
> >> The problem with the opt-out flag though is that one will need to check
> >> every single driver out there to see whether it stills behave correctly
> >> if its hardware is suddently put behind a IOMMU.
> > 
> > That's actually my default assumption :-) On ARM platforms at least, for
> > many devices, whether an IOMMU is present or not is an integration
> > property, not a property of the device. The same way a device shouldn't
> > care about the exact bus topology that connects it to the CPU and memory,
> > it shouldn't care about whether an IOMMU is present on that bus, except
> > in special cases.
> 
> Agreed. This must work by default, or basically all arm64 machines are
> broken. At the moment, arm64 does not support IOMMUs properly and uses
> uses swiotlb instead, but it's a huge performance bottleneck. On arm32,
> very few systems need an IOMMU at the moment, but it's getting more common.
> 
> >> Doing it the other way (a flag that enables IOMMU if available) sounds
> >> safer to me.
> >> 
> >> What we have right now is a mechanism that basically makes it impossible
> >> to use the DMA API on many ARM platforms if ARM_DMA_USE_IOMMU is set
> >> (and I suspect it would also make the IOMMU unusable as well, without
> >> any way to fix things). This is quite concerning.
> >> 
> >> Even more concerning is that -rc5 is about to be released and we have
> >> in-tree drivers (Rockchip DRM) that are not working as they should
> >> because of this patch. Will, what is your plan to fix this? Do we have
> >> stuff that absolutely depends on this patch? If not, can we just revert
> >> it until all these issues are solved?
> 
> keystone, shmobile, mvebu and highbank all have PCI buses that are unable
> to access all of RAM, with different kinds of hacks to work around that.

But Will's series doesn't fix that, does it ? 

-- 
Regards,

Laurent Pinchart

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Thierry Reding]

On Fri, Jan 16, 2015 at 01:18:21AM +0200, Laurent Pinchart wrote:
> On Thursday 15 January 2015 11:12:17 Will Deacon wrote:
> > On Thu, Jan 15, 2015 at 08:28:44AM +0000, Thierry Reding wrote:
> > > On Wed, Jan 14, 2015 at 10:46:10AM +0000, Will Deacon wrote:
> > > > On Wed, Jan 14, 2015 at 09:00:24AM +0000, Alexandre Courbot wrote:
> > > [...]
> > > 
> > >>> 2) Say you want to use the IOMMU API in your driver, and have an iommu
> > >>> property in your device's DT node. If by chance your IOMMU is
> > >>> registered early, you will already have a mapping automatically
> > >>> created even before your probe function is called. Can this be
> > >>> avoided? Is it even safe?
> > >> 
> > >> Currently, I think you have to either teardown the ops manually or
> > >> return an error from of_xlate. Thierry was also looking at this sort of
> > >> thing, so it might be worth talking to him.
> > > 
> > > I already explained in earlier threads why I think this is a bad idea.
> > > It's completely unnatural for any driver to manually tear down something
> > > that it didn't want set up in the first place. It also means that you
> > > have to carefully audit any users of these IOMMU APIs to make sure that
> > > they do tear down. That doesn't sound like a good incremental approach,
> > > as evidenced by the breakage that Alex and Heiko have encountered.
> > 
> > Well, perhaps we hide that behind a get_iommu API or something. We *do*
> > need this manual teardown step to support things like VFIO, so it makes
> > sense to reuse it for other users too imo.
> > 
> > > The solution for me has been to completely side-step the issue and not
> > > register the IOMMU with the new mechanism at all. That is, there's no
> > > .of_xlate() implementation, which means that the ARM DMA API glue won't
> > > try to be smart and use the IOMMU in ways it's not meant to be used.
> 
> That will break when someone will want to use the same IOMMU type for devices 
> that use the DMA mapping API to hide the IOMMU. That might not be the case for 
> your IOMMU today, but it's pretty fragile, we need to fix it.

No, there's absolutely no issue here. It simply means that you can't do
this on Tegra. So far I'm not sure I even see an advantage in using the
IOMMU for devices that don't care about it anyway. Consider the example
of the SD/MMC or HDA. They typically allocate fairly small buffers, the
order of a single page typically. They can simply use memory handed out
by the CMA.

So as long as we don't add a .of_xlate() implementation or instantiate
via the IOMMU_OF_DECLARE() mechanism we simply don't support IOMMU-over-
DMA on Tegra.

Thierry
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 819 bytes
Desc: not available
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20150119/79b595a5/attachment.sig>

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Arnd Bergmann]

On Monday 19 January 2015 13:36:24 Thierry Reding wrote:
> On Fri, Jan 16, 2015 at 01:18:21AM +0200, Laurent Pinchart wrote:
> > On Thursday 15 January 2015 11:12:17 Will Deacon wrote:
> > > On Thu, Jan 15, 2015 at 08:28:44AM +0000, Thierry Reding wrote:
> > > > On Wed, Jan 14, 2015 at 10:46:10AM +0000, Will Deacon wrote:
> > > > > On Wed, Jan 14, 2015 at 09:00:24AM +0000, Alexandre Courbot wrote:
> > > > [...]
> > > > 
> > > >>> 2) Say you want to use the IOMMU API in your driver, and have an iommu
> > > >>> property in your device's DT node. If by chance your IOMMU is
> > > >>> registered early, you will already have a mapping automatically
> > > >>> created even before your probe function is called. Can this be
> > > >>> avoided? Is it even safe?
> > > >> 
> > > >> Currently, I think you have to either teardown the ops manually or
> > > >> return an error from of_xlate. Thierry was also looking at this sort of
> > > >> thing, so it might be worth talking to him.
> > > > 
> > > > I already explained in earlier threads why I think this is a bad idea.
> > > > It's completely unnatural for any driver to manually tear down something
> > > > that it didn't want set up in the first place. It also means that you
> > > > have to carefully audit any users of these IOMMU APIs to make sure that
> > > > they do tear down. That doesn't sound like a good incremental approach,
> > > > as evidenced by the breakage that Alex and Heiko have encountered.
> > > 
> > > Well, perhaps we hide that behind a get_iommu API or something. We *do*
> > > need this manual teardown step to support things like VFIO, so it makes
> > > sense to reuse it for other users too imo.
> > > 
> > > > The solution for me has been to completely side-step the issue and not
> > > > register the IOMMU with the new mechanism at all. That is, there's no
> > > > .of_xlate() implementation, which means that the ARM DMA API glue won't
> > > > try to be smart and use the IOMMU in ways it's not meant to be used.
> > 
> > That will break when someone will want to use the same IOMMU type for devices 
> > that use the DMA mapping API to hide the IOMMU. That might not be the case for 
> > your IOMMU today, but it's pretty fragile, we need to fix it.
> 
> No, there's absolutely no issue here. It simply means that you can't do
> this on Tegra. So far I'm not sure I even see an advantage in using the
> IOMMU for devices that don't care about it anyway. Consider the example
> of the SD/MMC or HDA. They typically allocate fairly small buffers, the
> order of a single page typically. They can simply use memory handed out
> by the CMA.
> 
> So as long as we don't add a .of_xlate() implementation or instantiate
> via the IOMMU_OF_DECLARE() mechanism we simply don't support IOMMU-over-
> DMA on Tegra.

It breaks as soon as you have a system with memory above the 4GB boundary,
which is the whole point of iommus for most users.

CMA does not work for streaming mappings, only for the coherent API.

	Arnd

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Thierry Reding]

On Mon, Jan 19, 2015 at 04:52:41PM +0100, Arnd Bergmann wrote:
> On Monday 19 January 2015 13:36:24 Thierry Reding wrote:
> > On Fri, Jan 16, 2015 at 01:18:21AM +0200, Laurent Pinchart wrote:
> > > On Thursday 15 January 2015 11:12:17 Will Deacon wrote:
> > > > On Thu, Jan 15, 2015 at 08:28:44AM +0000, Thierry Reding wrote:
> > > > > On Wed, Jan 14, 2015 at 10:46:10AM +0000, Will Deacon wrote:
> > > > > > On Wed, Jan 14, 2015 at 09:00:24AM +0000, Alexandre Courbot wrote:
> > > > > [...]
> > > > > 
> > > > >>> 2) Say you want to use the IOMMU API in your driver, and have an iommu
> > > > >>> property in your device's DT node. If by chance your IOMMU is
> > > > >>> registered early, you will already have a mapping automatically
> > > > >>> created even before your probe function is called. Can this be
> > > > >>> avoided? Is it even safe?
> > > > >> 
> > > > >> Currently, I think you have to either teardown the ops manually or
> > > > >> return an error from of_xlate. Thierry was also looking at this sort of
> > > > >> thing, so it might be worth talking to him.
> > > > > 
> > > > > I already explained in earlier threads why I think this is a bad idea.
> > > > > It's completely unnatural for any driver to manually tear down something
> > > > > that it didn't want set up in the first place. It also means that you
> > > > > have to carefully audit any users of these IOMMU APIs to make sure that
> > > > > they do tear down. That doesn't sound like a good incremental approach,
> > > > > as evidenced by the breakage that Alex and Heiko have encountered.
> > > > 
> > > > Well, perhaps we hide that behind a get_iommu API or something. We *do*
> > > > need this manual teardown step to support things like VFIO, so it makes
> > > > sense to reuse it for other users too imo.
> > > > 
> > > > > The solution for me has been to completely side-step the issue and not
> > > > > register the IOMMU with the new mechanism at all. That is, there's no
> > > > > .of_xlate() implementation, which means that the ARM DMA API glue won't
> > > > > try to be smart and use the IOMMU in ways it's not meant to be used.
> > > 
> > > That will break when someone will want to use the same IOMMU type for devices 
> > > that use the DMA mapping API to hide the IOMMU. That might not be the case for 
> > > your IOMMU today, but it's pretty fragile, we need to fix it.
> > 
> > No, there's absolutely no issue here. It simply means that you can't do
> > this on Tegra. So far I'm not sure I even see an advantage in using the
> > IOMMU for devices that don't care about it anyway. Consider the example
> > of the SD/MMC or HDA. They typically allocate fairly small buffers, the
> > order of a single page typically. They can simply use memory handed out
> > by the CMA.
> > 
> > So as long as we don't add a .of_xlate() implementation or instantiate
> > via the IOMMU_OF_DECLARE() mechanism we simply don't support IOMMU-over-
> > DMA on Tegra.
> 
> It breaks as soon as you have a system with memory above the 4GB boundary,
> which is the whole point of iommus for most users.

Why does it break? The IOMMU API simply gets a list of pages and gets
the physical addresses from those pages when it maps them to the IO
virtual addresses. How is .of_xlate() or of_iommu_configure() related?

> CMA does not work for streaming mappings, only for the coherent API.

Why not? And if it doesn't I'm not sure we currently care on Tegra since
we've gotten away with using CMA just fine so far.

Thierry
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 819 bytes
Desc: not available
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20150119/00f9d081/attachment.sig>

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Arnd Bergmann]

On Monday 19 January 2015 17:21:14 Thierry Reding wrote:
> On Mon, Jan 19, 2015 at 04:52:41PM +0100, Arnd Bergmann wrote:
> > On Monday 19 January 2015 13:36:24 Thierry Reding wrote:
> > > On Fri, Jan 16, 2015 at 01:18:21AM +0200, Laurent Pinchart wrote:
> > > > On Thursday 15 January 2015 11:12:17 Will Deacon wrote:
> > > > > On Thu, Jan 15, 2015 at 08:28:44AM +0000, Thierry Reding wrote:
> > > > > > On Wed, Jan 14, 2015 at 10:46:10AM +0000, Will Deacon wrote:
> > > > > > > On Wed, Jan 14, 2015 at 09:00:24AM +0000, Alexandre Courbot wrote:
> > > > > > [...]
> > > > > > 
> > > > > >>> 2) Say you want to use the IOMMU API in your driver, and have an iommu
> > > > > >>> property in your device's DT node. If by chance your IOMMU is
> > > > > >>> registered early, you will already have a mapping automatically
> > > > > >>> created even before your probe function is called. Can this be
> > > > > >>> avoided? Is it even safe?
> > > > > >> 
> > > > > >> Currently, I think you have to either teardown the ops manually or
> > > > > >> return an error from of_xlate. Thierry was also looking at this sort of
> > > > > >> thing, so it might be worth talking to him.
> > > > > > 
> > > > > > I already explained in earlier threads why I think this is a bad idea.
> > > > > > It's completely unnatural for any driver to manually tear down something
> > > > > > that it didn't want set up in the first place. It also means that you
> > > > > > have to carefully audit any users of these IOMMU APIs to make sure that
> > > > > > they do tear down. That doesn't sound like a good incremental approach,
> > > > > > as evidenced by the breakage that Alex and Heiko have encountered.
> > > > > 
> > > > > Well, perhaps we hide that behind a get_iommu API or something. We *do*
> > > > > need this manual teardown step to support things like VFIO, so it makes
> > > > > sense to reuse it for other users too imo.
> > > > > 
> > > > > > The solution for me has been to completely side-step the issue and not
> > > > > > register the IOMMU with the new mechanism at all. That is, there's no
> > > > > > .of_xlate() implementation, which means that the ARM DMA API glue won't
> > > > > > try to be smart and use the IOMMU in ways it's not meant to be used.
> > > > 
> > > > That will break when someone will want to use the same IOMMU type for devices 
> > > > that use the DMA mapping API to hide the IOMMU. That might not be the case for 
> > > > your IOMMU today, but it's pretty fragile, we need to fix it.
> > > 
> > > No, there's absolutely no issue here. It simply means that you can't do
> > > this on Tegra. So far I'm not sure I even see an advantage in using the
> > > IOMMU for devices that don't care about it anyway. Consider the example
> > > of the SD/MMC or HDA. They typically allocate fairly small buffers, the
> > > order of a single page typically. They can simply use memory handed out
> > > by the CMA.
> > > 
> > > So as long as we don't add a .of_xlate() implementation or instantiate
> > > via the IOMMU_OF_DECLARE() mechanism we simply don't support IOMMU-over-
> > > DMA on Tegra.
> > 
> > It breaks as soon as you have a system with memory above the 4GB boundary,
> > which is the whole point of iommus for most users.
> 
> Why does it break? The IOMMU API simply gets a list of pages and gets
> the physical addresses from those pages when it maps them to the IO
> virtual addresses. How is .of_xlate() or of_iommu_configure() related?

Because almost no drivers use the IOMMU API, and they expect the dma-mapping
API to do the right thing independent of the hardware configuration.

> > CMA does not work for streaming mappings, only for the coherent API.
> 
> Why not? And if it doesn't I'm not sure we currently care on Tegra since
> we've gotten away with using CMA just fine so far.

CMA is used as the backing for the dma_alloc_* API. It's impossible to
use it for streaming data that originates from arbitrary page cache
pages, like a send() operation on a user space network socket, or
writing to disk.

	Arnd

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Laurent Pinchart]

Hi Thierry,

On Monday 19 January 2015 17:21:14 Thierry Reding wrote:
> On Mon, Jan 19, 2015 at 04:52:41PM +0100, Arnd Bergmann wrote:
> > On Monday 19 January 2015 13:36:24 Thierry Reding wrote:
> >> On Fri, Jan 16, 2015 at 01:18:21AM +0200, Laurent Pinchart wrote:
> >>> On Thursday 15 January 2015 11:12:17 Will Deacon wrote:
> >>>> On Thu, Jan 15, 2015 at 08:28:44AM +0000, Thierry Reding wrote:

[...]

> >>>>> The solution for me has been to completely side-step the issue and
> >>>>> not register the IOMMU with the new mechanism at all. That is,
> >>>>> there's no .of_xlate() implementation, which means that the ARM
> >>>>> DMA API glue won't try to be smart and use the IOMMU in ways it's
> >>>>> not meant to be used.
> >>> 
> >>> That will break when someone will want to use the same IOMMU type for
> >>> devices that use the DMA mapping API to hide the IOMMU. That might
> >>> not be the case for your IOMMU today, but it's pretty fragile, we
> >>> need to fix it.
> >> 
> >> No, there's absolutely no issue here. It simply means that you can't do
> >> this on Tegra. So far I'm not sure I even see an advantage in using the
> >> IOMMU for devices that don't care about it anyway. Consider the example
> >> of the SD/MMC or HDA. They typically allocate fairly small buffers, the
> >> order of a single page typically. They can simply use memory handed out
> >> by the CMA.
> >> 
> >> So as long as we don't add a .of_xlate() implementation or instantiate
> >> via the IOMMU_OF_DECLARE() mechanism we simply don't support IOMMU-over-
> >> DMA on Tegra.
> > 
> > It breaks as soon as you have a system with memory above the 4GB boundary,
> > which is the whole point of iommus for most users.
> 
> Why does it break? The IOMMU API simply gets a list of pages and gets
> the physical addresses from those pages when it maps them to the IO
> virtual addresses. How is .of_xlate() or of_iommu_configure() related?

Arnd's point was that using the IOMMU for devices that deal with small buffers 
is mandatory in case those devices only support a 32-bit DMA address space, 
and the system has memory over 4GB that you want to use for DMA purpose. 
That's the case of Renesas' platforms with LPAE support for instance.

Another use case for IOMMUs is device isolation, which is, depending on your 
use cases, something you might want for devices that are not IOMMU-aware.

It should also be noted that even DRM drivers don't need to be explicit IOMMU 
users. The rcar-du driver allocates large frame buffers through the DMA 
mapping API, without caring about whether an IOMMU is present or not.

> > CMA does not work for streaming mappings, only for the coherent API.
> 
> Why not? And if it doesn't I'm not sure we currently care on Tegra since
> we've gotten away with using CMA just fine so far.

-- 
Regards,

Laurent Pinchart

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Thierry Reding]

On Fri, Jan 16, 2015 at 01:18:21AM +0200, Laurent Pinchart wrote:
[...]
> The second way is to implement a mechanism to let drivers signal that they 
> want to handle DMA mappings themselves. As the mappings need in the general 
> case to be created before the probe function is called we can't signal this by 
> calling a function in probe(). A new flag field for struct device_driver is a 
> possible solution. This would however require delaying the creation of DMA 
> mappings until right before probe time. Attaching to the IOMMU could be pushed 
> to right before probe() as well, which would have the added benefit of making 
> IOMMU driver implementable as real platform drivers.

Right. This is a pretty important point, too. One of the things that
we've been working on is suspend/resume. Now if you don't have a struct
device you can't easily implement suspend/resume. You'd have to play
tricks like using syscore_ops, which then leads to potentially problems
with suspend/resume ordering. It also means we have to keep around
global variables for driver-private data because there's no struct
device to attach it to.

By properly encoding the dependencies via deferred probe we get the
proper ordering and we can use the regular driver model with all the
goodies that we've come up with over the years.

Thierry
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 819 bytes
Desc: not available
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20150119/094c3214/attachment-0001.sig>

[PATCH v6 8/8] arm: dma-mapping: plumb our iommu mapping ops into arch_setup_dma_ops

[Laurent Pinchart]

Hi Thierry,

On Monday 19 January 2015 13:49:36 Thierry Reding wrote:
> On Fri, Jan 16, 2015 at 01:18:21AM +0200, Laurent Pinchart wrote:
> [...]
> 
> > The second way is to implement a mechanism to let drivers signal that they
> > want to handle DMA mappings themselves. As the mappings need in the
> > general case to be created before the probe function is called we can't
> > signal this by calling a function in probe(). A new flag field for struct
> > device_driver is a possible solution. This would however require delaying
> > the creation of DMA mappings until right before probe time. Attaching to
> > the IOMMU could be pushed to right before probe() as well, which would
> > have the added benefit of making IOMMU driver implementable as real
> > platform drivers.
> 
> Right. This is a pretty important point, too. One of the things that
> we've been working on is suspend/resume. Now if you don't have a struct
> device you can't easily implement suspend/resume. You'd have to play
> tricks like using syscore_ops, which then leads to potentially problems
> with suspend/resume ordering. It also means we have to keep around
> global variables for driver-private data because there's no struct
> device to attach it to.
> 
> By properly encoding the dependencies via deferred probe we get the
> proper ordering and we can use the regular driver model with all the
> goodies that we've come up with over the years.

Marek's patch set to port the Exynos IOMMU driver on Will's patches uses 
of_platform_device_create() in the init handler registered with 
IOMMU_OF_DECLARE() to create a platform device for the IOMMU. I've initially 
considered this as a dubious hack, but on the other hand it avoids modifying 
the whole IOMMU driver to get rid of struct device, making it easier to move 
to a deferred probe approach later if needed (and possible and desired).

-- 
Regards,

Laurent Pinchart

[PATCH v6 0/8] Introduce automatic DMA configuration for IOMMU masters

[Olof Johansson]

On Mon, Dec 1, 2014 at 8:57 AM, Will Deacon <will.deacon@arm.com> wrote:
> Hello again,
>
> This is version 6 of the patches previously posted here:
>
>   RFCv1: http://lists.infradead.org/pipermail/linux-arm-kernel/2014-August/283023.html
>   RFCv2: http://lists.infradead.org/pipermail/linux-arm-kernel/2014-September/283752.html
>   RFCv3: http://lists.infradead.org/pipermail/linux-arm-kernel/2014-September/287031.html
>   RFCv4: http://lists.infradead.org/pipermail/linux-arm-kernel/2014-November/302711.html
>      v5: http://lists.infradead.org/pipermail/linux-arm-kernel/2014-November/307213.html
>
> The only change since v5 is the addition of acks from various maintainers.
>
> Now that the ARM bits have rmk's ack and the IOMMU bits have joro's ack,
> I think this is good for merging via the arm-soc tree.
>
> Please let me know if a pull request would be preferable.


Hi,

Yes, please collect the acks from the discussion in the last day or so
and send a pull request (and include Robin's patch, obviously).


-Olof

[PATCH v6 0/8] Introduce automatic DMA configuration for IOMMU masters

[Will Deacon]

On Fri, Dec 05, 2014 at 07:12:24AM +0000, Olof Johansson wrote:
> On Mon, Dec 1, 2014 at 8:57 AM, Will Deacon <will.deacon@arm.com> wrote:
> > Hello again,
> >
> > This is version 6 of the patches previously posted here:
> >
> >   RFCv1: http://lists.infradead.org/pipermail/linux-arm-kernel/2014-August/283023.html
> >   RFCv2: http://lists.infradead.org/pipermail/linux-arm-kernel/2014-September/283752.html
> >   RFCv3: http://lists.infradead.org/pipermail/linux-arm-kernel/2014-September/287031.html
> >   RFCv4: http://lists.infradead.org/pipermail/linux-arm-kernel/2014-November/302711.html
> >      v5: http://lists.infradead.org/pipermail/linux-arm-kernel/2014-November/307213.html
> >
> > The only change since v5 is the addition of acks from various maintainers.
> >
> > Now that the ARM bits have rmk's ack and the IOMMU bits have joro's ack,
> > I think this is good for merging via the arm-soc tree.
> >
> > Please let me know if a pull request would be preferable.
> 
> 
> Hi,
> 
> Yes, please collect the acks from the discussion in the last day or so
> and send a pull request (and include Robin's patch, obviously).

Sure, I'm just working out which of Robin's patches to take (if I don't hear
otherwise, it will be the one with Grant's ack on it).

Will

[PATCH/RFC] iommu/ipmmu-vmsa: Use DT-based instantiation

[Laurent Pinchart]

Signed-off-by: Laurent Pinchart <laurent.pinchart+renesas@ideasonboard.com>
---
 drivers/iommu/ipmmu-vmsa.c | 243 +++++++++++++++++----------------------------
 1 file changed, 93 insertions(+), 150 deletions(-)

Hi Will,

This patch is a proof of concept of the Renesas IPMMU-VMSA driver integration
with the DMA mapping core. The code is based on your "[PATCH v6 0/8] Introduce
automatic DMA configuration for IOMMU masters" series.

I'm not totally happy with the result, especially with the platform device
instantiation in the init function, and the ill-defined relationship of the
add_device and of_xlate callbacks.

Comments are welcome.

diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
index 0e7a70e3a3d3..1bb3ba1e68b0 100644
--- a/drivers/iommu/ipmmu-vmsa.c
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -17,6 +17,8 @@
 #include <linux/iommu.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/of_iommu.h>
+#include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/sizes.h>
 #include <linux/slab.h>
@@ -27,7 +29,6 @@
 struct ipmmu_vmsa_device {
 	struct device *dev;
 	void __iomem *base;
-	struct list_head list;
 
 	unsigned int num_utlbs;
 
@@ -49,9 +50,6 @@ struct ipmmu_vmsa_archdata {
 	unsigned int num_utlbs;
 };
 
-static DEFINE_SPINLOCK(ipmmu_devices_lock);
-static LIST_HEAD(ipmmu_devices);
-
 #define TLB_LOOP_TIMEOUT		100	/* 100us */
 
 /* -----------------------------------------------------------------------------
@@ -1004,117 +1002,16 @@ static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain,
 	return __pfn_to_phys(pte_pfn(pte)) | (iova & ~PAGE_MASK);
 }
 
-static int ipmmu_find_utlbs(struct ipmmu_vmsa_device *mmu, struct device *dev,
-			    unsigned int **_utlbs)
-{
-	unsigned int *utlbs;
-	unsigned int i;
-	int count;
-
-	count = of_count_phandle_with_args(dev->of_node, "iommus",
-					   "#iommu-cells");
-	if (count < 0)
-		return -EINVAL;
-
-	utlbs = kcalloc(count, sizeof(*utlbs), GFP_KERNEL);
-	if (!utlbs)
-		return -ENOMEM;
-
-	for (i = 0; i < count; ++i) {
-		struct of_phandle_args args;
-		int ret;
-
-		ret = of_parse_phandle_with_args(dev->of_node, "iommus",
-						 "#iommu-cells", i, &args);
-		if (ret < 0)
-			goto error;
-
-		of_node_put(args.np);
-
-		if (args.np != mmu->dev->of_node || args.args_count != 1)
-			goto error;
-
-		utlbs[i] = args.args[0];
-	}
-
-	*_utlbs = utlbs;
-
-	return count;
-
-error:
-	kfree(utlbs);
-	return -EINVAL;
-}
-
 static int ipmmu_add_device(struct device *dev)
 {
-	struct ipmmu_vmsa_archdata *archdata;
+	struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu;
 	struct ipmmu_vmsa_device *mmu;
-	struct iommu_group *group = NULL;
-	unsigned int *utlbs = NULL;
-	unsigned int i;
-	int num_utlbs = 0;
 	int ret;
 
-	if (dev->archdata.iommu) {
-		dev_warn(dev, "IOMMU driver already assigned to device %s\n",
-			 dev_name(dev));
-		return -EINVAL;
-	}
-
-	/* Find the master corresponding to the device. */
-	spin_lock(&ipmmu_devices_lock);
-
-	list_for_each_entry(mmu, &ipmmu_devices, list) {
-		num_utlbs = ipmmu_find_utlbs(mmu, dev, &utlbs);
-		if (num_utlbs) {
-			/*
-			 * TODO Take a reference to the MMU to protect
-			 * against device removal.
-			 */
-			break;
-		}
-	}
-
-	spin_unlock(&ipmmu_devices_lock);
-
-	if (num_utlbs <= 0)
+	if (!archdata)
 		return -ENODEV;
 
-	for (i = 0; i < num_utlbs; ++i) {
-		if (utlbs[i] >= mmu->num_utlbs) {
-			ret = -EINVAL;
-			goto error;
-		}
-	}
-
-	/* Create a device group and add the device to it. */
-	group = iommu_group_alloc();
-	if (IS_ERR(group)) {
-		dev_err(dev, "Failed to allocate IOMMU group\n");
-		ret = PTR_ERR(group);
-		goto error;
-	}
-
-	ret = iommu_group_add_device(group, dev);
-	iommu_group_put(group);
-
-	if (ret < 0) {
-		dev_err(dev, "Failed to add device to IPMMU group\n");
-		group = NULL;
-		goto error;
-	}
-
-	archdata = kzalloc(sizeof(*archdata), GFP_KERNEL);
-	if (!archdata) {
-		ret = -ENOMEM;
-		goto error;
-	}
-
-	archdata->mmu = mmu;
-	archdata->utlbs = utlbs;
-	archdata->num_utlbs = num_utlbs;
-	dev->archdata.iommu = archdata;
+	mmu = archdata->mmu;
 
 	/*
 	 * Create the ARM mapping, used by the ARM DMA mapping core to allocate
@@ -1132,34 +1029,24 @@ static int ipmmu_add_device(struct device *dev)
 						   SZ_1G, SZ_2G);
 		if (IS_ERR(mapping)) {
 			dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n");
-			ret = PTR_ERR(mapping);
-			goto error;
+			return PTR_ERR(mapping);
 		}
 
 		mmu->mapping = mapping;
 	}
 
-	/* Attach the ARM VA mapping to the device. */
+	/*
+	 * Detach the device from the default ARM VA mapping and attach it to
+	 * our private mapping.
+	 */
+	arm_iommu_detach_device(dev);
 	ret = arm_iommu_attach_device(dev, mmu->mapping);
 	if (ret < 0) {
 		dev_err(dev, "Failed to attach device to VA mapping\n");
-		goto error;
+		return ret;
 	}
 
 	return 0;
-
-error:
-	arm_iommu_release_mapping(mmu->mapping);
-
-	kfree(dev->archdata.iommu);
-	kfree(utlbs);
-
-	dev->archdata.iommu = NULL;
-
-	if (!IS_ERR_OR_NULL(group))
-		iommu_group_remove_device(dev);
-
-	return ret;
 }
 
 static void ipmmu_remove_device(struct device *dev)
@@ -1175,6 +1062,68 @@ static void ipmmu_remove_device(struct device *dev)
 	dev->archdata.iommu = NULL;
 }
 
+static int ipmmu_of_xlate(struct device *dev, struct of_phandle_args *args)
+{
+	struct ipmmu_vmsa_archdata *archdata;
+	struct ipmmu_vmsa_device *mmu;
+	struct platform_device *pdev;
+	unsigned int num_utlbs;
+	unsigned int *utlbs;
+	unsigned int utlb;
+
+	/* Find the master corresponding to the device. */
+	pdev = of_find_device_by_node(args->np);
+	if (!pdev) {
+		dev_dbg(dev, "%s: ipmmu pdev not found\n", __func__);
+		return -ENODEV;
+	}
+
+	mmu = platform_get_drvdata(pdev);
+	if (!mmu) {
+		dev_dbg(dev, "%s: ipmmu not found\n", __func__);
+		return -ENODEV;
+	}
+
+	/* Allocate arch data if not already done. */
+	if (!dev->archdata.iommu) {
+		dev->archdata.iommu = kzalloc(sizeof(*archdata), GFP_KERNEL);
+		if (!dev->archdata.iommu)
+			return -ENOMEM;
+	}
+
+	archdata = dev->archdata.iommu;
+	archdata->mmu = mmu;
+
+	/*
+	 * We don't support handling a device through different IOMMU
+	 * instances.
+	 */
+	if (archdata->mmu && archdata->mmu != mmu) {
+		dev_warn(dev, "IOMMU driver already assigned to device %s\n",
+			 dev_name(dev));
+		return -EINVAL;
+	}
+
+	/* Validate and store the microTLB number. */
+	utlb = args->args[0];
+	if (utlb >= mmu->num_utlbs) {
+		dev_dbg(dev, "%s: invalid utlb %u\n", __func__, utlb);
+		return -EINVAL;
+	}
+
+	num_utlbs = archdata->num_utlbs + 1;
+	utlbs = krealloc(archdata->utlbs, num_utlbs * sizeof(*utlbs),
+			 GFP_KERNEL);
+	if (utlbs == NULL)
+		return -ENOMEM;
+	utlbs[archdata->num_utlbs] = utlb;
+
+	archdata->utlbs = utlbs;
+	archdata->num_utlbs = num_utlbs;
+
+	return 0;
+}
+
 static const struct iommu_ops ipmmu_ops = {
 	.domain_init = ipmmu_domain_init,
 	.domain_destroy = ipmmu_domain_destroy,
@@ -1185,6 +1134,7 @@ static const struct iommu_ops ipmmu_ops = {
 	.iova_to_phys = ipmmu_iova_to_phys,
 	.add_device = ipmmu_add_device,
 	.remove_device = ipmmu_remove_device,
+	.of_xlate = ipmmu_of_xlate,
 	.pgsize_bitmap = SZ_2M | SZ_64K | SZ_4K,
 };
 
@@ -1249,10 +1199,6 @@ static int ipmmu_probe(struct platform_device *pdev)
 	 * ipmmu_init() after the probe function returns.
 	 */
 
-	spin_lock(&ipmmu_devices_lock);
-	list_add(&mmu->list, &ipmmu_devices);
-	spin_unlock(&ipmmu_devices_lock);
-
 	platform_set_drvdata(pdev, mmu);
 
 	return 0;
@@ -1262,10 +1208,6 @@ static int ipmmu_remove(struct platform_device *pdev)
 {
 	struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev);
 
-	spin_lock(&ipmmu_devices_lock);
-	list_del(&mmu->list);
-	spin_unlock(&ipmmu_devices_lock);
-
 	arm_iommu_release_mapping(mmu->mapping);
 
 	ipmmu_device_reset(mmu);
@@ -1287,28 +1229,29 @@ static struct platform_driver ipmmu_driver = {
 	.remove	= ipmmu_remove,
 };
 
-static int __init ipmmu_init(void)
+static int __init ipmmu_of_setup(struct device_node *np)
 {
+	static bool init_done;
+	struct platform_device *pdev;
 	int ret;
 
-	ret = platform_driver_register(&ipmmu_driver);
-	if (ret < 0)
-		return ret;
+	if (!init_done) {
+		ret = platform_driver_register(&ipmmu_driver);
+		if (ret < 0)
+			return ret;
 
-	if (!iommu_present(&platform_bus_type))
-		bus_set_iommu(&platform_bus_type, &ipmmu_ops);
+		if (!iommu_present(&platform_bus_type))
+			bus_set_iommu(&platform_bus_type, &ipmmu_ops);
 
-	return 0;
-}
+		init_done = true;
+	}
 
-static void __exit ipmmu_exit(void)
-{
-	return platform_driver_unregister(&ipmmu_driver);
-}
+	pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root);
+	if (IS_ERR(pdev))
+		return PTR_ERR(pdev);
 
-subsys_initcall(ipmmu_init);
-module_exit(ipmmu_exit);
+	of_iommu_set_ops(np, &ipmmu_ops);
+	return 0;
+}
 
-MODULE_DESCRIPTION("IOMMU API for Renesas VMSA-compatible IPMMU");
-MODULE_AUTHOR("Laurent Pinchart <laurent.pinchart@ideasonboard.com>");
-MODULE_LICENSE("GPL v2");
+IOMMU_OF_DECLARE(ipmmu_vmsa_of, "renesas,ipmmu-vmsa", ipmmu_of_setup);
-- 
Regards,

Laurent Pinchart