[PATCH 0/3] ARM: iommu: tegra: Add initial Tegra IOMMU driver

[PATCH 0/3] ARM: iommu: tegra: Add initial Tegra IOMMU driver

[hdoyu-DDmLM1+adcrQT0dZR+AlfA]

From: Hiroshi DOYU <hdoyu-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>

Hi,

This patchset adds support for Tegra IOMMU driver for Tegra2/3.

Presently this driver is specific to Tegra platform, but some of
functionalities could be replaced with a generic IOMMU framework. This
is expected to ease finding similarities with different platforms,
with the intention of solving problems once in a generic framework
which everyone can use.

Originally this was developed under git://nv-tegra.nvidia.com/linux-2.6.git,
and they are cleaned up for upstreaming.

Hiroshi DOYU (3):
  ARM: iommu: tegra/common: Initial support for IOVMM driver
  ARM: iommu: tegra2: Initial support for GART driver
  ARM: iommu: tegra3: Initial support for SMMU driver

 arch/arm/mach-tegra/include/mach/iovmm.h |  283 +++++++
 drivers/iommu/Kconfig                    |   34 +
 drivers/iommu/Makefile                   |    3 +
 drivers/iommu/tegra-gart.c               |  357 ++++++++
 drivers/iommu/tegra-iovmm.c              |  936 ++++++++++++++++++++
 drivers/iommu/tegra-smmu.c               | 1358 ++++++++++++++++++++++++++++++
 6 files changed, 2971 insertions(+), 0 deletions(-)
 create mode 100644 arch/arm/mach-tegra/include/mach/iovmm.h
 create mode 100644 drivers/iommu/tegra-gart.c
 create mode 100644 drivers/iommu/tegra-iovmm.c
 create mode 100644 drivers/iommu/tegra-smmu.c

[PATCH 1/3] ARM: iommu: tegra/common: Initial support for IOVMM driver

[hdoyu-DDmLM1+adcrQT0dZR+AlfA]

From: Hiroshi DOYU <hdoyu-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>

This is the tegra specific IOMMU framework, independent of H/W. H/W
dependent modules are to be registered to this framework so that this
can support different IOMMU H/Ws among Tegra generations, Tegra2/GART,
and Tegra3/SMMU H/Ws.

Most of this part could be replaced with a generic IOMMU
framework. This is expected to ease finding similarities with
different platforms, with the intention of solving problems once in a
generic framework which everyone can use.

Signed-off-by: Hiroshi DOYU <hdoyu-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
Cc: Hiro Sugawara <hsugawara-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
Cc: Krishna Reddy <vdumpa-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
---
 arch/arm/mach-tegra/include/mach/iovmm.h |  283 +++++++++
 drivers/iommu/Kconfig                    |    6 +
 drivers/iommu/Makefile                   |    1 +
 drivers/iommu/tegra-iovmm.c              |  936 ++++++++++++++++++++++++++++++
 4 files changed, 1226 insertions(+), 0 deletions(-)
 create mode 100644 arch/arm/mach-tegra/include/mach/iovmm.h
 create mode 100644 drivers/iommu/tegra-iovmm.c

diff --git a/arch/arm/mach-tegra/include/mach/iovmm.h b/arch/arm/mach-tegra/include/mach/iovmm.h
new file mode 100644
index 0000000..6fd0bb6
--- /dev/null
+++ b/arch/arm/mach-tegra/include/mach/iovmm.h
@@ -0,0 +1,283 @@
+/*
+ * Copyright (c) 2010-2011, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed i the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include <linux/list.h>
+#include <linux/platform_device.h>
+#include <linux/miscdevice.h>
+#include <linux/rbtree.h>
+#include <linux/rwsem.h>
+#include <linux/spinlock.h>
+#include <linux/types.h>
+
+#ifndef _MACH_TEGRA_IOVMM_H_
+#define _MACH_TEGRA_IOVMM_H_
+
+typedef u32 tegra_iovmm_addr_t;
+
+struct tegra_iovmm_device_ops;
+
+/* each I/O virtual memory manager unit should register a device with
+ * the iovmm system
+ */
+struct tegra_iovmm_device {
+	struct tegra_iovmm_device_ops	*ops;
+	const char			*name;
+	struct list_head		list;
+	int				pgsize_bits;
+};
+
+/* tegra_iovmm_domain serves a purpose analagous to mm_struct as defined in
+ * <linux/mm_types.h> - it defines a virtual address space within which
+ * tegra_iovmm_areas can be created.
+ */
+struct tegra_iovmm_domain {
+	atomic_t		clients;
+	atomic_t		locks;
+	spinlock_t		block_lock;
+	unsigned long		flags;
+	wait_queue_head_t	delay_lock;  /* when lock_client fails */
+	struct rw_semaphore	map_lock;
+	struct rb_root		all_blocks;  /* ordered by address */
+	struct rb_root		free_blocks; /* ordered by size */
+	struct tegra_iovmm_device *dev;
+};
+
+/* tegra_iovmm_client is analagous to an individual task in the task group
+ * which owns an mm_struct.
+ */
+
+struct iovmm_share_group;
+
+struct tegra_iovmm_client {
+	const char			*name;
+	unsigned long			flags;
+	struct iovmm_share_group	*group;
+	struct tegra_iovmm_domain	*domain;
+	struct miscdevice		*misc_dev;
+	struct list_head		list;
+};
+
+/* tegra_iovmm_area serves a purpose analagous to vm_area_struct as defined
+ * in <linux/mm_types.h> - it defines a virtual memory area which can be
+ * mapped to physical memory by a client-provided mapping function. */
+
+struct tegra_iovmm_area {
+	struct tegra_iovmm_domain	*domain;
+	tegra_iovmm_addr_t		iovm_start;
+	size_t				iovm_length;
+	pgprot_t			pgprot;
+	struct tegra_iovmm_area_ops	*ops;
+};
+
+struct tegra_iovmm_device_ops {
+	/* maps a VMA using the page residency functions provided by the VMA */
+	int (*map)(struct tegra_iovmm_domain *domain,
+		struct tegra_iovmm_area *io_vma);
+	/* marks all PTEs in a VMA as invalid; decommits the virtual addres
+	 * space (potentially freeing PDEs when decommit is true.) */
+	void (*unmap)(struct tegra_iovmm_domain *domain,
+		struct tegra_iovmm_area *io_vma, bool decommit);
+	void (*map_pfn)(struct tegra_iovmm_domain *domain,
+		struct tegra_iovmm_area *io_vma,
+		tegra_iovmm_addr_t offs, unsigned long pfn);
+	/* ensures that a domain is resident in the hardware's mapping region
+	 * so that it may be used by a client */
+	int (*lock_domain)(struct tegra_iovmm_domain *domain,
+		struct tegra_iovmm_client *client);
+	void (*unlock_domain)(struct tegra_iovmm_domain *domain,
+		struct tegra_iovmm_client *client);
+	/* allocates a vmm_domain for the specified client; may return the same
+	 * domain for multiple clients */
+	struct tegra_iovmm_domain* (*alloc_domain)(
+		struct tegra_iovmm_device *dev,
+		struct tegra_iovmm_client *client);
+	void (*free_domain)(struct tegra_iovmm_domain *domain,
+		struct tegra_iovmm_client *client);
+	int (*suspend)(struct tegra_iovmm_device *dev);
+	void (*resume)(struct tegra_iovmm_device *dev);
+};
+
+struct tegra_iovmm_area_ops {
+	/* ensures that the page of data starting at the specified offset
+	 * from the start of the iovma is resident and pinned for use by
+	 * DMA, returns the system pfn, or an invalid pfn if the
+	 * operation fails. */
+	unsigned long (*lock_makeresident)(struct tegra_iovmm_area *area,
+		tegra_iovmm_addr_t offs);
+	/* called when the page is unmapped from the I/O VMA */
+	void (*release)(struct tegra_iovmm_area *area, tegra_iovmm_addr_t offs);
+};
+
+#ifdef CONFIG_TEGRA_IOVMM
+/* called by clients to allocate an I/O VMM client mapping context which
+ * will be shared by all clients in the same share_group */
+struct tegra_iovmm_client *tegra_iovmm_alloc_client(const char *name,
+	const char *share_group, struct miscdevice *misc_dev);
+
+size_t tegra_iovmm_get_vm_size(struct tegra_iovmm_client *client);
+
+void tegra_iovmm_free_client(struct tegra_iovmm_client *client);
+
+/* called by clients to ensure that their mapping context is resident
+ * before performing any DMA operations addressing I/O VMM regions.
+ * client_lock may return -EINTR. */
+int tegra_iovmm_client_lock(struct tegra_iovmm_client *client);
+int tegra_iovmm_client_trylock(struct tegra_iovmm_client *client);
+
+/* called by clients after DMA operations are complete */
+void tegra_iovmm_client_unlock(struct tegra_iovmm_client *client);
+
+/* called by clients to allocate a new iovmm_area and reserve I/O virtual
+ * address space for it. if ops is NULL, clients should subsequently call
+ * tegra_iovmm_vm_map_pages and/or tegra_iovmm_vm_insert_pfn to explicitly
+ * map the I/O virtual address to an OS-allocated page or physical address,
+ * respectively. VM operations may be called before this call returns */
+struct tegra_iovmm_area *tegra_iovmm_create_vm(
+	struct tegra_iovmm_client *client, struct tegra_iovmm_area_ops *ops,
+	size_t size, size_t align, pgprot_t pgprot, unsigned long iovm_start);
+
+/* called by clients to "zap" an iovmm_area, and replace all mappings
+ * in it with invalid ones, without freeing the virtual address range */
+void tegra_iovmm_zap_vm(struct tegra_iovmm_area *vm);
+
+/* after zapping a demand-loaded iovmm_area, the client should unzap it
+ * to allow the VMM device to remap the page range. */
+void tegra_iovmm_unzap_vm(struct tegra_iovmm_area *vm);
+
+/* called by clients to return an iovmm_area to the free pool for the domain */
+void tegra_iovmm_free_vm(struct tegra_iovmm_area *vm);
+
+/* returns size of largest free iovm block */
+size_t tegra_iovmm_get_max_free(struct tegra_iovmm_client *client);
+
+/* called by client software to map the page-aligned I/O address vaddr to
+ * a specific physical address pfn. I/O VMA should have been created with
+ * a NULL tegra_iovmm_area_ops structure. */
+void tegra_iovmm_vm_insert_pfn(struct tegra_iovmm_area *area,
+	tegra_iovmm_addr_t vaddr, unsigned long pfn);
+
+/* called by clients to return the iovmm_area containing addr, or NULL if
+ * addr has not been allocated. caller should call tegra_iovmm_put_area when
+ * finished using the returned pointer */
+struct tegra_iovmm_area *tegra_iovmm_find_area_get(
+	struct tegra_iovmm_client *client, tegra_iovmm_addr_t addr);
+
+struct tegra_iovmm_area *tegra_iovmm_area_get(struct tegra_iovmm_area *vm);
+void tegra_iovmm_area_put(struct tegra_iovmm_area *vm);
+
+/* called by drivers to initialize a tegra_iovmm_domain structure */
+int tegra_iovmm_domain_init(struct tegra_iovmm_domain *domain,
+	struct tegra_iovmm_device *dev, tegra_iovmm_addr_t start,
+	tegra_iovmm_addr_t end);
+
+/* called by drivers to register an I/O VMM device with the system */
+int tegra_iovmm_register(struct tegra_iovmm_device *dev);
+
+/* called by drivers to remove an I/O VMM device from the system */
+int tegra_iovmm_unregister(struct tegra_iovmm_device *dev);
+
+#else /* CONFIG_TEGRA_IOVMM */
+
+static inline struct tegra_iovmm_client *tegra_iovmm_alloc_client(
+	const char *name, const char *share_group, struct miscdevice *misc_dev)
+{
+	return NULL;
+}
+
+static inline size_t tegra_iovmm_get_vm_size(struct tegra_iovmm_client *client)
+{
+	return 0;
+}
+
+static inline void tegra_iovmm_free_client(struct tegra_iovmm_client *client)
+{}
+
+static inline int tegra_iovmm_client_lock(struct tegra_iovmm_client *client)
+{
+	return 0;
+}
+
+static inline int tegra_iovmm_client_trylock(struct tegra_iovmm_client *client)
+{
+	return 0;
+}
+
+static inline void tegra_iovmm_client_unlock(struct tegra_iovmm_client *client)
+{}
+
+static inline struct tegra_iovmm_area *tegra_iovmm_create_vm(
+	struct tegra_iovmm_client *client, struct tegra_iovmm_area_ops *ops,
+	size_t size, size_t align, pgprot_t pgprot, unsigned long iovm_start)
+{
+	return NULL;
+}
+
+static inline void tegra_iovmm_zap_vm(struct tegra_iovmm_area *vm) { }
+
+static inline void tegra_iovmm_unzap_vm(struct tegra_iovmm_area *vm) { }
+
+static inline void tegra_iovmm_free_vm(struct tegra_iovmm_area *vm) { }
+static inline size_t tegra_iovmm_get_max_free(struct tegra_iovmm_client *client)
+{
+	return 0;
+}
+
+static inline void tegra_iovmm_vm_insert_pfn(struct tegra_iovmm_area *area,
+	tegra_iovmm_addr_t vaddr, unsigned long pfn) { }
+
+static inline struct tegra_iovmm_area *tegra_iovmm_find_area_get(
+	struct tegra_iovmm_client *client, tegra_iovmm_addr_t addr)
+{
+	return NULL;
+}
+
+static inline struct tegra_iovmm_area *tegra_iovmm_area_get(
+	struct tegra_iovmm_area *vm)
+{
+	return NULL;
+}
+
+static inline void tegra_iovmm_area_put(struct tegra_iovmm_area *vm) { }
+
+static inline int tegra_iovmm_domain_init(struct tegra_iovmm_domain *domain,
+	struct tegra_iovmm_device *dev, tegra_iovmm_addr_t start,
+	tegra_iovmm_addr_t end)
+{
+	return 0;
+}
+
+static inline int tegra_iovmm_register(struct tegra_iovmm_device *dev)
+{
+	return 0;
+}
+
+static inline int tegra_iovmm_unregister(struct tegra_iovmm_device *dev)
+{
+	return 0;
+}
+
+static inline int tegra_iovmm_suspend(void)
+{
+	return 0;
+}
+
+static inline void tegra_iovmm_resume(void) { }
+#endif /* CONFIG_TEGRA_IOVMM */
+
+
+#endif
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 5414253..487d1ee 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -131,4 +131,10 @@ config OMAP_IOMMU_DEBUG
 
          Say N unless you know you need this.
 
+
+# Tegra IOMMU support
+
+config TEGRA_IOVMM
+	bool
+
 endif # IOMMU_SUPPORT
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 2f44487..365eefb 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -7,3 +7,4 @@ obj-$(CONFIG_IRQ_REMAP) += intr_remapping.o
 obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o
 obj-$(CONFIG_OMAP_IOVMM) += omap-iovmm.o
 obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o
+obj-$(CONFIG_TEGRA_IOVMM) += tegra-iovmm.o
diff --git a/drivers/iommu/tegra-iovmm.c b/drivers/iommu/tegra-iovmm.c
new file mode 100644
index 0000000..0f2d996
--- /dev/null
+++ b/drivers/iommu/tegra-iovmm.c
@@ -0,0 +1,936 @@
+/*
+ * Tegra I/O VM manager
+ *
+ * Copyright (c) 2010-2011, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include <linux/kernel.h>
+#include <linux/spinlock.h>
+#include <linux/proc_fs.h>
+#include <linux/sched.h>
+#include <linux/string.h>
+#include <linux/slab.h>
+#include <linux/syscore_ops.h>
+
+#include <mach/iovmm.h>
+
+/*
+ * after the best-fit block is located, the remaining pages not needed
+ * for the allocation will be split into a new free block if the
+ * number of remaining pages is >= MIN_SPLIT_PAGE.
+ */
+#define MIN_SPLIT_PAGE		4
+#define MIN_SPLIT_BYTES(_d)	(MIN_SPLIT_PAGE << (_d)->dev->pgsize_bits)
+#define NO_SPLIT(m)		((m) < MIN_SPLIT_BYTES(domain))
+#define DO_SPLIT(m)		((m) >= MIN_SPLIT_BYTES(domain))
+
+#define iovmm_start(_b)		((_b)->vm_area.iovm_start)
+#define iovmm_length(_b)	((_b)->vm_area.iovm_length)
+#define iovmm_end(_b)		(iovmm_start(_b) + iovmm_length(_b))
+
+/* flags for the block */
+#define BK_free		0 /* indicates free mappings */
+#define BK_map_dirty	1 /* used by demand-loaded mappings */
+
+/* flags for the client */
+#define CL_locked	0
+
+/* flags for the domain */
+#define DM_map_dirty	0
+
+struct tegra_iovmm_block {
+	struct tegra_iovmm_area vm_area;
+	tegra_iovmm_addr_t	start;
+	size_t			length;
+	atomic_t		ref;
+	unsigned long		flags;
+	unsigned long		poison;
+	struct rb_node		free_node;
+	struct rb_node		all_node;
+};
+
+struct iovmm_share_group {
+	const char			*name;
+	struct tegra_iovmm_domain	*domain;
+	struct list_head		client_list;
+	struct list_head		group_list;
+	spinlock_t			lock; /* for client_list */
+};
+
+static LIST_HEAD(iovmm_devices);
+static LIST_HEAD(iovmm_groups);
+static DEFINE_MUTEX(iovmm_group_list_lock);
+static struct kmem_cache *iovmm_cache;
+
+#define SIMALIGN(b, a)	(((b)->start % (a)) ? ((a) - ((b)->start % (a))) : 0)
+
+size_t tegra_iovmm_get_max_free(struct tegra_iovmm_client *client)
+{
+	struct rb_node *n;
+	struct tegra_iovmm_block *b;
+	struct tegra_iovmm_domain *domain = client->domain;
+	tegra_iovmm_addr_t max_free = 0;
+
+	spin_lock(&domain->block_lock);
+	n = rb_first(&domain->all_blocks);
+	while (n) {
+		b = rb_entry(n, struct tegra_iovmm_block, all_node);
+		n = rb_next(n);
+		if (test_bit(BK_free, &b->flags)) {
+			max_free = max_t(tegra_iovmm_addr_t,
+				max_free, iovmm_length(b));
+		}
+	}
+	spin_unlock(&domain->block_lock);
+	return max_free;
+}
+
+
+static void tegra_iovmm_block_stats(struct tegra_iovmm_domain *domain,
+	unsigned int *num_blocks, unsigned int *num_free,
+	tegra_iovmm_addr_t *total, size_t *total_free, size_t *max_free)
+{
+	struct rb_node *n;
+	struct tegra_iovmm_block *b;
+
+	*num_blocks = 0;
+	*num_free = 0;
+	*total = 0;
+	*total_free = 0;
+	*max_free = 0;
+
+	spin_lock(&domain->block_lock);
+	n = rb_first(&domain->all_blocks);
+	while (n) {
+		b = rb_entry(n, struct tegra_iovmm_block, all_node);
+		n = rb_next(n);
+		(*num_blocks)++;
+		*total += b->length;
+		if (test_bit(BK_free, &b->flags)) {
+			(*num_free)++;
+			*total_free += b->length;
+			*max_free = max_t(size_t, *max_free, b->length);
+		}
+	}
+	spin_unlock(&domain->block_lock);
+}
+
+static int tegra_iovmm_read_proc(char *page, char **start, off_t off,
+	int count, int *eof, void *data)
+{
+	struct iovmm_share_group *grp;
+	size_t max_free, total_free, total;
+	unsigned int num, num_free;
+
+	int len = 0;
+
+	mutex_lock(&iovmm_group_list_lock);
+	len += snprintf(page + len, count - len, "\ngroups\n");
+	if (list_empty(&iovmm_groups))
+		len += snprintf(page + len, count - len, "\t<empty>\n");
+	else {
+		list_for_each_entry(grp, &iovmm_groups, group_list) {
+			len += snprintf(page + len, count - len,
+					"\t%s (device: %s)\n",
+				(grp->name) ? grp->name : "<unnamed>",
+				grp->domain->dev->name);
+			tegra_iovmm_block_stats(grp->domain, &num,
+				&num_free, &total, &total_free, &max_free);
+			total >>= 10;
+			total_free >>= 10;
+			max_free >>= 10;
+			len += snprintf(page + len, count - len,
+				"\t\tsize: %uKiB free: %uKiB "
+				"largest: %uKiB (%u free / %u total blocks)\n",
+				total, total_free, max_free, num_free, num);
+		}
+	}
+	mutex_unlock(&iovmm_group_list_lock);
+
+	*eof = 1;
+	return len;
+}
+
+static void iovmm_block_put(struct tegra_iovmm_block *b)
+{
+	BUG_ON(b->poison);
+	BUG_ON(atomic_read(&b->ref) == 0);
+	if (!atomic_dec_return(&b->ref)) {
+		b->poison = 0xa5a5a5a5;
+		kmem_cache_free(iovmm_cache, b);
+	}
+}
+
+static void iovmm_free_block(struct tegra_iovmm_domain *domain,
+	struct tegra_iovmm_block *block)
+{
+	struct tegra_iovmm_block *pred = NULL; /* address-order predecessor */
+	struct tegra_iovmm_block *succ = NULL; /* address-order successor */
+	struct rb_node **p;
+	struct rb_node *parent = NULL, *temp;
+	int pred_free = 0, succ_free = 0;
+
+	iovmm_block_put(block);
+
+	spin_lock(&domain->block_lock);
+	temp = rb_prev(&block->all_node);
+	if (temp)
+		pred = rb_entry(temp, struct tegra_iovmm_block, all_node);
+	temp = rb_next(&block->all_node);
+	if (temp)
+		succ = rb_entry(temp, struct tegra_iovmm_block, all_node);
+
+	if (pred)
+		pred_free = test_bit(BK_free, &pred->flags);
+	if (succ)
+		succ_free = test_bit(BK_free, &succ->flags);
+
+	if (pred_free && succ_free) {
+		pred->length += block->length;
+		pred->length += succ->length;
+		rb_erase(&block->all_node, &domain->all_blocks);
+		rb_erase(&succ->all_node, &domain->all_blocks);
+		rb_erase(&succ->free_node, &domain->free_blocks);
+		rb_erase(&pred->free_node, &domain->free_blocks);
+		iovmm_block_put(block);
+		iovmm_block_put(succ);
+		block = pred;
+	} else if (pred_free) {
+		pred->length += block->length;
+		rb_erase(&block->all_node, &domain->all_blocks);
+		rb_erase(&pred->free_node, &domain->free_blocks);
+		iovmm_block_put(block);
+		block = pred;
+	} else if (succ_free) {
+		block->length += succ->length;
+		rb_erase(&succ->all_node, &domain->all_blocks);
+		rb_erase(&succ->free_node, &domain->free_blocks);
+		iovmm_block_put(succ);
+	}
+
+	p = &domain->free_blocks.rb_node;
+	while (*p) {
+		struct tegra_iovmm_block *b;
+		parent = *p;
+		b = rb_entry(parent, struct tegra_iovmm_block, free_node);
+		if (block->length >= b->length)
+			p = &parent->rb_right;
+		else
+			p = &parent->rb_left;
+	}
+	rb_link_node(&block->free_node, parent, p);
+	rb_insert_color(&block->free_node, &domain->free_blocks);
+	set_bit(BK_free, &block->flags);
+	spin_unlock(&domain->block_lock);
+}
+
+/* if the best-fit block is larger than the requested size, a remainder
+ * block will be created and inserted into the free list in its place.
+ * since all free blocks are stored in two trees the new block needs to be
+ * linked into both. */
+static struct tegra_iovmm_block *iovmm_split_free_block(
+	struct tegra_iovmm_domain *domain,
+	struct tegra_iovmm_block *block, unsigned long size)
+{
+	struct rb_node **p;
+	struct rb_node *parent = NULL;
+	struct tegra_iovmm_block *rem;
+	struct tegra_iovmm_block *b;
+
+	rem = kmem_cache_zalloc(iovmm_cache, GFP_KERNEL);
+	if (!rem)
+		return NULL;
+
+	spin_lock(&domain->block_lock);
+	p = &domain->free_blocks.rb_node;
+
+	rem->start  = block->start + size;
+	rem->length = block->length - size;
+	atomic_set(&rem->ref, 1);
+	block->length = size;
+
+	while (*p) {
+		parent = *p;
+		b = rb_entry(parent, struct tegra_iovmm_block, free_node);
+		if (rem->length >= b->length)
+			p = &parent->rb_right;
+		else
+			p = &parent->rb_left;
+	}
+	set_bit(BK_free, &rem->flags);
+	rb_link_node(&rem->free_node, parent, p);
+	rb_insert_color(&rem->free_node, &domain->free_blocks);
+
+	p = &domain->all_blocks.rb_node;
+	parent = NULL;
+	while (*p) {
+		parent = *p;
+		b = rb_entry(parent, struct tegra_iovmm_block, all_node);
+		if (rem->start >= b->start)
+			p = &parent->rb_right;
+		else
+			p = &parent->rb_left;
+	}
+	rb_link_node(&rem->all_node, parent, p);
+	rb_insert_color(&rem->all_node, &domain->all_blocks);
+
+	return rem;
+}
+
+static int iovmm_block_splitting;
+static struct tegra_iovmm_block *iovmm_alloc_block(
+	struct tegra_iovmm_domain *domain, size_t size, size_t align)
+{
+	struct rb_node *n;
+	struct tegra_iovmm_block *b, *best;
+	size_t simalign;
+	unsigned long page_size = 1 << domain->dev->pgsize_bits;
+
+	BUG_ON(!size);
+
+	size = round_up(size, page_size);
+	align = round_up(align, page_size);
+	for (;;) {
+		spin_lock(&domain->block_lock);
+		if (!iovmm_block_splitting)
+			break;
+		spin_unlock(&domain->block_lock);
+		schedule();
+	}
+	n = domain->free_blocks.rb_node;
+	best = NULL;
+	while (n) {
+		tegra_iovmm_addr_t aligned_start, block_ceil;
+
+		b = rb_entry(n, struct tegra_iovmm_block, free_node);
+		simalign = SIMALIGN(b, align);
+		aligned_start = b->start + simalign;
+		block_ceil = b->start + b->length;
+
+		if (block_ceil >= aligned_start + size) {
+			/* Block has enough size */
+			best = b;
+			if (NO_SPLIT(simalign) &&
+				NO_SPLIT(block_ceil - (aligned_start + size)))
+				break;
+			n = n->rb_left;
+		} else {
+			n = n->rb_right;
+		}
+	}
+	if (!best) {
+		spin_unlock(&domain->block_lock);
+		return NULL;
+	}
+
+	simalign = SIMALIGN(best, align);
+	if (DO_SPLIT(simalign)) {
+		iovmm_block_splitting = 1;
+		spin_unlock(&domain->block_lock);
+
+		/* Split off misalignment */
+		b = best;
+		best = iovmm_split_free_block(domain, b, simalign);
+		if (best)
+			simalign = 0;
+		else
+			best = b;
+	}
+
+	/* Unfree designed block */
+	rb_erase(&best->free_node, &domain->free_blocks);
+	clear_bit(BK_free, &best->flags);
+	atomic_inc(&best->ref);
+
+	iovmm_start(best) = best->start + simalign;
+	iovmm_length(best) = size;
+
+	if (DO_SPLIT((best->start + best->length) - iovmm_end(best))) {
+		iovmm_block_splitting = 1;
+		spin_unlock(&domain->block_lock);
+
+		/* Split off excess */
+		(void)iovmm_split_free_block(domain, best, size + simalign);
+	}
+
+	iovmm_block_splitting = 0;
+	spin_unlock(&domain->block_lock);
+
+	return best;
+}
+
+static struct tegra_iovmm_block *iovmm_allocate_vm(
+	struct tegra_iovmm_domain *domain, size_t size,
+	size_t align, unsigned long iovm_start)
+{
+	struct rb_node *n;
+	struct tegra_iovmm_block *b, *best;
+	unsigned long page_size = 1 << domain->dev->pgsize_bits;
+
+	BUG_ON(iovm_start % align);
+	BUG_ON(!size);
+
+	size = round_up(size, page_size);
+	for (;;) {
+		spin_lock(&domain->block_lock);
+		if (!iovmm_block_splitting)
+			break;
+		spin_unlock(&domain->block_lock);
+		schedule();
+	}
+
+	n = rb_first(&domain->free_blocks);
+	best = NULL;
+	while (n) {
+		b = rb_entry(n, struct tegra_iovmm_block, free_node);
+		if ((b->start <= iovm_start) &&
+		     (b->start + b->length) >= (iovm_start + size)) {
+			best = b;
+			break;
+		}
+		n = rb_next(n);
+	}
+
+	if (!best)
+		goto fail;
+
+	/* split the mem before iovm_start. */
+	if (DO_SPLIT(iovm_start - best->start)) {
+		iovmm_block_splitting = 1;
+		spin_unlock(&domain->block_lock);
+		best = iovmm_split_free_block(domain, best,
+			(iovm_start - best->start));
+	}
+	if (!best)
+		goto fail;
+
+	/* remove the desired block from free list. */
+	rb_erase(&best->free_node, &domain->free_blocks);
+	clear_bit(BK_free, &best->flags);
+	atomic_inc(&best->ref);
+
+	iovmm_start(best) = iovm_start;
+	iovmm_length(best) = size;
+
+	BUG_ON(best->start > iovmm_start(best));
+	BUG_ON((best->start + best->length) < iovmm_end(best));
+	/* split the mem after iovm_start+size. */
+	if (DO_SPLIT(best->start + best->length - iovmm_end(best))) {
+		iovmm_block_splitting = 1;
+		spin_unlock(&domain->block_lock);
+		(void)iovmm_split_free_block(domain, best,
+			(iovmm_start(best) - best->start + size));
+	}
+fail:
+	iovmm_block_splitting = 0;
+	spin_unlock(&domain->block_lock);
+	return best;
+}
+
+int tegra_iovmm_domain_init(struct tegra_iovmm_domain *domain,
+	struct tegra_iovmm_device *dev, tegra_iovmm_addr_t start,
+	tegra_iovmm_addr_t end)
+{
+	struct tegra_iovmm_block *b;
+	unsigned long page_size = 1 << dev->pgsize_bits;
+
+	b = kmem_cache_zalloc(iovmm_cache, GFP_KERNEL);
+	if (!b)
+		return -ENOMEM;
+
+	domain->dev = dev;
+
+	atomic_set(&domain->clients, 0);
+	atomic_set(&domain->locks, 0);
+	atomic_set(&b->ref, 1);
+	spin_lock_init(&domain->block_lock);
+	init_rwsem(&domain->map_lock);
+	init_waitqueue_head(&domain->delay_lock);
+
+	b->start  = round_up(start, page_size);
+	b->length = round_down(end, page_size) - b->start;
+
+	set_bit(BK_free, &b->flags);
+	rb_link_node(&b->free_node, NULL, &domain->free_blocks.rb_node);
+	rb_insert_color(&b->free_node, &domain->free_blocks);
+	rb_link_node(&b->all_node, NULL, &domain->all_blocks.rb_node);
+	rb_insert_color(&b->all_node, &domain->all_blocks);
+
+	return 0;
+}
+
+/* If iovm_start != 0, tries to allocate specified iova block if it is free.
+ * if it is not free, it fails.
+ */
+struct tegra_iovmm_area *tegra_iovmm_create_vm(
+	struct tegra_iovmm_client *client, struct tegra_iovmm_area_ops *ops,
+	size_t size, size_t align, pgprot_t pgprot, unsigned long iovm_start)
+{
+	struct tegra_iovmm_block *b;
+	struct tegra_iovmm_domain *domain;
+
+	if (!client)
+		return NULL;
+
+	domain = client->domain;
+
+	if (iovm_start)
+		b = iovmm_allocate_vm(domain, size, align, iovm_start);
+	else
+		b = iovmm_alloc_block(domain, size, align);
+	if (!b)
+		return NULL;
+
+	b->vm_area.domain = domain;
+	b->vm_area.pgprot = pgprot;
+	b->vm_area.ops = ops;
+
+	down_read(&b->vm_area.domain->map_lock);
+	if (ops && !test_bit(CL_locked, &client->flags)) {
+		set_bit(BK_map_dirty, &b->flags);
+		set_bit(DM_map_dirty, &client->domain->flags);
+	} else if (ops) {
+		if (domain->dev->ops->map(domain, &b->vm_area))
+			pr_err("%s failed to map locked domain\n", __func__);
+	}
+	up_read(&b->vm_area.domain->map_lock);
+
+	return &b->vm_area;
+}
+
+void tegra_iovmm_vm_insert_pfn(struct tegra_iovmm_area *vm,
+	tegra_iovmm_addr_t vaddr, unsigned long pfn)
+{
+	struct tegra_iovmm_domain *domain = vm->domain;
+	BUG_ON(vaddr & ((1<<domain->dev->pgsize_bits)-1));
+	BUG_ON(vaddr >= vm->iovm_start + vm->iovm_length);
+	BUG_ON(vaddr < vm->iovm_start);
+	BUG_ON(vm->ops);
+
+	domain->dev->ops->map_pfn(domain, vm, vaddr, pfn);
+}
+
+void tegra_iovmm_zap_vm(struct tegra_iovmm_area *vm)
+{
+	struct tegra_iovmm_block *b;
+	struct tegra_iovmm_domain *domain;
+
+	b = container_of(vm, struct tegra_iovmm_block, vm_area);
+	domain = vm->domain;
+	/*
+	 * if the vm area mapping was deferred, don't unmap it since
+	 * the memory for the page tables it uses may not be allocated
+	 */
+	down_read(&domain->map_lock);
+	if (!test_and_clear_bit(BK_map_dirty, &b->flags))
+		domain->dev->ops->unmap(domain, vm, false);
+	up_read(&domain->map_lock);
+}
+
+void tegra_iovmm_unzap_vm(struct tegra_iovmm_area *vm)
+{
+	struct tegra_iovmm_block *b;
+	struct tegra_iovmm_domain *domain;
+
+	b = container_of(vm, struct tegra_iovmm_block, vm_area);
+	domain = vm->domain;
+	if (!vm->ops)
+		return;
+
+	down_read(&domain->map_lock);
+	if (vm->ops) {
+		if (atomic_read(&domain->locks))
+			domain->dev->ops->map(domain, vm);
+		else {
+			set_bit(BK_map_dirty, &b->flags);
+			set_bit(DM_map_dirty, &domain->flags);
+		}
+	}
+	up_read(&domain->map_lock);
+}
+
+void tegra_iovmm_free_vm(struct tegra_iovmm_area *vm)
+{
+	struct tegra_iovmm_block *b;
+	struct tegra_iovmm_domain *domain;
+
+	if (!vm)
+		return;
+
+	b = container_of(vm, struct tegra_iovmm_block, vm_area);
+	domain = vm->domain;
+	down_read(&domain->map_lock);
+	if (!test_and_clear_bit(BK_map_dirty, &b->flags))
+		domain->dev->ops->unmap(domain, vm, true);
+	iovmm_free_block(domain, b);
+	up_read(&domain->map_lock);
+}
+
+struct tegra_iovmm_area *tegra_iovmm_area_get(struct tegra_iovmm_area *vm)
+{
+	struct tegra_iovmm_block *b;
+
+	BUG_ON(!vm);
+	b = container_of(vm, struct tegra_iovmm_block, vm_area);
+
+	atomic_inc(&b->ref);
+	return &b->vm_area;
+}
+
+void tegra_iovmm_area_put(struct tegra_iovmm_area *vm)
+{
+	struct tegra_iovmm_block *b;
+	BUG_ON(!vm);
+	b = container_of(vm, struct tegra_iovmm_block, vm_area);
+	iovmm_block_put(b);
+}
+
+struct tegra_iovmm_area *tegra_iovmm_find_area_get(
+	struct tegra_iovmm_client *client, tegra_iovmm_addr_t addr)
+{
+	struct rb_node *n;
+	struct tegra_iovmm_block *b = NULL;
+
+	if (!client)
+		return NULL;
+
+	spin_lock(&client->domain->block_lock);
+	n = client->domain->all_blocks.rb_node;
+
+	while (n) {
+		b = rb_entry(n, struct tegra_iovmm_block, all_node);
+		if (iovmm_start(b) <= addr && addr <= iovmm_end(b)) {
+			if (test_bit(BK_free, &b->flags))
+				b = NULL;
+			break;
+		}
+		if (addr > iovmm_start(b))
+			n = n->rb_right;
+		else
+			n = n->rb_left;
+		b = NULL;
+	}
+	if (b)
+		atomic_inc(&b->ref);
+	spin_unlock(&client->domain->block_lock);
+	if (!b)
+		return NULL;
+	return &b->vm_area;
+}
+
+static int _iovmm_client_lock(struct tegra_iovmm_client *client)
+{
+	struct tegra_iovmm_device *dev;
+	struct tegra_iovmm_domain *domain;
+	int v;
+
+	if (unlikely(!client))
+		return -ENODEV;
+
+	if (unlikely(test_bit(CL_locked, &client->flags))) {
+		pr_err("attempting to relock client %s\n", client->name);
+		return 0;
+	}
+
+	domain = client->domain;
+	dev = domain->dev;
+	down_write(&domain->map_lock);
+	v = atomic_inc_return(&domain->locks);
+	/* if the device doesn't export the lock_domain function, the device
+	 * must guarantee that any valid domain will be locked. */
+	if (v == 1 && dev->ops->lock_domain) {
+		if (dev->ops->lock_domain(domain, client)) {
+			atomic_dec(&domain->locks);
+			up_write(&domain->map_lock);
+			return -EAGAIN;
+		}
+	}
+	if (test_and_clear_bit(DM_map_dirty, &domain->flags)) {
+		struct rb_node *n;
+		struct tegra_iovmm_block *b;
+
+		spin_lock(&domain->block_lock);
+		n = rb_first(&domain->all_blocks);
+		while (n) {
+			b = rb_entry(n, struct tegra_iovmm_block, all_node);
+			n = rb_next(n);
+			if (test_bit(BK_free, &b->flags))
+				continue;
+
+			if (test_and_clear_bit(BK_map_dirty, &b->flags)) {
+				if (!b->vm_area.ops) {
+					pr_err("%s: "
+					       "vm_area ops must exist for lazy maps\n",
+					       __func__);
+					continue;
+				}
+				dev->ops->map(domain, &b->vm_area);
+			}
+		}
+	}
+	set_bit(CL_locked, &client->flags);
+	up_write(&domain->map_lock);
+	return 0;
+}
+
+int tegra_iovmm_client_trylock(struct tegra_iovmm_client *client)
+{
+	return _iovmm_client_lock(client);
+}
+
+int tegra_iovmm_client_lock(struct tegra_iovmm_client *client)
+{
+	int ret;
+
+	if (!client)
+		return -ENODEV;
+
+	ret = wait_event_interruptible(client->domain->delay_lock,
+		_iovmm_client_lock(client) != -EAGAIN);
+
+	if (ret == -ERESTARTSYS)
+		return -EINTR;
+
+	return ret;
+}
+
+void tegra_iovmm_client_unlock(struct tegra_iovmm_client *client)
+{
+	struct tegra_iovmm_device *dev;
+	struct tegra_iovmm_domain *domain;
+	int do_wake = 0;
+
+	if (!client)
+		return;
+
+	if (!test_and_clear_bit(CL_locked, &client->flags)) {
+		pr_err("unlocking unlocked client %s\n", client->name);
+		return;
+	}
+
+	domain = client->domain;
+	dev = domain->dev;
+	down_write(&domain->map_lock);
+	if (!atomic_dec_return(&domain->locks)) {
+		if (dev->ops->unlock_domain)
+			dev->ops->unlock_domain(domain, client);
+		do_wake = 1;
+	}
+	up_write(&domain->map_lock);
+	if (do_wake)
+		wake_up(&domain->delay_lock);
+}
+
+size_t tegra_iovmm_get_vm_size(struct tegra_iovmm_client *client)
+{
+	struct tegra_iovmm_domain *domain;
+	struct rb_node *n;
+	struct tegra_iovmm_block *b;
+	size_t size = 0;
+
+	if (!client)
+		return 0;
+
+	domain = client->domain;
+
+	spin_lock(&domain->block_lock);
+	n = rb_first(&domain->all_blocks);
+	while (n) {
+		b = rb_entry(n, struct tegra_iovmm_block, all_node);
+		n = rb_next(n);
+		size += b->length;
+	}
+	spin_unlock(&domain->block_lock);
+
+	return size;
+}
+
+void tegra_iovmm_free_client(struct tegra_iovmm_client *client)
+{
+	struct tegra_iovmm_device *dev;
+	struct tegra_iovmm_domain *domain;
+
+	if (!client)
+		return;
+
+	BUG_ON(!client->domain || !client->domain->dev);
+
+	domain = client->domain;
+	dev = domain->dev;
+
+	if (test_and_clear_bit(CL_locked, &client->flags)) {
+		pr_err("freeing locked client %s\n", client->name);
+		if (!atomic_dec_return(&domain->locks)) {
+			down_write(&domain->map_lock);
+			if (dev->ops->unlock_domain)
+				dev->ops->unlock_domain(domain, client);
+			up_write(&domain->map_lock);
+			wake_up(&domain->delay_lock);
+		}
+	}
+	mutex_lock(&iovmm_group_list_lock);
+	if (!atomic_dec_return(&domain->clients))
+		if (dev->ops->free_domain)
+			dev->ops->free_domain(domain, client);
+	list_del(&client->list);
+	if (list_empty(&client->group->client_list)) {
+		list_del(&client->group->group_list);
+		kfree(client->group->name);
+		kfree(client->group);
+	}
+	kfree(client->name);
+	kfree(client);
+	mutex_unlock(&iovmm_group_list_lock);
+}
+
+struct tegra_iovmm_client *tegra_iovmm_alloc_client(const char *name,
+	const char *share_group, struct miscdevice *misc_dev)
+{
+	struct tegra_iovmm_client *c = kzalloc(sizeof(*c), GFP_KERNEL);
+	struct iovmm_share_group *grp = NULL;
+	struct tegra_iovmm_device *dev;
+
+	if (!c)
+		return NULL;
+	c->name = kstrdup(name, GFP_KERNEL);
+	if (!c->name)
+		goto fail;
+	c->misc_dev = misc_dev;
+
+	mutex_lock(&iovmm_group_list_lock);
+	if (share_group) {
+		list_for_each_entry(grp, &iovmm_groups, group_list) {
+			if (grp->name && !strcmp(grp->name, share_group))
+				break;
+		}
+	}
+	if (!grp || strcmp(grp->name, share_group)) {
+		grp = kzalloc(sizeof(*grp), GFP_KERNEL);
+		if (!grp)
+			goto fail_lock;
+		grp->name =
+			(share_group) ? kstrdup(share_group, GFP_KERNEL) : NULL;
+		if (share_group && !grp->name) {
+			kfree(grp);
+			goto fail_lock;
+		}
+		list_for_each_entry(dev, &iovmm_devices, list) {
+			grp->domain = dev->ops->alloc_domain(dev, c);
+			if (grp->domain)
+				break;
+		}
+		if (!grp->domain) {
+			pr_err("%s: alloc_domain failed for %s\n",
+				__func__, c->name);
+			dump_stack();
+			kfree(grp->name);
+			kfree(grp);
+			grp = NULL;
+			goto fail_lock;
+		}
+		spin_lock_init(&grp->lock);
+		INIT_LIST_HEAD(&grp->client_list);
+		list_add_tail(&grp->group_list, &iovmm_groups);
+	}
+
+	atomic_inc(&grp->domain->clients);
+	c->group = grp;
+	c->domain = grp->domain;
+	spin_lock(&grp->lock);
+	list_add_tail(&c->list, &grp->client_list);
+	spin_unlock(&grp->lock);
+	mutex_unlock(&iovmm_group_list_lock);
+	return c;
+
+fail_lock:
+	mutex_unlock(&iovmm_group_list_lock);
+fail:
+	if (c)
+		kfree(c->name);
+	kfree(c);
+	return NULL;
+}
+
+int tegra_iovmm_register(struct tegra_iovmm_device *dev)
+{
+	BUG_ON(!dev);
+	mutex_lock(&iovmm_group_list_lock);
+	if (list_empty(&iovmm_devices)) {
+		iovmm_cache = KMEM_CACHE(tegra_iovmm_block, 0);
+		if (!iovmm_cache) {
+			pr_err("%s: failed to make kmem cache\n", __func__);
+			mutex_unlock(&iovmm_group_list_lock);
+			return -ENOMEM;
+		}
+		create_proc_read_entry("iovmminfo", S_IRUGO, NULL,
+			tegra_iovmm_read_proc, NULL);
+	}
+	list_add_tail(&dev->list, &iovmm_devices);
+	mutex_unlock(&iovmm_group_list_lock);
+	pr_info("%s: added %s\n", __func__, dev->name);
+	return 0;
+}
+
+int tegra_iovmm_unregister(struct tegra_iovmm_device *dev)
+{
+	mutex_lock(&iovmm_group_list_lock);
+	list_del(&dev->list);
+	mutex_unlock(&iovmm_group_list_lock);
+	return 0;
+}
+
+static int tegra_iovmm_suspend(void)
+{
+	int rc = 0;
+	struct tegra_iovmm_device *dev;
+
+	list_for_each_entry(dev, &iovmm_devices, list) {
+
+		if (!dev->ops->suspend)
+			continue;
+
+		rc = dev->ops->suspend(dev);
+		if (rc) {
+			pr_err("%s: %s suspend returned %d\n",
+			       __func__, dev->name, rc);
+			return rc;
+		}
+	}
+	return 0;
+}
+
+static void tegra_iovmm_resume(void)
+{
+	struct tegra_iovmm_device *dev;
+
+	list_for_each_entry(dev, &iovmm_devices, list) {
+		if (dev->ops->resume)
+			dev->ops->resume(dev);
+	}
+}
+
+static struct syscore_ops tegra_iovmm_syscore_ops = {
+	.suspend = tegra_iovmm_suspend,
+	.resume = tegra_iovmm_resume,
+};
+
+static __init int tegra_iovmm_syscore_init(void)
+{
+	register_syscore_ops(&tegra_iovmm_syscore_ops);
+	return 0;
+}
+subsys_initcall(tegra_iovmm_syscore_init);
-- 
1.7.0.4

Re: [PATCH 1/3] ARM: iommu: tegra/common: Initial support for IOVMM driver

[Thierry Reding]

[-- Attachment #1: Type: text/plain, Size: 42203 bytes --]

I'm not very knowledgeable about IOMMUs in general, so my comments are more
about general style.

* hdoyu-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org wrote:
> From: Hiroshi DOYU <hdoyu-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> 
> This is the tegra specific IOMMU framework, independent of H/W. H/W

You should keep the spelling of "Tegra" consistent.

> dependent modules are to be registered to this framework so that this
> can support different IOMMU H/Ws among Tegra generations, Tegra2/GART,
> and Tegra3/SMMU H/Ws.
> 
> Most of this part could be replaced with a generic IOMMU
> framework. This is expected to ease finding similarities with
> different platforms, with the intention of solving problems once in a
> generic framework which everyone can use.
> 
> Signed-off-by: Hiroshi DOYU <hdoyu-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> Cc: Hiro Sugawara <hsugawara-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> Cc: Krishna Reddy <vdumpa-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> ---
>  arch/arm/mach-tegra/include/mach/iovmm.h |  283 +++++++++
>  drivers/iommu/Kconfig                    |    6 +
>  drivers/iommu/Makefile                   |    1 +
>  drivers/iommu/tegra-iovmm.c              |  936 ++++++++++++++++++++++++++++++
>  4 files changed, 1226 insertions(+), 0 deletions(-)
>  create mode 100644 arch/arm/mach-tegra/include/mach/iovmm.h
>  create mode 100644 drivers/iommu/tegra-iovmm.c
> 
> diff --git a/arch/arm/mach-tegra/include/mach/iovmm.h b/arch/arm/mach-tegra/include/mach/iovmm.h
> new file mode 100644
> index 0000000..6fd0bb6
> --- /dev/null
> +++ b/arch/arm/mach-tegra/include/mach/iovmm.h
> @@ -0,0 +1,283 @@
> +/*
> + * Copyright (c) 2010-2011, NVIDIA Corporation.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundation; either version 2 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed i the hope that it will be useful, but WITHOUT
> + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
> + * more details.
> + *
> + * You should have received a copy of the GNU General Public License along
> + * with this program; if not, write to the Free Software Foundation, Inc.,
> + * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
> + */
> +
> +#include <linux/list.h>
> +#include <linux/platform_device.h>
> +#include <linux/miscdevice.h>
> +#include <linux/rbtree.h>
> +#include <linux/rwsem.h>
> +#include <linux/spinlock.h>
> +#include <linux/types.h>
> +
> +#ifndef _MACH_TEGRA_IOVMM_H_
> +#define _MACH_TEGRA_IOVMM_H_
> +
> +typedef u32 tegra_iovmm_addr_t;
> +
> +struct tegra_iovmm_device_ops;
> +
> +/* each I/O virtual memory manager unit should register a device with
> + * the iovmm system
> + */

Generally, multi-line comments have the starting /* and ending */ on separate
lines. Also since this API is public it may be better to document it using
kerneldoc.

> +struct tegra_iovmm_device {
> +	struct tegra_iovmm_device_ops	*ops;
> +	const char			*name;
> +	struct list_head		list;
> +	int				pgsize_bits;
> +};
> +
> +/* tegra_iovmm_domain serves a purpose analagous to mm_struct as defined in
> + * <linux/mm_types.h> - it defines a virtual address space within which
> + * tegra_iovmm_areas can be created.
> + */
> +struct tegra_iovmm_domain {
> +	atomic_t		clients;
> +	atomic_t		locks;
> +	spinlock_t		block_lock;
> +	unsigned long		flags;
> +	wait_queue_head_t	delay_lock;  /* when lock_client fails */
> +	struct rw_semaphore	map_lock;
> +	struct rb_root		all_blocks;  /* ordered by address */
> +	struct rb_root		free_blocks; /* ordered by size */
> +	struct tegra_iovmm_device *dev;
> +};
> +
> +/* tegra_iovmm_client is analagous to an individual task in the task group
> + * which owns an mm_struct.
> + */
> +
> +struct iovmm_share_group;
> +
> +struct tegra_iovmm_client {
> +	const char			*name;
> +	unsigned long			flags;
> +	struct iovmm_share_group	*group;
> +	struct tegra_iovmm_domain	*domain;
> +	struct miscdevice		*misc_dev;
> +	struct list_head		list;
> +};
> +
> +/* tegra_iovmm_area serves a purpose analagous to vm_area_struct as defined
> + * in <linux/mm_types.h> - it defines a virtual memory area which can be
> + * mapped to physical memory by a client-provided mapping function. */
> +
> +struct tegra_iovmm_area {
> +	struct tegra_iovmm_domain	*domain;
> +	tegra_iovmm_addr_t		iovm_start;
> +	size_t				iovm_length;
> +	pgprot_t			pgprot;
> +	struct tegra_iovmm_area_ops	*ops;
> +};
> +
> +struct tegra_iovmm_device_ops {
> +	/* maps a VMA using the page residency functions provided by the VMA */
> +	int (*map)(struct tegra_iovmm_domain *domain,
> +		struct tegra_iovmm_area *io_vma);
> +	/* marks all PTEs in a VMA as invalid; decommits the virtual addres
> +	 * space (potentially freeing PDEs when decommit is true.) */
> +	void (*unmap)(struct tegra_iovmm_domain *domain,
> +		struct tegra_iovmm_area *io_vma, bool decommit);
> +	void (*map_pfn)(struct tegra_iovmm_domain *domain,
> +		struct tegra_iovmm_area *io_vma,
> +		tegra_iovmm_addr_t offs, unsigned long pfn);
> +	/* ensures that a domain is resident in the hardware's mapping region
> +	 * so that it may be used by a client */
> +	int (*lock_domain)(struct tegra_iovmm_domain *domain,
> +		struct tegra_iovmm_client *client);
> +	void (*unlock_domain)(struct tegra_iovmm_domain *domain,
> +		struct tegra_iovmm_client *client);
> +	/* allocates a vmm_domain for the specified client; may return the same
> +	 * domain for multiple clients */
> +	struct tegra_iovmm_domain* (*alloc_domain)(
> +		struct tegra_iovmm_device *dev,
> +		struct tegra_iovmm_client *client);
> +	void (*free_domain)(struct tegra_iovmm_domain *domain,
> +		struct tegra_iovmm_client *client);
> +	int (*suspend)(struct tegra_iovmm_device *dev);
> +	void (*resume)(struct tegra_iovmm_device *dev);
> +};
> +
> +struct tegra_iovmm_area_ops {
> +	/* ensures that the page of data starting at the specified offset
> +	 * from the start of the iovma is resident and pinned for use by
> +	 * DMA, returns the system pfn, or an invalid pfn if the
> +	 * operation fails. */
> +	unsigned long (*lock_makeresident)(struct tegra_iovmm_area *area,
> +		tegra_iovmm_addr_t offs);
> +	/* called when the page is unmapped from the I/O VMA */
> +	void (*release)(struct tegra_iovmm_area *area, tegra_iovmm_addr_t offs);
> +};
> +
> +#ifdef CONFIG_TEGRA_IOVMM
> +/* called by clients to allocate an I/O VMM client mapping context which
> + * will be shared by all clients in the same share_group */
> +struct tegra_iovmm_client *tegra_iovmm_alloc_client(const char *name,
> +	const char *share_group, struct miscdevice *misc_dev);
> +
> +size_t tegra_iovmm_get_vm_size(struct tegra_iovmm_client *client);
> +
> +void tegra_iovmm_free_client(struct tegra_iovmm_client *client);
> +
> +/* called by clients to ensure that their mapping context is resident
> + * before performing any DMA operations addressing I/O VMM regions.
> + * client_lock may return -EINTR. */
> +int tegra_iovmm_client_lock(struct tegra_iovmm_client *client);
> +int tegra_iovmm_client_trylock(struct tegra_iovmm_client *client);
> +
> +/* called by clients after DMA operations are complete */
> +void tegra_iovmm_client_unlock(struct tegra_iovmm_client *client);
> +
> +/* called by clients to allocate a new iovmm_area and reserve I/O virtual
> + * address space for it. if ops is NULL, clients should subsequently call
> + * tegra_iovmm_vm_map_pages and/or tegra_iovmm_vm_insert_pfn to explicitly
> + * map the I/O virtual address to an OS-allocated page or physical address,
> + * respectively. VM operations may be called before this call returns */
> +struct tegra_iovmm_area *tegra_iovmm_create_vm(
> +	struct tegra_iovmm_client *client, struct tegra_iovmm_area_ops *ops,
> +	size_t size, size_t align, pgprot_t pgprot, unsigned long iovm_start);
> +
> +/* called by clients to "zap" an iovmm_area, and replace all mappings
> + * in it with invalid ones, without freeing the virtual address range */
> +void tegra_iovmm_zap_vm(struct tegra_iovmm_area *vm);
> +
> +/* after zapping a demand-loaded iovmm_area, the client should unzap it
> + * to allow the VMM device to remap the page range. */
> +void tegra_iovmm_unzap_vm(struct tegra_iovmm_area *vm);
> +
> +/* called by clients to return an iovmm_area to the free pool for the domain */
> +void tegra_iovmm_free_vm(struct tegra_iovmm_area *vm);
> +
> +/* returns size of largest free iovm block */
> +size_t tegra_iovmm_get_max_free(struct tegra_iovmm_client *client);
> +
> +/* called by client software to map the page-aligned I/O address vaddr to
> + * a specific physical address pfn. I/O VMA should have been created with
> + * a NULL tegra_iovmm_area_ops structure. */
> +void tegra_iovmm_vm_insert_pfn(struct tegra_iovmm_area *area,
> +	tegra_iovmm_addr_t vaddr, unsigned long pfn);
> +
> +/* called by clients to return the iovmm_area containing addr, or NULL if
> + * addr has not been allocated. caller should call tegra_iovmm_put_area when
> + * finished using the returned pointer */
> +struct tegra_iovmm_area *tegra_iovmm_find_area_get(
> +	struct tegra_iovmm_client *client, tegra_iovmm_addr_t addr);

The comment mentions tegra_iovmm_put_area() but the function declared below
is called tegra_iovmm_area_put().

> +
> +struct tegra_iovmm_area *tegra_iovmm_area_get(struct tegra_iovmm_area *vm);
> +void tegra_iovmm_area_put(struct tegra_iovmm_area *vm);
> +
> +/* called by drivers to initialize a tegra_iovmm_domain structure */
> +int tegra_iovmm_domain_init(struct tegra_iovmm_domain *domain,
> +	struct tegra_iovmm_device *dev, tegra_iovmm_addr_t start,
> +	tegra_iovmm_addr_t end);
> +
> +/* called by drivers to register an I/O VMM device with the system */
> +int tegra_iovmm_register(struct tegra_iovmm_device *dev);
> +
> +/* called by drivers to remove an I/O VMM device from the system */
> +int tegra_iovmm_unregister(struct tegra_iovmm_device *dev);
> +
> +#else /* CONFIG_TEGRA_IOVMM */
> +
> +static inline struct tegra_iovmm_client *tegra_iovmm_alloc_client(
> +	const char *name, const char *share_group, struct miscdevice *misc_dev)
> +{
> +	return NULL;
> +}
> +
> +static inline size_t tegra_iovmm_get_vm_size(struct tegra_iovmm_client *client)
> +{
> +	return 0;
> +}
> +
> +static inline void tegra_iovmm_free_client(struct tegra_iovmm_client *client)
> +{}

For readability, the braces should probably go on separate lines.

> +
> +static inline int tegra_iovmm_client_lock(struct tegra_iovmm_client *client)
> +{
> +	return 0;
> +}
> +
> +static inline int tegra_iovmm_client_trylock(struct tegra_iovmm_client *client)
> +{
> +	return 0;
> +}
> +
> +static inline void tegra_iovmm_client_unlock(struct tegra_iovmm_client *client)
> +{}
> +

Here as well.

> +static inline struct tegra_iovmm_area *tegra_iovmm_create_vm(
> +	struct tegra_iovmm_client *client, struct tegra_iovmm_area_ops *ops,
> +	size_t size, size_t align, pgprot_t pgprot, unsigned long iovm_start)
> +{
> +	return NULL;
> +}
> +
> +static inline void tegra_iovmm_zap_vm(struct tegra_iovmm_area *vm) { }
> +
> +static inline void tegra_iovmm_unzap_vm(struct tegra_iovmm_area *vm) { }
> +
> +static inline void tegra_iovmm_free_vm(struct tegra_iovmm_area *vm) { }

And here.

> +static inline size_t tegra_iovmm_get_max_free(struct tegra_iovmm_client *client)
> +{
> +	return 0;
> +}
> +
> +static inline void tegra_iovmm_vm_insert_pfn(struct tegra_iovmm_area *area,
> +	tegra_iovmm_addr_t vaddr, unsigned long pfn) { }
> +

Here.

> +static inline struct tegra_iovmm_area *tegra_iovmm_find_area_get(
> +	struct tegra_iovmm_client *client, tegra_iovmm_addr_t addr)
> +{
> +	return NULL;
> +}
> +
> +static inline struct tegra_iovmm_area *tegra_iovmm_area_get(
> +	struct tegra_iovmm_area *vm)
> +{
> +	return NULL;
> +}
> +
> +static inline void tegra_iovmm_area_put(struct tegra_iovmm_area *vm) { }
> +

Here.

> +static inline int tegra_iovmm_domain_init(struct tegra_iovmm_domain *domain,
> +	struct tegra_iovmm_device *dev, tegra_iovmm_addr_t start,
> +	tegra_iovmm_addr_t end)
> +{
> +	return 0;
> +}
> +
> +static inline int tegra_iovmm_register(struct tegra_iovmm_device *dev)
> +{
> +	return 0;
> +}
> +
> +static inline int tegra_iovmm_unregister(struct tegra_iovmm_device *dev)
> +{
> +	return 0;
> +}
> +
> +static inline int tegra_iovmm_suspend(void)
> +{
> +	return 0;
> +}
> +
> +static inline void tegra_iovmm_resume(void) { }
> +#endif /* CONFIG_TEGRA_IOVMM */

And here. Preferably with a blank line between the closing } and the #endif.

> +
> +

One of these is gratuitous.

> +#endif
> diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
> index 5414253..487d1ee 100644
> --- a/drivers/iommu/Kconfig
> +++ b/drivers/iommu/Kconfig
> @@ -131,4 +131,10 @@ config OMAP_IOMMU_DEBUG
>  
>           Say N unless you know you need this.
>  
> +

This one also.

> +# Tegra IOMMU support
> +
> +config TEGRA_IOVMM
> +	bool
> +
>  endif # IOMMU_SUPPORT
> diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
> index 2f44487..365eefb 100644
> --- a/drivers/iommu/Makefile
> +++ b/drivers/iommu/Makefile
> @@ -7,3 +7,4 @@ obj-$(CONFIG_IRQ_REMAP) += intr_remapping.o
>  obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o
>  obj-$(CONFIG_OMAP_IOVMM) += omap-iovmm.o
>  obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o
> +obj-$(CONFIG_TEGRA_IOVMM) += tegra-iovmm.o
> diff --git a/drivers/iommu/tegra-iovmm.c b/drivers/iommu/tegra-iovmm.c
> new file mode 100644
> index 0000000..0f2d996
> --- /dev/null
> +++ b/drivers/iommu/tegra-iovmm.c
> @@ -0,0 +1,936 @@
> +/*
> + * Tegra I/O VM manager
> + *
> + * Copyright (c) 2010-2011, NVIDIA Corporation.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundation; either version 2 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful, but WITHOUT
> + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
> + * more details.
> + *
> + * You should have received a copy of the GNU General Public License along
> + * with this program; if not, write to the Free Software Foundation, Inc.,
> + * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
> + */
> +
> +#include <linux/kernel.h>
> +#include <linux/spinlock.h>
> +#include <linux/proc_fs.h>
> +#include <linux/sched.h>
> +#include <linux/string.h>
> +#include <linux/slab.h>
> +#include <linux/syscore_ops.h>
> +
> +#include <mach/iovmm.h>
> +
> +/*
> + * after the best-fit block is located, the remaining pages not needed
> + * for the allocation will be split into a new free block if the
> + * number of remaining pages is >= MIN_SPLIT_PAGE.
> + */
> +#define MIN_SPLIT_PAGE		4
> +#define MIN_SPLIT_BYTES(_d)	(MIN_SPLIT_PAGE << (_d)->dev->pgsize_bits)
> +#define NO_SPLIT(m)		((m) < MIN_SPLIT_BYTES(domain))
> +#define DO_SPLIT(m)		((m) >= MIN_SPLIT_BYTES(domain))
> +
> +#define iovmm_start(_b)		((_b)->vm_area.iovm_start)
> +#define iovmm_length(_b)	((_b)->vm_area.iovm_length)
> +#define iovmm_end(_b)		(iovmm_start(_b) + iovmm_length(_b))
> +
> +/* flags for the block */
> +#define BK_free		0 /* indicates free mappings */
> +#define BK_map_dirty	1 /* used by demand-loaded mappings */
> +
> +/* flags for the client */
> +#define CL_locked	0
> +
> +/* flags for the domain */
> +#define DM_map_dirty	0

These flags should be all uppercase.

> +
> +struct tegra_iovmm_block {
> +	struct tegra_iovmm_area vm_area;
> +	tegra_iovmm_addr_t	start;
> +	size_t			length;
> +	atomic_t		ref;
> +	unsigned long		flags;
> +	unsigned long		poison;
> +	struct rb_node		free_node;
> +	struct rb_node		all_node;
> +};
> +
> +struct iovmm_share_group {
> +	const char			*name;
> +	struct tegra_iovmm_domain	*domain;
> +	struct list_head		client_list;
> +	struct list_head		group_list;
> +	spinlock_t			lock; /* for client_list */
> +};
> +
> +static LIST_HEAD(iovmm_devices);
> +static LIST_HEAD(iovmm_groups);
> +static DEFINE_MUTEX(iovmm_group_list_lock);
> +static struct kmem_cache *iovmm_cache;
> +
> +#define SIMALIGN(b, a)	(((b)->start % (a)) ? ((a) - ((b)->start % (a))) : 0)
> +
> +size_t tegra_iovmm_get_max_free(struct tegra_iovmm_client *client)
> +{
> +	struct rb_node *n;
> +	struct tegra_iovmm_block *b;
> +	struct tegra_iovmm_domain *domain = client->domain;
> +	tegra_iovmm_addr_t max_free = 0;
> +
> +	spin_lock(&domain->block_lock);
> +	n = rb_first(&domain->all_blocks);
> +	while (n) {
> +		b = rb_entry(n, struct tegra_iovmm_block, all_node);
> +		n = rb_next(n);
> +		if (test_bit(BK_free, &b->flags)) {
> +			max_free = max_t(tegra_iovmm_addr_t,
> +				max_free, iovmm_length(b));
> +		}
> +	}
> +	spin_unlock(&domain->block_lock);
> +	return max_free;
> +}
> +
> +
> +static void tegra_iovmm_block_stats(struct tegra_iovmm_domain *domain,
> +	unsigned int *num_blocks, unsigned int *num_free,
> +	tegra_iovmm_addr_t *total, size_t *total_free, size_t *max_free)
> +{
> +	struct rb_node *n;
> +	struct tegra_iovmm_block *b;
> +
> +	*num_blocks = 0;
> +	*num_free = 0;
> +	*total = 0;
> +	*total_free = 0;
> +	*max_free = 0;
> +
> +	spin_lock(&domain->block_lock);
> +	n = rb_first(&domain->all_blocks);
> +	while (n) {
> +		b = rb_entry(n, struct tegra_iovmm_block, all_node);
> +		n = rb_next(n);
> +		(*num_blocks)++;
> +		*total += b->length;
> +		if (test_bit(BK_free, &b->flags)) {
> +			(*num_free)++;
> +			*total_free += b->length;
> +			*max_free = max_t(size_t, *max_free, b->length);
> +		}
> +	}
> +	spin_unlock(&domain->block_lock);
> +}
> +
> +static int tegra_iovmm_read_proc(char *page, char **start, off_t off,
> +	int count, int *eof, void *data)
> +{
> +	struct iovmm_share_group *grp;
> +	size_t max_free, total_free, total;
> +	unsigned int num, num_free;
> +
> +	int len = 0;
> +
> +	mutex_lock(&iovmm_group_list_lock);
> +	len += snprintf(page + len, count - len, "\ngroups\n");
> +	if (list_empty(&iovmm_groups))
> +		len += snprintf(page + len, count - len, "\t<empty>\n");
> +	else {
> +		list_for_each_entry(grp, &iovmm_groups, group_list) {
> +			len += snprintf(page + len, count - len,
> +					"\t%s (device: %s)\n",
> +				(grp->name) ? grp->name : "<unnamed>",
> +				grp->domain->dev->name);
> +			tegra_iovmm_block_stats(grp->domain, &num,
> +				&num_free, &total, &total_free, &max_free);
> +			total >>= 10;
> +			total_free >>= 10;
> +			max_free >>= 10;
> +			len += snprintf(page + len, count - len,
> +				"\t\tsize: %uKiB free: %uKiB "
> +				"largest: %uKiB (%u free / %u total blocks)\n",
> +				total, total_free, max_free, num_free, num);
> +		}
> +	}
> +	mutex_unlock(&iovmm_group_list_lock);
> +
> +	*eof = 1;
> +	return len;
> +}
> +
> +static void iovmm_block_put(struct tegra_iovmm_block *b)
> +{
> +	BUG_ON(b->poison);
> +	BUG_ON(atomic_read(&b->ref) == 0);
> +	if (!atomic_dec_return(&b->ref)) {
> +		b->poison = 0xa5a5a5a5;
> +		kmem_cache_free(iovmm_cache, b);
> +	}
> +}
> +
> +static void iovmm_free_block(struct tegra_iovmm_domain *domain,
> +	struct tegra_iovmm_block *block)
> +{
> +	struct tegra_iovmm_block *pred = NULL; /* address-order predecessor */
> +	struct tegra_iovmm_block *succ = NULL; /* address-order successor */
> +	struct rb_node **p;
> +	struct rb_node *parent = NULL, *temp;
> +	int pred_free = 0, succ_free = 0;
> +
> +	iovmm_block_put(block);
> +
> +	spin_lock(&domain->block_lock);
> +	temp = rb_prev(&block->all_node);
> +	if (temp)
> +		pred = rb_entry(temp, struct tegra_iovmm_block, all_node);
> +	temp = rb_next(&block->all_node);
> +	if (temp)
> +		succ = rb_entry(temp, struct tegra_iovmm_block, all_node);
> +
> +	if (pred)
> +		pred_free = test_bit(BK_free, &pred->flags);
> +	if (succ)
> +		succ_free = test_bit(BK_free, &succ->flags);
> +
> +	if (pred_free && succ_free) {
> +		pred->length += block->length;
> +		pred->length += succ->length;
> +		rb_erase(&block->all_node, &domain->all_blocks);
> +		rb_erase(&succ->all_node, &domain->all_blocks);
> +		rb_erase(&succ->free_node, &domain->free_blocks);
> +		rb_erase(&pred->free_node, &domain->free_blocks);
> +		iovmm_block_put(block);
> +		iovmm_block_put(succ);
> +		block = pred;
> +	} else if (pred_free) {
> +		pred->length += block->length;
> +		rb_erase(&block->all_node, &domain->all_blocks);
> +		rb_erase(&pred->free_node, &domain->free_blocks);
> +		iovmm_block_put(block);
> +		block = pred;
> +	} else if (succ_free) {
> +		block->length += succ->length;
> +		rb_erase(&succ->all_node, &domain->all_blocks);
> +		rb_erase(&succ->free_node, &domain->free_blocks);
> +		iovmm_block_put(succ);
> +	}
> +
> +	p = &domain->free_blocks.rb_node;
> +	while (*p) {
> +		struct tegra_iovmm_block *b;
> +		parent = *p;
> +		b = rb_entry(parent, struct tegra_iovmm_block, free_node);
> +		if (block->length >= b->length)
> +			p = &parent->rb_right;
> +		else
> +			p = &parent->rb_left;
> +	}
> +	rb_link_node(&block->free_node, parent, p);
> +	rb_insert_color(&block->free_node, &domain->free_blocks);
> +	set_bit(BK_free, &block->flags);
> +	spin_unlock(&domain->block_lock);
> +}
> +
> +/* if the best-fit block is larger than the requested size, a remainder
> + * block will be created and inserted into the free list in its place.
> + * since all free blocks are stored in two trees the new block needs to be
> + * linked into both. */
> +static struct tegra_iovmm_block *iovmm_split_free_block(
> +	struct tegra_iovmm_domain *domain,
> +	struct tegra_iovmm_block *block, unsigned long size)
> +{
> +	struct rb_node **p;
> +	struct rb_node *parent = NULL;
> +	struct tegra_iovmm_block *rem;
> +	struct tegra_iovmm_block *b;
> +
> +	rem = kmem_cache_zalloc(iovmm_cache, GFP_KERNEL);
> +	if (!rem)
> +		return NULL;
> +
> +	spin_lock(&domain->block_lock);
> +	p = &domain->free_blocks.rb_node;
> +
> +	rem->start  = block->start + size;
> +	rem->length = block->length - size;
> +	atomic_set(&rem->ref, 1);
> +	block->length = size;
> +
> +	while (*p) {
> +		parent = *p;
> +		b = rb_entry(parent, struct tegra_iovmm_block, free_node);
> +		if (rem->length >= b->length)
> +			p = &parent->rb_right;
> +		else
> +			p = &parent->rb_left;
> +	}
> +	set_bit(BK_free, &rem->flags);
> +	rb_link_node(&rem->free_node, parent, p);
> +	rb_insert_color(&rem->free_node, &domain->free_blocks);
> +
> +	p = &domain->all_blocks.rb_node;
> +	parent = NULL;
> +	while (*p) {
> +		parent = *p;
> +		b = rb_entry(parent, struct tegra_iovmm_block, all_node);
> +		if (rem->start >= b->start)
> +			p = &parent->rb_right;
> +		else
> +			p = &parent->rb_left;
> +	}
> +	rb_link_node(&rem->all_node, parent, p);
> +	rb_insert_color(&rem->all_node, &domain->all_blocks);
> +
> +	return rem;
> +}
> +
> +static int iovmm_block_splitting;

Could this be moved into the tegra_iovmm_domain structure?

> +static struct tegra_iovmm_block *iovmm_alloc_block(
> +	struct tegra_iovmm_domain *domain, size_t size, size_t align)
> +{
> +	struct rb_node *n;
> +	struct tegra_iovmm_block *b, *best;
> +	size_t simalign;
> +	unsigned long page_size = 1 << domain->dev->pgsize_bits;
> +
> +	BUG_ON(!size);
> +
> +	size = round_up(size, page_size);
> +	align = round_up(align, page_size);
> +	for (;;) {
> +		spin_lock(&domain->block_lock);
> +		if (!iovmm_block_splitting)
> +			break;
> +		spin_unlock(&domain->block_lock);
> +		schedule();
> +	}
> +	n = domain->free_blocks.rb_node;
> +	best = NULL;
> +	while (n) {
> +		tegra_iovmm_addr_t aligned_start, block_ceil;
> +
> +		b = rb_entry(n, struct tegra_iovmm_block, free_node);
> +		simalign = SIMALIGN(b, align);
> +		aligned_start = b->start + simalign;
> +		block_ceil = b->start + b->length;
> +
> +		if (block_ceil >= aligned_start + size) {
> +			/* Block has enough size */
> +			best = b;
> +			if (NO_SPLIT(simalign) &&
> +				NO_SPLIT(block_ceil - (aligned_start + size)))
> +				break;
> +			n = n->rb_left;
> +		} else {
> +			n = n->rb_right;
> +		}
> +	}
> +	if (!best) {
> +		spin_unlock(&domain->block_lock);
> +		return NULL;
> +	}
> +
> +	simalign = SIMALIGN(best, align);
> +	if (DO_SPLIT(simalign)) {
> +		iovmm_block_splitting = 1;
> +		spin_unlock(&domain->block_lock);
> +
> +		/* Split off misalignment */
> +		b = best;
> +		best = iovmm_split_free_block(domain, b, simalign);
> +		if (best)
> +			simalign = 0;
> +		else
> +			best = b;
> +	}
> +
> +	/* Unfree designed block */
> +	rb_erase(&best->free_node, &domain->free_blocks);
> +	clear_bit(BK_free, &best->flags);
> +	atomic_inc(&best->ref);
> +
> +	iovmm_start(best) = best->start + simalign;
> +	iovmm_length(best) = size;
> +
> +	if (DO_SPLIT((best->start + best->length) - iovmm_end(best))) {
> +		iovmm_block_splitting = 1;
> +		spin_unlock(&domain->block_lock);
> +
> +		/* Split off excess */
> +		(void)iovmm_split_free_block(domain, best, size + simalign);
> +	}
> +
> +	iovmm_block_splitting = 0;
> +	spin_unlock(&domain->block_lock);
> +
> +	return best;
> +}
> +
> +static struct tegra_iovmm_block *iovmm_allocate_vm(
> +	struct tegra_iovmm_domain *domain, size_t size,
> +	size_t align, unsigned long iovm_start)
> +{
> +	struct rb_node *n;
> +	struct tegra_iovmm_block *b, *best;
> +	unsigned long page_size = 1 << domain->dev->pgsize_bits;
> +
> +	BUG_ON(iovm_start % align);
> +	BUG_ON(!size);
> +
> +	size = round_up(size, page_size);
> +	for (;;) {
> +		spin_lock(&domain->block_lock);
> +		if (!iovmm_block_splitting)
> +			break;
> +		spin_unlock(&domain->block_lock);
> +		schedule();
> +	}
> +
> +	n = rb_first(&domain->free_blocks);
> +	best = NULL;
> +	while (n) {
> +		b = rb_entry(n, struct tegra_iovmm_block, free_node);
> +		if ((b->start <= iovm_start) &&
> +		     (b->start + b->length) >= (iovm_start + size)) {
> +			best = b;
> +			break;
> +		}
> +		n = rb_next(n);
> +	}
> +
> +	if (!best)
> +		goto fail;
> +
> +	/* split the mem before iovm_start. */
> +	if (DO_SPLIT(iovm_start - best->start)) {
> +		iovmm_block_splitting = 1;
> +		spin_unlock(&domain->block_lock);
> +		best = iovmm_split_free_block(domain, best,
> +			(iovm_start - best->start));
> +	}
> +	if (!best)
> +		goto fail;
> +
> +	/* remove the desired block from free list. */
> +	rb_erase(&best->free_node, &domain->free_blocks);
> +	clear_bit(BK_free, &best->flags);
> +	atomic_inc(&best->ref);
> +
> +	iovmm_start(best) = iovm_start;
> +	iovmm_length(best) = size;
> +
> +	BUG_ON(best->start > iovmm_start(best));
> +	BUG_ON((best->start + best->length) < iovmm_end(best));
> +	/* split the mem after iovm_start+size. */
> +	if (DO_SPLIT(best->start + best->length - iovmm_end(best))) {
> +		iovmm_block_splitting = 1;
> +		spin_unlock(&domain->block_lock);
> +		(void)iovmm_split_free_block(domain, best,
> +			(iovmm_start(best) - best->start + size));
> +	}
> +fail:
> +	iovmm_block_splitting = 0;
> +	spin_unlock(&domain->block_lock);
> +	return best;
> +}
> +
> +int tegra_iovmm_domain_init(struct tegra_iovmm_domain *domain,
> +	struct tegra_iovmm_device *dev, tegra_iovmm_addr_t start,
> +	tegra_iovmm_addr_t end)
> +{
> +	struct tegra_iovmm_block *b;
> +	unsigned long page_size = 1 << dev->pgsize_bits;
> +
> +	b = kmem_cache_zalloc(iovmm_cache, GFP_KERNEL);
> +	if (!b)
> +		return -ENOMEM;
> +
> +	domain->dev = dev;
> +
> +	atomic_set(&domain->clients, 0);
> +	atomic_set(&domain->locks, 0);
> +	atomic_set(&b->ref, 1);
> +	spin_lock_init(&domain->block_lock);
> +	init_rwsem(&domain->map_lock);
> +	init_waitqueue_head(&domain->delay_lock);
> +
> +	b->start  = round_up(start, page_size);
> +	b->length = round_down(end, page_size) - b->start;
> +
> +	set_bit(BK_free, &b->flags);
> +	rb_link_node(&b->free_node, NULL, &domain->free_blocks.rb_node);
> +	rb_insert_color(&b->free_node, &domain->free_blocks);
> +	rb_link_node(&b->all_node, NULL, &domain->all_blocks.rb_node);
> +	rb_insert_color(&b->all_node, &domain->all_blocks);
> +
> +	return 0;
> +}
> +
> +/* If iovm_start != 0, tries to allocate specified iova block if it is free.
> + * if it is not free, it fails.
> + */
> +struct tegra_iovmm_area *tegra_iovmm_create_vm(
> +	struct tegra_iovmm_client *client, struct tegra_iovmm_area_ops *ops,
> +	size_t size, size_t align, pgprot_t pgprot, unsigned long iovm_start)
> +{
> +	struct tegra_iovmm_block *b;
> +	struct tegra_iovmm_domain *domain;
> +
> +	if (!client)
> +		return NULL;
> +
> +	domain = client->domain;
> +
> +	if (iovm_start)
> +		b = iovmm_allocate_vm(domain, size, align, iovm_start);
> +	else
> +		b = iovmm_alloc_block(domain, size, align);
> +	if (!b)
> +		return NULL;
> +
> +	b->vm_area.domain = domain;
> +	b->vm_area.pgprot = pgprot;
> +	b->vm_area.ops = ops;
> +
> +	down_read(&b->vm_area.domain->map_lock);
> +	if (ops && !test_bit(CL_locked, &client->flags)) {
> +		set_bit(BK_map_dirty, &b->flags);
> +		set_bit(DM_map_dirty, &client->domain->flags);
> +	} else if (ops) {
> +		if (domain->dev->ops->map(domain, &b->vm_area))
> +			pr_err("%s failed to map locked domain\n", __func__);
> +	}
> +	up_read(&b->vm_area.domain->map_lock);
> +
> +	return &b->vm_area;
> +}
> +
> +void tegra_iovmm_vm_insert_pfn(struct tegra_iovmm_area *vm,
> +	tegra_iovmm_addr_t vaddr, unsigned long pfn)
> +{
> +	struct tegra_iovmm_domain *domain = vm->domain;

Does this need a check for !vm?

> +	BUG_ON(vaddr & ((1<<domain->dev->pgsize_bits)-1));
> +	BUG_ON(vaddr >= vm->iovm_start + vm->iovm_length);
> +	BUG_ON(vaddr < vm->iovm_start);
> +	BUG_ON(vm->ops);
> +
> +	domain->dev->ops->map_pfn(domain, vm, vaddr, pfn);
> +}
> +
> +void tegra_iovmm_zap_vm(struct tegra_iovmm_area *vm)
> +{
> +	struct tegra_iovmm_block *b;
> +	struct tegra_iovmm_domain *domain;
> +
> +	b = container_of(vm, struct tegra_iovmm_block, vm_area);
> +	domain = vm->domain;

Same.

> +	/*
> +	 * if the vm area mapping was deferred, don't unmap it since
> +	 * the memory for the page tables it uses may not be allocated
> +	 */
> +	down_read(&domain->map_lock);
> +	if (!test_and_clear_bit(BK_map_dirty, &b->flags))
> +		domain->dev->ops->unmap(domain, vm, false);
> +	up_read(&domain->map_lock);
> +}
> +
> +void tegra_iovmm_unzap_vm(struct tegra_iovmm_area *vm)
> +{
> +	struct tegra_iovmm_block *b;
> +	struct tegra_iovmm_domain *domain;
> +
> +	b = container_of(vm, struct tegra_iovmm_block, vm_area);
> +	domain = vm->domain;

Same.

> +	if (!vm->ops)
> +		return;
> +
> +	down_read(&domain->map_lock);
> +	if (vm->ops) {
> +		if (atomic_read(&domain->locks))
> +			domain->dev->ops->map(domain, vm);
> +		else {
> +			set_bit(BK_map_dirty, &b->flags);
> +			set_bit(DM_map_dirty, &domain->flags);
> +		}
> +	}
> +	up_read(&domain->map_lock);
> +}
> +
> +void tegra_iovmm_free_vm(struct tegra_iovmm_area *vm)
> +{
> +	struct tegra_iovmm_block *b;
> +	struct tegra_iovmm_domain *domain;
> +
> +	if (!vm)
> +		return;
> +
> +	b = container_of(vm, struct tegra_iovmm_block, vm_area);
> +	domain = vm->domain;
> +	down_read(&domain->map_lock);
> +	if (!test_and_clear_bit(BK_map_dirty, &b->flags))
> +		domain->dev->ops->unmap(domain, vm, true);
> +	iovmm_free_block(domain, b);
> +	up_read(&domain->map_lock);
> +}

I believe it is more common to initialize b to container_of(...) in the same
line as the declaration. Since container_of() doesn't actually do any
dereferencing, it is safe to check for !vm later.

> +
> +struct tegra_iovmm_area *tegra_iovmm_area_get(struct tegra_iovmm_area *vm)
> +{
> +	struct tegra_iovmm_block *b;
> +
> +	BUG_ON(!vm);
> +	b = container_of(vm, struct tegra_iovmm_block, vm_area);
> +
> +	atomic_inc(&b->ref);
> +	return &b->vm_area;
> +}
> +
> +void tegra_iovmm_area_put(struct tegra_iovmm_area *vm)
> +{
> +	struct tegra_iovmm_block *b;
> +	BUG_ON(!vm);
> +	b = container_of(vm, struct tegra_iovmm_block, vm_area);
> +	iovmm_block_put(b);
> +}
> +
> +struct tegra_iovmm_area *tegra_iovmm_find_area_get(
> +	struct tegra_iovmm_client *client, tegra_iovmm_addr_t addr)
> +{
> +	struct rb_node *n;
> +	struct tegra_iovmm_block *b = NULL;
> +
> +	if (!client)
> +		return NULL;
> +
> +	spin_lock(&client->domain->block_lock);
> +	n = client->domain->all_blocks.rb_node;
> +
> +	while (n) {
> +		b = rb_entry(n, struct tegra_iovmm_block, all_node);
> +		if (iovmm_start(b) <= addr && addr <= iovmm_end(b)) {
> +			if (test_bit(BK_free, &b->flags))
> +				b = NULL;
> +			break;
> +		}
> +		if (addr > iovmm_start(b))
> +			n = n->rb_right;
> +		else
> +			n = n->rb_left;
> +		b = NULL;
> +	}
> +	if (b)
> +		atomic_inc(&b->ref);
> +	spin_unlock(&client->domain->block_lock);
> +	if (!b)
> +		return NULL;
> +	return &b->vm_area;
> +}
> +
> +static int _iovmm_client_lock(struct tegra_iovmm_client *client)
> +{
> +	struct tegra_iovmm_device *dev;
> +	struct tegra_iovmm_domain *domain;
> +	int v;
> +
> +	if (unlikely(!client))
> +		return -ENODEV;
> +
> +	if (unlikely(test_bit(CL_locked, &client->flags))) {
> +		pr_err("attempting to relock client %s\n", client->name);
> +		return 0;
> +	}
> +
> +	domain = client->domain;
> +	dev = domain->dev;
> +	down_write(&domain->map_lock);
> +	v = atomic_inc_return(&domain->locks);
> +	/* if the device doesn't export the lock_domain function, the device
> +	 * must guarantee that any valid domain will be locked. */
> +	if (v == 1 && dev->ops->lock_domain) {
> +		if (dev->ops->lock_domain(domain, client)) {
> +			atomic_dec(&domain->locks);
> +			up_write(&domain->map_lock);
> +			return -EAGAIN;
> +		}
> +	}
> +	if (test_and_clear_bit(DM_map_dirty, &domain->flags)) {
> +		struct rb_node *n;
> +		struct tegra_iovmm_block *b;
> +
> +		spin_lock(&domain->block_lock);
> +		n = rb_first(&domain->all_blocks);
> +		while (n) {
> +			b = rb_entry(n, struct tegra_iovmm_block, all_node);
> +			n = rb_next(n);
> +			if (test_bit(BK_free, &b->flags))
> +				continue;
> +
> +			if (test_and_clear_bit(BK_map_dirty, &b->flags)) {
> +				if (!b->vm_area.ops) {
> +					pr_err("%s: "
> +					       "vm_area ops must exist for lazy maps\n",
> +					       __func__);
> +					continue;
> +				}
> +				dev->ops->map(domain, &b->vm_area);
> +			}
> +		}
> +	}
> +	set_bit(CL_locked, &client->flags);
> +	up_write(&domain->map_lock);
> +	return 0;
> +}
> +
> +int tegra_iovmm_client_trylock(struct tegra_iovmm_client *client)
> +{
> +	return _iovmm_client_lock(client);
> +}
> +
> +int tegra_iovmm_client_lock(struct tegra_iovmm_client *client)
> +{
> +	int ret;
> +
> +	if (!client)
> +		return -ENODEV;
> +
> +	ret = wait_event_interruptible(client->domain->delay_lock,
> +		_iovmm_client_lock(client) != -EAGAIN);
> +
> +	if (ret == -ERESTARTSYS)
> +		return -EINTR;
> +
> +	return ret;
> +}
> +
> +void tegra_iovmm_client_unlock(struct tegra_iovmm_client *client)
> +{
> +	struct tegra_iovmm_device *dev;
> +	struct tegra_iovmm_domain *domain;
> +	int do_wake = 0;
> +
> +	if (!client)
> +		return;
> +
> +	if (!test_and_clear_bit(CL_locked, &client->flags)) {
> +		pr_err("unlocking unlocked client %s\n", client->name);
> +		return;
> +	}
> +
> +	domain = client->domain;
> +	dev = domain->dev;
> +	down_write(&domain->map_lock);
> +	if (!atomic_dec_return(&domain->locks)) {
> +		if (dev->ops->unlock_domain)
> +			dev->ops->unlock_domain(domain, client);
> +		do_wake = 1;
> +	}
> +	up_write(&domain->map_lock);
> +	if (do_wake)
> +		wake_up(&domain->delay_lock);
> +}
> +
> +size_t tegra_iovmm_get_vm_size(struct tegra_iovmm_client *client)
> +{
> +	struct tegra_iovmm_domain *domain;
> +	struct rb_node *n;
> +	struct tegra_iovmm_block *b;
> +	size_t size = 0;
> +
> +	if (!client)
> +		return 0;
> +
> +	domain = client->domain;
> +
> +	spin_lock(&domain->block_lock);
> +	n = rb_first(&domain->all_blocks);
> +	while (n) {
> +		b = rb_entry(n, struct tegra_iovmm_block, all_node);
> +		n = rb_next(n);
> +		size += b->length;
> +	}
> +	spin_unlock(&domain->block_lock);
> +
> +	return size;
> +}
> +
> +void tegra_iovmm_free_client(struct tegra_iovmm_client *client)
> +{
> +	struct tegra_iovmm_device *dev;
> +	struct tegra_iovmm_domain *domain;
> +
> +	if (!client)
> +		return;
> +
> +	BUG_ON(!client->domain || !client->domain->dev);
> +
> +	domain = client->domain;
> +	dev = domain->dev;
> +
> +	if (test_and_clear_bit(CL_locked, &client->flags)) {
> +		pr_err("freeing locked client %s\n", client->name);
> +		if (!atomic_dec_return(&domain->locks)) {
> +			down_write(&domain->map_lock);
> +			if (dev->ops->unlock_domain)
> +				dev->ops->unlock_domain(domain, client);
> +			up_write(&domain->map_lock);
> +			wake_up(&domain->delay_lock);
> +		}
> +	}
> +	mutex_lock(&iovmm_group_list_lock);
> +	if (!atomic_dec_return(&domain->clients))
> +		if (dev->ops->free_domain)
> +			dev->ops->free_domain(domain, client);
> +	list_del(&client->list);
> +	if (list_empty(&client->group->client_list)) {
> +		list_del(&client->group->group_list);
> +		kfree(client->group->name);
> +		kfree(client->group);
> +	}
> +	kfree(client->name);
> +	kfree(client);
> +	mutex_unlock(&iovmm_group_list_lock);
> +}
> +
> +struct tegra_iovmm_client *tegra_iovmm_alloc_client(const char *name,
> +	const char *share_group, struct miscdevice *misc_dev)
> +{
> +	struct tegra_iovmm_client *c = kzalloc(sizeof(*c), GFP_KERNEL);
> +	struct iovmm_share_group *grp = NULL;
> +	struct tegra_iovmm_device *dev;
> +
> +	if (!c)
> +		return NULL;
> +	c->name = kstrdup(name, GFP_KERNEL);
> +	if (!c->name)
> +		goto fail;
> +	c->misc_dev = misc_dev;
> +
> +	mutex_lock(&iovmm_group_list_lock);
> +	if (share_group) {
> +		list_for_each_entry(grp, &iovmm_groups, group_list) {
> +			if (grp->name && !strcmp(grp->name, share_group))
> +				break;
> +		}
> +	}
> +	if (!grp || strcmp(grp->name, share_group)) {
> +		grp = kzalloc(sizeof(*grp), GFP_KERNEL);
> +		if (!grp)
> +			goto fail_lock;
> +		grp->name =
> +			(share_group) ? kstrdup(share_group, GFP_KERNEL) : NULL;

Parens are not required around "share_group".

> +		if (share_group && !grp->name) {
> +			kfree(grp);
> +			goto fail_lock;
> +		}
> +		list_for_each_entry(dev, &iovmm_devices, list) {
> +			grp->domain = dev->ops->alloc_domain(dev, c);
> +			if (grp->domain)
> +				break;
> +		}
> +		if (!grp->domain) {
> +			pr_err("%s: alloc_domain failed for %s\n",
> +				__func__, c->name);
> +			dump_stack();
> +			kfree(grp->name);
> +			kfree(grp);
> +			grp = NULL;
> +			goto fail_lock;
> +		}
> +		spin_lock_init(&grp->lock);
> +		INIT_LIST_HEAD(&grp->client_list);
> +		list_add_tail(&grp->group_list, &iovmm_groups);
> +	}
> +
> +	atomic_inc(&grp->domain->clients);
> +	c->group = grp;
> +	c->domain = grp->domain;
> +	spin_lock(&grp->lock);
> +	list_add_tail(&c->list, &grp->client_list);
> +	spin_unlock(&grp->lock);
> +	mutex_unlock(&iovmm_group_list_lock);
> +	return c;
> +
> +fail_lock:
> +	mutex_unlock(&iovmm_group_list_lock);
> +fail:
> +	if (c)
> +		kfree(c->name);
> +	kfree(c);
> +	return NULL;
> +}
> +
> +int tegra_iovmm_register(struct tegra_iovmm_device *dev)
> +{
> +	BUG_ON(!dev);
> +	mutex_lock(&iovmm_group_list_lock);
> +	if (list_empty(&iovmm_devices)) {
> +		iovmm_cache = KMEM_CACHE(tegra_iovmm_block, 0);
> +		if (!iovmm_cache) {
> +			pr_err("%s: failed to make kmem cache\n", __func__);
> +			mutex_unlock(&iovmm_group_list_lock);
> +			return -ENOMEM;
> +		}
> +		create_proc_read_entry("iovmminfo", S_IRUGO, NULL,
> +			tegra_iovmm_read_proc, NULL);
> +	}
> +	list_add_tail(&dev->list, &iovmm_devices);
> +	mutex_unlock(&iovmm_group_list_lock);
> +	pr_info("%s: added %s\n", __func__, dev->name);
> +	return 0;
> +}
> +
> +int tegra_iovmm_unregister(struct tegra_iovmm_device *dev)
> +{
> +	mutex_lock(&iovmm_group_list_lock);
> +	list_del(&dev->list);
> +	mutex_unlock(&iovmm_group_list_lock);
> +	return 0;
> +}
> +
> +static int tegra_iovmm_suspend(void)
> +{
> +	int rc = 0;
> +	struct tegra_iovmm_device *dev;
> +
> +	list_for_each_entry(dev, &iovmm_devices, list) {
> +

Gratuitous newline.

> +		if (!dev->ops->suspend)
> +			continue;
> +
> +		rc = dev->ops->suspend(dev);
> +		if (rc) {
> +			pr_err("%s: %s suspend returned %d\n",
> +			       __func__, dev->name, rc);
> +			return rc;
> +		}
> +	}
> +	return 0;
> +}
> +
> +static void tegra_iovmm_resume(void)
> +{
> +	struct tegra_iovmm_device *dev;
> +
> +	list_for_each_entry(dev, &iovmm_devices, list) {
> +		if (dev->ops->resume)
> +			dev->ops->resume(dev);
> +	}
> +}
> +
> +static struct syscore_ops tegra_iovmm_syscore_ops = {
> +	.suspend = tegra_iovmm_suspend,
> +	.resume = tegra_iovmm_resume,
> +};
> +
> +static __init int tegra_iovmm_syscore_init(void)
> +{
> +	register_syscore_ops(&tegra_iovmm_syscore_ops);
> +	return 0;
> +}
> +subsys_initcall(tegra_iovmm_syscore_init);
> -- 
> 1.7.0.4
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-tegra" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 
> 

[-- Attachment #2: Type: application/pgp-signature, Size: 198 bytes --]

Re: [PATCH 1/3] ARM: iommu: tegra/common: Initial support for IOVMM driver

[Hiroshi Doyu]

Hi Thierry,

From: Thierry Reding <thierry.reding-RM9K5IK7kjKj5M59NBduVrNAH6kLmebB@public.gmane.org>
Subject: Re: [PATCH 1/3] ARM: iommu: tegra/common: Initial support for IOVMM driver
Date: Thu, 17 Nov 2011 21:32:02 +0100
Message-ID: <20111117203202.GB20889-RM9K5IK7kjIQXX3q8xo1gnVAuStQJXxyR5q1nwbD4aMs9pC9oP6+/A@public.gmane.org>

> * PGP Signed by an unknown key
> 
> I'm not very knowledgeable about IOMMUs in general, so my comments are more
> about general style.

Thank you for your review. Mostly I'll take them in the next version
of patchset.

> * hdoyu-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org wrote:
> > From: Hiroshi DOYU <hdoyu-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> >
> > This is the tegra specific IOMMU framework, independent of H/W. H/W
> 
> You should keep the spelling of "Tegra" consistent.
> 
> > dependent modules are to be registered to this framework so that this
> > can support different IOMMU H/Ws among Tegra generations, Tegra2/GART,
> > and Tegra3/SMMU H/Ws.
> >
> > Most of this part could be replaced with a generic IOMMU
> > framework. This is expected to ease finding similarities with
> > different platforms, with the intention of solving problems once in a
> > generic framework which everyone can use.
> >
> > Signed-off-by: Hiroshi DOYU <hdoyu-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> > Cc: Hiro Sugawara <hsugawara-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> > Cc: Krishna Reddy <vdumpa-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> > ---
> >  arch/arm/mach-tegra/include/mach/iovmm.h |  283 +++++++++
> >  drivers/iommu/Kconfig                    |    6 +
> >  drivers/iommu/Makefile                   |    1 +
> >  drivers/iommu/tegra-iovmm.c              |  936 ++++++++++++++++++++++++++++++
> >  4 files changed, 1226 insertions(+), 0 deletions(-)
> >  create mode 100644 arch/arm/mach-tegra/include/mach/iovmm.h
> >  create mode 100644 drivers/iommu/tegra-iovmm.c
> >
> > diff --git a/arch/arm/mach-tegra/include/mach/iovmm.h b/arch/arm/mach-tegra/include/mach/iovmm.h
> > new file mode 100644
> > index 0000000..6fd0bb6
> > --- /dev/null
> > +++ b/arch/arm/mach-tegra/include/mach/iovmm.h
> > @@ -0,0 +1,283 @@
> > +/*
> > + * Copyright (c) 2010-2011, NVIDIA Corporation.
> > + *
> > + * This program is free software; you can redistribute it and/or modify
> > + * it under the terms of the GNU General Public License as published by
> > + * the Free Software Foundation; either version 2 of the License, or
> > + * (at your option) any later version.
> > + *
> > + * This program is distributed i the hope that it will be useful, but WITHOUT
> > + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> > + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
> > + * more details.
> > + *
> > + * You should have received a copy of the GNU General Public License along
> > + * with this program; if not, write to the Free Software Foundation, Inc.,
> > + * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
> > + */
> > +
> > +#include <linux/list.h>
> > +#include <linux/platform_device.h>
> > +#include <linux/miscdevice.h>
> > +#include <linux/rbtree.h>
> > +#include <linux/rwsem.h>
> > +#include <linux/spinlock.h>
> > +#include <linux/types.h>
> > +
> > +#ifndef _MACH_TEGRA_IOVMM_H_
> > +#define _MACH_TEGRA_IOVMM_H_
> > +
> > +typedef u32 tegra_iovmm_addr_t;
> > +
> > +struct tegra_iovmm_device_ops;
> > +
> > +/* each I/O virtual memory manager unit should register a device with
> > + * the iovmm system
> > + */
> 
> Generally, multi-line comments have the starting /* and ending */ on separate
> lines. Also since this API is public it may be better to document it using
> kerneldoc.

I'll leave this kerneldoc out for a while since those public API may
be replaced with geneic IOMMU ones.

[PATCH 2/3] ARM: iommu: tegra2: Initial support for GART driver

[hdoyu-DDmLM1+adcrQT0dZR+AlfA]

From: Hiroshi DOYU <hdoyu-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>

Tegra 2 IOMMU H/W dependent part, GART (Graphics Address Relocation
Table). This is one of the tegra_iovmm_device to register to the upper
tegra IOVMM framework. This supports only single virtual address
space(tegra_iovmm_domain), and manages a simple 1-to-1 mapping
H/W translation pagetable.

Signed-off-by: Hiroshi DOYU <hdoyu-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
Cc: Hiro Sugawara <hsugawara-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
Cc: Krishna Reddy <vdumpa-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
---
 drivers/iommu/Kconfig      |   10 ++
 drivers/iommu/Makefile     |    1 +
 drivers/iommu/tegra-gart.c |  357 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 368 insertions(+), 0 deletions(-)
 create mode 100644 drivers/iommu/tegra-gart.c

diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 487d1ee..7f342e8 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -133,6 +133,16 @@ config OMAP_IOMMU_DEBUG
 
 
 # Tegra IOMMU support
+config TEGRA_IOVMM_GART
+	bool "Enable I/O virtual memory manager for GART"
+	depends on ARCH_TEGRA_2x_SOC
+	default y
+	select TEGRA_IOVMM
+	help
+	  Enables support for remapping discontiguous physical memory
+	  shared with the operating system into contiguous I/O virtual
+	  space through the GART (Graphics Address Relocation Table)
+	  hardware included on Tegra SoCs.
 
 config TEGRA_IOVMM
 	bool
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 365eefb..4b61b05 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -8,3 +8,4 @@ obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o
 obj-$(CONFIG_OMAP_IOVMM) += omap-iovmm.o
 obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o
 obj-$(CONFIG_TEGRA_IOVMM) += tegra-iovmm.o
+obj-$(CONFIG_TEGRA_IOVMM_GART) += tegra-gart.o
diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c
new file mode 100644
index 0000000..24f893b
--- /dev/null
+++ b/drivers/iommu/tegra-gart.c
@@ -0,0 +1,357 @@
+/*
+ * Tegra I/O VMM implementation for GART devices in Tegra and Tegra 2 series
+ * systems-on-a-chip.
+ *
+ * Copyright (c) 2010-2011, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/mm.h>
+#include <linux/io.h>
+
+#include <asm/cacheflush.h>
+
+#include <mach/iovmm.h>
+
+#define GART_CONFIG		0x24
+#define GART_ENTRY_ADDR		0x28
+#define GART_ENTRY_DATA		0x2c
+
+#define VMM_NAME "iovmm-gart"
+#define DRIVER_NAME "tegra_gart"
+
+#define GART_PAGE_SHIFT		12
+#define GART_PAGE_SIZE		(1 << GART_PAGE_SHIFT)
+#define GART_PAGE_MASK		(~(GART_PAGE_SIZE - 1))
+
+struct gart_device {
+	void __iomem		*regs;
+	u32			*savedata;
+	u32			page_count; /* total remappable size */
+	tegra_iovmm_addr_t	iovmm_base; /* offset to apply to vmm_area */
+	spinlock_t		pte_lock;
+	struct tegra_iovmm_device iovmm;
+	struct tegra_iovmm_domain domain;
+	bool			enable;
+};
+
+static inline void __gart_set_pte(struct gart_device *gart,
+				tegra_iovmm_addr_t offs, u32 pte)
+{
+	writel(offs, gart->regs + GART_ENTRY_ADDR);
+	writel(pte, gart->regs + GART_ENTRY_DATA);
+	/*
+	 * Any interaction between any block on PPSB and a block on
+	 * APB or AHB must have these barriers to ensure the APB/AHB
+	 * bus transaction is complete before initiating activity on
+	 * the PPSB block.
+	 */
+	wmb();
+}
+
+static inline void gart_set_pte(struct gart_device *gart,
+				tegra_iovmm_addr_t offs, u32 pte)
+{
+	spin_lock(&gart->pte_lock);
+
+	__gart_set_pte(gart, offs, pte);
+
+	spin_unlock(&gart->pte_lock);
+}
+
+static int gart_map(struct tegra_iovmm_domain *, struct tegra_iovmm_area *);
+static void gart_unmap(struct tegra_iovmm_domain *,
+	struct tegra_iovmm_area *, bool);
+static void gart_map_pfn(struct tegra_iovmm_domain *,
+	struct tegra_iovmm_area *, tegra_iovmm_addr_t, unsigned long);
+static struct tegra_iovmm_domain *gart_alloc_domain(
+	struct tegra_iovmm_device *, struct tegra_iovmm_client *);
+
+static int gart_probe(struct platform_device *);
+static int gart_remove(struct platform_device *);
+static int gart_suspend(struct tegra_iovmm_device *dev);
+static void gart_resume(struct tegra_iovmm_device *dev);
+
+
+static struct tegra_iovmm_device_ops tegra_iovmm_gart_ops = {
+	.map		= gart_map,
+	.unmap		= gart_unmap,
+	.map_pfn	= gart_map_pfn,
+	.alloc_domain	= gart_alloc_domain,
+	.suspend	= gart_suspend,
+	.resume		= gart_resume,
+};
+
+static struct platform_driver tegra_iovmm_gart_drv = {
+	.probe		= gart_probe,
+	.remove		= gart_remove,
+	.driver		= {
+		.name	= DRIVER_NAME,
+	},
+};
+
+static int gart_suspend(struct tegra_iovmm_device *dev)
+{
+	struct gart_device *gart = container_of(dev, struct gart_device, iovmm);
+	unsigned int i;
+	unsigned long reg;
+
+	if (!gart)
+		return -ENODEV;
+
+	if (!gart->enable)
+		return 0;
+
+	spin_lock(&gart->pte_lock);
+	reg = gart->iovmm_base;
+	for (i = 0; i < gart->page_count; i++) {
+		writel(reg, gart->regs + GART_ENTRY_ADDR);
+		gart->savedata[i] = readl(gart->regs + GART_ENTRY_DATA);
+		dmb();
+		reg += GART_PAGE_SIZE;
+	}
+	spin_unlock(&gart->pte_lock);
+	return 0;
+}
+
+static void do_gart_setup(struct gart_device *gart, const u32 *data)
+{
+	unsigned long reg;
+	unsigned int i;
+
+	reg = gart->iovmm_base;
+	for (i = 0; i < gart->page_count; i++) {
+		__gart_set_pte(gart, reg, data ? data[i] : 0);
+		reg += GART_PAGE_SIZE;
+	}
+
+	writel(1, gart->regs + GART_CONFIG);
+}
+
+static void gart_resume(struct tegra_iovmm_device *dev)
+{
+	struct gart_device *gart = container_of(dev, struct gart_device, iovmm);
+
+	if (!gart || !gart->enable || (gart->enable && !gart->savedata))
+		return;
+
+	spin_lock(&gart->pte_lock);
+	do_gart_setup(gart, gart->savedata);
+	spin_unlock(&gart->pte_lock);
+}
+
+static int gart_remove(struct platform_device *pdev)
+{
+	struct gart_device *gart = platform_get_drvdata(pdev);
+
+	if (!gart)
+		return 0;
+
+	if (gart->enable)
+		writel(0, gart->regs + GART_CONFIG);
+
+	gart->enable = 0;
+	platform_set_drvdata(pdev, NULL);
+	tegra_iovmm_unregister(&gart->iovmm);
+	if (gart->savedata)
+		vfree(gart->savedata);
+	if (gart->regs)
+		iounmap(gart->regs);
+	kfree(gart);
+	return 0;
+}
+
+static int gart_probe(struct platform_device *pdev)
+{
+	struct gart_device *gart;
+	struct resource *res, *res_remap;
+	void __iomem *gart_regs;
+	int e;
+
+	if (!pdev) {
+		pr_err(DRIVER_NAME ": platform_device required\n");
+		return -ENODEV;
+	}
+
+	if (PAGE_SHIFT != GART_PAGE_SHIFT) {
+		pr_err(DRIVER_NAME ": GART and CPU page size must match\n");
+		return -ENXIO;
+	}
+
+	/* the GART memory aperture is required */
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	res_remap = platform_get_resource(pdev, IORESOURCE_MEM, 1);
+
+	if (!res || !res_remap) {
+		pr_err(DRIVER_NAME ": GART memory aperture expected\n");
+		return -ENXIO;
+	}
+	gart = kzalloc(sizeof(*gart), GFP_KERNEL);
+	if (!gart) {
+		pr_err(DRIVER_NAME ": failed to allocate tegra_iovmm_device\n");
+		return -ENOMEM;
+	}
+
+	gart_regs = ioremap_wc(res->start, res->end - res->start + 1);
+	if (!gart_regs) {
+		pr_err(DRIVER_NAME ": failed to remap GART registers\n");
+		e = -ENXIO;
+		goto fail;
+	}
+
+	gart->iovmm.name = VMM_NAME;
+	gart->iovmm.ops = &tegra_iovmm_gart_ops;
+	gart->iovmm.pgsize_bits = GART_PAGE_SHIFT;
+	spin_lock_init(&gart->pte_lock);
+
+	platform_set_drvdata(pdev, gart);
+
+	e = tegra_iovmm_register(&gart->iovmm);
+	if (e)
+		goto fail;
+
+	e = tegra_iovmm_domain_init(&gart->domain, &gart->iovmm,
+		(tegra_iovmm_addr_t)res_remap->start,
+		(tegra_iovmm_addr_t)res_remap->end + 1);
+	if (e)
+		goto fail;
+
+	gart->regs = gart_regs;
+	gart->iovmm_base = (tegra_iovmm_addr_t)res_remap->start;
+	gart->page_count = res_remap->end - res_remap->start + 1;
+	gart->page_count >>= GART_PAGE_SHIFT;
+
+	gart->savedata = vmalloc(sizeof(u32) * gart->page_count);
+	if (!gart->savedata) {
+		pr_err(DRIVER_NAME ": failed to allocate context save area\n");
+		e = -ENOMEM;
+		goto fail;
+	}
+
+	spin_lock(&gart->pte_lock);
+
+	do_gart_setup(gart, NULL);
+	gart->enable = 1;
+
+	spin_unlock(&gart->pte_lock);
+	return 0;
+
+fail:
+	if (gart_regs)
+		iounmap(gart_regs);
+	if (gart && gart->savedata)
+		vfree(gart->savedata);
+	kfree(gart);
+	return e;
+}
+
+static int __devinit gart_init(void)
+{
+	return platform_driver_register(&tegra_iovmm_gart_drv);
+}
+
+static void __exit gart_exit(void)
+{
+	return platform_driver_unregister(&tegra_iovmm_gart_drv);
+}
+
+#define GART_PTE(_pfn) (0x80000000ul | ((_pfn) << PAGE_SHIFT))
+
+
+static int gart_map(struct tegra_iovmm_domain *domain,
+	struct tegra_iovmm_area *iovma)
+{
+	struct gart_device *gart =
+		container_of(domain, struct gart_device, domain);
+	unsigned long gart_page, count;
+	unsigned int i;
+
+	gart_page = iovma->iovm_start;
+	count = iovma->iovm_length >> GART_PAGE_SHIFT;
+
+	for (i = 0; i < count; i++) {
+		unsigned long pfn;
+
+		pfn = iovma->ops->lock_makeresident(iovma, i << PAGE_SHIFT);
+		if (!pfn_valid(pfn))
+			goto fail;
+
+		gart_set_pte(gart, gart_page, GART_PTE(pfn));
+		gart_page += GART_PAGE_SIZE;
+	}
+
+	return 0;
+
+fail:
+	spin_lock(&gart->pte_lock);
+	while (i--) {
+		iovma->ops->release(iovma, i << PAGE_SHIFT);
+		gart_page -= GART_PAGE_SIZE;
+		__gart_set_pte(gart, gart_page, 0);
+	}
+	spin_unlock(&gart->pte_lock);
+
+	return -ENOMEM;
+}
+
+static void gart_unmap(struct tegra_iovmm_domain *domain,
+	struct tegra_iovmm_area *iovma, bool decommit)
+{
+	struct gart_device *gart =
+		container_of(domain, struct gart_device, domain);
+	unsigned long gart_page, count;
+	unsigned int i;
+
+	count = iovma->iovm_length >> GART_PAGE_SHIFT;
+	gart_page = iovma->iovm_start;
+
+	spin_lock(&gart->pte_lock);
+	for (i = 0; i < count; i++) {
+		if (iovma->ops && iovma->ops->release)
+			iovma->ops->release(iovma, i << PAGE_SHIFT);
+
+		__gart_set_pte(gart, gart_page, 0);
+		gart_page += GART_PAGE_SIZE;
+	}
+	spin_unlock(&gart->pte_lock);
+}
+
+static void gart_map_pfn(struct tegra_iovmm_domain *domain,
+	struct tegra_iovmm_area *iovma, tegra_iovmm_addr_t offs,
+	unsigned long pfn)
+{
+	struct gart_device *gart =
+		container_of(domain, struct gart_device, domain);
+
+	BUG_ON(!pfn_valid(pfn));
+
+	gart_set_pte(gart, offs, GART_PTE(pfn));
+}
+
+static struct tegra_iovmm_domain *gart_alloc_domain(
+	struct tegra_iovmm_device *dev, struct tegra_iovmm_client *client)
+{
+	struct gart_device *gart = container_of(dev, struct gart_device, iovmm);
+	return &gart->domain;
+}
+
+subsys_initcall(gart_init);
+module_exit(gart_exit);
-- 
1.7.0.4

Re: [PATCH 2/3] ARM: iommu: tegra2: Initial support for GART driver

[Thierry Reding]

[-- Attachment #1: Type: text/plain, Size: 13855 bytes --]

* hdoyu-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org wrote:
> From: Hiroshi DOYU <hdoyu-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> 
> Tegra 2 IOMMU H/W dependent part, GART (Graphics Address Relocation
> Table). This is one of the tegra_iovmm_device to register to the upper
> tegra IOVMM framework. This supports only single virtual address
> space(tegra_iovmm_domain), and manages a simple 1-to-1 mapping
> H/W translation pagetable.

I know I'm nitpicking, but this has inconsistent spelling for "Tegra" again.

> Signed-off-by: Hiroshi DOYU <hdoyu-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> Cc: Hiro Sugawara <hsugawara-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> Cc: Krishna Reddy <vdumpa-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org>
> ---
>  drivers/iommu/Kconfig      |   10 ++
>  drivers/iommu/Makefile     |    1 +
>  drivers/iommu/tegra-gart.c |  357 ++++++++++++++++++++++++++++++++++++++++++++
>  3 files changed, 368 insertions(+), 0 deletions(-)
>  create mode 100644 drivers/iommu/tegra-gart.c
> 
> diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
> index 487d1ee..7f342e8 100644
> --- a/drivers/iommu/Kconfig
> +++ b/drivers/iommu/Kconfig
> @@ -133,6 +133,16 @@ config OMAP_IOMMU_DEBUG
>  
>  
>  # Tegra IOMMU support
> +config TEGRA_IOVMM_GART
> +	bool "Enable I/O virtual memory manager for GART"
> +	depends on ARCH_TEGRA_2x_SOC
> +	default y
> +	select TEGRA_IOVMM
> +	help
> +	  Enables support for remapping discontiguous physical memory
> +	  shared with the operating system into contiguous I/O virtual
> +	  space through the GART (Graphics Address Relocation Table)
> +	  hardware included on Tegra SoCs.
>  
>  config TEGRA_IOVMM
>  	bool

I think you should move TEGRA_IOVMM_GART below TEGRA_IOVMM. Also, is it not
possible to build this as a module?

> diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
> index 365eefb..4b61b05 100644
> --- a/drivers/iommu/Makefile
> +++ b/drivers/iommu/Makefile
> @@ -8,3 +8,4 @@ obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o
>  obj-$(CONFIG_OMAP_IOVMM) += omap-iovmm.o
>  obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o
>  obj-$(CONFIG_TEGRA_IOVMM) += tegra-iovmm.o
> +obj-$(CONFIG_TEGRA_IOVMM_GART) += tegra-gart.o
> diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c
> new file mode 100644
> index 0000000..24f893b
> --- /dev/null
> +++ b/drivers/iommu/tegra-gart.c
> @@ -0,0 +1,357 @@
> +/*
> + * Tegra I/O VMM implementation for GART devices in Tegra and Tegra 2 series
> + * systems-on-a-chip.
> + *
> + * Copyright (c) 2010-2011, NVIDIA Corporation.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundation; either version 2 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful, but WITHOUT
> + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
> + * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
> + * more details.
> + *
> + * You should have received a copy of the GNU General Public License along
> + * with this program; if not, write to the Free Software Foundation, Inc.,
> + * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
> + */
> +
> +#include <linux/module.h>
> +#include <linux/platform_device.h>
> +#include <linux/spinlock.h>
> +#include <linux/slab.h>
> +#include <linux/vmalloc.h>
> +#include <linux/mm.h>
> +#include <linux/io.h>
> +
> +#include <asm/cacheflush.h>
> +
> +#include <mach/iovmm.h>
> +
> +#define GART_CONFIG		0x24
> +#define GART_ENTRY_ADDR		0x28
> +#define GART_ENTRY_DATA		0x2c
> +
> +#define VMM_NAME "iovmm-gart"
> +#define DRIVER_NAME "tegra_gart"
> +
> +#define GART_PAGE_SHIFT		12
> +#define GART_PAGE_SIZE		(1 << GART_PAGE_SHIFT)
> +#define GART_PAGE_MASK		(~(GART_PAGE_SIZE - 1))
> +
> +struct gart_device {
> +	void __iomem		*regs;
> +	u32			*savedata;
> +	u32			page_count; /* total remappable size */
> +	tegra_iovmm_addr_t	iovmm_base; /* offset to apply to vmm_area */
> +	spinlock_t		pte_lock;
> +	struct tegra_iovmm_device iovmm;
> +	struct tegra_iovmm_domain domain;
> +	bool			enable;
> +};
> +
> +static inline void __gart_set_pte(struct gart_device *gart,
> +				tegra_iovmm_addr_t offs, u32 pte)
> +{
> +	writel(offs, gart->regs + GART_ENTRY_ADDR);
> +	writel(pte, gart->regs + GART_ENTRY_DATA);
> +	/*
> +	 * Any interaction between any block on PPSB and a block on
> +	 * APB or AHB must have these barriers to ensure the APB/AHB
> +	 * bus transaction is complete before initiating activity on
> +	 * the PPSB block.
> +	 */
> +	wmb();
> +}
> +
> +static inline void gart_set_pte(struct gart_device *gart,
> +				tegra_iovmm_addr_t offs, u32 pte)
> +{
> +	spin_lock(&gart->pte_lock);
> +
> +	__gart_set_pte(gart, offs, pte);
> +
> +	spin_unlock(&gart->pte_lock);
> +}
> +
> +static int gart_map(struct tegra_iovmm_domain *, struct tegra_iovmm_area *);
> +static void gart_unmap(struct tegra_iovmm_domain *,
> +	struct tegra_iovmm_area *, bool);
> +static void gart_map_pfn(struct tegra_iovmm_domain *,
> +	struct tegra_iovmm_area *, tegra_iovmm_addr_t, unsigned long);
> +static struct tegra_iovmm_domain *gart_alloc_domain(
> +	struct tegra_iovmm_device *, struct tegra_iovmm_client *);
> +
> +static int gart_probe(struct platform_device *);
> +static int gart_remove(struct platform_device *);
> +static int gart_suspend(struct tegra_iovmm_device *dev);
> +static void gart_resume(struct tegra_iovmm_device *dev);
> +
> +
> +static struct tegra_iovmm_device_ops tegra_iovmm_gart_ops = {
> +	.map		= gart_map,
> +	.unmap		= gart_unmap,
> +	.map_pfn	= gart_map_pfn,
> +	.alloc_domain	= gart_alloc_domain,
> +	.suspend	= gart_suspend,
> +	.resume		= gart_resume,
> +};
> +
> +static struct platform_driver tegra_iovmm_gart_drv = {
> +	.probe		= gart_probe,
> +	.remove		= gart_remove,
> +	.driver		= {
> +		.name	= DRIVER_NAME,
> +	},
> +};
> +

You should probably move both of these structures to the end of the file so
you can get rid of the prototypes above.

> +static int gart_suspend(struct tegra_iovmm_device *dev)
> +{
> +	struct gart_device *gart = container_of(dev, struct gart_device, iovmm);
> +	unsigned int i;
> +	unsigned long reg;
> +
> +	if (!gart)
> +		return -ENODEV;

gart (or dev for that matter) can never actually be NULL here.

> +
> +	if (!gart->enable)
> +		return 0;
> +
> +	spin_lock(&gart->pte_lock);
> +	reg = gart->iovmm_base;
> +	for (i = 0; i < gart->page_count; i++) {
> +		writel(reg, gart->regs + GART_ENTRY_ADDR);
> +		gart->savedata[i] = readl(gart->regs + GART_ENTRY_DATA);
> +		dmb();
> +		reg += GART_PAGE_SIZE;
> +	}
> +	spin_unlock(&gart->pte_lock);
> +	return 0;
> +}
> +
> +static void do_gart_setup(struct gart_device *gart, const u32 *data)
> +{
> +	unsigned long reg;
> +	unsigned int i;
> +
> +	reg = gart->iovmm_base;
> +	for (i = 0; i < gart->page_count; i++) {
> +		__gart_set_pte(gart, reg, data ? data[i] : 0);
> +		reg += GART_PAGE_SIZE;
> +	}
> +
> +	writel(1, gart->regs + GART_CONFIG);
> +}
> +
> +static void gart_resume(struct tegra_iovmm_device *dev)
> +{
> +	struct gart_device *gart = container_of(dev, struct gart_device, iovmm);
> +
> +	if (!gart || !gart->enable || (gart->enable && !gart->savedata))
> +		return;

Same here.

> +
> +	spin_lock(&gart->pte_lock);
> +	do_gart_setup(gart, gart->savedata);
> +	spin_unlock(&gart->pte_lock);
> +}
> +
> +static int gart_remove(struct platform_device *pdev)
> +{
> +	struct gart_device *gart = platform_get_drvdata(pdev);
> +
> +	if (!gart)
> +		return 0;
> +

I don't think this can happen here either.

> +	if (gart->enable)
> +		writel(0, gart->regs + GART_CONFIG);
> +
> +	gart->enable = 0;
> +	platform_set_drvdata(pdev, NULL);
> +	tegra_iovmm_unregister(&gart->iovmm);
> +	if (gart->savedata)
> +		vfree(gart->savedata);
> +	if (gart->regs)
> +		iounmap(gart->regs);
> +	kfree(gart);
> +	return 0;
> +}
> +
> +static int gart_probe(struct platform_device *pdev)
> +{
> +	struct gart_device *gart;
> +	struct resource *res, *res_remap;
> +	void __iomem *gart_regs;
> +	int e;
> +
> +	if (!pdev) {
> +		pr_err(DRIVER_NAME ": platform_device required\n");
> +		return -ENODEV;
> +	}

This check can be dropped.

> +
> +	if (PAGE_SHIFT != GART_PAGE_SHIFT) {
> +		pr_err(DRIVER_NAME ": GART and CPU page size must match\n");
> +		return -ENXIO;
> +	}

This should probably be checked at build time using BUILD_BUG_ON().

> +
> +	/* the GART memory aperture is required */
> +	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
> +	res_remap = platform_get_resource(pdev, IORESOURCE_MEM, 1);
> +
> +	if (!res || !res_remap) {
> +		pr_err(DRIVER_NAME ": GART memory aperture expected\n");
> +		return -ENXIO;
> +	}
> +	gart = kzalloc(sizeof(*gart), GFP_KERNEL);
> +	if (!gart) {
> +		pr_err(DRIVER_NAME ": failed to allocate tegra_iovmm_device\n");
> +		return -ENOMEM;
> +	}
> +
> +	gart_regs = ioremap_wc(res->start, res->end - res->start + 1);

resource_size(res)

> +	if (!gart_regs) {
> +		pr_err(DRIVER_NAME ": failed to remap GART registers\n");
> +		e = -ENXIO;
> +		goto fail;
> +	}
> +
> +	gart->iovmm.name = VMM_NAME;
> +	gart->iovmm.ops = &tegra_iovmm_gart_ops;
> +	gart->iovmm.pgsize_bits = GART_PAGE_SHIFT;
> +	spin_lock_init(&gart->pte_lock);
> +
> +	platform_set_drvdata(pdev, gart);
> +
> +	e = tegra_iovmm_register(&gart->iovmm);
> +	if (e)
> +		goto fail;
> +
> +	e = tegra_iovmm_domain_init(&gart->domain, &gart->iovmm,
> +		(tegra_iovmm_addr_t)res_remap->start,
> +		(tegra_iovmm_addr_t)res_remap->end + 1);

Why "end + 1"?

> +	if (e)
> +		goto fail;
> +
> +	gart->regs = gart_regs;
> +	gart->iovmm_base = (tegra_iovmm_addr_t)res_remap->start;
> +	gart->page_count = res_remap->end - res_remap->start + 1;

resource_size(res_remap)

> +	gart->page_count >>= GART_PAGE_SHIFT;
> +
> +	gart->savedata = vmalloc(sizeof(u32) * gart->page_count);
> +	if (!gart->savedata) {
> +		pr_err(DRIVER_NAME ": failed to allocate context save area\n");
> +		e = -ENOMEM;
> +		goto fail;
> +	}
> +
> +	spin_lock(&gart->pte_lock);
> +
> +	do_gart_setup(gart, NULL);
> +	gart->enable = 1;
> +
> +	spin_unlock(&gart->pte_lock);

Is the lock required here? Nothing else should be using the device until
after the probe function returns 0.

Also, wouldn't it be safer to move this complete setup code before the call
to tegra_iovmm_register()?

> +	return 0;
> +
> +fail:
> +	if (gart_regs)
> +		iounmap(gart_regs);
> +	if (gart && gart->savedata)
> +		vfree(gart->savedata);
> +	kfree(gart);
> +	return e;
> +}
> +
> +static int __devinit gart_init(void)
> +{
> +	return platform_driver_register(&tegra_iovmm_gart_drv);
> +}
> +
> +static void __exit gart_exit(void)
> +{
> +	return platform_driver_unregister(&tegra_iovmm_gart_drv);
> +}

Gratuitous "return".

> +
> +#define GART_PTE(_pfn) (0x80000000ul | ((_pfn) << PAGE_SHIFT))
> +
> +
> +static int gart_map(struct tegra_iovmm_domain *domain,
> +	struct tegra_iovmm_area *iovma)
> +{
> +	struct gart_device *gart =
> +		container_of(domain, struct gart_device, domain);
> +	unsigned long gart_page, count;
> +	unsigned int i;
> +
> +	gart_page = iovma->iovm_start;
> +	count = iovma->iovm_length >> GART_PAGE_SHIFT;
> +
> +	for (i = 0; i < count; i++) {
> +		unsigned long pfn;
> +
> +		pfn = iovma->ops->lock_makeresident(iovma, i << PAGE_SHIFT);
> +		if (!pfn_valid(pfn))
> +			goto fail;
> +
> +		gart_set_pte(gart, gart_page, GART_PTE(pfn));
> +		gart_page += GART_PAGE_SIZE;
> +	}
> +
> +	return 0;
> +
> +fail:
> +	spin_lock(&gart->pte_lock);
> +	while (i--) {
> +		iovma->ops->release(iovma, i << PAGE_SHIFT);
> +		gart_page -= GART_PAGE_SIZE;
> +		__gart_set_pte(gart, gart_page, 0);
> +	}
> +	spin_unlock(&gart->pte_lock);
> +
> +	return -ENOMEM;
> +}
> +
> +static void gart_unmap(struct tegra_iovmm_domain *domain,
> +	struct tegra_iovmm_area *iovma, bool decommit)
> +{
> +	struct gart_device *gart =
> +		container_of(domain, struct gart_device, domain);
> +	unsigned long gart_page, count;
> +	unsigned int i;
> +
> +	count = iovma->iovm_length >> GART_PAGE_SHIFT;
> +	gart_page = iovma->iovm_start;
> +
> +	spin_lock(&gart->pte_lock);
> +	for (i = 0; i < count; i++) {
> +		if (iovma->ops && iovma->ops->release)
> +			iovma->ops->release(iovma, i << PAGE_SHIFT);
> +
> +		__gart_set_pte(gart, gart_page, 0);
> +		gart_page += GART_PAGE_SIZE;
> +	}
> +	spin_unlock(&gart->pte_lock);
> +}
> +
> +static void gart_map_pfn(struct tegra_iovmm_domain *domain,
> +	struct tegra_iovmm_area *iovma, tegra_iovmm_addr_t offs,
> +	unsigned long pfn)
> +{
> +	struct gart_device *gart =
> +		container_of(domain, struct gart_device, domain);
> +
> +	BUG_ON(!pfn_valid(pfn));
> +
> +	gart_set_pte(gart, offs, GART_PTE(pfn));
> +}
> +
> +static struct tegra_iovmm_domain *gart_alloc_domain(
> +	struct tegra_iovmm_device *dev, struct tegra_iovmm_client *client)
> +{
> +	struct gart_device *gart = container_of(dev, struct gart_device, iovmm);
> +	return &gart->domain;
> +}

All of these functions should go before the platform_driver code.

> +
> +subsys_initcall(gart_init);

module_init()?

> +module_exit(gart_exit);
> -- 
> 1.7.0.4
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-tegra" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 
> 

[-- Attachment #2: Type: application/pgp-signature, Size: 198 bytes --]

Re: [PATCH 0/3] ARM: iommu: tegra: Add initial Tegra IOMMU driver

[KyongHo Cho]

On Thu, Nov 17, 2011 at 8:01 PM,  <hdoyu-DDmLM1+adcrQT0dZR+AlfA@public.gmane.org> wrote:
>
> Hiroshi DOYU (3):
>  ARM: iommu: tegra/common: Initial support for IOVMM driver
>  ARM: iommu: tegra2: Initial support for GART driver
>  ARM: iommu: tegra3: Initial support for SMMU driver
>

Hi.

I found that those patches are not IOMMU API implementations.
I wonder why they need to be located in drivers/iommu?

Regards,

  KyongHo

Re: [PATCH 0/3] ARM: iommu: tegra: Add initial Tegra IOMMU driver

[Joerg Roedel]

On Fri, Nov 18, 2011 at 07:19:53PM +0900, KyongHo Cho wrote:

> I found that those patches are not IOMMU API implementations.
> I wonder why they need to be located in drivers/iommu?

The current effort goes into creating a generic framework for IOMMUs.
These patches create their own framework which is totally the wrong
direction, they even implement their own version of an IOMMU-API.

I object against merging them until they are converted to the generic
IOMMU-API.


        Joerg

Re: [PATCH 0/3] ARM: iommu: tegra: Add initial Tegra IOMMU driver

[Hiroshi Doyu]

Hi Joerg,

On Fri, 18 Nov 2011 11:43:54 +0100
Joerg Roedel <joro-zLv9SwRftAIdnm+yROfE0A@public.gmane.org> wrote:

> On Fri, Nov 18, 2011 at 07:19:53PM +0900, KyongHo Cho wrote:
> 
> > I found that those patches are not IOMMU API implementations.
> > I wonder why they need to be located in drivers/iommu?
> 
> The current effort goes into creating a generic framework for IOMMUs.
> These patches create their own framework which is totally the wrong
> direction, they even implement their own version of an IOMMU-API.
> 
> I object against merging them until they are converted to the generic
> IOMMU-API.

Ok, I'll in v2 of this patchset.
That was a plan, mentioned in the cover letter of this patchset.

[PATCH 3/3] ARM: iommu: tegra3: Initial support for SMMU driver

[hdoyu]

From: Hiroshi DOYU <hdoyu@nvidia.com>

Tegra 3 IOMMU H/W dependent part, SMMU (System Memory Management
Unit). This is one of the tegra_iovmm_device to register to the upper
tegra IOVMM framework. This supports multiple virtual address
spaces(tegra_iovmm_domain x4), and manages 2 level H/W translation
pagetable.

Signed-off-by: Hiroshi DOYU <hdoyu@nvidia.com>
Cc: Hiro Sugawara <hsugawara@nvidia.com>
Cc: Krishna Reddy <vdumpa@nvidia.com>
---
 drivers/iommu/Kconfig      |   18 +
 drivers/iommu/Makefile     |    1 +
 drivers/iommu/tegra-smmu.c | 1358 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 1377 insertions(+), 0 deletions(-)
 create mode 100644 drivers/iommu/tegra-smmu.c

diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 7f342e8..3f67732 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -144,6 +144,24 @@ config TEGRA_IOVMM_GART
 	  space through the GART (Graphics Address Relocation Table)
 	  hardware included on Tegra SoCs.
 
+config TEGRA_IOVMM_SMMU
+	bool "Enable I/O virtual memory manager for SMMU"
+	depends on ARCH_TEGRA_3x_SOC
+	default y
+	select TEGRA_IOVMM
+	help
+	  Enables support for remapping discontiguous physical memory
+	  shared with the operating system into contiguous I/O virtual
+	  space through the SMMU (System Memory Management Unit)
+	  hardware included on Tegra SoCs.
+
+config TEGRA_IOVMM_SMMU_SYSFS
+	bool "Enable SMMU register access for debugging"
+	depends on TEGRA_IOVMM_SMMU
+	default n
+	help
+	  Enables SMMU register access through /sys/devices/smmu/* files.
+
 config TEGRA_IOVMM
 	bool
 
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 4b61b05..fa822ed 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -9,3 +9,4 @@ obj-$(CONFIG_OMAP_IOVMM) += omap-iovmm.o
 obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o
 obj-$(CONFIG_TEGRA_IOVMM) += tegra-iovmm.o
 obj-$(CONFIG_TEGRA_IOVMM_GART) += tegra-gart.o
+obj-$(CONFIG_TEGRA_IOVMM_SMMU) += tegra-smmu.o
diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
new file mode 100644
index 0000000..d1f7b1f
--- /dev/null
+++ b/drivers/iommu/tegra-smmu.c
@@ -0,0 +1,1358 @@
+/*
+ * Tegra I/O VMM implementation for SMMU devices for Tegra 3 series
+ * systems-on-a-chip.
+ *
+ * Copyright (c) 2010-2011, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+#include <linux/slab.h>
+#include <linux/vmalloc.h>
+#include <linux/mm.h>
+#include <linux/pagemap.h>
+#include <linux/sysfs.h>
+#include <linux/device.h>
+#include <linux/sched.h>
+#include <linux/io.h>
+
+#include <asm/page.h>
+#include <asm/cacheflush.h>
+
+#include <mach/iovmm.h>
+#include <mach/iomap.h>
+
+#define SMMU_CONFIG				0x10
+#define CONFIG_SMMU_ENABLE_DISABLE		0
+#define CONFIG_SMMU_ENABLE_ENABLE		1
+
+#define TLB_CONFIG				0x14
+#define TLB_CONFIG_TLB_STATS__MASK		(1 << 31)
+#define TLB_CONFIG_TLB_STATS__ENABLE		(1 << 31)
+#define TLB_CONFIG_TLB_HIT_UNDER_MISS__ENABLE	(1 << 29)
+#define TLB_CONFIG_TLB_ACTIVE_LINES__VALUE	0x10
+#define TLB_CONFIG_RESET_VAL			0x20000010
+
+#define PTC_CONFIG				0x18
+#define PTC_CONFIG_PTC_STATS__MASK		(1 << 31)
+#define PTC_CONFIG_PTC_STATS__ENABLE		(1 << 31)
+#define PTC_CONFIG_PTC_CACHE__ENABLE		(1 << 29)
+#define PTC_CONFIG_PTC_INDEX_MAP__PATTERN	0x3f
+#define PTC_CONFIG_RESET_VAL			0x2000003f
+
+#define PTB_ASID				0x1c
+#define PTB_ASID_CURRENT_ASID_SHIFT		0
+
+#define PTB_DATA				0x20
+#define PTB_DATA_RESET_VAL			0
+#define PTB_DATA_ASID_NONSECURE_SHIFT		29
+#define PTB_DATA_ASID_WRITABLE_SHIFT		30
+#define PTB_DATA_ASID_READABLE_SHIFT		31
+
+#define TLB_FLUSH				0x30
+#define TLB_FLUSH_TLB_FLUSH_VA_MATCH_ALL	0
+#define TLB_FLUSH_TLB_FLUSH_VA_MATCH_SECTION	2
+#define TLB_FLUSH_TLB_FLUSH_VA_MATCH_GROUP	3
+#define TLB_FLUSH_TLB_FLUSH_ASID_SHIFT		29
+#define TLB_FLUSH_TLB_FLUSH_ASID_MATCH_DISABLE	0
+#define TLB_FLUSH_TLB_FLUSH_ASID_MATCH_ENABLE	1
+#define TLB_FLUSH_TLB_FLUSH_ASID_MATCH_SHIFT	31
+
+#define PTC_FLUSH				0x34
+#define PTC_FLUSH_PTC_FLUSH_TYPE_ALL		0
+#define PTC_FLUSH_PTC_FLUSH_TYPE_ADR		1
+#define PTC_FLUSH_PTC_FLUSH_ADR_SHIFT		4
+
+#define ASID_SECURITY				0x38
+
+#define STATS_TLB_HIT_COUNT			0x1f0
+#define STATS_TLB_MISS_COUNT			0x1f4
+#define STATS_PTC_HIT_COUNT			0x1f8
+#define STATS_PTC_MISS_COUNT			0x1fc
+
+#define TRANSLATION_ENABLE_0			0x228
+#define TRANSLATION_ENABLE_1			0x22c
+#define TRANSLATION_ENABLE_2			0x230
+
+#define AFI_ASID              0x238   /* PCIE */
+#define AVPC_ASID             0x23c   /* AVP */
+#define DC_ASID               0x240   /* Display controller */
+#define DCB_ASID              0x244   /* Display controller B */
+#define EPP_ASID              0x248   /* Encoder pre-processor */
+#define G2_ASID               0x24c   /* 2D engine */
+#define HC_ASID               0x250   /* Host1x */
+#define HDA_ASID              0x254   /* High-def audio */
+#define ISP_ASID              0x258   /* Image signal processor */
+#define MPE_ASID              0x264   /* MPEG encoder */
+#define NV_ASID               0x268   /* (3D) */
+#define NV2_ASID              0x26c   /* (3D) */
+#define PPCS_ASID             0x270   /* AHB */
+#define SATA_ASID             0x278   /* SATA */
+#define VDE_ASID              0x27c   /* Video decoder */
+#define VI_ASID               0x280   /* Video input */
+
+#define SMMU_PDE_NEXT_SHIFT		28
+
+/* AHB Arbiter Registers */
+#define XBAR_CTRL				0xe0
+#define XBAR_CTRL_SMMU_INIT_DONE_DONE		1
+#define XBAR_CTRL_SMMU_INIT_DONE_SHIFT	17
+
+#define NUM_ASIDS	4
+#define TLB_FLUSH_TLB_FLUSH_VA_SECTION__MASK		0xffc00000
+#define TLB_FLUSH_TLB_FLUSH_VA_SECTION__SHIFT	12 /* right shift */
+#define TLB_FLUSH_TLB_FLUSH_VA_GROUP__MASK		0xffffc000
+#define TLB_FLUSH_TLB_FLUSH_VA_GROUP__SHIFT	12 /* right shift */
+#define TLB_FLUSH_TLB_FLUSH_VA(iova, which)	\
+	((((iova) & TLB_FLUSH_TLB_FLUSH_VA_##which##__MASK) >> \
+		TLB_FLUSH_TLB_FLUSH_VA_##which##__SHIFT) |	\
+	TLB_FLUSH_TLB_FLUSH_VA_MATCH_##which)
+#define PTB_ASID_CURRENT_ASID(n)	\
+		((n) << PTB_ASID_CURRENT_ASID_SHIFT)
+#define TLB_FLUSH_TLB_FLUSH_ASID_MATCH_disable		\
+		(TLB_FLUSH_TLB_FLUSH_ASID_MATCH_DISABLE <<	\
+			TLB_FLUSH_TLB_FLUSH_ASID_MATCH_SHIFT)
+#define TLB_FLUSH_TLB_FLUSH_ASID_MATCH__ENABLE		\
+		(TLB_FLUSH_TLB_FLUSH_ASID_MATCH_ENABLE <<	\
+			TLB_FLUSH_TLB_FLUSH_ASID_MATCH_SHIFT)
+
+#define VMM_NAME "iovmm-smmu"
+#define DRIVER_NAME "tegra_smmu"
+
+#define SMMU_PAGE_SHIFT 12
+#define SMMU_PAGE_SIZE	(1 << SMMU_PAGE_SHIFT)
+
+#define SMMU_PDIR_COUNT	1024
+#define SMMU_PDIR_SIZE	(sizeof(unsigned long) * SMMU_PDIR_COUNT)
+#define SMMU_PTBL_COUNT	1024
+#define SMMU_PTBL_SIZE	(sizeof(unsigned long) * SMMU_PTBL_COUNT)
+#define SMMU_PDIR_SHIFT	12
+#define SMMU_PDE_SHIFT	12
+#define SMMU_PTE_SHIFT	12
+#define SMMU_PFN_MASK	0x000fffff
+
+#define SMMU_ADDR_TO_PFN(addr)	((addr) >> 12)
+#define SMMU_ADDR_TO_PDN(addr)	((addr) >> 22)
+#define SMMU_PDN_TO_ADDR(addr)	((pdn) << 22)
+
+#define _READABLE	(1 << PTB_DATA_ASID_READABLE_SHIFT)
+#define _WRITABLE	(1 << PTB_DATA_ASID_WRITABLE_SHIFT)
+#define _NONSECURE	(1 << PTB_DATA_ASID_NONSECURE_SHIFT)
+#define _PDE_NEXT	(1 << SMMU_PDE_NEXT_SHIFT)
+#define _MASK_ATTR	(_READABLE | _WRITABLE | _NONSECURE)
+
+#define _PDIR_ATTR	(_READABLE | _WRITABLE | _NONSECURE)
+
+#define _PDE_ATTR	(_READABLE | _WRITABLE | _NONSECURE)
+#define _PDE_ATTR_N	(_PDE_ATTR | _PDE_NEXT)
+#define _PDE_VACANT(pdn)	(((pdn) << 10) | _PDE_ATTR)
+
+#define _PTE_ATTR	(_READABLE | _WRITABLE | _NONSECURE)
+#define _PTE_VACANT(addr)	(((addr) >> SMMU_PAGE_SHIFT) | _PTE_ATTR)
+
+#define SMMU_MK_PDIR(page, attr)	\
+		((page_to_phys(page) >> SMMU_PDIR_SHIFT) | (attr))
+#define SMMU_MK_PDE(page, attr)		\
+		(unsigned long)((page_to_phys(page) >> SMMU_PDE_SHIFT) | (attr))
+#define SMMU_EX_PTBL_PAGE(pde)		\
+		pfn_to_page((unsigned long)(pde) & SMMU_PFN_MASK)
+#define SMMU_PFN_TO_PTE(pfn, attr)	(unsigned long)((pfn) | (attr))
+
+#define SMMU_ASID_ENABLE(asid)	((asid) | (1 << 31))
+#define SMMU_ASID_DISABLE	0
+#define SMMU_ASID_ASID(n)	((n) & ~SMMU_ASID_ENABLE(0))
+
+/* Keep this as a "natural" enumeration (no assignments) */
+enum smmu_hwclient {
+	HWC_AFI,
+	HWC_AVPC,
+	HWC_DC,
+	HWC_DCB,
+	HWC_EPP,
+	HWC_G2,
+	HWC_HC,
+	HWC_HDA,
+	HWC_ISP,
+	HWC_MPE,
+	HWC_NV,
+	HWC_NV2,
+	HWC_PPCS,
+	HWC_SATA,
+	HWC_VDE,
+	HWC_VI,
+
+	HWC_COUNT
+};
+
+struct smmu_hwc_state {
+	unsigned long reg;
+	unsigned long enable_disable;
+};
+
+/* Hardware client mapping initializer */
+#define HWC_INIT(client)	\
+	[HWC_##client] = {client##_ASID, SMMU_ASID_DISABLE},
+
+static const struct smmu_hwc_state smmu_hwc_state_init[] = {
+	HWC_INIT(AFI)
+	HWC_INIT(AVPC)
+	HWC_INIT(DC)
+	HWC_INIT(DCB)
+	HWC_INIT(EPP)
+	HWC_INIT(G2)
+	HWC_INIT(HC)
+	HWC_INIT(HDA)
+	HWC_INIT(ISP)
+	HWC_INIT(MPE)
+	HWC_INIT(NV)
+	HWC_INIT(NV2)
+	HWC_INIT(PPCS)
+	HWC_INIT(SATA)
+	HWC_INIT(VDE)
+	HWC_INIT(VI)
+};
+
+
+struct domain_hwc_map {
+	const char *dev_name;
+	const enum smmu_hwclient *hwcs;
+	const unsigned int nr_hwcs;
+};
+
+/* Enable all hardware clients for SMMU translation */
+static const enum smmu_hwclient nvmap_hwcs[] = {
+	HWC_AFI,
+	HWC_AVPC,
+	HWC_DC,
+	HWC_DCB,
+	HWC_EPP,
+	HWC_G2,
+	HWC_HC,
+	HWC_HDA,
+	HWC_ISP,
+	HWC_MPE,
+	HWC_NV,
+	HWC_NV2,
+	HWC_PPCS,
+	HWC_SATA,
+	HWC_VDE,
+	HWC_VI
+};
+
+static const struct domain_hwc_map smmu_hwc_map[] = {
+	{
+		.dev_name = "nvmap",
+		.hwcs = nvmap_hwcs,
+		.nr_hwcs = ARRAY_SIZE(nvmap_hwcs),
+	},
+};
+
+/*
+ * Per address space
+ */
+struct smmu_as {
+	struct smmu_device	*smmu;	/* back pointer to container */
+	unsigned int		asid;
+	const struct domain_hwc_map	*hwclients;
+	struct mutex	lock;	/* for pagetable */
+	struct tegra_iovmm_domain domain;
+	struct page	*pdir_page;
+	unsigned long	pdir_attr;
+	unsigned long	pde_attr;
+	unsigned long	pte_attr;
+	unsigned int	*pte_count;
+	struct device	sysfs_dev;
+	int		sysfs_use_count;
+};
+
+/*
+ * Per SMMU device
+ */
+struct smmu_device {
+	void __iomem	*regs, *regs_ahbarb;
+	tegra_iovmm_addr_t	iovmm_base;	/* remappable base address */
+	unsigned long	page_count;		/* total remappable size */
+	spinlock_t	lock;
+	char		*name;
+	struct tegra_iovmm_device iovmm_dev;
+	int		num_ases;
+	struct smmu_as	*as;			/* Run-time allocated array */
+	struct smmu_hwc_state	hwc_state[HWC_COUNT];
+	struct device	sysfs_dev;
+	int		sysfs_use_count;
+	bool		enable;
+	struct page *avp_vector_page;	/* dummy page shared by all AS's */
+
+	/*
+	 * Register image savers for suspend/resume
+	 */
+	unsigned long translation_enable_0;
+	unsigned long translation_enable_1;
+	unsigned long translation_enable_2;
+	unsigned long asid_security;
+
+	unsigned long lowest_asid;	/* Variables for hardware testing */
+	unsigned long debug_asid;
+	unsigned long signature_pid;	/* For debugging aid */
+};
+
+
+/*
+ *	SMMU/AHB register accessors
+ */
+static inline u32 smmu_read(struct smmu_device *smmu, size_t offs)
+{
+	return readl(smmu->regs + offs);
+}
+static inline void smmu_write(struct smmu_device *smmu, u32 val, size_t offs)
+{
+	writel(val, smmu->regs + offs);
+}
+
+static inline u32 ahb_read(struct smmu_device *smmu, size_t offs)
+{
+	return readl(smmu->regs_ahbarb + offs);
+}
+static inline void ahb_write(struct smmu_device *smmu, u32 val, size_t offs)
+{
+	writel(val, smmu->regs_ahbarb + offs);
+}
+
+
+#define VA_PAGE_TO_PA(va, page)	\
+	(page_to_phys(page) + ((unsigned long)(va) & ~PAGE_MASK))
+
+#define FLUSH_CPU_DCACHE(va, page, size)	\
+	do {	\
+		unsigned long _pa_ = VA_PAGE_TO_PA(va, page);		\
+		__cpuc_flush_dcache_area((void *)(va), (size_t)(size));	\
+		outer_flush_range(_pa_, _pa_+(size_t)(size));		\
+	} while (0)
+
+/*
+ * Any interaction between any block on PPSB and a block on APB or AHB
+ * must have these read-back barriers to ensure the APB/AHB bus
+ * transaction is complete before initiating activity on the PPSB
+ * block.
+ */
+#define FLUSH_SMMU_REGS(smmu)	\
+	do { wmb(); smmu_read(smmu, SMMU_CONFIG); } while (0)
+
+/*
+ * Flush all TLB entries and all PTC entries
+ * Caller must lock smmu
+ */
+static void smmu_flush_regs(struct smmu_device *smmu, int enable)
+{
+	u32 l;
+
+	smmu_write(smmu, PTC_FLUSH_PTC_FLUSH_TYPE_ALL, PTC_FLUSH);
+	FLUSH_SMMU_REGS(smmu);
+	l = TLB_FLUSH_TLB_FLUSH_VA_MATCH_ALL |
+		TLB_FLUSH_TLB_FLUSH_ASID_MATCH_disable;
+	smmu_write(smmu, l, TLB_FLUSH);
+
+	if (enable)
+		smmu_write(smmu, CONFIG_SMMU_ENABLE_ENABLE, SMMU_CONFIG);
+	FLUSH_SMMU_REGS(smmu);
+}
+
+static void smmu_setup_regs(struct smmu_device *smmu)
+{
+	int i, asid;
+	u32 l;
+
+	/* Set/restore page directory for each AS */
+	for (asid = 0; asid < smmu->num_ases; asid++) {
+		struct smmu_as *as = &smmu->as[asid];
+
+		smmu_write(smmu, PTB_ASID_CURRENT_ASID(as->asid), PTB_ASID);
+		l = as->pdir_page ?
+			SMMU_MK_PDIR(as->pdir_page, as->pdir_attr) :
+			PTB_DATA_RESET_VAL;
+		smmu_write(smmu, l, PTB_DATA);
+	}
+
+	/* Set/restore ASID for each hardware client */
+	for (i = 0; i < HWC_COUNT; i++) {
+		struct smmu_hwc_state *hwcst = &smmu->hwc_state[i];
+		smmu_write(smmu, hwcst->enable_disable, hwcst->reg);
+	}
+
+	smmu_write(smmu, smmu->translation_enable_0, TRANSLATION_ENABLE_0);
+	smmu_write(smmu, smmu->translation_enable_1, TRANSLATION_ENABLE_1);
+	smmu_write(smmu, smmu->translation_enable_2, TRANSLATION_ENABLE_2);
+	smmu_write(smmu, smmu->asid_security, ASID_SECURITY);
+	smmu_write(smmu, TLB_CONFIG_RESET_VAL, TLB_CONFIG);
+	smmu_write(smmu, PTC_CONFIG_RESET_VAL, PTC_CONFIG);
+
+	smmu_flush_regs(smmu, 1);
+
+	l = ahb_read(smmu, XBAR_CTRL);
+	l |= XBAR_CTRL_SMMU_INIT_DONE_DONE <<
+		XBAR_CTRL_SMMU_INIT_DONE_SHIFT;
+	ahb_write(smmu, l, XBAR_CTRL);
+}
+
+static int smmu_suspend(struct tegra_iovmm_device *dev)
+{
+	struct smmu_device *smmu =
+		container_of(dev, struct smmu_device, iovmm_dev);
+
+	smmu->translation_enable_0 = smmu_read(smmu, TRANSLATION_ENABLE_0);
+	smmu->translation_enable_1 = smmu_read(smmu, TRANSLATION_ENABLE_1);
+	smmu->translation_enable_2 = smmu_read(smmu, TRANSLATION_ENABLE_2);
+	smmu->asid_security = smmu_read(smmu, ASID_SECURITY);
+
+	return 0;
+}
+
+static void smmu_resume(struct tegra_iovmm_device *dev)
+{
+	struct smmu_device *smmu =
+		container_of(dev, struct smmu_device, iovmm_dev);
+
+	if (!smmu->enable)
+		return;
+
+	spin_lock(&smmu->lock);
+	smmu_setup_regs(smmu);
+	spin_unlock(&smmu->lock);
+}
+
+static void flush_ptc_and_tlb(struct smmu_device *smmu,
+		struct smmu_as *as, unsigned long iova,
+		unsigned long *pte, struct page *ptpage, int is_pde)
+{
+	u32 l;
+	unsigned long tlb_flush_va = is_pde
+			?  TLB_FLUSH_TLB_FLUSH_VA(iova, SECTION)
+			:  TLB_FLUSH_TLB_FLUSH_VA(iova, GROUP);
+
+	l = PTC_FLUSH_PTC_FLUSH_TYPE_ADR | VA_PAGE_TO_PA(pte, ptpage);
+	smmu_write(smmu, l, PTC_FLUSH);
+	FLUSH_SMMU_REGS(smmu);
+	l = tlb_flush_va |
+		TLB_FLUSH_TLB_FLUSH_ASID_MATCH__ENABLE |
+		(as->asid << TLB_FLUSH_TLB_FLUSH_ASID_SHIFT);
+	smmu_write(smmu, l, TLB_FLUSH);
+	FLUSH_SMMU_REGS(smmu);
+}
+
+static void free_ptbl(struct smmu_as *as, unsigned long iova)
+{
+	unsigned long pdn = SMMU_ADDR_TO_PDN(iova);
+	unsigned long *pdir = (unsigned long *)kmap(as->pdir_page);
+
+	if (pdir[pdn] != _PDE_VACANT(pdn)) {
+		pr_debug("%s:%d pdn=%lx\n", __func__, __LINE__, pdn);
+
+		ClearPageReserved(SMMU_EX_PTBL_PAGE(pdir[pdn]));
+		__free_page(SMMU_EX_PTBL_PAGE(pdir[pdn]));
+		pdir[pdn] = _PDE_VACANT(pdn);
+		FLUSH_CPU_DCACHE(&pdir[pdn], as->pdir_page, sizeof pdir[pdn]);
+		flush_ptc_and_tlb(as->smmu, as, iova, &pdir[pdn],
+				as->pdir_page, 1);
+	}
+	kunmap(as->pdir_page);
+}
+
+static void free_pdir(struct smmu_as *as)
+{
+	if (as->pdir_page) {
+		unsigned addr = as->smmu->iovmm_base;
+		int count = as->smmu->page_count;
+
+		while (count-- > 0) {
+			free_ptbl(as, addr);
+			addr += SMMU_PAGE_SIZE * SMMU_PTBL_COUNT;
+		}
+		ClearPageReserved(as->pdir_page);
+		__free_page(as->pdir_page);
+		as->pdir_page = NULL;
+		kfree(as->pte_count);
+		as->pte_count = NULL;
+	}
+}
+
+static int smmu_remove(struct platform_device *pdev)
+{
+	struct smmu_device *smmu = platform_get_drvdata(pdev);
+
+	if (!smmu)
+		return 0;
+
+	if (smmu->enable) {
+		smmu_write(smmu, CONFIG_SMMU_ENABLE_DISABLE, SMMU_CONFIG);
+		smmu->enable = 0;
+	}
+	platform_set_drvdata(pdev, NULL);
+
+	if (smmu->as) {
+		int asid;
+
+		for (asid = 0; asid < smmu->num_ases; asid++)
+			free_pdir(&smmu->as[asid]);
+		kfree(smmu->as);
+	}
+
+	if (smmu->avp_vector_page)
+		__free_page(smmu->avp_vector_page);
+	if (smmu->regs)
+		iounmap(smmu->regs);
+	if (smmu->regs_ahbarb)
+		iounmap(smmu->regs_ahbarb);
+	tegra_iovmm_unregister(&smmu->iovmm_dev);
+	kfree(smmu);
+	return 0;
+}
+
+/*
+ * Maps PTBL for given iova and returns the PTE address
+ * Caller must unmap the mapped PTBL returned in *ptbl_page_p
+ */
+static unsigned long *locate_pte(struct smmu_as *as,
+		unsigned long iova, bool allocate,
+		struct page **ptbl_page_p,
+		unsigned int **pte_counter)
+{
+	unsigned long ptn = SMMU_ADDR_TO_PFN(iova);
+	unsigned long pdn = SMMU_ADDR_TO_PDN(iova);
+	unsigned long *pdir = kmap(as->pdir_page);
+	unsigned long *ptbl;
+
+	if (pdir[pdn] != _PDE_VACANT(pdn)) {
+		/* Mapped entry table already exists */
+		*ptbl_page_p = SMMU_EX_PTBL_PAGE(pdir[pdn]);
+		ptbl = kmap(*ptbl_page_p);
+	} else if (!allocate) {
+		kunmap(as->pdir_page);
+		return NULL;
+	} else {
+		/* Vacant - allocate a new page table */
+		pr_debug("%s:%d new PTBL pdn=%lx\n", __func__, __LINE__, pdn);
+
+		*ptbl_page_p = alloc_page(GFP_KERNEL | __GFP_DMA);
+		if (!*ptbl_page_p) {
+			kunmap(as->pdir_page);
+			pr_err(DRIVER_NAME
+			": failed to allocate tegra_iovmm_device page table\n");
+			return NULL;
+		}
+		SetPageReserved(*ptbl_page_p);
+		ptbl = (unsigned long *)kmap(*ptbl_page_p);
+		{
+			int pn;
+			unsigned long addr = SMMU_PDN_TO_ADDR(pdn);
+			for (pn = 0; pn < SMMU_PTBL_COUNT;
+				pn++, addr += SMMU_PAGE_SIZE) {
+				ptbl[pn] = _PTE_VACANT(addr);
+			}
+		}
+		FLUSH_CPU_DCACHE(ptbl, *ptbl_page_p, SMMU_PTBL_SIZE);
+		pdir[pdn] = SMMU_MK_PDE(*ptbl_page_p,
+				as->pde_attr | _PDE_NEXT);
+		FLUSH_CPU_DCACHE(&pdir[pdn], as->pdir_page, sizeof pdir[pdn]);
+		flush_ptc_and_tlb(as->smmu, as, iova, &pdir[pdn],
+				as->pdir_page, 1);
+	}
+	*pte_counter = &as->pte_count[pdn];
+
+	kunmap(as->pdir_page);
+	return &ptbl[ptn % SMMU_PTBL_COUNT];
+}
+
+static void put_signature(struct smmu_as *as,
+			unsigned long addr, unsigned long pfn)
+{
+	if (as->smmu->signature_pid == current->pid) {
+		struct page *page = pfn_to_page(pfn);
+		unsigned long *vaddr = kmap(page);
+		if (vaddr) {
+			vaddr[0] = addr;
+			vaddr[1] = pfn << PAGE_SHIFT;
+			FLUSH_CPU_DCACHE(vaddr, page, sizeof(vaddr[0]) * 2);
+			kunmap(page);
+		}
+	}
+}
+
+static int smmu_map(struct tegra_iovmm_domain *domain,
+		struct tegra_iovmm_area *iovma)
+{
+	struct smmu_as *as = container_of(domain, struct smmu_as, domain);
+	unsigned long addr = iovma->iovm_start;
+	unsigned long pcount = iovma->iovm_length >> SMMU_PAGE_SHIFT;
+	int i;
+
+	pr_debug("%s:%d iova=%lx asid=%d\n", __func__, __LINE__,
+		 addr, as - as->smmu->as);
+
+	for (i = 0; i < pcount; i++) {
+		unsigned long pfn;
+		unsigned long *pte;
+		unsigned int *pte_counter;
+		struct page *ptpage;
+
+		pfn = iovma->ops->lock_makeresident(iovma, i << PAGE_SHIFT);
+		if (!pfn_valid(pfn))
+			goto fail;
+
+		mutex_lock(&as->lock);
+
+		pte = locate_pte(as, addr, true, &ptpage, &pte_counter);
+		if (!pte)
+			goto fail2;
+
+		pr_debug("%s:%d iova=%lx pfn=%lx asid=%d\n",
+			 __func__, __LINE__, addr, pfn, as - as->smmu->as);
+
+		if (*pte == _PTE_VACANT(addr))
+			(*pte_counter)++;
+		*pte = SMMU_PFN_TO_PTE(pfn, as->pte_attr);
+		if (unlikely((*pte == _PTE_VACANT(addr))))
+			(*pte_counter)--;
+		FLUSH_CPU_DCACHE(pte, ptpage, sizeof *pte);
+		flush_ptc_and_tlb(as->smmu, as, addr, pte, ptpage, 0);
+		kunmap(ptpage);
+		mutex_unlock(&as->lock);
+		put_signature(as, addr, pfn);
+		addr += SMMU_PAGE_SIZE;
+	}
+	return 0;
+
+fail:
+	mutex_lock(&as->lock);
+fail2:
+
+	while (i-- > 0) {
+		unsigned long *pte;
+		unsigned int *pte_counter;
+		struct page *page;
+
+		iovma->ops->release(iovma, i<<PAGE_SHIFT);
+		addr -= SMMU_PAGE_SIZE;
+		pte = locate_pte(as, addr, false, &page, &pte_counter);
+		if (pte) {
+			if (*pte != _PTE_VACANT(addr)) {
+				*pte = _PTE_VACANT(addr);
+				FLUSH_CPU_DCACHE(pte, page, sizeof *pte);
+				flush_ptc_and_tlb(as->smmu, as, addr, pte,
+						page, 0);
+				kunmap(page);
+				if (!--(*pte_counter))
+					free_ptbl(as, addr);
+			} else {
+				kunmap(page);
+			}
+		}
+	}
+	mutex_unlock(&as->lock);
+	return -ENOMEM;
+}
+
+static void smmu_unmap(struct tegra_iovmm_domain *domain,
+	struct tegra_iovmm_area *iovma, bool decommit)
+{
+	struct smmu_as *as = container_of(domain, struct smmu_as, domain);
+	unsigned long addr = iovma->iovm_start;
+	unsigned int pcount = iovma->iovm_length >> SMMU_PAGE_SHIFT;
+	unsigned int i, *pte_counter;
+
+	pr_debug("%s:%d iova=%lx asid=%d\n", __func__, __LINE__,
+		 addr, as - as->smmu->as);
+
+	mutex_lock(&as->lock);
+	for (i = 0; i < pcount; i++) {
+		unsigned long *pte;
+		struct page *page;
+
+		if (iovma->ops && iovma->ops->release)
+			iovma->ops->release(iovma, i << PAGE_SHIFT);
+
+		pte = locate_pte(as, addr, false, &page, &pte_counter);
+		if (pte) {
+			if (*pte != _PTE_VACANT(addr)) {
+				*pte = _PTE_VACANT(addr);
+				FLUSH_CPU_DCACHE(pte, page, sizeof *pte);
+				flush_ptc_and_tlb(as->smmu, as, addr, pte,
+						page, 0);
+				kunmap(page);
+				if (!--(*pte_counter) && decommit) {
+					free_ptbl(as, addr);
+					smmu_flush_regs(as->smmu, 0);
+				}
+			}
+		}
+		addr += SMMU_PAGE_SIZE;
+	}
+	mutex_unlock(&as->lock);
+}
+
+static void smmu_map_pfn(struct tegra_iovmm_domain *domain,
+	struct tegra_iovmm_area *iovma, tegra_iovmm_addr_t addr,
+	unsigned long pfn)
+{
+	struct smmu_as *as = container_of(domain, struct smmu_as, domain);
+	struct smmu_device *smmu = as->smmu;
+	unsigned long *pte;
+	unsigned int *pte_counter;
+	struct page *ptpage;
+
+	pr_debug("%s:%d iova=%lx pfn=%lx asid=%d\n", __func__, __LINE__,
+		 (unsigned long)addr, pfn, as - as->smmu->as);
+
+	BUG_ON(!pfn_valid(pfn));
+	mutex_lock(&as->lock);
+	pte = locate_pte(as, addr, true, &ptpage, &pte_counter);
+	if (pte) {
+		if (*pte == _PTE_VACANT(addr))
+			(*pte_counter)++;
+		*pte = SMMU_PFN_TO_PTE(pfn, as->pte_attr);
+		if (unlikely((*pte == _PTE_VACANT(addr))))
+			(*pte_counter)--;
+		FLUSH_CPU_DCACHE(pte, ptpage, sizeof *pte);
+		flush_ptc_and_tlb(smmu, as, addr, pte, ptpage, 0);
+		kunmap(ptpage);
+		put_signature(as, addr, pfn);
+	}
+	mutex_unlock(&as->lock);
+}
+
+/*
+ * Caller must lock/unlock as
+ */
+static int alloc_pdir(struct smmu_as *as)
+{
+	unsigned long *pdir;
+	int pdn;
+	u32 l;
+	struct smmu_device *smmu = as->smmu;
+
+	if (as->pdir_page)
+		return 0;
+
+	as->pte_count = kzalloc(sizeof(as->pte_count[0]) * SMMU_PDIR_COUNT,
+				GFP_KERNEL);
+	if (!as->pte_count) {
+		pr_err(DRIVER_NAME
+		": failed to allocate tegra_iovmm_device PTE cunters\n");
+		return -ENOMEM;
+	}
+	as->pdir_page = alloc_page(GFP_KERNEL | __GFP_DMA);
+	if (!as->pdir_page) {
+		pr_err(DRIVER_NAME
+		": failed to allocate tegra_iovmm_device page directory\n");
+		kfree(as->pte_count);
+		as->pte_count = NULL;
+		return -ENOMEM;
+	}
+	SetPageReserved(as->pdir_page);
+	pdir = kmap(as->pdir_page);
+
+	for (pdn = 0; pdn < SMMU_PDIR_COUNT; pdn++)
+		pdir[pdn] = _PDE_VACANT(pdn);
+	FLUSH_CPU_DCACHE(pdir, as->pdir_page, SMMU_PDIR_SIZE);
+	l = PTC_FLUSH_PTC_FLUSH_TYPE_ADR | VA_PAGE_TO_PA(pdir, as->pdir_page);
+	smmu_write(smmu, l, PTC_FLUSH);
+	FLUSH_SMMU_REGS(as->smmu);
+	l = TLB_FLUSH_TLB_FLUSH_VA_MATCH_ALL |
+		TLB_FLUSH_TLB_FLUSH_ASID_MATCH__ENABLE |
+		(as->asid << TLB_FLUSH_TLB_FLUSH_ASID_SHIFT);
+	smmu_write(smmu, l, TLB_FLUSH);
+	FLUSH_SMMU_REGS(as->smmu);
+	kunmap(as->pdir_page);
+
+	return 0;
+}
+
+static void _sysfs_create(struct smmu_as *as, struct device *sysfs_parent);
+
+/*
+ * Allocate resources for an AS
+ *	TODO: split into "alloc" and "lock"
+ */
+static struct tegra_iovmm_domain *smmu_alloc_domain(
+	struct tegra_iovmm_device *dev, struct tegra_iovmm_client *client)
+{
+	struct smmu_device *smmu =
+		container_of(dev, struct smmu_device, iovmm_dev);
+	struct smmu_as *as = NULL;
+	const struct domain_hwc_map *map = NULL;
+	int asid, i;
+
+	/* Look for a free AS */
+	for  (asid = smmu->lowest_asid; asid < smmu->num_ases; asid++) {
+		mutex_lock(&smmu->as[asid].lock);
+		if (!smmu->as[asid].hwclients) {
+			as = &smmu->as[asid];
+			break;
+		}
+		mutex_unlock(&smmu->as[asid].lock);
+	}
+
+	if (!as) {
+		pr_err(DRIVER_NAME ": no free AS\n");
+		return NULL;
+	}
+
+	if (alloc_pdir(as) < 0)
+		goto bad3;
+
+	/* Look for a matching hardware client group */
+	for (i = 0; ARRAY_SIZE(smmu_hwc_map); i++) {
+		if (!strcmp(smmu_hwc_map[i].dev_name, client->misc_dev->name)) {
+			map = &smmu_hwc_map[i];
+			break;
+		}
+	}
+
+	if (!map) {
+		pr_err(DRIVER_NAME ": no SMMU resource for %s (%s)\n",
+			client->name, client->misc_dev->name);
+		goto bad2;
+	}
+
+	spin_lock(&smmu->lock);
+	/* Update PDIR register */
+	smmu_write(smmu, PTB_ASID_CURRENT_ASID(as->asid), PTB_ASID);
+	smmu_write(smmu, SMMU_MK_PDIR(as->pdir_page, as->pdir_attr), PTB_DATA);
+	FLUSH_SMMU_REGS(smmu);
+
+	/* Put each hardware client in the group into the address space */
+	for (i = 0; i < map->nr_hwcs; i++) {
+		struct smmu_hwc_state *hwcst = &smmu->hwc_state[map->hwcs[i]];
+
+		/* Is the hardware client busy? */
+		if (hwcst->enable_disable != SMMU_ASID_DISABLE &&
+			hwcst->enable_disable != SMMU_ASID_ENABLE(as->asid)) {
+			pr_err(DRIVER_NAME
+				": HW 0x%lx busy for ASID %ld (client!=%s)\n",
+				hwcst->reg,
+				SMMU_ASID_ASID(hwcst->enable_disable),
+				client->name);
+			goto bad;
+		}
+		hwcst->enable_disable = SMMU_ASID_ENABLE(as->asid);
+		smmu_write(smmu, hwcst->enable_disable, hwcst->reg);
+	}
+	FLUSH_SMMU_REGS(smmu);
+	spin_unlock(&smmu->lock);
+	as->hwclients = map;
+	_sysfs_create(as, client->misc_dev->this_device);
+	mutex_unlock(&as->lock);
+
+	/* Reserve "page zero" for AVP vectors using a common dummy page */
+	smmu_map_pfn(&as->domain, NULL, 0,
+		page_to_phys(as->smmu->avp_vector_page) >> SMMU_PAGE_SHIFT);
+	return &as->domain;
+
+bad:
+	/* Reset hardware clients that have been enabled */
+	while (--i >= 0) {
+		struct smmu_hwc_state *hwcst = &smmu->hwc_state[map->hwcs[i]];
+
+		hwcst->enable_disable = SMMU_ASID_DISABLE;
+		smmu_write(smmu, hwcst->enable_disable, hwcst->reg);
+	}
+	FLUSH_SMMU_REGS(smmu);
+	spin_unlock(&as->smmu->lock);
+bad2:
+	free_pdir(as);
+bad3:
+	mutex_unlock(&as->lock);
+	return NULL;
+
+}
+
+/*
+ * Release resources for an AS
+ *	TODO: split into "unlock" and "free"
+ */
+static void smmu_free_domain(
+	struct tegra_iovmm_domain *domain, struct tegra_iovmm_client *client)
+{
+	struct smmu_as *as = container_of(domain, struct smmu_as, domain);
+	struct smmu_device *smmu = as->smmu;
+	const struct domain_hwc_map *map = NULL;
+	int i;
+
+	mutex_lock(&as->lock);
+	map = as->hwclients;
+
+	spin_lock(&smmu->lock);
+	for (i = 0; i < map->nr_hwcs; i++) {
+		struct smmu_hwc_state *hwcst = &smmu->hwc_state[map->hwcs[i]];
+
+		hwcst->enable_disable = SMMU_ASID_DISABLE;
+		smmu_write(smmu, SMMU_ASID_DISABLE, hwcst->reg);
+	}
+	FLUSH_SMMU_REGS(smmu);
+	spin_unlock(&smmu->lock);
+
+	as->hwclients = NULL;
+	if (as->pdir_page) {
+		spin_lock(&smmu->lock);
+		smmu_write(smmu, PTB_ASID_CURRENT_ASID(as->asid), PTB_ASID);
+		smmu_write(smmu, PTB_DATA_RESET_VAL, PTB_DATA);
+		FLUSH_SMMU_REGS(smmu);
+		spin_unlock(&smmu->lock);
+
+		free_pdir(as);
+	}
+	mutex_unlock(&as->lock);
+}
+
+static struct tegra_iovmm_device_ops tegra_iovmm_smmu_ops = {
+	.map = smmu_map,
+	.unmap = smmu_unmap,
+	.map_pfn = smmu_map_pfn,
+	.alloc_domain = smmu_alloc_domain,
+	.free_domain = smmu_free_domain,
+	.suspend = smmu_suspend,
+	.resume = smmu_resume,
+};
+
+static int smmu_probe(struct platform_device *pdev)
+{
+	struct smmu_device *smmu;
+	struct resource *regs, *regs2, *window;
+	int e, asid;
+
+	if (!pdev) {
+		pr_err(DRIVER_NAME ": platform_device required\n");
+		return -ENODEV;
+	}
+
+	if (PAGE_SHIFT != SMMU_PAGE_SHIFT) {
+		pr_err(DRIVER_NAME ": SMMU and CPU page sizes must match\n");
+		return -ENXIO;
+	}
+
+	if (ARRAY_SIZE(smmu_hwc_state_init) != HWC_COUNT) {
+		pr_err(DRIVER_NAME
+			": sizeof smmu_hwc_state_init != enum smmu_hwclient\n");
+		return -ENXIO;
+	}
+
+	regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mc");
+	regs2 = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ahbarb");
+	window = platform_get_resource_byname(pdev, IORESOURCE_MEM, "window");
+	if (!regs || !regs2 || !window) {
+		pr_err(DRIVER_NAME ": No SMMU resources\n");
+		return -ENODEV;
+	}
+
+	smmu = kzalloc(sizeof(*smmu), GFP_KERNEL);
+	if (!smmu) {
+		pr_err(DRIVER_NAME ": failed to allocate smmu_device\n");
+		return -ENOMEM;
+	}
+
+	smmu->num_ases = NUM_ASIDS;
+	smmu->iovmm_base = (tegra_iovmm_addr_t)window->start;
+	smmu->page_count = (window->end + 1 - window->start) >> SMMU_PAGE_SHIFT;
+	smmu->regs = ioremap(regs->start, regs->end + 1 - regs->start);
+	smmu->regs_ahbarb =
+		ioremap(regs2->start, regs2->end + 1 - regs2->start);
+	if (!smmu->regs || !smmu->regs_ahbarb) {
+		pr_err(DRIVER_NAME ": failed to remap SMMU registers\n");
+		e = -ENXIO;
+		goto fail;
+	}
+
+	smmu->translation_enable_0 = ~0;
+	smmu->translation_enable_1 = ~0;
+	smmu->translation_enable_2 = ~0;
+	smmu->asid_security        = 0;
+
+	memcpy(smmu->hwc_state, smmu_hwc_state_init, sizeof(smmu->hwc_state));
+
+	smmu->iovmm_dev.name = VMM_NAME;
+	smmu->iovmm_dev.ops = &tegra_iovmm_smmu_ops;
+	smmu->iovmm_dev.pgsize_bits = SMMU_PAGE_SHIFT;
+
+	e = tegra_iovmm_register(&smmu->iovmm_dev);
+	if (e)
+		goto fail;
+
+	smmu->as = kzalloc(sizeof(smmu->as[0]) * smmu->num_ases, GFP_KERNEL);
+	if (!smmu->as) {
+		pr_err(DRIVER_NAME ": failed to allocate smmu_as\n");
+		e = -ENOMEM;
+		goto fail;
+	}
+
+	/* Initialize address space structure array */
+	for (asid = 0; asid < smmu->num_ases; asid++) {
+		struct smmu_as *as = &smmu->as[asid];
+
+		as->smmu = smmu;
+		as->asid = asid;
+		as->pdir_attr = _PDIR_ATTR;
+		as->pde_attr  = _PDE_ATTR;
+		as->pte_attr  = _PTE_ATTR;
+
+		mutex_init(&as->lock);
+
+		e = tegra_iovmm_domain_init(&as->domain, &smmu->iovmm_dev,
+			smmu->iovmm_base,
+			smmu->iovmm_base +
+				(smmu->page_count << SMMU_PAGE_SHIFT));
+		if (e)
+			goto fail;
+	}
+	spin_lock_init(&smmu->lock);
+	smmu_setup_regs(smmu);
+	smmu->enable = 1;
+	platform_set_drvdata(pdev, smmu);
+
+	smmu->avp_vector_page = alloc_page(GFP_KERNEL);
+	if (!smmu->avp_vector_page)
+		goto fail;
+	return 0;
+
+fail:
+	if (smmu->avp_vector_page)
+		__free_page(smmu->avp_vector_page);
+	if (smmu->regs)
+		iounmap(smmu->regs);
+	if (smmu->regs_ahbarb)
+		iounmap(smmu->regs_ahbarb);
+	if (smmu && smmu->as) {
+		for (asid = 0; asid < smmu->num_ases; asid++) {
+			if (smmu->as[asid].pdir_page) {
+				ClearPageReserved(smmu->as[asid].pdir_page);
+				__free_page(smmu->as[asid].pdir_page);
+			}
+		}
+		kfree(smmu->as);
+	}
+	kfree(smmu);
+	return e;
+}
+
+static struct platform_driver tegra_iovmm_smmu_drv = {
+	.probe = smmu_probe,
+	.remove = smmu_remove,
+	.driver = {
+		.name = DRIVER_NAME,
+	},
+};
+
+static int __devinit smmu_init(void)
+{
+	return platform_driver_register(&tegra_iovmm_smmu_drv);
+}
+
+static void __exit smmu_exit(void)
+{
+	return platform_driver_unregister(&tegra_iovmm_smmu_drv);
+}
+
+subsys_initcall(smmu_init);
+module_exit(smmu_exit);
+
+/*
+ * SMMU-global sysfs interface for debugging
+ */
+static ssize_t _sysfs_show_reg(struct device *d,
+				struct device_attribute *da, char *buf);
+static ssize_t _sysfs_store_reg(struct device *d,
+				struct device_attribute *da, const char *buf,
+				size_t count);
+
+#define _NAME_MAP(_name)	{	\
+	.name = __stringify(_name),	\
+	.offset = _name,		\
+	.dev_attr = __ATTR(_name, S_IRUGO | S_IWUSR,	\
+			_sysfs_show_reg, _sysfs_store_reg)	\
+}
+
+static
+struct _reg_name_map {
+	const char *name;
+	unsigned	offset;
+	struct device_attribute	dev_attr;
+} _smmu_reg_name_map[] = {
+	_NAME_MAP(SMMU_CONFIG),
+	_NAME_MAP(TLB_CONFIG),
+	_NAME_MAP(PTC_CONFIG),
+	_NAME_MAP(PTB_ASID),
+	_NAME_MAP(PTB_DATA),
+	_NAME_MAP(TLB_FLUSH),
+	_NAME_MAP(PTC_FLUSH),
+	_NAME_MAP(ASID_SECURITY),
+	_NAME_MAP(STATS_TLB_HIT_COUNT),
+	_NAME_MAP(STATS_TLB_MISS_COUNT),
+	_NAME_MAP(STATS_PTC_HIT_COUNT),
+	_NAME_MAP(STATS_PTC_MISS_COUNT),
+	_NAME_MAP(TRANSLATION_ENABLE_0),
+	_NAME_MAP(TRANSLATION_ENABLE_1),
+	_NAME_MAP(TRANSLATION_ENABLE_2),
+	_NAME_MAP(AFI_ASID),
+	_NAME_MAP(AVPC_ASID),
+	_NAME_MAP(DC_ASID),
+	_NAME_MAP(DCB_ASID),
+	_NAME_MAP(EPP_ASID),
+	_NAME_MAP(G2_ASID),
+	_NAME_MAP(HC_ASID),
+	_NAME_MAP(HDA_ASID),
+	_NAME_MAP(ISP_ASID),
+	_NAME_MAP(MPE_ASID),
+	_NAME_MAP(NV_ASID),
+	_NAME_MAP(NV2_ASID),
+	_NAME_MAP(PPCS_ASID),
+	_NAME_MAP(SATA_ASID),
+	_NAME_MAP(VDE_ASID),
+	_NAME_MAP(VI_ASID),
+};
+
+static ssize_t lookup_reg(struct device_attribute *da)
+{
+	int i;
+	for (i = 0; i < ARRAY_SIZE(_smmu_reg_name_map); i++) {
+		if (!strcmp(_smmu_reg_name_map[i].name, da->attr.name))
+			return _smmu_reg_name_map[i].offset;
+	}
+	return -ENODEV;
+}
+
+static ssize_t _sysfs_show_reg(struct device *d,
+					struct device_attribute *da, char *buf)
+{
+	struct smmu_device *smmu =
+		container_of(d, struct smmu_device, sysfs_dev);
+	ssize_t offset = lookup_reg(da);
+
+	if (offset < 0)
+		return offset;
+	return sprintf(buf, "%08lx\n", (unsigned long)smmu_read(smmu, offset));
+}
+
+static ssize_t _sysfs_store_reg(struct device *d,
+			struct device_attribute *da,
+			const char *buf, size_t count)
+{
+	struct smmu_device *smmu =
+		container_of(d, struct smmu_device, sysfs_dev);
+	ssize_t offset = lookup_reg(da);
+	u32 value;
+	int err;
+
+	if (offset < 0)
+		return offset;
+
+	err = kstrtou32(buf, 16, &value);
+	if (err)
+		return err;
+
+#ifdef CONFIG_TEGRA_IOVMM_SMMU_SYSFS
+	smmu_write(smmu, value, offset);
+#else
+	/* Allow writing to reg only for TLB/PTC stats enabling/disabling */
+	{
+		unsigned long mask = 0;
+		switch (offset) {
+		case TLB_CONFIG:
+			mask = TLB_CONFIG_TLB_STATS__MASK;
+			break;
+		case PTC_CONFIG:
+			mask = PTC_CONFIG_PTC_STATS__MASK;
+			break;
+		default:
+			break;
+		}
+
+		if (mask) {
+			unsigned long currval = smmu_read(smmu, offset);
+			currval &= ~mask;
+			value &= mask;
+			value |= currval;
+			smmu_write(smmu, value, offset);
+		}
+	}
+#endif
+	return count;
+}
+
+static ssize_t _sysfs_show_smmu(struct device *d,
+				struct device_attribute *da, char *buf)
+{
+	struct smmu_device *smmu =
+		container_of(d, struct smmu_device, sysfs_dev);
+	ssize_t	rv = 0;
+
+	rv += sprintf(buf + rv , "      regs: %p\n", smmu->regs);
+	rv += sprintf(buf + rv , "iovmm_base: %p\n", (void *)smmu->iovmm_base);
+	rv += sprintf(buf + rv , "page_count: %lx\n", smmu->page_count);
+	rv += sprintf(buf + rv , "  num_ases: %d\n", smmu->num_ases);
+	rv += sprintf(buf + rv , "        as: %p\n", smmu->as);
+	rv += sprintf(buf + rv , "    enable: %s\n",
+			smmu->enable ? "yes" : "no");
+	return rv;
+}
+
+static struct device_attribute _attr_show_smmu
+		 = __ATTR(show_smmu, S_IRUGO, _sysfs_show_smmu, NULL);
+
+#define _SYSFS_SHOW_VALUE(name, field, fmt)		\
+static ssize_t _sysfs_show_##name(struct device *d,	\
+	struct device_attribute *da, char *buf)		\
+{							\
+	struct smmu_device *smmu =			\
+		container_of(d, struct smmu_device, sysfs_dev);	\
+	ssize_t rv = 0;					\
+	rv += sprintf(buf + rv, fmt "\n", smmu->field);	\
+	return rv;					\
+}
+
+static void (*_sysfs_null_callback)(struct smmu_device *, unsigned long *) =
+	NULL;
+
+#define _SYSFS_SET_VALUE_DO(name, field, base, ceil, callback)	\
+static ssize_t _sysfs_set_##name(struct device *d,		\
+		struct device_attribute *da, const char *buf, size_t count) \
+{								\
+	int err;						\
+	u32 value;						\
+	struct smmu_device *smmu =				\
+		container_of(d, struct smmu_device, sysfs_dev);	\
+	err = kstrtou32(buf, base, &value);			\
+	if (err)						\
+		return err;					\
+	if (0 <= value && value < ceil) {			\
+		smmu->field = value;				\
+		if (callback)					\
+			callback(smmu, &smmu->field);		\
+	}							\
+	return count;						\
+}
+#ifdef CONFIG_TEGRA_IOVMM_SMMU_SYSFS
+#define _SYSFS_SET_VALUE	_SYSFS_SET_VALUE_DO
+#else
+#define _SYSFS_SET_VALUE(name, field, base, ceil, callback)	\
+static ssize_t _sysfs_set_##name(struct device *d,		\
+		struct device_attribute *da, const char *buf, size_t count) \
+{								\
+	return count;						\
+}
+#endif
+
+_SYSFS_SHOW_VALUE(lowest_asid, lowest_asid, "%lu")
+_SYSFS_SET_VALUE(lowest_asid, lowest_asid, 10,
+		NUM_ASIDS, _sysfs_null_callback)
+_SYSFS_SHOW_VALUE(debug_asid, debug_asid, "%lu")
+_SYSFS_SET_VALUE(debug_asid, debug_asid, 10,
+		NUM_ASIDS, _sysfs_null_callback)
+_SYSFS_SHOW_VALUE(signature_pid, signature_pid, "%lu")
+_SYSFS_SET_VALUE_DO(signature_pid, signature_pid, 10, PID_MAX_LIMIT + 1,
+		_sysfs_null_callback)
+
+#ifdef CONFIG_TEGRA_IOVMM_SMMU_SYSFS
+static void _sysfs_mask_attr(struct smmu_device *smmu, unsigned long *field)
+{
+	*field &= _MASK_ATTR;
+}
+
+static void _sysfs_mask_pdir_attr(struct smmu_device *smmu,
+	unsigned long *field)
+{
+	unsigned long pdir;
+
+	_sysfs_mask_attr(smmu, field);
+	smmu_write(smmu, PTB_ASID_CURRENT_ASID(smmu->debug_asid), PTB_ASID);
+	pdir = smmu_read(smmu, PTB_DATA);
+	pdir &= ~_MASK_ATTR;
+	pdir |= *field;
+	smmu_write(smmu, pdir, PTB_DATA);
+	FLUSH_SMMU_REGS(smmu);
+}
+
+static void (*_sysfs_mask_attr_callback)(struct smmu_device *,
+				unsigned long *field) = &_sysfs_mask_attr;
+static void (*_sysfs_mask_pdir_attr_callback)(struct smmu_device *,
+				unsigned long *field) = &_sysfs_mask_pdir_attr;
+#endif
+
+_SYSFS_SHOW_VALUE(pdir_attr, as[smmu->debug_asid].pdir_attr, "%lx")
+_SYSFS_SET_VALUE(pdir_attr, as[smmu->debug_asid].pdir_attr, 16,
+		_PDIR_ATTR + 1, _sysfs_mask_pdir_attr_callback)
+_SYSFS_SHOW_VALUE(pde_attr, as[smmu->debug_asid].pde_attr, "%lx")
+_SYSFS_SET_VALUE(pde_attr, as[smmu->debug_asid].pde_attr, 16,
+		_PDE_ATTR + 1, _sysfs_mask_attr_callback)
+_SYSFS_SHOW_VALUE(pte_attr, as[smmu->debug_asid].pte_attr, "%lx")
+_SYSFS_SET_VALUE(pte_attr, as[smmu->debug_asid].pte_attr, 16,
+		_PTE_ATTR + 1, _sysfs_mask_attr_callback)
+
+static struct device_attribute _attr_values[] = {
+	__ATTR(lowest_asid, S_IRUGO | S_IWUSR,
+		_sysfs_show_lowest_asid, _sysfs_set_lowest_asid),
+	__ATTR(debug_asid, S_IRUGO | S_IWUSR,
+		_sysfs_show_debug_asid, _sysfs_set_debug_asid),
+	__ATTR(signature_pid, S_IRUGO | S_IWUSR,
+		_sysfs_show_signature_pid, _sysfs_set_signature_pid),
+
+	__ATTR(pdir_attr, S_IRUGO | S_IWUSR,
+		_sysfs_show_pdir_attr, _sysfs_set_pdir_attr),
+	__ATTR(pde_attr, S_IRUGO | S_IWUSR,
+		_sysfs_show_pde_attr, _sysfs_set_pde_attr),
+	__ATTR(pte_attr, S_IRUGO | S_IWUSR,
+		_sysfs_show_pte_attr, _sysfs_set_pte_attr),
+};
+
+static struct attribute *_smmu_attrs[
+	ARRAY_SIZE(_smmu_reg_name_map) + ARRAY_SIZE(_attr_values) + 3];
+static struct attribute_group _smmu_attr_group = {
+	.attrs = _smmu_attrs
+};
+
+static void _sysfs_smmu(struct smmu_device *smmu, struct device *parent)
+{
+	int i, j;
+
+	if (smmu->sysfs_use_count++ > 0)
+		return;
+	for (i = 0; i < ARRAY_SIZE(_smmu_reg_name_map); i++)
+		_smmu_attrs[i] = &_smmu_reg_name_map[i].dev_attr.attr;
+	for (j = 0; j < ARRAY_SIZE(_attr_values); j++)
+		_smmu_attrs[i++] = &_attr_values[j].attr;
+	_smmu_attrs[i++] = &_attr_show_smmu.attr;
+	_smmu_attrs[i] = NULL;
+
+	dev_set_name(&smmu->sysfs_dev, "smmu");
+	smmu->sysfs_dev.parent = parent;
+	smmu->sysfs_dev.driver = NULL;
+	smmu->sysfs_dev.release = NULL;
+	if (device_register(&smmu->sysfs_dev)) {
+		pr_err("%s: failed to register smmu_sysfs_dev\n", __func__);
+		smmu->sysfs_use_count--;
+		return;
+	}
+	if (sysfs_create_group(&smmu->sysfs_dev.kobj, &_smmu_attr_group)) {
+		pr_err("%s: failed to create group for smmu_sysfs_dev\n",
+			__func__);
+		smmu->sysfs_use_count--;
+		return;
+	}
+}
+
+static void _sysfs_create(struct smmu_as *as, struct device *parent)
+{
+	_sysfs_smmu(as->smmu, parent);
+}
-- 
1.7.0.4