[PATCH v10 3/3] iommu/exynos: Add iommu driver for Exynos Platforms

[PATCH v10 3/3] iommu/exynos: Add iommu driver for Exynos Platforms

[KyongHo Cho]

HAALgBjAGgAbwBAAHMAYQBtAHMAdQBuAGcALgBjAG8AbQA=;Tue,
 06 Mar 2012 08:31:29 GMT;WwBQAEEAVABDAEgAIAB2ADEAMAAgADMALwAzAF0AIABpAG8AbQBtAHUALwBlAHgAeQBuAG8AcwA6ACAAQQBkAGQAIABpAG8AbQBtAHUAIABkAHIAaQB2AGUAcgAgAGYAbwByACAARQB4AHkAbgBvAHMAIABQAGwAYQB0AGYAbwByAG0AcwA=
x-cr-puzzleid: {CF0AAF04-8639-4D69-B4E1-BF71EA1B0A70}

This is the System MMU driver and IOMMU API implementation for
Exynos SOC platforms. Exynos platforms has more than 10 System
MMUs dedicated for each multimedia accellerators.

The System MMU driver is already in arc/arm/plat-s5p but it is
moved to drivers/iommu due to Ohad Ben-Cohen gathered IOMMU drivers
there

Any device driver in Exynos platforms that needs to control its
System MMU must call platform_set_sysmmu() to inform System MMU
driver who will control it.
platform_set_sysmmu() is defined in <mach/sysmmu.h>

Cc: Joerg Roedel <joerg.roedel@amd.com>
Cc: Kukjin Kim <kgene.kim@samsung.com>
Signed-off-by: KyongHo Cho <pullip.cho@samsung.com>
---
 drivers/iommu/Kconfig        |   21 +
 drivers/iommu/Makefile       |    1 +
 drivers/iommu/exynos-iommu.c | 1088 ++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 1110 insertions(+), 0 deletions(-)
 create mode 100644 drivers/iommu/exynos-iommu.c

diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index 6bea696..25d3eed 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -142,4 +142,25 @@ config OMAP_IOMMU_DEBUG
 
          Say N unless you know you need this.
 
+config EXYNOS_IOMMU
+	bool "Exynos IOMMU Support"
+	depends on EXYNOS_DEV_SYSMMU
+	select IOMMU_API
+	help
+	  Support for the IOMMU(System MMU) of Samsung Exynos application
+	  processor family. This enables H/W multimedia accellerators to see
+	  non-linear physical memory chunks as a linear memory in their
+	  address spaces
+
+	  If unsure, say N here.
+
+config EXYNOS_IOMMU_DEBUG
+	bool "Debugging log for Exynos IOMMU"
+	depends on EXYNOS_IOMMU
+	help
+	  Select this to see the detailed log message that shows what
+	  happens in the IOMMU driver
+
+	  Say N unless you need kernel log message for IOMMU debugging
+
 endif # IOMMU_SUPPORT
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 0e36b49..5a8fd97 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -8,3 +8,4 @@ obj-$(CONFIG_IRQ_REMAP) += intr_remapping.o
 obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o
 obj-$(CONFIG_OMAP_IOVMM) += omap-iovmm.o
 obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o
+obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o
diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
new file mode 100644
index 0000000..4c1c48a
--- /dev/null
+++ b/drivers/iommu/exynos-iommu.c
@@ -0,0 +1,1088 @@
+/* linux/drivers/iommu/exynos_iommu.c
+ *
+ * Copyright (c) 2011 Samsung Electronics Co., Ltd.
+ *		http://www.samsung.com
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifdef CONFIG_EXYNOS_IOMMU_DEBUG
+#define DEBUG
+#endif
+
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/pm_runtime.h>
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/mm.h>
+#include <linux/iommu.h>
+#include <linux/errno.h>
+#include <linux/list.h>
+#include <linux/memblock.h>
+#include <linux/export.h>
+
+#include <asm/cacheflush.h>
+#include <asm/pgtable.h>
+
+#include <mach/sysmmu.h>
+
+#define S5P_MMU_CTRL			0x000
+#define S5P_MMU_CFG			0x004
+#define S5P_MMU_STATUS			0x008
+#define S5P_MMU_FLUSH			0x00C
+#define S5P_PT_BASE_ADDR		0x014
+#define S5P_INT_STATUS			0x018
+#define S5P_INT_CLEAR			0x01C
+#define S5P_PAGE_FAULT_ADDR		0x024
+#define S5P_AW_FAULT_ADDR		0x028
+#define S5P_AR_FAULT_ADDR		0x02C
+#define S5P_DEFAULT_SLAVE_ADDR		0x030
+#define S5P_MMU_VERSION			0x034
+#define S5P_PB0_SADDR			0x04C
+#define S5P_PB0_EADDR			0x050
+#define S5P_PB1_SADDR			0x054
+#define S5P_PB1_EADDR			0x058
+
+#define SECT_ORDER 20
+#define LPAGE_ORDER 16
+#define SPAGE_ORDER 12
+
+#define SECT_SIZE (1 << SECT_ORDER)
+#define LPAGE_SIZE (1 << LPAGE_ORDER)
+#define SPAGE_SIZE (1 << SPAGE_ORDER)
+
+#define SECT_MASK (~(SECT_SIZE - 1))
+#define LPAGE_MASK (~(LPAGE_SIZE - 1))
+#define SPAGE_MASK (~(SPAGE_SIZE - 1))
+
+#define lv1ent_fault(sent) (((*(sent) & 3) == 0) || ((*(sent) & 3) == 3))
+#define lv1ent_page(sent) ((*(sent) & 3) == 1)
+#define lv1ent_section(sent) ((*(sent) & 3) == 2)
+
+#define lv2ent_fault(pent) ((*(pent) & 3) == 0)
+#define lv2ent_small(pent) ((*(pent) & 2) == 2)
+#define lv2ent_large(pent) ((*(pent) & 3) == 1)
+
+#define section_phys(sent) (*(sent) & SECT_MASK)
+#define section_offs(iova) ((iova) & 0xFFFFF)
+#define lpage_phys(pent) (*(pent) & LPAGE_MASK)
+#define lpage_offs(iova) ((iova) & 0xFFFF)
+#define spage_phys(pent) (*(pent) & SPAGE_MASK)
+#define spage_offs(iova) ((iova) & 0xFFF)
+
+#define lv1ent_offset(iova) ((iova) >> SECT_ORDER)
+#define lv2ent_offset(iova) (((iova) & 0xFF000) >> SPAGE_ORDER)
+
+#define NUM_LV1ENTRIES 4096
+#define NUM_LV2ENTRIES 256
+
+#define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(long))
+
+#define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE)
+
+#define lv2table_base(sent) (*(sent) & 0xFFFFFC00)
+
+#define mk_lv1ent_sect(pa) ((pa) | 2)
+#define mk_lv1ent_page(pa) ((pa) | 1)
+#define mk_lv2ent_lpage(pa) ((pa) | 1)
+#define mk_lv2ent_spage(pa) ((pa) | 2)
+
+#define CTRL_ENABLE	0x5
+#define CTRL_BLOCK	0x7
+#define CTRL_DISABLE	0x0
+
+#define REG_MMU_CTRL		0x000
+#define REG_MMU_CFG		0x004
+#define REG_MMU_STATUS		0x008
+#define REG_MMU_FLUSH		0x00C
+#define REG_MMU_FLUSH_ENTRY	0x010
+#define REG_PT_BASE_ADDR	0x014
+#define REG_INT_STATUS		0x018
+#define REG_INT_CLEAR		0x01C
+#define REG_MMU_VERSION		0x034
+
+#define REG_PAGE_FAULT_ADDR	0x024
+#define REG_AW_FAULT_ADDR	0x028
+#define REG_AR_FAULT_ADDR	0x02C
+#define REG_DEFAULT_SLAVE_ADDR	0x030
+
+static unsigned long *section_entry(unsigned long *pgtable, unsigned long iova)
+{
+	return pgtable + lv1ent_offset(iova);
+}
+
+static unsigned long *page_entry(unsigned long *sent, unsigned long iova)
+{
+	return (unsigned long *)__va(lv2table_base(sent)) + lv2ent_offset(iova);
+}
+
+enum EXYNOS_SYSMMU_INTERRUPT_TYPE {
+	SYSMMU_PAGEFAULT,
+	SYSMMU_AR_MULTIHIT,
+	SYSMMU_AW_MULTIHIT,
+	SYSMMU_BUSERROR,
+	SYSMMU_AR_SECURITY,
+	SYSMMU_AR_ACCESS,
+	SYSMMU_AW_SECURITY,
+	SYSMMU_AW_PROTECTION, /* 7 */
+	SYSMMU_FAULT_UNKNOWN,
+	SYSMMU_FAULTS_NUM
+};
+
+typedef int (*sysmmu_fault_handler_t)(enum EXYNOS_SYSMMU_INTERRUPT_TYPE itype,
+			unsigned long pgtable_base, unsigned long fault_addr);
+
+static unsigned short fault_reg_offset[SYSMMU_FAULTS_NUM] = {
+	REG_PAGE_FAULT_ADDR,
+	REG_AR_FAULT_ADDR,
+	REG_AW_FAULT_ADDR,
+	REG_DEFAULT_SLAVE_ADDR,
+	REG_AR_FAULT_ADDR,
+	REG_AR_FAULT_ADDR,
+	REG_AW_FAULT_ADDR,
+	REG_AW_FAULT_ADDR
+};
+
+static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = {
+	"PAGE FAULT",
+	"AR MULTI-HIT FAULT",
+	"AW MULTI-HIT FAULT",
+	"BUS ERROR",
+	"AR SECURITY PROTECTION FAULT",
+	"AR ACCESS PROTECTION FAULT",
+	"AW SECURITY PROTECTION FAULT",
+	"AW ACCESS PROTECTION FAULT",
+	"UNKNOWN FAULT"
+};
+
+/* We does not consider super section mapping (16MB) */
+struct iommu_client {
+	struct list_head node;
+	struct device *dev;
+	int refcnt;
+};
+
+struct exynos_iommu_domain {
+	struct list_head clients; /* list of iommu_client */
+	unsigned long *pgtable; /* lv1 page table, 16KB */
+	short *lv2entcnt; /* free lv2 entry counter for each section */
+	spinlock_t lock; /* lock for this structure and attached iommu_client */
+	spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */
+};
+
+struct sysmmu_drvdata {
+	struct device *sysmmu;
+	char *dbgname;
+	int nsfrs;
+	void __iomem **sfrbases;
+	struct clk *clk[2];
+	bool active;
+	rwlock_t lock;
+	struct iommu_domain *domain;
+	sysmmu_fault_handler_t fault_handler;
+	unsigned long pgtable;
+	struct iommu_client client;
+};
+
+static bool set_sysmmu_active(struct sysmmu_drvdata *data)
+{
+	/* return true if the System MMU was not active previously
+	   and it needs to be initialized */
+	if (WARN_ON_ONCE(data->active))
+		return false;
+	data->active = true;
+	return true;
+}
+
+static bool set_sysmmu_inactive(struct sysmmu_drvdata *data)
+{
+	/* return true if the System MMU is needed to be disabled */
+	if (WARN_ON_ONCE(!data->active))
+		return false;
+	data->active = false;
+	return true;
+}
+
+static bool is_sysmmu_active(struct sysmmu_drvdata *data)
+{
+	return data->active;
+}
+
+static void sysmmu_block(void __iomem *sfrbase)
+{
+	__raw_writel(CTRL_BLOCK, sfrbase + REG_MMU_CTRL);
+}
+
+static void sysmmu_unblock(void __iomem *sfrbase)
+{
+	__raw_writel(CTRL_ENABLE, sfrbase + REG_MMU_CTRL);
+}
+
+static void __sysmmu_tlb_invalidate(void __iomem *sfrbase)
+{
+	__raw_writel(0x1, sfrbase + REG_MMU_FLUSH);
+}
+
+static void __sysmmu_tlb_invalidate_entry(void __iomem *sfrbase,
+						unsigned long iova)
+{
+	__raw_writel((iova & SPAGE_MASK) | 1, sfrbase + REG_MMU_FLUSH_ENTRY);
+}
+
+static void __sysmmu_set_ptbase(void __iomem *sfrbase,
+				       unsigned long pgd)
+{
+	__raw_writel(0x1, sfrbase + REG_MMU_CFG); /* 16KB LV1, LRU */
+	__raw_writel(pgd, sfrbase + REG_PT_BASE_ADDR);
+
+	__sysmmu_tlb_invalidate(sfrbase);
+}
+
+static void __sysmmu_set_prefbuf(void __iomem *sfrbase, unsigned long base,
+						unsigned long size, int idx)
+{
+	__raw_writel(base, sfrbase + S5P_PB0_SADDR + idx * 8);
+	__raw_writel(size - 1 + base,  sfrbase + S5P_PB0_EADDR + idx * 8);
+}
+
+void exynos_sysmmu_set_prefbuf(struct device *dev,
+				unsigned long base0, unsigned long size0,
+				unsigned long base1, unsigned long size1)
+{
+	struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
+	unsigned long flags;
+	int i;
+
+	BUG_ON((base0 + size0) <= base0);
+	BUG_ON((size1 > 0) && ((base1 + size1) <= base1));
+
+	read_lock_irqsave(&data->lock, flags);
+	if (!is_sysmmu_active(data))
+		goto finish;
+
+	for (i = 0; i < data->nsfrs; i++) {
+		if ((readl(data->sfrbases[i] + S5P_MMU_VERSION) >> 28) == 3) {
+			sysmmu_block(data->sfrbases[i]);
+
+			if (size1 == 0) {
+				if (size0 <= SZ_128K) {
+					base1 = base0;
+					size1 = size0;
+				} else {
+					size1 = size0 -
+						ALIGN(size0 / 2, SZ_64K);
+					size0 = size0 - size1;
+					base1 = base0 + size0;
+				}
+			}
+
+			__sysmmu_set_prefbuf(
+					data->sfrbases[i], base0, size0, 0);
+			__sysmmu_set_prefbuf(
+					data->sfrbases[i], base1, size1, 1);
+
+			sysmmu_unblock(data->sfrbases[i]);
+		}
+	}
+finish:
+	read_unlock_irqrestore(&data->lock, flags);
+}
+
+static void __set_fault_handler(struct sysmmu_drvdata *data,
+					sysmmu_fault_handler_t handler)
+{
+	unsigned long flags;
+
+	write_lock_irqsave(&data->lock, flags);
+	data->fault_handler = handler;
+	write_unlock_irqrestore(&data->lock, flags);
+}
+
+void exynos_sysmmu_set_fault_handler(struct device *dev,
+					sysmmu_fault_handler_t handler)
+{
+	struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
+
+	__set_fault_handler(data, handler);
+}
+
+static int default_fault_handler(enum EXYNOS_SYSMMU_INTERRUPT_TYPE itype,
+		     unsigned long pgtable_base, unsigned long fault_addr)
+{
+	unsigned long *ent;
+
+	if ((itype >= SYSMMU_FAULTS_NUM) || (itype < SYSMMU_PAGEFAULT))
+		itype = SYSMMU_FAULT_UNKNOWN;
+
+	pr_err("%s occured@0x%lx(Page table base: 0x%lx)\n",
+			sysmmu_fault_name[itype], fault_addr, pgtable_base);
+
+	ent = section_entry(__va(pgtable_base), fault_addr);
+	pr_err("\tLv1 entry: 0x%lx\n", *ent);
+
+	if (lv1ent_page(ent)) {
+		ent = page_entry(ent, fault_addr);
+		pr_err("\t Lv2 entry: 0x%lx\n", *ent);
+	}
+
+	pr_err("Generating Kernel OOPS... because it is unrecoverable.\n");
+
+	BUG();
+
+	return 0;
+}
+
+static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
+{
+	/* SYSMMU is in blocked when interrupt occurred. */
+	struct sysmmu_drvdata *data = dev_id;
+	struct resource *irqres;
+	struct platform_device *pdev;
+	enum EXYNOS_SYSMMU_INTERRUPT_TYPE itype;
+	unsigned long addr = -1;
+
+	int i, ret = -ENOSYS;
+
+	read_lock(&data->lock);
+
+	WARN_ON(!is_sysmmu_active(data));
+
+	pdev = to_platform_device(data->sysmmu);
+	for (i = 0; i < pdev->num_resources; i++) {
+		irqres = platform_get_resource(pdev, IORESOURCE_IRQ, i);
+		if (irqres && ((int)irqres->start == irq))
+			break;
+	}
+
+	if (i == pdev->num_resources) {
+		itype = SYSMMU_FAULT_UNKNOWN;
+	} else {
+		i /= 2;
+
+		itype = (enum EXYNOS_SYSMMU_INTERRUPT_TYPE)
+			__ffs(__raw_readl(data->sfrbases[i] + REG_INT_STATUS));
+		if (WARN_ON(!((itype >= 0) && (itype < SYSMMU_FAULT_UNKNOWN))))
+			itype = SYSMMU_FAULT_UNKNOWN;
+		else
+			addr = __raw_readl(
+				data->sfrbases[i] + fault_reg_offset[itype]);
+	}
+
+	if (data->domain)
+		ret = report_iommu_fault(data->domain, data->client.dev,
+				addr, itype);
+
+	if ((ret == -ENOSYS) && data->fault_handler) {
+		unsigned long base = data->pgtable;
+		if (itype != SYSMMU_FAULT_UNKNOWN)
+			base = __raw_readl(
+					data->sfrbases[i] + REG_PT_BASE_ADDR);
+		ret = data->fault_handler(itype, base, addr);
+	}
+
+	if (!ret && (itype != SYSMMU_FAULT_UNKNOWN))
+		__raw_writel(1 << itype, data->sfrbases[i] + REG_INT_CLEAR);
+	else
+		dev_dbg(data->sysmmu, "(%s) %s is not handled.\n",
+				data->dbgname, sysmmu_fault_name[itype]);
+
+	if (itype != SYSMMU_FAULT_UNKNOWN)
+		sysmmu_unblock(data->sfrbases[i]);
+
+	read_unlock(&data->lock);
+
+	return IRQ_HANDLED;
+}
+
+static bool __sysmmu_disable(struct sysmmu_drvdata *data)
+{
+	unsigned long flags;
+	bool disabled = false;
+	int i;
+
+	write_lock_irqsave(&data->lock, flags);
+
+	if (!set_sysmmu_inactive(data))
+		goto finish;
+
+	for (i = 0; i < data->nsfrs; i++) {
+		__raw_writel(CTRL_DISABLE, data->sfrbases[i] + REG_MMU_CTRL);
+		if (data->clk[1])
+			clk_disable(data->clk[1]);
+		if (data->clk[0])
+			clk_disable(data->clk[0]);
+		disabled = true;
+		data->pgtable = 0;
+		data->domain = NULL;
+		pm_runtime_put(data->sysmmu);
+	}
+
+finish:
+	write_unlock_irqrestore(&data->lock, flags);
+
+	return disabled;
+}
+
+/* __exynos_sysmmu_enable: Enables System MMU
+ *
+ * returns -error if an error occurred and System MMU is not enabled,
+ * 0 if the System MMU has been just enabled and 1 if System MMU was already
+ * enabled before.
+ */
+static int __exynos_sysmmu_enable(struct device *dev, unsigned long pgtable,
+						struct iommu_domain *domain)
+{
+	int i, ret = 0;
+	unsigned long flags;
+	struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
+
+	BUG_ON(!memblock_is_memory(pgtable));
+
+	ret = pm_runtime_get_sync(data->sysmmu);
+	if (ret < 0)
+		return ret;
+
+	write_lock_irqsave(&data->lock, flags);
+
+	if (!set_sysmmu_active(data)) {
+		/* RPM of sysmmu won't be refcounted */
+		if (ret > 0)
+			pm_runtime_put(data->sysmmu);
+
+		if (WARN_ON(pgtable != data->pgtable)) {
+			set_sysmmu_inactive(data);
+			ret = -EBUSY;
+		}
+
+		ret = 1;
+		dev_dbg(data->sysmmu, "(%s) Already enabled\n", data->dbgname);
+		goto finish;
+	}
+
+	if (data->clk[0])
+		clk_enable(data->clk[0]);
+	if (data->clk[1])
+		clk_enable(data->clk[1]);
+
+	data->pgtable = pgtable;
+
+	for (i = 0; i < data->nsfrs; i++) {
+		__sysmmu_set_ptbase(data->sfrbases[i], pgtable);
+
+		if ((readl(data->sfrbases[i] + S5P_MMU_VERSION) >> 28) == 3) {
+			/* System MMU version is 3.x */
+			__raw_writel((1 << 12) | (2 << 28),
+					data->sfrbases[i] + S5P_MMU_CFG);
+			__sysmmu_set_prefbuf(data->sfrbases[i], 0, -1, 0);
+			__sysmmu_set_prefbuf(data->sfrbases[i], 0, -1, 1);
+		}
+
+		__raw_writel(CTRL_ENABLE, data->sfrbases[i] + REG_MMU_CTRL);
+	}
+
+	data->domain = domain;
+
+	dev_dbg(data->sysmmu, "(%s) Enabled\n", data->dbgname);
+finish:
+	write_unlock_irqrestore(&data->lock, flags);
+
+	if (ret < 0) {
+		__sysmmu_disable(data);
+		dev_dbg(data->sysmmu, "(%s) Failed to enable\n", data->dbgname);
+	}
+
+	return ret;
+}
+
+static struct iommu_client *__get_enabled_iommu_client(
+				struct iommu_domain *domain, struct device *dev)
+{
+	struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
+	struct exynos_iommu_domain *priv = domain->priv;
+	int ret;
+
+	ret = __exynos_sysmmu_enable(dev, __pa(priv->pgtable), domain);
+	if (ret < 0)
+		return ERR_PTR(ret);
+	else if (ret == 1)
+		return ERR_PTR(-EBUSY);
+
+	INIT_LIST_HEAD(&data->client.node);
+	data->client.dev = dev;
+	data->client.refcnt = 1;
+
+	return &data->client;
+}
+
+int exynos_sysmmu_enable(struct device *dev, unsigned long pgtable)
+{
+	return __exynos_sysmmu_enable(dev, pgtable, NULL);
+}
+
+void exynos_sysmmu_disable(struct device *dev)
+{
+	struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
+	bool disabled;
+
+	disabled = __sysmmu_disable(data);
+	dev_dbg(data->sysmmu,
+			disabled ? "(%s) Disabled.\n" :
+				"(%s) Deactivation request ignorred.\n",
+			data->dbgname);
+}
+
+static void sysmmu_tlb_invalidate_entry(struct device *dev, unsigned long iova)
+{
+	unsigned long flags;
+	struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
+
+	read_lock_irqsave(&data->lock, flags);
+
+	if (is_sysmmu_active(data)) {
+		int i;
+		for (i = 0; i < data->nsfrs; i++) {
+			sysmmu_block(data->sfrbases[i]);
+			__sysmmu_tlb_invalidate_entry(data->sfrbases[i], iova);
+			sysmmu_unblock(data->sfrbases[i]);
+		}
+	} else {
+		dev_dbg(data->sysmmu,
+			"(%s) Disabled. Skipping invalidating TLB.\n",
+			data->dbgname);
+	}
+
+	read_unlock_irqrestore(&data->lock, flags);
+}
+
+void exynos_sysmmu_tlb_invalidate(struct device *dev)
+{
+	unsigned long flags;
+	struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
+
+	read_lock_irqsave(&data->lock, flags);
+
+	if (is_sysmmu_active(data)) {
+		int i;
+		for (i = 0; i < data->nsfrs; i++) {
+			sysmmu_block(data->sfrbases[i]);
+			__sysmmu_tlb_invalidate(data->sfrbases[i]);
+			sysmmu_unblock(data->sfrbases[i]);
+		}
+	} else {
+		dev_dbg(data->sysmmu,
+			"(%s) Disabled. Skipping invalidating TLB.\n",
+			data->dbgname);
+	}
+
+	read_unlock_irqrestore(&data->lock, flags);
+}
+
+static int exynos_sysmmu_probe(struct platform_device *pdev)
+{
+	int i, ret;
+	struct device *dev;
+	struct sysmmu_drvdata *data;
+
+	dev = &pdev->dev;
+
+	data = kzalloc(sizeof(*data), GFP_KERNEL);
+	if (!data) {
+		dev_dbg(dev, "Not enough memory\n");
+		ret = -ENOMEM;
+		goto err_alloc;
+	}
+
+	ret = dev_set_drvdata(dev, data);
+	if (ret) {
+		dev_dbg(dev, "Unabled to initialize driver data\n");
+		goto err_init;
+	}
+
+	data->nsfrs = pdev->num_resources / 2;
+	data->sfrbases = kmalloc(sizeof(*data->sfrbases) * data->nsfrs,
+								GFP_KERNEL);
+	if (data->sfrbases == NULL) {
+		dev_dbg(dev, "Not enough memory\n");
+		ret = -ENOMEM;
+		goto err_init;
+	}
+
+	for (i = 0; i < data->nsfrs; i++) {
+		struct resource *res;
+		res = platform_get_resource(pdev, IORESOURCE_MEM, i);
+		if (!res) {
+			dev_dbg(dev, "Unable to find IOMEM region\n");
+			ret = -ENOENT;
+			goto err_res;
+		}
+
+		data->sfrbases[i] = ioremap(res->start, resource_size(res));
+		if (!data->sfrbases[i]) {
+			dev_dbg(dev, "Unable to map IOMEM @ PA:%#x\n",
+							res->start);
+			ret = -ENOENT;
+			goto err_res;
+		}
+	}
+
+	for (i = 0; i < data->nsfrs; i++) {
+		ret = platform_get_irq(pdev, i);
+		if (ret <= 0) {
+			dev_dbg(dev, "Unable to find IRQ resource\n");
+			goto err_irq;
+		}
+
+		ret = request_irq(ret, exynos_sysmmu_irq, 0,
+					dev_name(dev), data);
+		if (ret) {
+			dev_dbg(dev, "Unabled to register interrupt handler\n");
+			goto err_irq;
+		}
+	}
+
+	if (dev_get_platdata(dev)) {
+		char *deli, *beg;
+		struct sysmmu_platform_data *platdata = dev_get_platdata(dev);
+
+		beg = platdata->clockname;
+
+		for (deli = beg; (*deli != '\0') && (*deli != ','); deli++)
+			/* NOTHING */;
+
+		if (*deli == '\0')
+			deli = NULL;
+		else
+			*deli = '\0';
+
+		data->clk[0] = clk_get(dev, beg);
+		if (IS_ERR(data->clk[0])) {
+			data->clk[0] = NULL;
+			dev_dbg(dev, "No clock descriptor registered\n");
+		}
+
+		if (data->clk[0] && deli) {
+			*deli = ',';
+			data->clk[1] = clk_get(dev, deli + 1);
+			if (IS_ERR(data->clk[1]))
+				data->clk[1] = NULL;
+		}
+
+		data->dbgname = platdata->dbgname;
+	}
+
+	data->sysmmu = dev;
+	rwlock_init(&data->lock);
+
+	__set_fault_handler(data, &default_fault_handler);
+
+	if (dev->parent)
+		pm_runtime_enable(dev);
+
+	dev_dbg(dev, "(%s) Initialized\n", data->dbgname);
+	return 0;
+err_irq:
+	while (i-- > 0) {
+		int irq;
+
+		irq = platform_get_irq(pdev, i);
+		free_irq(irq, data);
+	}
+err_res:
+	while (data->nsfrs-- > 0)
+		iounmap(data->sfrbases[data->nsfrs]);
+	kfree(data->sfrbases);
+err_init:
+	kfree(data);
+err_alloc:
+	dev_err(dev, "Failed to initialize\n");
+	return ret;
+}
+
+static struct platform_driver exynos_sysmmu_driver = {
+	.probe		= exynos_sysmmu_probe,
+	.driver		= {
+		.owner		= THIS_MODULE,
+		.name		= "exynos-sysmmu",
+	}
+};
+
+static inline void pgtable_flush(void *vastart, void *vaend)
+{
+	dmac_flush_range(vastart, vaend);
+	outer_flush_range(virt_to_phys(vastart),
+				virt_to_phys(vaend));
+}
+
+static int exynos_iommu_domain_init(struct iommu_domain *domain)
+{
+	struct exynos_iommu_domain *priv;
+
+	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->pgtable = (unsigned long *)__get_free_pages(
+						GFP_KERNEL | __GFP_ZERO, 2);
+	if (!priv->pgtable)
+		goto err_pgtable;
+
+	priv->lv2entcnt = (short *)__get_free_pages(
+						GFP_KERNEL | __GFP_ZERO, 1);
+	if (!priv->lv2entcnt)
+		goto err_counter;
+
+	pgtable_flush(priv->pgtable, priv->pgtable + NUM_LV1ENTRIES);
+
+	spin_lock_init(&priv->lock);
+	spin_lock_init(&priv->pgtablelock);
+	INIT_LIST_HEAD(&priv->clients);
+
+	domain->priv = priv;
+	return 0;
+
+err_counter:
+	free_pages((unsigned long)priv->pgtable, 2);
+err_pgtable:
+	kfree(priv);
+	return -ENOMEM;
+}
+
+static void exynos_iommu_domain_destroy(struct iommu_domain *domain)
+{
+	struct exynos_iommu_domain *priv = domain->priv;
+	struct list_head *pos, *n;
+	unsigned long flags;
+	int i;
+
+	WARN_ON(!list_empty(&priv->clients));
+
+	spin_lock_irqsave(&priv->lock, flags);
+
+	list_for_each_safe(pos, n, &priv->clients) {
+		struct iommu_client *client;
+
+		client = list_entry(pos, struct iommu_client, node);
+		exynos_sysmmu_disable(client->dev);
+		kfree(client);
+	}
+
+	spin_unlock_irqrestore(&priv->lock, flags);
+
+	for (i = 0; i < NUM_LV1ENTRIES; i++)
+		if (lv1ent_page(priv->pgtable + i))
+			kfree(__va(lv2table_base(priv->pgtable + i)));
+
+	free_pages((unsigned long)priv->pgtable, 2);
+	free_pages((unsigned long)priv->lv2entcnt, 1);
+	kfree(domain->priv);
+	domain->priv = NULL;
+}
+
+static int exynos_iommu_attach_device(struct iommu_domain *domain,
+				   struct device *dev)
+{
+	struct exynos_iommu_domain *priv = domain->priv;
+	struct iommu_client *client = NULL;
+	struct list_head *pos;
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->lock, flags);
+
+	list_for_each(pos, &priv->clients) {
+		struct iommu_client *cur;
+
+		cur = list_entry(pos, struct iommu_client, node);
+		if (cur->dev == dev) {
+			client = cur;
+			break;
+		}
+	}
+
+	if (client != NULL)
+		client->refcnt++;
+
+	spin_unlock_irqrestore(&priv->lock, flags);
+
+	if (client != NULL) {
+		dev_dbg(dev, "%s: IOMMU with pgtable 0x%lx already attached\n",
+					__func__, __pa(priv->pgtable));
+		return 0;
+	}
+
+	client = __get_enabled_iommu_client(domain, dev);
+	if (IS_ERR(client))
+		return PTR_ERR(client);
+
+	spin_lock_irqsave(&priv->lock, flags);
+	list_add_tail(&client->node, &priv->clients);
+	spin_unlock_irqrestore(&priv->lock, flags);
+
+	dev_dbg(dev, "%s: Attached new IOMMU with pgtable 0x%lx\n", __func__,
+						__pa(priv->pgtable));
+	return 0;
+}
+
+static void exynos_iommu_detach_device(struct iommu_domain *domain,
+				    struct device *dev)
+{
+	struct exynos_iommu_domain *priv = domain->priv;
+	struct iommu_client *client = NULL;
+	struct list_head *pos;
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->lock, flags);
+
+	list_for_each(pos, &priv->clients) {
+		struct iommu_client *cur;
+
+		cur = list_entry(pos, struct iommu_client, node);
+		if (cur->dev == dev) {
+			cur->refcnt--;
+			client = cur;
+			break;
+		}
+	}
+
+	spin_unlock_irqrestore(&priv->lock, flags);
+
+	if (WARN_ON(client == NULL))
+		return;
+
+	if (client->refcnt > 0) {
+		dev_dbg(dev,
+			"IOMMU is still attached %d times with pgtable %#lx\n",
+					client->refcnt, __pa(priv->pgtable));
+		return;
+	}
+
+	list_del(&client->node);
+	exynos_sysmmu_disable(client->dev);
+	dev_dbg(dev, "%s: Detached IOMMU with pgtable 0x%lx\n", __func__,
+						__pa(priv->pgtable));
+}
+
+static unsigned long *alloc_lv2entry(unsigned long *sent, unsigned long iova,
+					short *pgcounter)
+{
+	if (lv1ent_fault(sent)) {
+		unsigned long *pent;
+
+		pent = kzalloc(LV2TABLE_SIZE, GFP_ATOMIC);
+		BUG_ON((unsigned long)pent & (LV2TABLE_SIZE - 1));
+		if (!pent)
+			return NULL;
+
+		*sent = mk_lv1ent_page(__pa(pent));
+		*pgcounter = NUM_LV2ENTRIES;
+		pgtable_flush(pent, pent + NUM_LV2ENTRIES);
+		pgtable_flush(sent, sent + 1);
+	}
+
+	return page_entry(sent, iova);
+}
+
+static int lv1set_section(unsigned long *sent, phys_addr_t paddr, short *pgcnt)
+{
+	if (lv1ent_section(sent))
+		return -EADDRINUSE;
+
+	if (lv1ent_page(sent)) {
+		if (*pgcnt != NUM_LV2ENTRIES)
+			return -EADDRINUSE;
+
+		kfree(page_entry(sent, 0));
+
+		*pgcnt = 0;
+	}
+
+	*sent = mk_lv1ent_sect(paddr);
+
+	pgtable_flush(sent, sent + 1);
+
+	return 0;
+}
+
+static int lv2set_page(unsigned long *pent, phys_addr_t paddr, size_t size,
+								short *pgcnt)
+{
+	if (size == SPAGE_SIZE) {
+		if (!lv2ent_fault(pent))
+			return -EADDRINUSE;
+
+		*pent = mk_lv2ent_spage(paddr);
+		pgtable_flush(pent, pent + 1);
+		*pgcnt -= 1;
+	} else { /* size == LPAGE_SIZE */
+		int i;
+		for (i = 0; i < SPAGES_PER_LPAGE; i++, pent++) {
+			if (!lv2ent_fault(pent)) {
+				memset(pent, 0, sizeof(*pent) * i);
+				return -EADDRINUSE;
+			}
+
+			*pent = mk_lv2ent_lpage(paddr);
+		}
+		pgtable_flush(pent - SPAGES_PER_LPAGE, pent);
+		*pgcnt -= SPAGES_PER_LPAGE;
+	}
+
+	return 0;
+}
+
+static int exynos_iommu_map(struct iommu_domain *domain, unsigned long iova,
+			 phys_addr_t paddr, size_t size, int prot)
+{
+	struct exynos_iommu_domain *priv = domain->priv;
+	unsigned long *entry;
+	unsigned long flags;
+	int ret = -ENOMEM;
+
+	BUG_ON(priv->pgtable == NULL);
+
+	spin_lock_irqsave(&priv->pgtablelock, flags);
+
+	entry = section_entry(priv->pgtable, iova);
+
+	if (size == SECT_SIZE) {
+		ret = lv1set_section(entry, paddr,
+					&priv->lv2entcnt[lv1ent_offset(iova)]);
+	} else {
+		unsigned long *pent;
+
+		pent = alloc_lv2entry(entry, iova,
+					&priv->lv2entcnt[lv1ent_offset(iova)]);
+
+		if (!pent)
+			ret = -ENOMEM;
+		else
+			ret = lv2set_page(pent, paddr, size,
+					&priv->lv2entcnt[lv1ent_offset(iova)]);
+	}
+
+	if (ret) {
+		pr_debug("%s: Failed to map iova 0x%lx/0x%x bytes\n",
+							__func__, iova, size);
+	}
+
+	spin_unlock_irqrestore(&priv->pgtablelock, flags);
+
+	return ret;
+}
+
+static size_t exynos_iommu_unmap(struct iommu_domain *domain,
+					       unsigned long iova, size_t size)
+{
+	struct exynos_iommu_domain *priv = domain->priv;
+	struct iommu_client *client;
+	unsigned long flags;
+	unsigned long *ent;
+
+	BUG_ON(priv->pgtable == NULL);
+
+	spin_lock_irqsave(&priv->pgtablelock, flags);
+
+	ent = section_entry(priv->pgtable, iova);
+
+	if (lv1ent_section(ent)) {
+		BUG_ON(size < SECT_SIZE);
+
+		*ent = 0;
+		pgtable_flush(ent, ent + 1);
+		size = SECT_SIZE;
+		goto done;
+	}
+
+	if (unlikely(lv1ent_fault(ent))) {
+		if (size > SECT_SIZE)
+			size = SECT_SIZE;
+		goto done;
+	}
+
+	/* lv1ent_page(sent) == true here */
+
+	ent = page_entry(ent, iova);
+
+	if (unlikely(lv2ent_fault(ent))) {
+		size = SPAGE_SIZE;
+		goto done;
+	}
+
+	if (lv2ent_small(ent)) {
+		*ent = 0;
+		size = SPAGE_SIZE;
+		priv->lv2entcnt[lv1ent_offset(iova)] -= 1;
+		goto done;
+	}
+
+	/* lv1ent_large(ent) == true here */
+	BUG_ON(size < LPAGE_SIZE);
+
+	memset(ent, 0, sizeof(*ent) * SPAGES_PER_LPAGE);
+
+	size = LPAGE_SIZE;
+	priv->lv2entcnt[lv1ent_offset(iova)] -= SPAGES_PER_LPAGE;
+done:
+	list_for_each_entry(client, &priv->clients, node)
+		sysmmu_tlb_invalidate_entry(client->dev, iova);
+
+	spin_unlock_irqrestore(&priv->pgtablelock, flags);
+
+	return size;
+}
+
+static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain,
+					  unsigned long iova)
+{
+	struct exynos_iommu_domain *priv = domain->priv;
+	unsigned long *entry;
+	unsigned long flags;
+	phys_addr_t phys = -1;
+
+	spin_lock_irqsave(&priv->pgtablelock, flags);
+
+	entry = section_entry(priv->pgtable, iova);
+
+	if (lv1ent_section(entry)) {
+		phys = section_phys(entry) + section_offs(iova);
+	} else if (lv1ent_page(entry)) {
+		entry = page_entry(entry, iova);
+
+		if (lv2ent_large(entry))
+			phys = lpage_phys(entry) + lpage_offs(iova);
+		else if (lv2ent_small(entry))
+			phys = spage_phys(entry) + spage_offs(iova);
+	}
+
+	spin_unlock_irqrestore(&priv->pgtablelock, flags);
+
+	return phys;
+}
+
+static struct iommu_ops exynos_iommu_ops = {
+	.domain_init = &exynos_iommu_domain_init,
+	.domain_destroy = &exynos_iommu_domain_destroy,
+	.attach_dev = &exynos_iommu_attach_device,
+	.detach_dev = &exynos_iommu_detach_device,
+	.map = &exynos_iommu_map,
+	.unmap = &exynos_iommu_unmap,
+	.iova_to_phys = &exynos_iommu_iova_to_phys,
+	.pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE,
+};
+
+static int __init exynos_iommu_init(void)
+{
+	int ret;
+
+	ret = platform_driver_register(&exynos_sysmmu_driver);
+
+	if (ret == 0)
+		bus_set_iommu(&platform_bus_type, &exynos_iommu_ops);
+
+	return ret;
+}
+arch_initcall(exynos_iommu_init);
-- 
1.7.1

[PATCH v10 3/3] iommu/exynos: Add iommu driver for Exynos Platforms

[Kyungmin Park]

Hi,

some not critical comments.

Thank you,
Kyungmin Park

On Tue, Mar 6, 2012 at 5:31 PM, KyongHo Cho <pullip.cho@samsung.com> wrote:
> HAALgBjAGgAbwBAAHMAYQBtAHMAdQBuAGcALgBjAG8AbQA=;Tue,
> ?06 Mar 2012 08:31:29 GMT;WwBQAEEAVABDAEgAIAB2ADEAMAAgADMALwAzAF0AIABpAG8AbQBtAHUALwBlAHgAeQBuAG8AcwA6ACAAQQBkAGQAIABpAG8AbQBtAHUAIABkAHIAaQB2AGUAcgAgAGYAbwByACAARQB4AHkAbgBvAHMAIABQAGwAYQB0AGYAbwByAG0AcwA=
> x-cr-puzzleid: {CF0AAF04-8639-4D69-B4E1-BF71EA1B0A70}
What's this?
>
> This is the System MMU driver and IOMMU API implementation for
> Exynos SOC platforms. Exynos platforms has more than 10 System
> MMUs dedicated for each multimedia accellerators.
>
> The System MMU driver is already in arc/arm/plat-s5p but it is
> moved to drivers/iommu due to Ohad Ben-Cohen gathered IOMMU drivers
> there
>
> Any device driver in Exynos platforms that needs to control its
> System MMU must call platform_set_sysmmu() to inform System MMU
> driver who will control it.
> platform_set_sysmmu() is defined in <mach/sysmmu.h>
Does it really required? if we can remove it. it can be compiled
without arch dependency.
>
> Cc: Joerg Roedel <joerg.roedel@amd.com>
> Cc: Kukjin Kim <kgene.kim@samsung.com>
> Signed-off-by: KyongHo Cho <pullip.cho@samsung.com>
> ---
> ?drivers/iommu/Kconfig ? ? ? ?| ? 21 +
> ?drivers/iommu/Makefile ? ? ? | ? ?1 +
> ?drivers/iommu/exynos-iommu.c | 1088 ++++++++++++++++++++++++++++++++++++++++++
> ?3 files changed, 1110 insertions(+), 0 deletions(-)
> ?create mode 100644 drivers/iommu/exynos-iommu.c
>
> diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
> index 6bea696..25d3eed 100644
> --- a/drivers/iommu/Kconfig
> +++ b/drivers/iommu/Kconfig
> @@ -142,4 +142,25 @@ config OMAP_IOMMU_DEBUG
>
> ? ? ? ? ?Say N unless you know you need this.
>
> +config EXYNOS_IOMMU
> + ? ? ? bool "Exynos IOMMU Support"
> + ? ? ? depends on EXYNOS_DEV_SYSMMU
> + ? ? ? select IOMMU_API
> + ? ? ? help
> + ? ? ? ? Support for the IOMMU(System MMU) of Samsung Exynos application
> + ? ? ? ? processor family. This enables H/W multimedia accellerators to see
> + ? ? ? ? non-linear physical memory chunks as a linear memory in their
> + ? ? ? ? address spaces
> +
> + ? ? ? ? If unsure, say N here.
> +
> +config EXYNOS_IOMMU_DEBUG
> + ? ? ? bool "Debugging log for Exynos IOMMU"
> + ? ? ? depends on EXYNOS_IOMMU
> + ? ? ? help
> + ? ? ? ? Select this to see the detailed log message that shows what
> + ? ? ? ? happens in the IOMMU driver
> +
> + ? ? ? ? Say N unless you need kernel log message for IOMMU debugging
> +
> ?endif # IOMMU_SUPPORT
> diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
> index 0e36b49..5a8fd97 100644
> --- a/drivers/iommu/Makefile
> +++ b/drivers/iommu/Makefile
> @@ -8,3 +8,4 @@ obj-$(CONFIG_IRQ_REMAP) += intr_remapping.o
> ?obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o
> ?obj-$(CONFIG_OMAP_IOVMM) += omap-iovmm.o
> ?obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o
> +obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o
> diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
> new file mode 100644
> index 0000000..4c1c48a
> --- /dev/null
> +++ b/drivers/iommu/exynos-iommu.c
> @@ -0,0 +1,1088 @@
> +/* linux/drivers/iommu/exynos_iommu.c
> + *
> + * Copyright (c) 2011 Samsung Electronics Co., Ltd.
> + * ? ? ? ? ? ? http://www.samsung.com
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + */
> +
> +#ifdef CONFIG_EXYNOS_IOMMU_DEBUG
> +#define DEBUG
> +#endif
If it's used only at here. you can add it at Makefile.

ifeq ($(CONFIG_EXYNOS_IOMMU_DEBUG),y)
         CFLAGS-exynos-iommu        += -DDEBUG
endif

Of course you can select as your preference.
> +
> +#include <linux/io.h>
> +#include <linux/interrupt.h>
> +#include <linux/platform_device.h>
> +#include <linux/slab.h>
> +#include <linux/pm_runtime.h>
> +#include <linux/clk.h>
> +#include <linux/err.h>
> +#include <linux/mm.h>
> +#include <linux/iommu.h>
> +#include <linux/errno.h>
> +#include <linux/list.h>
> +#include <linux/memblock.h>
> +#include <linux/export.h>
> +
> +#include <asm/cacheflush.h>
> +#include <asm/pgtable.h>
> +
> +#include <mach/sysmmu.h>
> +
> +#define S5P_MMU_CTRL ? ? ? ? ? ? ? ? ? 0x000
> +#define S5P_MMU_CFG ? ? ? ? ? ? ? ? ? ?0x004
> +#define S5P_MMU_STATUS ? ? ? ? ? ? ? ? 0x008
> +#define S5P_MMU_FLUSH ? ? ? ? ? ? ? ? ?0x00C
> +#define S5P_PT_BASE_ADDR ? ? ? ? ? ? ? 0x014
> +#define S5P_INT_STATUS ? ? ? ? ? ? ? ? 0x018
> +#define S5P_INT_CLEAR ? ? ? ? ? ? ? ? ?0x01C
> +#define S5P_PAGE_FAULT_ADDR ? ? ? ? ? ?0x024
> +#define S5P_AW_FAULT_ADDR ? ? ? ? ? ? ?0x028
> +#define S5P_AR_FAULT_ADDR ? ? ? ? ? ? ?0x02C
> +#define S5P_DEFAULT_SLAVE_ADDR ? ? ? ? 0x030
> +#define S5P_MMU_VERSION ? ? ? ? ? ? ? ? ? ? ? ?0x034
> +#define S5P_PB0_SADDR ? ? ? ? ? ? ? ? ?0x04C
> +#define S5P_PB0_EADDR ? ? ? ? ? ? ? ? ?0x050
> +#define S5P_PB1_SADDR ? ? ? ? ? ? ? ? ?0x054
> +#define S5P_PB1_EADDR ? ? ? ? ? ? ? ? ?0x058
> +
> +#define SECT_ORDER 20
> +#define LPAGE_ORDER 16
> +#define SPAGE_ORDER 12
> +
> +#define SECT_SIZE (1 << SECT_ORDER)
> +#define LPAGE_SIZE (1 << LPAGE_ORDER)
> +#define SPAGE_SIZE (1 << SPAGE_ORDER)
> +
> +#define SECT_MASK (~(SECT_SIZE - 1))
> +#define LPAGE_MASK (~(LPAGE_SIZE - 1))
> +#define SPAGE_MASK (~(SPAGE_SIZE - 1))
> +
> +#define lv1ent_fault(sent) (((*(sent) & 3) == 0) || ((*(sent) & 3) == 3))
> +#define lv1ent_page(sent) ((*(sent) & 3) == 1)
> +#define lv1ent_section(sent) ((*(sent) & 3) == 2)
> +
> +#define lv2ent_fault(pent) ((*(pent) & 3) == 0)
> +#define lv2ent_small(pent) ((*(pent) & 2) == 2)
#define lv2ent_small(pent) ((*(pent) & 3) == 2)?
> +#define lv2ent_large(pent) ((*(pent) & 3) == 1)
> +
> +#define section_phys(sent) (*(sent) & SECT_MASK)
> +#define section_offs(iova) ((iova) & 0xFFFFF)
> +#define lpage_phys(pent) (*(pent) & LPAGE_MASK)
> +#define lpage_offs(iova) ((iova) & 0xFFFF)
> +#define spage_phys(pent) (*(pent) & SPAGE_MASK)
> +#define spage_offs(iova) ((iova) & 0xFFF)
> +
> +#define lv1ent_offset(iova) ((iova) >> SECT_ORDER)
> +#define lv2ent_offset(iova) (((iova) & 0xFF000) >> SPAGE_ORDER)
> +
> +#define NUM_LV1ENTRIES 4096
> +#define NUM_LV2ENTRIES 256
> +
> +#define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(long))
> +
> +#define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE)
> +
> +#define lv2table_base(sent) (*(sent) & 0xFFFFFC00)
> +
> +#define mk_lv1ent_sect(pa) ((pa) | 2)
> +#define mk_lv1ent_page(pa) ((pa) | 1)
> +#define mk_lv2ent_lpage(pa) ((pa) | 1)
> +#define mk_lv2ent_spage(pa) ((pa) | 2)
> +
> +#define CTRL_ENABLE ? ?0x5
> +#define CTRL_BLOCK ? ? 0x7
> +#define CTRL_DISABLE ? 0x0
> +
> +#define REG_MMU_CTRL ? ? ? ? ? 0x000
> +#define REG_MMU_CFG ? ? ? ? ? ?0x004
> +#define REG_MMU_STATUS ? ? ? ? 0x008
> +#define REG_MMU_FLUSH ? ? ? ? ?0x00C
> +#define REG_MMU_FLUSH_ENTRY ? ?0x010
> +#define REG_PT_BASE_ADDR ? ? ? 0x014
> +#define REG_INT_STATUS ? ? ? ? 0x018
> +#define REG_INT_CLEAR ? ? ? ? ?0x01C
> +#define REG_MMU_VERSION ? ? ? ? ? ? ? ?0x034
Please place it in order.
> +
> +#define REG_PAGE_FAULT_ADDR ? ?0x024
> +#define REG_AW_FAULT_ADDR ? ? ?0x028
> +#define REG_AR_FAULT_ADDR ? ? ?0x02C
> +#define REG_DEFAULT_SLAVE_ADDR 0x030
> +
> +static unsigned long *section_entry(unsigned long *pgtable, unsigned long iova)
> +{
> + ? ? ? return pgtable + lv1ent_offset(iova);
> +}
> +
> +static unsigned long *page_entry(unsigned long *sent, unsigned long iova)
> +{
> + ? ? ? return (unsigned long *)__va(lv2table_base(sent)) + lv2ent_offset(iova);
> +}
> +
> +enum EXYNOS_SYSMMU_INTERRUPT_TYPE {
> + ? ? ? SYSMMU_PAGEFAULT,
> + ? ? ? SYSMMU_AR_MULTIHIT,
> + ? ? ? SYSMMU_AW_MULTIHIT,
> + ? ? ? SYSMMU_BUSERROR,
> + ? ? ? SYSMMU_AR_SECURITY,
> + ? ? ? SYSMMU_AR_ACCESS,
> + ? ? ? SYSMMU_AW_SECURITY,
> + ? ? ? SYSMMU_AW_PROTECTION, /* 7 */
> + ? ? ? SYSMMU_FAULT_UNKNOWN,
> + ? ? ? SYSMMU_FAULTS_NUM
missing comma, "SYSMMU_FAULTS_NUM,"
> +};
> +
> +typedef int (*sysmmu_fault_handler_t)(enum EXYNOS_SYSMMU_INTERRUPT_TYPE itype,
> + ? ? ? ? ? ? ? ? ? ? ? unsigned long pgtable_base, unsigned long fault_addr);
> +
> +static unsigned short fault_reg_offset[SYSMMU_FAULTS_NUM] = {
> + ? ? ? REG_PAGE_FAULT_ADDR,
> + ? ? ? REG_AR_FAULT_ADDR,
> + ? ? ? REG_AW_FAULT_ADDR,
> + ? ? ? REG_DEFAULT_SLAVE_ADDR,
> + ? ? ? REG_AR_FAULT_ADDR,
> + ? ? ? REG_AR_FAULT_ADDR,
> + ? ? ? REG_AW_FAULT_ADDR,
> + ? ? ? REG_AW_FAULT_ADDR
ditto.
> +};
> +
> +static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = {
> + ? ? ? "PAGE FAULT",
> + ? ? ? "AR MULTI-HIT FAULT",
> + ? ? ? "AW MULTI-HIT FAULT",
> + ? ? ? "BUS ERROR",
> + ? ? ? "AR SECURITY PROTECTION FAULT",
> + ? ? ? "AR ACCESS PROTECTION FAULT",
> + ? ? ? "AW SECURITY PROTECTION FAULT",
> + ? ? ? "AW ACCESS PROTECTION FAULT",
> + ? ? ? "UNKNOWN FAULT"
ditto.
> +};
> +
> +/* We does not consider super section mapping (16MB) */
> +struct iommu_client {
> + ? ? ? struct list_head node;
> + ? ? ? struct device *dev;
> + ? ? ? int refcnt;
> +};
> +
> +struct exynos_iommu_domain {
> + ? ? ? struct list_head clients; /* list of iommu_client */
> + ? ? ? unsigned long *pgtable; /* lv1 page table, 16KB */
> + ? ? ? short *lv2entcnt; /* free lv2 entry counter for each section */
> + ? ? ? spinlock_t lock; /* lock for this structure and attached iommu_client */
> + ? ? ? spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */
> +};
> +
> +struct sysmmu_drvdata {
> + ? ? ? struct device *sysmmu;
> + ? ? ? char *dbgname;
> + ? ? ? int nsfrs;
> + ? ? ? void __iomem **sfrbases;
> + ? ? ? struct clk *clk[2];
> + ? ? ? bool active;
> + ? ? ? rwlock_t lock;
> + ? ? ? struct iommu_domain *domain;
> + ? ? ? sysmmu_fault_handler_t fault_handler;
> + ? ? ? unsigned long pgtable;
> + ? ? ? struct iommu_client client;
> +};
> +
> +static bool set_sysmmu_active(struct sysmmu_drvdata *data)
> +{
> + ? ? ? /* return true if the System MMU was not active previously
> + ? ? ? ? ?and it needs to be initialized */
> + ? ? ? if (WARN_ON_ONCE(data->active))
> + ? ? ? ? ? ? ? return false;
> + ? ? ? data->active = true;
> + ? ? ? return true;
> +}
> +
> +static bool set_sysmmu_inactive(struct sysmmu_drvdata *data)
> +{
> + ? ? ? /* return true if the System MMU is needed to be disabled */
> + ? ? ? if (WARN_ON_ONCE(!data->active))
> + ? ? ? ? ? ? ? return false;
> + ? ? ? data->active = false;
> + ? ? ? return true;
> +}
> +
> +static bool is_sysmmu_active(struct sysmmu_drvdata *data)
> +{
> + ? ? ? return data->active;
> +}
> +
> +static void sysmmu_block(void __iomem *sfrbase)
> +{
> + ? ? ? __raw_writel(CTRL_BLOCK, sfrbase + REG_MMU_CTRL);
> +}
> +
> +static void sysmmu_unblock(void __iomem *sfrbase)
> +{
> + ? ? ? __raw_writel(CTRL_ENABLE, sfrbase + REG_MMU_CTRL);
> +}
> +
> +static void __sysmmu_tlb_invalidate(void __iomem *sfrbase)
> +{
> + ? ? ? __raw_writel(0x1, sfrbase + REG_MMU_FLUSH);
> +}
> +
> +static void __sysmmu_tlb_invalidate_entry(void __iomem *sfrbase,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? unsigned long iova)
> +{
> + ? ? ? __raw_writel((iova & SPAGE_MASK) | 1, sfrbase + REG_MMU_FLUSH_ENTRY);
> +}
> +
> +static void __sysmmu_set_ptbase(void __iomem *sfrbase,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?unsigned long pgd)
> +{
> + ? ? ? __raw_writel(0x1, sfrbase + REG_MMU_CFG); /* 16KB LV1, LRU */
> + ? ? ? __raw_writel(pgd, sfrbase + REG_PT_BASE_ADDR);
> +
> + ? ? ? __sysmmu_tlb_invalidate(sfrbase);
> +}
> +
> +static void __sysmmu_set_prefbuf(void __iomem *sfrbase, unsigned long base,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? unsigned long size, int idx)
> +{
> + ? ? ? __raw_writel(base, sfrbase + S5P_PB0_SADDR + idx * 8);
> + ? ? ? __raw_writel(size - 1 + base, ?sfrbase + S5P_PB0_EADDR + idx * 8);
> +}
> +
> +void exynos_sysmmu_set_prefbuf(struct device *dev,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? unsigned long base0, unsigned long size0,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? unsigned long base1, unsigned long size1)
> +{
> + ? ? ? struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
> + ? ? ? unsigned long flags;
> + ? ? ? int i;
> +
> + ? ? ? BUG_ON((base0 + size0) <= base0);
> + ? ? ? BUG_ON((size1 > 0) && ((base1 + size1) <= base1));
It's bogus check. it's impossible. Do you think overflow case?
> +
> + ? ? ? read_lock_irqsave(&data->lock, flags);
> + ? ? ? if (!is_sysmmu_active(data))
> + ? ? ? ? ? ? ? goto finish;
> +
> + ? ? ? for (i = 0; i < data->nsfrs; i++) {
> + ? ? ? ? ? ? ? if ((readl(data->sfrbases[i] + S5P_MMU_VERSION) >> 28) == 3) {
> + ? ? ? ? ? ? ? ? ? ? ? sysmmu_block(data->sfrbases[i]);
> +
> + ? ? ? ? ? ? ? ? ? ? ? if (size1 == 0) {
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? if (size0 <= SZ_128K) {
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? base1 = base0;
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? size1 = size0;
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? } else {
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? size1 = size0 -
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ALIGN(size0 / 2, SZ_64K);
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? size0 = size0 - size1;
Umm I mis-understand this codes at first. it think size1 has 0
already. so it's meaning-less but now it re-use the size1 above.
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? base1 = base0 + size0;
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? }
> + ? ? ? ? ? ? ? ? ? ? ? }
> +
> + ? ? ? ? ? ? ? ? ? ? ? __sysmmu_set_prefbuf(
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? data->sfrbases[i], base0, size0, 0);
> + ? ? ? ? ? ? ? ? ? ? ? __sysmmu_set_prefbuf(
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? data->sfrbases[i], base1, size1, 1);
> +
> + ? ? ? ? ? ? ? ? ? ? ? sysmmu_unblock(data->sfrbases[i]);
> + ? ? ? ? ? ? ? }
> + ? ? ? }
> +finish:
> + ? ? ? read_unlock_irqrestore(&data->lock, flags);
> +}
> +
> +static void __set_fault_handler(struct sysmmu_drvdata *data,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? sysmmu_fault_handler_t handler)
> +{
> + ? ? ? unsigned long flags;
> +
> + ? ? ? write_lock_irqsave(&data->lock, flags);
> + ? ? ? data->fault_handler = handler;
> + ? ? ? write_unlock_irqrestore(&data->lock, flags);
> +}
> +
> +void exynos_sysmmu_set_fault_handler(struct device *dev,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? sysmmu_fault_handler_t handler)
> +{
> + ? ? ? struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
> +
> + ? ? ? __set_fault_handler(data, handler);
> +}
> +
> +static int default_fault_handler(enum EXYNOS_SYSMMU_INTERRUPT_TYPE itype,
Can you use the small letter instead of large one? do you see codes like this?
> + ? ? ? ? ? ? ? ? ? ?unsigned long pgtable_base, unsigned long fault_addr)
> +{
> + ? ? ? unsigned long *ent;
> +
> + ? ? ? if ((itype >= SYSMMU_FAULTS_NUM) || (itype < SYSMMU_PAGEFAULT))
> + ? ? ? ? ? ? ? itype = SYSMMU_FAULT_UNKNOWN;
> +
> + ? ? ? pr_err("%s occured at 0x%lx(Page table base: 0x%lx)\n",
> + ? ? ? ? ? ? ? ? ? ? ? sysmmu_fault_name[itype], fault_addr, pgtable_base);
> +
> + ? ? ? ent = section_entry(__va(pgtable_base), fault_addr);
> + ? ? ? pr_err("\tLv1 entry: 0x%lx\n", *ent);
> +
> + ? ? ? if (lv1ent_page(ent)) {
> + ? ? ? ? ? ? ? ent = page_entry(ent, fault_addr);
> + ? ? ? ? ? ? ? pr_err("\t Lv2 entry: 0x%lx\n", *ent);
> + ? ? ? }
> +
> + ? ? ? pr_err("Generating Kernel OOPS... because it is unrecoverable.\n");
> +
> + ? ? ? BUG();
> +
> + ? ? ? return 0;
> +}
> +
> +static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
> +{
> + ? ? ? /* SYSMMU is in blocked when interrupt occurred. */
> + ? ? ? struct sysmmu_drvdata *data = dev_id;
> + ? ? ? struct resource *irqres;
> + ? ? ? struct platform_device *pdev;
> + ? ? ? enum EXYNOS_SYSMMU_INTERRUPT_TYPE itype;
> + ? ? ? unsigned long addr = -1;
> +
> + ? ? ? int i, ret = -ENOSYS;
> +
> + ? ? ? read_lock(&data->lock);
> +
> + ? ? ? WARN_ON(!is_sysmmu_active(data));
> +
> + ? ? ? pdev = to_platform_device(data->sysmmu);
> + ? ? ? for (i = 0; i < pdev->num_resources; i++) {
> + ? ? ? ? ? ? ? irqres = platform_get_resource(pdev, IORESOURCE_IRQ, i);
> + ? ? ? ? ? ? ? if (irqres && ((int)irqres->start == irq))
> + ? ? ? ? ? ? ? ? ? ? ? break;
> + ? ? ? }
> +
> + ? ? ? if (i == pdev->num_resources) {
> + ? ? ? ? ? ? ? itype = SYSMMU_FAULT_UNKNOWN;
> + ? ? ? } else {
> + ? ? ? ? ? ? ? i /= 2;
> +
> + ? ? ? ? ? ? ? itype = (enum EXYNOS_SYSMMU_INTERRUPT_TYPE)
> + ? ? ? ? ? ? ? ? ? ? ? __ffs(__raw_readl(data->sfrbases[i] + REG_INT_STATUS));
> + ? ? ? ? ? ? ? if (WARN_ON(!((itype >= 0) && (itype < SYSMMU_FAULT_UNKNOWN))))
> + ? ? ? ? ? ? ? ? ? ? ? itype = SYSMMU_FAULT_UNKNOWN;
> + ? ? ? ? ? ? ? else
> + ? ? ? ? ? ? ? ? ? ? ? addr = __raw_readl(
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? data->sfrbases[i] + fault_reg_offset[itype]);
> + ? ? ? }
> +
> + ? ? ? if (data->domain)
> + ? ? ? ? ? ? ? ret = report_iommu_fault(data->domain, data->client.dev,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? addr, itype);
> +
> + ? ? ? if ((ret == -ENOSYS) && data->fault_handler) {
> + ? ? ? ? ? ? ? unsigned long base = data->pgtable;
> + ? ? ? ? ? ? ? if (itype != SYSMMU_FAULT_UNKNOWN)
> + ? ? ? ? ? ? ? ? ? ? ? base = __raw_readl(
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? data->sfrbases[i] + REG_PT_BASE_ADDR);
> + ? ? ? ? ? ? ? ret = data->fault_handler(itype, base, addr);
> + ? ? ? }
> +
> + ? ? ? if (!ret && (itype != SYSMMU_FAULT_UNKNOWN))
> + ? ? ? ? ? ? ? __raw_writel(1 << itype, data->sfrbases[i] + REG_INT_CLEAR);
> + ? ? ? else
> + ? ? ? ? ? ? ? dev_dbg(data->sysmmu, "(%s) %s is not handled.\n",
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? data->dbgname, sysmmu_fault_name[itype]);
> +
> + ? ? ? if (itype != SYSMMU_FAULT_UNKNOWN)
> + ? ? ? ? ? ? ? sysmmu_unblock(data->sfrbases[i]);
> +
> + ? ? ? read_unlock(&data->lock);
> +
> + ? ? ? return IRQ_HANDLED;
> +}
> +
> +static bool __sysmmu_disable(struct sysmmu_drvdata *data)
> +{
> + ? ? ? unsigned long flags;
> + ? ? ? bool disabled = false;
> + ? ? ? int i;
> +
> + ? ? ? write_lock_irqsave(&data->lock, flags);
> +
> + ? ? ? if (!set_sysmmu_inactive(data))
> + ? ? ? ? ? ? ? goto finish;
> +
> + ? ? ? for (i = 0; i < data->nsfrs; i++) {
> + ? ? ? ? ? ? ? __raw_writel(CTRL_DISABLE, data->sfrbases[i] + REG_MMU_CTRL);
> + ? ? ? ? ? ? ? if (data->clk[1])
> + ? ? ? ? ? ? ? ? ? ? ? clk_disable(data->clk[1]);
> + ? ? ? ? ? ? ? if (data->clk[0])
> + ? ? ? ? ? ? ? ? ? ? ? clk_disable(data->clk[0]);
> + ? ? ? ? ? ? ? disabled = true;
> + ? ? ? ? ? ? ? data->pgtable = 0;
> + ? ? ? ? ? ? ? data->domain = NULL;
> + ? ? ? ? ? ? ? pm_runtime_put(data->sysmmu);
> + ? ? ? }
> +
> +finish:
> + ? ? ? write_unlock_irqrestore(&data->lock, flags);
> +
> + ? ? ? return disabled;
> +}
> +
> +/* __exynos_sysmmu_enable: Enables System MMU
> + *
> + * returns -error if an error occurred and System MMU is not enabled,
> + * 0 if the System MMU has been just enabled and 1 if System MMU was already
> + * enabled before.
> + */
> +static int __exynos_sysmmu_enable(struct device *dev, unsigned long pgtable,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? struct iommu_domain *domain)
> +{
> + ? ? ? int i, ret = 0;
> + ? ? ? unsigned long flags;
> + ? ? ? struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
> +
> + ? ? ? BUG_ON(!memblock_is_memory(pgtable));
> +
> + ? ? ? ret = pm_runtime_get_sync(data->sysmmu);
> + ? ? ? if (ret < 0)
> + ? ? ? ? ? ? ? return ret;
> +
> + ? ? ? write_lock_irqsave(&data->lock, flags);
> +
> + ? ? ? if (!set_sysmmu_active(data)) {
> + ? ? ? ? ? ? ? /* RPM of sysmmu won't be refcounted */
> + ? ? ? ? ? ? ? if (ret > 0)
> + ? ? ? ? ? ? ? ? ? ? ? pm_runtime_put(data->sysmmu);
> +
> + ? ? ? ? ? ? ? if (WARN_ON(pgtable != data->pgtable)) {
> + ? ? ? ? ? ? ? ? ? ? ? set_sysmmu_inactive(data);
> + ? ? ? ? ? ? ? ? ? ? ? ret = -EBUSY;
> + ? ? ? ? ? ? ? }
> +
> + ? ? ? ? ? ? ? ret = 1;
> + ? ? ? ? ? ? ? dev_dbg(data->sysmmu, "(%s) Already enabled\n", data->dbgname);
> + ? ? ? ? ? ? ? goto finish;
> + ? ? ? }
> +
> + ? ? ? if (data->clk[0])
> + ? ? ? ? ? ? ? clk_enable(data->clk[0]);
> + ? ? ? if (data->clk[1])
> + ? ? ? ? ? ? ? clk_enable(data->clk[1]);
> +
> + ? ? ? data->pgtable = pgtable;
> +
> + ? ? ? for (i = 0; i < data->nsfrs; i++) {
> + ? ? ? ? ? ? ? __sysmmu_set_ptbase(data->sfrbases[i], pgtable);
> +
> + ? ? ? ? ? ? ? if ((readl(data->sfrbases[i] + S5P_MMU_VERSION) >> 28) == 3) {
> + ? ? ? ? ? ? ? ? ? ? ? /* System MMU version is 3.x */
> + ? ? ? ? ? ? ? ? ? ? ? __raw_writel((1 << 12) | (2 << 28),
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? data->sfrbases[i] + S5P_MMU_CFG);
> + ? ? ? ? ? ? ? ? ? ? ? __sysmmu_set_prefbuf(data->sfrbases[i], 0, -1, 0);
> + ? ? ? ? ? ? ? ? ? ? ? __sysmmu_set_prefbuf(data->sfrbases[i], 0, -1, 1);
> + ? ? ? ? ? ? ? }
> +
> + ? ? ? ? ? ? ? __raw_writel(CTRL_ENABLE, data->sfrbases[i] + REG_MMU_CTRL);
> + ? ? ? }
> +
> + ? ? ? data->domain = domain;
> +
> + ? ? ? dev_dbg(data->sysmmu, "(%s) Enabled\n", data->dbgname);
> +finish:
> + ? ? ? write_unlock_irqrestore(&data->lock, flags);
> +
> + ? ? ? if (ret < 0) {
> + ? ? ? ? ? ? ? __sysmmu_disable(data);
> + ? ? ? ? ? ? ? dev_dbg(data->sysmmu, "(%s) Failed to enable\n", data->dbgname);
> + ? ? ? }
> +
> + ? ? ? return ret;
> +}
> +
> +static struct iommu_client *__get_enabled_iommu_client(
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? struct iommu_domain *domain, struct device *dev)
> +{
> + ? ? ? struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
> + ? ? ? struct exynos_iommu_domain *priv = domain->priv;
> + ? ? ? int ret;
> +
> + ? ? ? ret = __exynos_sysmmu_enable(dev, __pa(priv->pgtable), domain);
> + ? ? ? if (ret < 0)
> + ? ? ? ? ? ? ? return ERR_PTR(ret);
> + ? ? ? else if (ret == 1)
> + ? ? ? ? ? ? ? return ERR_PTR(-EBUSY);
> +
> + ? ? ? INIT_LIST_HEAD(&data->client.node);
> + ? ? ? data->client.dev = dev;
> + ? ? ? data->client.refcnt = 1;
> +
> + ? ? ? return &data->client;
> +}
> +
> +int exynos_sysmmu_enable(struct device *dev, unsigned long pgtable)
> +{
> + ? ? ? return __exynos_sysmmu_enable(dev, pgtable, NULL);
> +}
> +
> +void exynos_sysmmu_disable(struct device *dev)
> +{
> + ? ? ? struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
> + ? ? ? bool disabled;
> +
> + ? ? ? disabled = __sysmmu_disable(data);
> + ? ? ? dev_dbg(data->sysmmu,
> + ? ? ? ? ? ? ? ? ? ? ? disabled ? "(%s) Disabled.\n" :
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? "(%s) Deactivation request ignorred.\n",
> + ? ? ? ? ? ? ? ? ? ? ? data->dbgname);
> +}
> +
> +static void sysmmu_tlb_invalidate_entry(struct device *dev, unsigned long iova)
> +{
> + ? ? ? unsigned long flags;
> + ? ? ? struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
> +
> + ? ? ? read_lock_irqsave(&data->lock, flags);
> +
> + ? ? ? if (is_sysmmu_active(data)) {
> + ? ? ? ? ? ? ? int i;
> + ? ? ? ? ? ? ? for (i = 0; i < data->nsfrs; i++) {
> + ? ? ? ? ? ? ? ? ? ? ? sysmmu_block(data->sfrbases[i]);
> + ? ? ? ? ? ? ? ? ? ? ? __sysmmu_tlb_invalidate_entry(data->sfrbases[i], iova);
> + ? ? ? ? ? ? ? ? ? ? ? sysmmu_unblock(data->sfrbases[i]);
> + ? ? ? ? ? ? ? }
> + ? ? ? } else {
> + ? ? ? ? ? ? ? dev_dbg(data->sysmmu,
> + ? ? ? ? ? ? ? ? ? ? ? "(%s) Disabled. Skipping invalidating TLB.\n",
> + ? ? ? ? ? ? ? ? ? ? ? data->dbgname);
> + ? ? ? }
> +
> + ? ? ? read_unlock_irqrestore(&data->lock, flags);
> +}
> +
> +void exynos_sysmmu_tlb_invalidate(struct device *dev)
> +{
> + ? ? ? unsigned long flags;
> + ? ? ? struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
> +
> + ? ? ? read_lock_irqsave(&data->lock, flags);
> +
> + ? ? ? if (is_sysmmu_active(data)) {
> + ? ? ? ? ? ? ? int i;
> + ? ? ? ? ? ? ? for (i = 0; i < data->nsfrs; i++) {
> + ? ? ? ? ? ? ? ? ? ? ? sysmmu_block(data->sfrbases[i]);
> + ? ? ? ? ? ? ? ? ? ? ? __sysmmu_tlb_invalidate(data->sfrbases[i]);
> + ? ? ? ? ? ? ? ? ? ? ? sysmmu_unblock(data->sfrbases[i]);
> + ? ? ? ? ? ? ? }
> + ? ? ? } else {
> + ? ? ? ? ? ? ? dev_dbg(data->sysmmu,
> + ? ? ? ? ? ? ? ? ? ? ? "(%s) Disabled. Skipping invalidating TLB.\n",
> + ? ? ? ? ? ? ? ? ? ? ? data->dbgname);
> + ? ? ? }
> +
> + ? ? ? read_unlock_irqrestore(&data->lock, flags);
> +}
> +
> +static int exynos_sysmmu_probe(struct platform_device *pdev)
> +{
> + ? ? ? int i, ret;
> + ? ? ? struct device *dev;
> + ? ? ? struct sysmmu_drvdata *data;
> +
> + ? ? ? dev = &pdev->dev;
> +
> + ? ? ? data = kzalloc(sizeof(*data), GFP_KERNEL);
> + ? ? ? if (!data) {
> + ? ? ? ? ? ? ? dev_dbg(dev, "Not enough memory\n");
> + ? ? ? ? ? ? ? ret = -ENOMEM;
> + ? ? ? ? ? ? ? goto err_alloc;
> + ? ? ? }
> +
> + ? ? ? ret = dev_set_drvdata(dev, data);
> + ? ? ? if (ret) {
> + ? ? ? ? ? ? ? dev_dbg(dev, "Unabled to initialize driver data\n");
> + ? ? ? ? ? ? ? goto err_init;
> + ? ? ? }
> +
> + ? ? ? data->nsfrs = pdev->num_resources / 2;
> + ? ? ? data->sfrbases = kmalloc(sizeof(*data->sfrbases) * data->nsfrs,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? GFP_KERNEL);
> + ? ? ? if (data->sfrbases == NULL) {
> + ? ? ? ? ? ? ? dev_dbg(dev, "Not enough memory\n");
> + ? ? ? ? ? ? ? ret = -ENOMEM;
> + ? ? ? ? ? ? ? goto err_init;
> + ? ? ? }
> +
> + ? ? ? for (i = 0; i < data->nsfrs; i++) {
> + ? ? ? ? ? ? ? struct resource *res;
> + ? ? ? ? ? ? ? res = platform_get_resource(pdev, IORESOURCE_MEM, i);
> + ? ? ? ? ? ? ? if (!res) {
> + ? ? ? ? ? ? ? ? ? ? ? dev_dbg(dev, "Unable to find IOMEM region\n");
> + ? ? ? ? ? ? ? ? ? ? ? ret = -ENOENT;
> + ? ? ? ? ? ? ? ? ? ? ? goto err_res;
> + ? ? ? ? ? ? ? }
> +
> + ? ? ? ? ? ? ? data->sfrbases[i] = ioremap(res->start, resource_size(res));
> + ? ? ? ? ? ? ? if (!data->sfrbases[i]) {
> + ? ? ? ? ? ? ? ? ? ? ? dev_dbg(dev, "Unable to map IOMEM @ PA:%#x\n",
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? res->start);
> + ? ? ? ? ? ? ? ? ? ? ? ret = -ENOENT;
> + ? ? ? ? ? ? ? ? ? ? ? goto err_res;
> + ? ? ? ? ? ? ? }
> + ? ? ? }
> +
> + ? ? ? for (i = 0; i < data->nsfrs; i++) {
> + ? ? ? ? ? ? ? ret = platform_get_irq(pdev, i);
> + ? ? ? ? ? ? ? if (ret <= 0) {
> + ? ? ? ? ? ? ? ? ? ? ? dev_dbg(dev, "Unable to find IRQ resource\n");
> + ? ? ? ? ? ? ? ? ? ? ? goto err_irq;
> + ? ? ? ? ? ? ? }
> +
> + ? ? ? ? ? ? ? ret = request_irq(ret, exynos_sysmmu_irq, 0,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? dev_name(dev), data);
> + ? ? ? ? ? ? ? if (ret) {
> + ? ? ? ? ? ? ? ? ? ? ? dev_dbg(dev, "Unabled to register interrupt handler\n");
> + ? ? ? ? ? ? ? ? ? ? ? goto err_irq;
> + ? ? ? ? ? ? ? }
> + ? ? ? }
> +
> + ? ? ? if (dev_get_platdata(dev)) {
> + ? ? ? ? ? ? ? char *deli, *beg;
> + ? ? ? ? ? ? ? struct sysmmu_platform_data *platdata = dev_get_platdata(dev);
> +
> + ? ? ? ? ? ? ? beg = platdata->clockname;
> +
> + ? ? ? ? ? ? ? for (deli = beg; (*deli != '\0') && (*deli != ','); deli++)
> + ? ? ? ? ? ? ? ? ? ? ? /* NOTHING */;
> +
> + ? ? ? ? ? ? ? if (*deli == '\0')
> + ? ? ? ? ? ? ? ? ? ? ? deli = NULL;
> + ? ? ? ? ? ? ? else
> + ? ? ? ? ? ? ? ? ? ? ? *deli = '\0';
> +
> + ? ? ? ? ? ? ? data->clk[0] = clk_get(dev, beg);
> + ? ? ? ? ? ? ? if (IS_ERR(data->clk[0])) {
> + ? ? ? ? ? ? ? ? ? ? ? data->clk[0] = NULL;
> + ? ? ? ? ? ? ? ? ? ? ? dev_dbg(dev, "No clock descriptor registered\n");
> + ? ? ? ? ? ? ? }
> +
> + ? ? ? ? ? ? ? if (data->clk[0] && deli) {
> + ? ? ? ? ? ? ? ? ? ? ? *deli = ',';
> + ? ? ? ? ? ? ? ? ? ? ? data->clk[1] = clk_get(dev, deli + 1);
> + ? ? ? ? ? ? ? ? ? ? ? if (IS_ERR(data->clk[1]))
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? data->clk[1] = NULL;
> + ? ? ? ? ? ? ? }
> +
> + ? ? ? ? ? ? ? data->dbgname = platdata->dbgname;
> + ? ? ? }
> +
> + ? ? ? data->sysmmu = dev;
> + ? ? ? rwlock_init(&data->lock);
> +
> + ? ? ? __set_fault_handler(data, &default_fault_handler);
> +
> + ? ? ? if (dev->parent)
> + ? ? ? ? ? ? ? pm_runtime_enable(dev);
> +
> + ? ? ? dev_dbg(dev, "(%s) Initialized\n", data->dbgname);
> + ? ? ? return 0;
> +err_irq:
> + ? ? ? while (i-- > 0) {
> + ? ? ? ? ? ? ? int irq;
> +
> + ? ? ? ? ? ? ? irq = platform_get_irq(pdev, i);
> + ? ? ? ? ? ? ? free_irq(irq, data);
> + ? ? ? }
> +err_res:
> + ? ? ? while (data->nsfrs-- > 0)
> + ? ? ? ? ? ? ? iounmap(data->sfrbases[data->nsfrs]);
> + ? ? ? kfree(data->sfrbases);
> +err_init:
> + ? ? ? kfree(data);
> +err_alloc:
> + ? ? ? dev_err(dev, "Failed to initialize\n");
> + ? ? ? return ret;
> +}
> +
> +static struct platform_driver exynos_sysmmu_driver = {
> + ? ? ? .probe ? ? ? ? ?= exynos_sysmmu_probe,
> + ? ? ? .driver ? ? ? ? = {
> + ? ? ? ? ? ? ? .owner ? ? ? ? ?= THIS_MODULE,
> + ? ? ? ? ? ? ? .name ? ? ? ? ? = "exynos-sysmmu",
> + ? ? ? }
> +};
> +
> +static inline void pgtable_flush(void *vastart, void *vaend)
> +{
> + ? ? ? dmac_flush_range(vastart, vaend);
> + ? ? ? outer_flush_range(virt_to_phys(vastart),
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? virt_to_phys(vaend));
> +}
> +
> +static int exynos_iommu_domain_init(struct iommu_domain *domain)
> +{
> + ? ? ? struct exynos_iommu_domain *priv;
> +
> + ? ? ? priv = kzalloc(sizeof(*priv), GFP_KERNEL);
> + ? ? ? if (!priv)
> + ? ? ? ? ? ? ? return -ENOMEM;
> +
> + ? ? ? priv->pgtable = (unsigned long *)__get_free_pages(
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? GFP_KERNEL | __GFP_ZERO, 2);
> + ? ? ? if (!priv->pgtable)
> + ? ? ? ? ? ? ? goto err_pgtable;
> +
> + ? ? ? priv->lv2entcnt = (short *)__get_free_pages(
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? GFP_KERNEL | __GFP_ZERO, 1);
> + ? ? ? if (!priv->lv2entcnt)
> + ? ? ? ? ? ? ? goto err_counter;
> +
> + ? ? ? pgtable_flush(priv->pgtable, priv->pgtable + NUM_LV1ENTRIES);
> +
> + ? ? ? spin_lock_init(&priv->lock);
> + ? ? ? spin_lock_init(&priv->pgtablelock);
> + ? ? ? INIT_LIST_HEAD(&priv->clients);
> +
> + ? ? ? domain->priv = priv;
> + ? ? ? return 0;
> +
> +err_counter:
> + ? ? ? free_pages((unsigned long)priv->pgtable, 2);
> +err_pgtable:
> + ? ? ? kfree(priv);
> + ? ? ? return -ENOMEM;
> +}
> +
> +static void exynos_iommu_domain_destroy(struct iommu_domain *domain)
> +{
> + ? ? ? struct exynos_iommu_domain *priv = domain->priv;
> + ? ? ? struct list_head *pos, *n;
> + ? ? ? unsigned long flags;
> + ? ? ? int i;
> +
> + ? ? ? WARN_ON(!list_empty(&priv->clients));
> +
> + ? ? ? spin_lock_irqsave(&priv->lock, flags);
> +
> + ? ? ? list_for_each_safe(pos, n, &priv->clients) {
> + ? ? ? ? ? ? ? struct iommu_client *client;
> +
> + ? ? ? ? ? ? ? client = list_entry(pos, struct iommu_client, node);
> + ? ? ? ? ? ? ? exynos_sysmmu_disable(client->dev);
> + ? ? ? ? ? ? ? kfree(client);
> + ? ? ? }
> +
> + ? ? ? spin_unlock_irqrestore(&priv->lock, flags);
> +
> + ? ? ? for (i = 0; i < NUM_LV1ENTRIES; i++)
> + ? ? ? ? ? ? ? if (lv1ent_page(priv->pgtable + i))
> + ? ? ? ? ? ? ? ? ? ? ? kfree(__va(lv2table_base(priv->pgtable + i)));
> +
> + ? ? ? free_pages((unsigned long)priv->pgtable, 2);
> + ? ? ? free_pages((unsigned long)priv->lv2entcnt, 1);
> + ? ? ? kfree(domain->priv);
> + ? ? ? domain->priv = NULL;
> +}
> +
> +static int exynos_iommu_attach_device(struct iommu_domain *domain,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?struct device *dev)
> +{
> + ? ? ? struct exynos_iommu_domain *priv = domain->priv;
> + ? ? ? struct iommu_client *client = NULL;
> + ? ? ? struct list_head *pos;
> + ? ? ? unsigned long flags;
> +
> + ? ? ? spin_lock_irqsave(&priv->lock, flags);
> +
> + ? ? ? list_for_each(pos, &priv->clients) {
> + ? ? ? ? ? ? ? struct iommu_client *cur;
> +
> + ? ? ? ? ? ? ? cur = list_entry(pos, struct iommu_client, node);
> + ? ? ? ? ? ? ? if (cur->dev == dev) {
> + ? ? ? ? ? ? ? ? ? ? ? client = cur;
> + ? ? ? ? ? ? ? ? ? ? ? break;
> + ? ? ? ? ? ? ? }
> + ? ? ? }
> +
> + ? ? ? if (client != NULL)
> + ? ? ? ? ? ? ? client->refcnt++;
> +
> + ? ? ? spin_unlock_irqrestore(&priv->lock, flags);
> +
> + ? ? ? if (client != NULL) {
> + ? ? ? ? ? ? ? dev_dbg(dev, "%s: IOMMU with pgtable 0x%lx already attached\n",
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? __func__, __pa(priv->pgtable));
> + ? ? ? ? ? ? ? return 0;
> + ? ? ? }
> +
> + ? ? ? client = __get_enabled_iommu_client(domain, dev);
> + ? ? ? if (IS_ERR(client))
> + ? ? ? ? ? ? ? return PTR_ERR(client);
> +
> + ? ? ? spin_lock_irqsave(&priv->lock, flags);
> + ? ? ? list_add_tail(&client->node, &priv->clients);
> + ? ? ? spin_unlock_irqrestore(&priv->lock, flags);
> +
> + ? ? ? dev_dbg(dev, "%s: Attached new IOMMU with pgtable 0x%lx\n", __func__,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? __pa(priv->pgtable));
> + ? ? ? return 0;
> +}
> +
> +static void exynos_iommu_detach_device(struct iommu_domain *domain,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? struct device *dev)
> +{
> + ? ? ? struct exynos_iommu_domain *priv = domain->priv;
> + ? ? ? struct iommu_client *client = NULL;
> + ? ? ? struct list_head *pos;
> + ? ? ? unsigned long flags;
> +
> + ? ? ? spin_lock_irqsave(&priv->lock, flags);
> +
> + ? ? ? list_for_each(pos, &priv->clients) {
> + ? ? ? ? ? ? ? struct iommu_client *cur;
> +
> + ? ? ? ? ? ? ? cur = list_entry(pos, struct iommu_client, node);
> + ? ? ? ? ? ? ? if (cur->dev == dev) {
> + ? ? ? ? ? ? ? ? ? ? ? cur->refcnt--;
> + ? ? ? ? ? ? ? ? ? ? ? client = cur;
> + ? ? ? ? ? ? ? ? ? ? ? break;
> + ? ? ? ? ? ? ? }
> + ? ? ? }
> +
> + ? ? ? spin_unlock_irqrestore(&priv->lock, flags);
> +
> + ? ? ? if (WARN_ON(client == NULL))
> + ? ? ? ? ? ? ? return;
> +
> + ? ? ? if (client->refcnt > 0) {
> + ? ? ? ? ? ? ? dev_dbg(dev,
> + ? ? ? ? ? ? ? ? ? ? ? "IOMMU is still attached %d times with pgtable %#lx\n",
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? client->refcnt, __pa(priv->pgtable));
> + ? ? ? ? ? ? ? return;
> + ? ? ? }
> +
> + ? ? ? list_del(&client->node);
> + ? ? ? exynos_sysmmu_disable(client->dev);
> + ? ? ? dev_dbg(dev, "%s: Detached IOMMU with pgtable 0x%lx\n", __func__,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? __pa(priv->pgtable));
> +}
> +
> +static unsigned long *alloc_lv2entry(unsigned long *sent, unsigned long iova,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? short *pgcounter)
> +{
> + ? ? ? if (lv1ent_fault(sent)) {
> + ? ? ? ? ? ? ? unsigned long *pent;
> +
> + ? ? ? ? ? ? ? pent = kzalloc(LV2TABLE_SIZE, GFP_ATOMIC);
> + ? ? ? ? ? ? ? BUG_ON((unsigned long)pent & (LV2TABLE_SIZE - 1));
> + ? ? ? ? ? ? ? if (!pent)
> + ? ? ? ? ? ? ? ? ? ? ? return NULL;
> +
> + ? ? ? ? ? ? ? *sent = mk_lv1ent_page(__pa(pent));
> + ? ? ? ? ? ? ? *pgcounter = NUM_LV2ENTRIES;
> + ? ? ? ? ? ? ? pgtable_flush(pent, pent + NUM_LV2ENTRIES);
> + ? ? ? ? ? ? ? pgtable_flush(sent, sent + 1);
> + ? ? ? }
> +
> + ? ? ? return page_entry(sent, iova);
> +}
> +
> +static int lv1set_section(unsigned long *sent, phys_addr_t paddr, short *pgcnt)
> +{
> + ? ? ? if (lv1ent_section(sent))
> + ? ? ? ? ? ? ? return -EADDRINUSE;
> +
> + ? ? ? if (lv1ent_page(sent)) {
> + ? ? ? ? ? ? ? if (*pgcnt != NUM_LV2ENTRIES)
> + ? ? ? ? ? ? ? ? ? ? ? return -EADDRINUSE;
> +
> + ? ? ? ? ? ? ? kfree(page_entry(sent, 0));
> +
> + ? ? ? ? ? ? ? *pgcnt = 0;
> + ? ? ? }
> +
> + ? ? ? *sent = mk_lv1ent_sect(paddr);
> +
> + ? ? ? pgtable_flush(sent, sent + 1);
> +
> + ? ? ? return 0;
> +}
> +
> +static int lv2set_page(unsigned long *pent, phys_addr_t paddr, size_t size,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? short *pgcnt)
> +{
> + ? ? ? if (size == SPAGE_SIZE) {
> + ? ? ? ? ? ? ? if (!lv2ent_fault(pent))
> + ? ? ? ? ? ? ? ? ? ? ? return -EADDRINUSE;
> +
> + ? ? ? ? ? ? ? *pent = mk_lv2ent_spage(paddr);
> + ? ? ? ? ? ? ? pgtable_flush(pent, pent + 1);
> + ? ? ? ? ? ? ? *pgcnt -= 1;
> + ? ? ? } else { /* size == LPAGE_SIZE */
> + ? ? ? ? ? ? ? int i;
> + ? ? ? ? ? ? ? for (i = 0; i < SPAGES_PER_LPAGE; i++, pent++) {
> + ? ? ? ? ? ? ? ? ? ? ? if (!lv2ent_fault(pent)) {
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? memset(pent, 0, sizeof(*pent) * i);
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? return -EADDRINUSE;
> + ? ? ? ? ? ? ? ? ? ? ? }
> +
> + ? ? ? ? ? ? ? ? ? ? ? *pent = mk_lv2ent_lpage(paddr);
> + ? ? ? ? ? ? ? }
> + ? ? ? ? ? ? ? pgtable_flush(pent - SPAGES_PER_LPAGE, pent);
> + ? ? ? ? ? ? ? *pgcnt -= SPAGES_PER_LPAGE;
> + ? ? ? }
> +
> + ? ? ? return 0;
> +}
> +
> +static int exynos_iommu_map(struct iommu_domain *domain, unsigned long iova,
> + ? ? ? ? ? ? ? ? ? ? ? ?phys_addr_t paddr, size_t size, int prot)
> +{
> + ? ? ? struct exynos_iommu_domain *priv = domain->priv;
> + ? ? ? unsigned long *entry;
> + ? ? ? unsigned long flags;
> + ? ? ? int ret = -ENOMEM;
> +
> + ? ? ? BUG_ON(priv->pgtable == NULL);
> +
> + ? ? ? spin_lock_irqsave(&priv->pgtablelock, flags);
> +
> + ? ? ? entry = section_entry(priv->pgtable, iova);
> +
> + ? ? ? if (size == SECT_SIZE) {
> + ? ? ? ? ? ? ? ret = lv1set_section(entry, paddr,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? &priv->lv2entcnt[lv1ent_offset(iova)]);
> + ? ? ? } else {
> + ? ? ? ? ? ? ? unsigned long *pent;
> +
> + ? ? ? ? ? ? ? pent = alloc_lv2entry(entry, iova,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? &priv->lv2entcnt[lv1ent_offset(iova)]);
> +
> + ? ? ? ? ? ? ? if (!pent)
> + ? ? ? ? ? ? ? ? ? ? ? ret = -ENOMEM;
> + ? ? ? ? ? ? ? else
> + ? ? ? ? ? ? ? ? ? ? ? ret = lv2set_page(pent, paddr, size,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? &priv->lv2entcnt[lv1ent_offset(iova)]);
> + ? ? ? }
> +
> + ? ? ? if (ret) {
> + ? ? ? ? ? ? ? pr_debug("%s: Failed to map iova 0x%lx/0x%x bytes\n",
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? __func__, iova, size);
> + ? ? ? }
> +
> + ? ? ? spin_unlock_irqrestore(&priv->pgtablelock, flags);
> +
> + ? ? ? return ret;
> +}
> +
> +static size_t exynos_iommu_unmap(struct iommu_domain *domain,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ?unsigned long iova, size_t size)
> +{
> + ? ? ? struct exynos_iommu_domain *priv = domain->priv;
> + ? ? ? struct iommu_client *client;
> + ? ? ? unsigned long flags;
> + ? ? ? unsigned long *ent;
> +
> + ? ? ? BUG_ON(priv->pgtable == NULL);
> +
> + ? ? ? spin_lock_irqsave(&priv->pgtablelock, flags);
> +
> + ? ? ? ent = section_entry(priv->pgtable, iova);
> +
> + ? ? ? if (lv1ent_section(ent)) {
> + ? ? ? ? ? ? ? BUG_ON(size < SECT_SIZE);
> +
> + ? ? ? ? ? ? ? *ent = 0;
> + ? ? ? ? ? ? ? pgtable_flush(ent, ent + 1);
> + ? ? ? ? ? ? ? size = SECT_SIZE;
> + ? ? ? ? ? ? ? goto done;
> + ? ? ? }
> +
> + ? ? ? if (unlikely(lv1ent_fault(ent))) {
> + ? ? ? ? ? ? ? if (size > SECT_SIZE)
> + ? ? ? ? ? ? ? ? ? ? ? size = SECT_SIZE;
> + ? ? ? ? ? ? ? goto done;
> + ? ? ? }
> +
> + ? ? ? /* lv1ent_page(sent) == true here */
> +
> + ? ? ? ent = page_entry(ent, iova);
> +
> + ? ? ? if (unlikely(lv2ent_fault(ent))) {
> + ? ? ? ? ? ? ? size = SPAGE_SIZE;
> + ? ? ? ? ? ? ? goto done;
> + ? ? ? }
> +
> + ? ? ? if (lv2ent_small(ent)) {
> + ? ? ? ? ? ? ? *ent = 0;
> + ? ? ? ? ? ? ? size = SPAGE_SIZE;
> + ? ? ? ? ? ? ? priv->lv2entcnt[lv1ent_offset(iova)] -= 1;
> + ? ? ? ? ? ? ? goto done;
> + ? ? ? }
> +
> + ? ? ? /* lv1ent_large(ent) == true here */
> + ? ? ? BUG_ON(size < LPAGE_SIZE);
> +
> + ? ? ? memset(ent, 0, sizeof(*ent) * SPAGES_PER_LPAGE);
> +
> + ? ? ? size = LPAGE_SIZE;
> + ? ? ? priv->lv2entcnt[lv1ent_offset(iova)] -= SPAGES_PER_LPAGE;
> +done:
> + ? ? ? list_for_each_entry(client, &priv->clients, node)
> + ? ? ? ? ? ? ? sysmmu_tlb_invalidate_entry(client->dev, iova);
> +
> + ? ? ? spin_unlock_irqrestore(&priv->pgtablelock, flags);
> +
> + ? ? ? return size;
> +}
> +
> +static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? unsigned long iova)
> +{
> + ? ? ? struct exynos_iommu_domain *priv = domain->priv;
> + ? ? ? unsigned long *entry;
> + ? ? ? unsigned long flags;
> + ? ? ? phys_addr_t phys = -1;
> +
> + ? ? ? spin_lock_irqsave(&priv->pgtablelock, flags);
> +
> + ? ? ? entry = section_entry(priv->pgtable, iova);
> +
> + ? ? ? if (lv1ent_section(entry)) {
> + ? ? ? ? ? ? ? phys = section_phys(entry) + section_offs(iova);
> + ? ? ? } else if (lv1ent_page(entry)) {
> + ? ? ? ? ? ? ? entry = page_entry(entry, iova);
> +
> + ? ? ? ? ? ? ? if (lv2ent_large(entry))
> + ? ? ? ? ? ? ? ? ? ? ? phys = lpage_phys(entry) + lpage_offs(iova);
> + ? ? ? ? ? ? ? else if (lv2ent_small(entry))
> + ? ? ? ? ? ? ? ? ? ? ? phys = spage_phys(entry) + spage_offs(iova);
> + ? ? ? }
> +
> + ? ? ? spin_unlock_irqrestore(&priv->pgtablelock, flags);
> +
> + ? ? ? return phys;
What's happened if phys has -1? I mean it can't trigger any condition at above.
> +}
> +
> +static struct iommu_ops exynos_iommu_ops = {
> + ? ? ? .domain_init = &exynos_iommu_domain_init,
> + ? ? ? .domain_destroy = &exynos_iommu_domain_destroy,
> + ? ? ? .attach_dev = &exynos_iommu_attach_device,
> + ? ? ? .detach_dev = &exynos_iommu_detach_device,
> + ? ? ? .map = &exynos_iommu_map,
> + ? ? ? .unmap = &exynos_iommu_unmap,
> + ? ? ? .iova_to_phys = &exynos_iommu_iova_to_phys,
> + ? ? ? .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE,
> +};
> +
> +static int __init exynos_iommu_init(void)
> +{
> + ? ? ? int ret;
> +
> + ? ? ? ret = platform_driver_register(&exynos_sysmmu_driver);
> +
> + ? ? ? if (ret == 0)
> + ? ? ? ? ? ? ? bus_set_iommu(&platform_bus_type, &exynos_iommu_ops);
> +
> + ? ? ? return ret;
> +}
> +arch_initcall(exynos_iommu_init);
> --
> 1.7.1
>
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo at vger.kernel.org
> More majordomo info at ?http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at ?http://www.tux.org/lkml/

[PATCH v10 3/3] iommu/exynos: Add iommu driver for Exynos Platforms

[KyongHo Cho]

On Tue, Mar 6, 2012 at 10:13 PM, Kyungmin Park <kmpark@infradead.org> wrote:
> On Tue, Mar 6, 2012 at 5:31 PM, KyongHo Cho <pullip.cho@samsung.com> wrote:
>> HAALgBjAGgAbwBAAHMAYQBtAHMAdQBuAGcALgBjAG8AbQA=;Tue,
>> ?06 Mar 2012 08:31:29 GMT;WwBQAEEAVABDAEgAIAB2ADEAMAAgADMALwAzAF0AIABpAG8AbQBtAHUALwBlAHgAeQBuAG8AcwA6ACAAQQBkAGQAIABpAG8AbQBtAHUAIABkAHIAaQB2AGUAcgAgAGYAbwByACAARQB4AHkAbgBvAHMAIABQAGwAYQB0AGYAbwByAG0AcwA=
>> x-cr-puzzleid: {CF0AAF04-8639-4D69-B4E1-BF71EA1B0A70}
> What's this?
I think it is inserted by Outlook and it was not found in the email editor
before sending this patch. I am unable to use sendmail due to security
reason in my company.
I will find a way not to insert the above message. Thank you.

>>
>> This is the System MMU driver and IOMMU API implementation for
>> Exynos SOC platforms. Exynos platforms has more than 10 System
>> MMUs dedicated for each multimedia accellerators.
>>
>> The System MMU driver is already in arc/arm/plat-s5p but it is
>> moved to drivers/iommu due to Ohad Ben-Cohen gathered IOMMU drivers
>> there
>>
>> Any device driver in Exynos platforms that needs to control its
>> System MMU must call platform_set_sysmmu() to inform System MMU
>> driver who will control it.
>> platform_set_sysmmu() is defined in <mach/sysmmu.h>
> Does it really required? if we can remove it. it can be compiled
> without arch dependency.
The function must be called while board initialization and all source files for
the purpose is placed in 'mach' directory. Thus device driver is not dependent
on arch except the definition of data structure conveyed in
device.platform_data.

Since our System MMUs are attached to each peripheral devices, it is required
to define relationships between System MMUs and peripheral devices.
platform_set_sysmmu() defines the relationship.

>> +
>> +#ifdef CONFIG_EXYNOS_IOMMU_DEBUG
>> +#define DEBUG
>> +#endif
> If it's used only at here. you can add it at Makefile.
>
> ifeq ($(CONFIG_EXYNOS_IOMMU_DEBUG),y)
> ? ? ? ? CFLAGS-exynos-iommu ? ? ? ?+= -DDEBUG
> endif
>
> Of course you can select as your preference.

Thank you for advice.

>> +
>> +#define lv2ent_fault(pent) ((*(pent) & 3) == 0)
>> +#define lv2ent_small(pent) ((*(pent) & 2) == 2)
> #define lv2ent_small(pent) ((*(pent) & 3) == 2)?

The last bit is 'don't-care'.
It is intentionally ignored in the above comparison.

>> +
>> +#define REG_MMU_CTRL ? ? ? ? ? 0x000
>> +#define REG_MMU_CFG ? ? ? ? ? ?0x004
>> +#define REG_MMU_STATUS ? ? ? ? 0x008
>> +#define REG_MMU_FLUSH ? ? ? ? ?0x00C
>> +#define REG_MMU_FLUSH_ENTRY ? ?0x010
>> +#define REG_PT_BASE_ADDR ? ? ? 0x014
>> +#define REG_INT_STATUS ? ? ? ? 0x018
>> +#define REG_INT_CLEAR ? ? ? ? ?0x01C
>> +#define REG_MMU_VERSION ? ? ? ? ? ? ? ?0x034
> Please place it in order.

I found that a mistake while defining the offsets of control registers.
I will fix it.

>> +
>> +enum EXYNOS_SYSMMU_INTERRUPT_TYPE {
>> + ? ? ? SYSMMU_PAGEFAULT,
>> + ? ? ? SYSMMU_AR_MULTIHIT,
>> + ? ? ? SYSMMU_AW_MULTIHIT,
>> + ? ? ? SYSMMU_BUSERROR,
>> + ? ? ? SYSMMU_AR_SECURITY,
>> + ? ? ? SYSMMU_AR_ACCESS,
>> + ? ? ? SYSMMU_AW_SECURITY,
>> + ? ? ? SYSMMU_AW_PROTECTION, /* 7 */
>> + ? ? ? SYSMMU_FAULT_UNKNOWN,
>> + ? ? ? SYSMMU_FAULTS_NUM
> missing comma, "SYSMMU_FAULTS_NUM,"
Yes it is. I don't think that it is needed to appending comma after
the last element
unless it is predictable that adding elements after the last element
in the future.

>> +};
>> +
>> +typedef int (*sysmmu_fault_handler_t)(enum EXYNOS_SYSMMU_INTERRUPT_TYPE itype,
>> + ? ? ? ? ? ? ? ? ? ? ? unsigned long pgtable_base, unsigned long fault_addr);
>> +
>> +static unsigned short fault_reg_offset[SYSMMU_FAULTS_NUM] = {
>> + ? ? ? REG_PAGE_FAULT_ADDR,
>> + ? ? ? REG_AR_FAULT_ADDR,
>> + ? ? ? REG_AW_FAULT_ADDR,
>> + ? ? ? REG_DEFAULT_SLAVE_ADDR,
>> + ? ? ? REG_AR_FAULT_ADDR,
>> + ? ? ? REG_AR_FAULT_ADDR,
>> + ? ? ? REG_AW_FAULT_ADDR,
>> + ? ? ? REG_AW_FAULT_ADDR
> ditto.
Answered above.

>> +};
>> +
>> +static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = {
>> + ? ? ? "PAGE FAULT",
>> + ? ? ? "AR MULTI-HIT FAULT",
>> + ? ? ? "AW MULTI-HIT FAULT",
>> + ? ? ? "BUS ERROR",
>> + ? ? ? "AR SECURITY PROTECTION FAULT",
>> + ? ? ? "AR ACCESS PROTECTION FAULT",
>> + ? ? ? "AW SECURITY PROTECTION FAULT",
>> + ? ? ? "AW ACCESS PROTECTION FAULT",
>> + ? ? ? "UNKNOWN FAULT"
> ditto.
Answered above.

>> +void exynos_sysmmu_set_prefbuf(struct device *dev,
>> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? unsigned long base0, unsigned long size0,
>> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? unsigned long base1, unsigned long size1)
>> +{
>> + ? ? ? struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
>> + ? ? ? unsigned long flags;
>> + ? ? ? int i;
>> +
>> + ? ? ? BUG_ON((base0 + size0) <= base0);
>> + ? ? ? BUG_ON((size1 > 0) && ((base1 + size1) <= base1));
> It's bogus check. it's impossible. Do you think overflow case?
Yes I do.
It is prepared to detect invalid parameters from device drivers.

>> +
>> + ? ? ? read_lock_irqsave(&data->lock, flags);
>> + ? ? ? if (!is_sysmmu_active(data))
>> + ? ? ? ? ? ? ? goto finish;
>> +
>> + ? ? ? for (i = 0; i < data->nsfrs; i++) {
>> + ? ? ? ? ? ? ? if ((readl(data->sfrbases[i] + S5P_MMU_VERSION) >> 28) == 3) {
>> + ? ? ? ? ? ? ? ? ? ? ? sysmmu_block(data->sfrbases[i]);
>> +
>> + ? ? ? ? ? ? ? ? ? ? ? if (size1 == 0) {
>> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? if (size0 <= SZ_128K) {
>> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? base1 = base0;
>> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? size1 = size0;
>> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? } else {
>> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? size1 = size0 -
>> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ALIGN(size0 / 2, SZ_64K);
>> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? size0 = size0 - size1;
> Umm I mis-understand this codes at first. it think size1 has 0
> already. so it's meaning-less but now it re-use the size1 above.

Let me explain how this function works.
This function can accepts 2 memory regions to specify in the registers
of System MMU.
It is simple a device driver defines 2 memory regions but this function allows
it to define just one memory region and specify second regions as zero.
If the second regions is zero, this function divides the first region
into 2 regions
and specifies those regions into the registers of System MMU.

>> +void exynos_sysmmu_set_fault_handler(struct device *dev,
>> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? sysmmu_fault_handler_t handler)
>> +{
>> + ? ? ? struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu);
>> +
>> + ? ? ? __set_fault_handler(data, handler);
>> +}
>> +
>> +static int default_fault_handler(enum EXYNOS_SYSMMU_INTERRUPT_TYPE itype,
> Can you use the small letter instead of large one? do you see codes like this?
Ok. I will make it lower case.

>> +static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain,
>> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? unsigned long iova)
>> +{
>> + ? ? ? struct exynos_iommu_domain *priv = domain->priv;
>> + ? ? ? unsigned long *entry;
>> + ? ? ? unsigned long flags;
>> + ? ? ? phys_addr_t phys = -1;
>> +
>> + ? ? ? spin_lock_irqsave(&priv->pgtablelock, flags);
>> +
>> + ? ? ? entry = section_entry(priv->pgtable, iova);
>> +
>> + ? ? ? if (lv1ent_section(entry)) {
>> + ? ? ? ? ? ? ? phys = section_phys(entry) + section_offs(iova);
>> + ? ? ? } else if (lv1ent_page(entry)) {
>> + ? ? ? ? ? ? ? entry = page_entry(entry, iova);
>> +
>> + ? ? ? ? ? ? ? if (lv2ent_large(entry))
>> + ? ? ? ? ? ? ? ? ? ? ? phys = lpage_phys(entry) + lpage_offs(iova);
>> + ? ? ? ? ? ? ? else if (lv2ent_small(entry))
>> + ? ? ? ? ? ? ? ? ? ? ? phys = spage_phys(entry) + spage_offs(iova);
>> + ? ? ? }
>> +
>> + ? ? ? spin_unlock_irqrestore(&priv->pgtablelock, flags);
>> +
>> + ? ? ? return phys;
> What's happened if phys has -1? I mean it can't trigger any condition at above.
This function just follows page tables and finding corresponding
physical address
for the given IO virtual address.
-1 (0xFFFFFFFF in 32bit system) is just an error value
if no translation information is found.

But I found most IOMMU drivers return 0 for the case.
I will change it to 0.

Thank you for kind review.

Cho KyongHo.

[PATCH v10 3/3] iommu/exynos: Add iommu driver for Exynos Platforms

[InKi Dae]

> +static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
> +{
> + ? ? ? /* SYSMMU is in blocked when interrupt occurred. */
> + ? ? ? struct sysmmu_drvdata *data = dev_id;
> + ? ? ? struct resource *irqres;
> + ? ? ? struct platform_device *pdev;
> + ? ? ? enum EXYNOS_SYSMMU_INTERRUPT_TYPE itype;

how about to change EXYNOS_SYSMMU_INTERRUPT_TYPE to small letter? just
for code clean.

> + ? ? ? unsigned long addr = -1;
> +
> + ? ? ? int i, ret = -ENOSYS;
> +
> + ? ? ? read_lock(&data->lock);
> +
> + ? ? ? WARN_ON(!is_sysmmu_active(data));
> +
> + ? ? ? pdev = to_platform_device(data->sysmmu);
> + ? ? ? for (i = 0; i < pdev->num_resources; i++) {
> + ? ? ? ? ? ? ? irqres = platform_get_resource(pdev, IORESOURCE_IRQ, i);
> + ? ? ? ? ? ? ? if (irqres && ((int)irqres->start == irq))
> + ? ? ? ? ? ? ? ? ? ? ? break;
> + ? ? ? }

is there any reason that it should get irq resources in interrupt
handler? if not so then how about to move it into probe()?

> +
> + ? ? ? if (i == pdev->num_resources) {
> + ? ? ? ? ? ? ? itype = SYSMMU_FAULT_UNKNOWN;
> + ? ? ? } else {
> + ? ? ? ? ? ? ? i /= 2;
> +
> + ? ? ? ? ? ? ? itype = (enum EXYNOS_SYSMMU_INTERRUPT_TYPE)

ditto.

> + ? ? ? ? ? ? ? ? ? ? ? __ffs(__raw_readl(data->sfrbases[i] + REG_INT_STATUS));
> + ? ? ? ? ? ? ? if (WARN_ON(!((itype >= 0) && (itype < SYSMMU_FAULT_UNKNOWN))))
> + ? ? ? ? ? ? ? ? ? ? ? itype = SYSMMU_FAULT_UNKNOWN;
> + ? ? ? ? ? ? ? else
> + ? ? ? ? ? ? ? ? ? ? ? addr = __raw_readl(
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? data->sfrbases[i] + fault_reg_offset[itype]);
> + ? ? ? }
> +

[PATCH v10 3/3] iommu/exynos: Add iommu driver for Exynos Platforms

[KyongHo Cho]

On Wed, Mar 7, 2012 at 2:28 PM, InKi Dae <daeinki@gmail.com> wrote:
>> +static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id)
>> +{
>> + ? ? ? /* SYSMMU is in blocked when interrupt occurred. */
>> + ? ? ? struct sysmmu_drvdata *data = dev_id;
>> + ? ? ? struct resource *irqres;
>> + ? ? ? struct platform_device *pdev;
>> + ? ? ? enum EXYNOS_SYSMMU_INTERRUPT_TYPE itype;
>
> how about to change EXYNOS_SYSMMU_INTERRUPT_TYPE to small letter? just
> for code clean.
>
I will. please refer to the my reply to Kyungmin Park.

>> + ? ? ? unsigned long addr = -1;
>> +
>> + ? ? ? int i, ret = -ENOSYS;
>> +
>> + ? ? ? read_lock(&data->lock);
>> +
>> + ? ? ? WARN_ON(!is_sysmmu_active(data));
>> +
>> + ? ? ? pdev = to_platform_device(data->sysmmu);
>> + ? ? ? for (i = 0; i < pdev->num_resources; i++) {
>> + ? ? ? ? ? ? ? irqres = platform_get_resource(pdev, IORESOURCE_IRQ, i);
>> + ? ? ? ? ? ? ? if (irqres && ((int)irqres->start == irq))
>> + ? ? ? ? ? ? ? ? ? ? ? break;
>> + ? ? ? }
>
> is there any reason that it should get irq resources in interrupt
> handler? if not so then how about to move it into probe()?
>
The platform device of ISP has a lot of IRQ sources.
The above for clause determines which IRQ source (System MMU)
generated the interrupt.

>> +
>> + ? ? ? if (i == pdev->num_resources) {
>> + ? ? ? ? ? ? ? itype = SYSMMU_FAULT_UNKNOWN;
>> + ? ? ? } else {
>> + ? ? ? ? ? ? ? i /= 2;
>> +
>> + ? ? ? ? ? ? ? itype = (enum EXYNOS_SYSMMU_INTERRUPT_TYPE)
>
> ditto.
>

Thank you.

Cho KyongHo.