[PATCH 0/4] Add support for page alloc w/ custom cache attributes

[PATCH 0/4] Add support for page alloc w/ custom cache attributes

[Gary King]

This series is an attempt to address the recent discussion regarding
dual-mapping of kernel pages with different cache attributes that
was spawned by the VCM patch series (and other IOMMU / SMMU patches),
to allow memory allocations for the SMMU to use any kernel-managed
page.

This adds 2 new APIs (enabled with the ARM_ATTRIB_ALLOCATOR config)
to allocate and free pages with caller-specified cache attributes.
When this config is enabled, the kernel's lowmem mapping is mapped
using PTEs rather than sections, so that the cache attributes for the
kernel's mapping can be updated when the page is allocated to match
the caller's request (and returned to pgprot_kernel when freed).

Additionally, this patch stack adds an additional memory type (inner-
writeback, outer-non-cacheable) that we (NVIDIA) have found to be
extremely beneficial in optimizing the drawing that Android's window
system (among others) uses.

I'm sending this out for comments on the overall direction. The APIs
are similar to functions that we implemented in the Tegra system MMU
driver.

- Gary

[PATCH 1/4] [ARM] mmu: add option to map lowmem with page mappings

[Gary King]

add a kernel configuration to map the kernel's lowmem pages using PTE
mappings, rather than the default behavior of 1MiB section mappings.
on ARMv7 processors, to support allocating pages with DMA-coherent
cache attributes, the cache attributes specified in the kernel's
mapping must match cache attributes specified for other mappings;
to ensure that this is the case, the kernel's attributes must be
specified on a per-page basis.

to avoid problems caused by the init_mm page table allocations exceeding
the available initial memory, when this config is enabled lowmem is
initially mapped using sections (matches current behavior), then remapped
using pages after bootmem is initialized

Signed-off-by: Gary King <gking@nvidia.com>
---
 arch/arm/mm/Kconfig |    9 +++++++
 arch/arm/mm/mmu.c   |   63 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+), 0 deletions(-)

diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index 101105e..176e815 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -837,3 +837,12 @@ config ARCH_HAS_BARRIERS
 	help
 	  This option allows the use of custom mandatory barriers
 	  included via the mach/barriers.h file.
+
+config ARCH_LOWMEM_IN_PTES
+	bool
+	help
+	  This option will map the kernel direct-mapped lowmem region
+	  using page table mappings rather than section mappings.
+
+config ARCH_USES_PG_UNCACHED
+	bool
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index 2858941..5818e83 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -247,7 +247,10 @@ static struct mem_type mem_types[] = {
 		.domain    = DOMAIN_USER,
 	},
 	[MT_MEMORY] = {
+		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
+				L_PTE_EXEC,
 		.prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
+		.prot_l1   = PMD_TYPE_TABLE,
 		.domain    = DOMAIN_KERNEL,
 	},
 	[MT_ROM] = {
@@ -463,6 +466,7 @@ static void __init build_mem_type_table(void)
 	mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask;
 	mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask;
 	mem_types[MT_MEMORY].prot_sect |= ecc_mask | cp->pmd;
+	mem_types[MT_MEMORY].prot_pte |= pgprot_kernel;
 	mem_types[MT_ROM].prot_sect |= cp->pmd;
 
 	switch (cp->pmd) {
@@ -506,6 +510,30 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
 	} while (pte++, addr += PAGE_SIZE, addr != end);
 }
 
+#ifdef CONFIG_ARCH_LOWMEM_IN_PTES
+static void __init realloc_init_pte(pmd_t *pmd, unsigned long addr,
+				    unsigned long end, unsigned long pfn,
+				    const struct mem_type *type)
+{
+	pte_t *pte, *ptep;
+
+	if ((pmd_val(*pmd) & PMD_TYPE_MASK) != PMD_TYPE_SECT)
+		return;
+
+	pte = alloc_bootmem_low_pages(2 * PTRS_PER_PTE * sizeof(pte_t));
+	if (WARN_ON(!pte))
+		return;
+
+	ptep = pte + PTRS_PER_PTE + __pte_index(addr);
+	do {
+		set_pte_ext(ptep, pfn_pte(pfn, __pgprot(type->prot_pte)), 0);
+		pfn++;
+	} while (ptep++, addr += PAGE_SIZE, addr != end);
+
+	__pmd_populate(pmd, __pa(pte) | type->prot_l1);
+}
+#endif
+
 static void __init alloc_init_section(pgd_t *pgd, unsigned long addr,
 				      unsigned long end, unsigned long phys,
 				      const struct mem_type *type)
@@ -1045,6 +1073,40 @@ static void __init map_lowmem(void)
 	}
 }
 
+static void __init remap_lowmem(void)
+{
+#ifdef CONFIG_ARCH_LOWMEM_IN_PTES
+	struct meminfo *mi = &meminfo;
+	const struct mem_type *type = &mem_types[MT_MEMORY];
+	int i;
+
+	for (i = 0; i < mi->nr_banks; i++) {
+		pgd_t *pgd;
+		unsigned long phys, addr, end;
+		struct membank *bank = &mi->bank[i];
+
+		if (bank->highmem)
+			continue;
+
+		phys = __pfn_to_phys(bank_pfn_start(bank));
+		addr = __phys_to_virt(bank_phys_start(bank));
+		end = addr + bank_phys_size(bank);
+
+		pgd = pgd_offset_k(addr);
+		do {
+			unsigned long next = pgd_addr_end(addr, end);
+			pmd_t *pmd = pmd_offset(pgd, addr);
+
+			realloc_init_pte(pmd, addr, next,
+					 __phys_to_pfn(phys), type);
+
+			phys += next - addr;
+			addr = next;
+		} while (pgd++, addr != end);
+	}
+#endif
+}
+
 static int __init meminfo_cmp(const void *_a, const void *_b)
 {
 	const struct membank *a = _a, *b = _b;
@@ -1067,6 +1129,7 @@ void __init paging_init(struct machine_desc *mdesc)
 	prepare_page_table();
 	map_lowmem();
 	bootmem_init();
+	remap_lowmem();
 	devicemaps_init(mdesc);
 	kmap_init();
 
-- 
1.7.0.4

[PATCH 2/4] [ARM] mm: add page allocator for customizing cache attributes

[Gary King]

ARM CPUs with speculative prefetching have undefined behaviors when the
same physical page is mapped to two different virtual addresses with
conflicting cache attributes.

since many recent systems include IOMMU functionality (i.e., remapping
of discontiguous physical pages into a virtually-contiguous address
range for I/O devices), it is desirable to support allocating any
available OS memory for use by the I/O devices. however, since many
systems do not support cache coherency between the CPU and DMA devices,
these devices are left with using DMA-coherent allocations from the OS
(which severely limits the benefit of an IOMMU) or performing cache
maintenance (which can be a severe performance loss, particularly on
systems with outer caches, compared to using DMA-coherent memory).

this change adds an API for allocating pages from the OS with specific
cache maintenance properties and ensures that the kernel's mapping
of the page reflects the desired cache attributes, in line with the
ARMv7 architectural requirements

since the kmap page properties are now page-dependent, to ensure that
highmem pages are always mapped with the desired attributes, kmap_prot
is implemented as a static inline function which expects that "page"
is declared as a struct page * variable in any function which uses
kmap_prot, so that the correct properties will be returned.

Signed-off-by: Gary King <gking@nvidia.com>
---
 arch/arm/include/asm/attrib_alloc.h |   57 ++++++++++++++++
 arch/arm/include/asm/highmem.h      |   12 ++++
 arch/arm/mm/Kconfig                 |   24 +++++++
 arch/arm/mm/Makefile                |    2 +
 arch/arm/mm/attrib_alloc.c          |  122 +++++++++++++++++++++++++++++++++++
 arch/arm/mm/dma-mapping.c           |   16 ++++-
 arch/arm/mm/flush.c                 |    9 +++
 arch/arm/mm/highmem.c               |    3 +-
 8 files changed, 243 insertions(+), 2 deletions(-)
 create mode 100644 arch/arm/include/asm/attrib_alloc.h
 create mode 100644 arch/arm/mm/attrib_alloc.c

diff --git a/arch/arm/include/asm/attrib_alloc.h b/arch/arm/include/asm/attrib_alloc.h
new file mode 100644
index 0000000..609939d
--- /dev/null
+++ b/arch/arm/include/asm/attrib_alloc.h
@@ -0,0 +1,57 @@
+/*
+ * arch/arm/include/asm/attrib_alloc.h
+ *
+ * Page allocator with custom cache attributes
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __ARCH_ARM_ATTRIB_ALLOC_H
+#define __ARCH_ARM_ATTRIB_ALLOC_H
+
+#include <linux/types.h>
+#include <asm/page.h>
+
+#ifdef CONFIG_ARM_ATTRIB_ALLOCATOR
+struct page *arm_attrib_alloc_pages_node(int nid, gfp_t gfp,
+					 unsigned int order, pgprot_t prot);
+
+void arm_attrib_free_pages(struct page *page, unsigned int order);
+#else
+static inline struct page *arm_attrib_alloc_pages_node(int, gfp_t,
+						       unsigned int, pgprot_t)
+{
+	return NULL;
+}
+
+static inline arm_attrib_free_pages(struct page *, unsigned int)
+{
+}
+#endif
+
+static inline struct page *arm_attrib_alloc_pages(gfp_t gfp,
+					  unsigned int order, pgprot_t prot)
+{
+	return arm_attrib_alloc_pages_node(-1, gfp, order, prot);
+}
+
+#define arm_attrib_alloc_page(gfp, prot)	\
+	arm_attrib_alloc_pages((gfp), 0, (prot))
+
+#define arm_attrib_free_page(page)	arm_attrib_free_pages((page), 0)
+
+#endif /* __ARCH_ARM_ATTRIB_ALLOC_H */
diff --git a/arch/arm/include/asm/highmem.h b/arch/arm/include/asm/highmem.h
index feb988a..d8c464c 100644
--- a/arch/arm/include/asm/highmem.h
+++ b/arch/arm/include/asm/highmem.h
@@ -9,7 +9,19 @@
 #define PKMAP_NR(virt)		(((virt) - PKMAP_BASE) >> PAGE_SHIFT)
 #define PKMAP_ADDR(nr)		(PKMAP_BASE + ((nr) << PAGE_SHIFT))
 
+#ifndef CONFIG_ARM_ATTRIB_ALLOCATOR
 #define kmap_prot		PAGE_KERNEL
+#else
+static inline pgprot_t kmap_prot_of(struct page *page)
+{
+	if (PageUncached(page)) {
+		return __pgprot_modify(PAGE_KERNEL, L_PTE_MT_MASK,
+				       page->private);
+	} else
+		return PAGE_KERNEL;
+}
+#define kmap_prot		kmap_prot_of(page)
+#endif
 
 #define flush_cache_kmaps() \
 	do { \
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index 176e815..4abe9ee 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -811,6 +811,30 @@ config ARM_L1_CACHE_SHIFT
 	default 6 if ARM_L1_CACHE_SHIFT_6
 	default 5
 
+config ARM_ATTRIB_ALLOCATOR
+	bool "Support custom cache attribute allocations in low memory"
+	select ARCH_LOWMEM_IN_PTES
+	select ARCH_USES_PG_UNCACHED
+	depends on MMU && !CPU_CACHE_VIVT
+	help
+	  Historically, the kernel has only reserved a small region
+	  of physical memory for uncached access, and relied on
+	  explicit cache maintenance for ensuring coherency between
+	  the CPU and DMA.
+
+	  However, many recent systems support mapping discontiguous
+	  physical pages into contiguous DMA addresses (so-called
+	  system MMUs). For some DMA clients (notably graphics and
+	  multimedia engines), performing explict cache maintenance
+	  between CPU and DMA mappings can be prohibitively expensive,
+	  and since ARMv7, mapping the same physical page with different
+	  cache attributes is disallowed and has unpredictable behavior.
+
+	  Say 'Y' here to include page allocation support with explicit
+	  cache attributes; on ARMv7 systems this will also force the
+	  kernel's page tables to be mapped using page tables rather
+	  than sections.
+
 config ARM_DMA_MEM_BUFFERABLE
 	bool "Use non-cacheable memory for DMA" if CPU_V6 && !CPU_V7
 	depends on !(MACH_REALVIEW_PB1176 || REALVIEW_EB_ARM11MP || \
diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile
index e8d34a8..ce803db 100644
--- a/arch/arm/mm/Makefile
+++ b/arch/arm/mm/Makefile
@@ -12,6 +12,8 @@ ifneq ($(CONFIG_MMU),y)
 obj-y				+= nommu.o
 endif
 
+obj-$(CONFIG_ARM_ATTRIB_ALLOCATOR) += attrib_alloc.o
+
 obj-$(CONFIG_MODULES)		+= proc-syms.o
 
 obj-$(CONFIG_ALIGNMENT_TRAP)	+= alignment.o
diff --git a/arch/arm/mm/attrib_alloc.c b/arch/arm/mm/attrib_alloc.c
new file mode 100644
index 0000000..c188772
--- /dev/null
+++ b/arch/arm/mm/attrib_alloc.c
@@ -0,0 +1,122 @@
+/*
+ * arch/arm/mm/attrib_alloc.c
+ *
+ * Page allocator with custom cache attributes
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include <linux/kernel.h>
+#include <linux/highmem.h>
+#include <linux/gfp.h>
+#include <linux/page-flags.h>
+#include <asm/tlbflush.h>
+#include <asm/cacheflush.h>
+#include <asm/pgtable.h>
+#include <asm/pgalloc.h>
+#include <asm/fixmap.h>
+#include <asm/outercache.h>
+#include <asm/attrib_alloc.h>
+#include "mm.h"
+
+#define pgprot_type(pte)	(pgprot_val(pte) & ~L_PTE_MT_MASK)
+
+static void update_kmap_pte(struct page *page, pgprot_t prot)
+{
+	unsigned long addr;
+	pte_t *pte;
+
+	addr = (unsigned long)kmap_high_get(page);
+	BUG_ON(!PageHighMem(page) || addr >= FIXADDR_START);
+	if (!addr)
+		return;
+
+	pte = &pkmap_page_table[PKMAP_NR(addr)];
+	set_pte_at(&init_mm, addr, pte, mk_pte(page, __pgprot(prot)));
+	flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
+	kunmap_high(page);
+}
+
+static void update_pte(struct page *page, pgprot_t prot)
+{
+	unsigned long addr = (unsigned long)page_address(page);
+	pgd_t *pgd = pgd_offset_k(addr);
+	pmd_t *pmd = pmd_offset(pgd, addr);
+	pte_t *pte;
+
+	BUG_ON(pmd_none(*pmd));
+	pte = pte_offset_kernel(pmd, addr);
+	set_pte_at(&init_mm, addr, pte, mk_pte(page, __pgprot(prot)));
+	flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
+}
+
+void arm_attrib_free_pages(struct page *page, unsigned int order)
+{
+	/* reset the page's mappings back to the standard kernel mappings
+	 * before returning it to the page allocator */
+	if (PageUncached(page)) {
+		struct page *loop;
+		unsigned int i;
+
+		for (i = 0, loop = page; i < (1 << order); i++, loop++) {
+
+			if (PageHighMem(loop))
+				update_kmap_pte(loop, pgprot_kernel);
+			else
+				update_pte(loop, pgprot_kernel);
+
+			ClearPageUncached(page);
+			set_page_private(page, 0);
+		}
+	}
+	__free_pages(page, order);
+}
+
+struct page *arm_attrib_alloc_pages_node(int nid, gfp_t gfp,
+					 unsigned int order, pgprot_t prot)
+{
+	struct page *page, *loop;
+	unsigned int i;
+	unsigned int type = pgprot_type(prot);
+
+	page = alloc_pages_node(nid, gfp, order);
+	/* if the requested cache attributes match the default value,
+	 * just return because no special handling will be needed for
+	 * this page */
+	if (!page || (type == pgprot_type(pgprot_kernel)))
+		return page;
+
+	for (i = 0, loop = page; i < (1 << order); i++, loop++) {
+		unsigned long phys = page_to_phys(page);
+		__flush_dcache_page(page_mapping(page), page);
+		outer_flush_range(phys, phys + PAGE_SIZE);
+
+		SetPageUncached(loop);
+		set_page_private(page, type);
+
+		/* even though a freshly-allocated highmem page shouldn't
+		 * be mapped, because the kmaps are flushed lazily, it
+		 * is possible that a mapping from an old kmap_high call
+		 * is still present, and its cache attributes need to
+		 * be updated to match the new expectations */
+		if (PageHighMem(loop))
+			update_kmap_pte(loop, prot);
+		else
+			update_pte(loop, prot);
+	}
+	return page;
+}
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index 9e7742f..fa94be5 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -485,6 +485,12 @@ void ___dma_page_cpu_to_dev(struct page *page, unsigned long off,
 {
 	unsigned long paddr;
 
+	if (PageUncached(page) &&
+	    !(page_private(page) == L_PTE_MT_WRITEBACK ||
+	      page_private(page) == L_PTE_MT_WRITEALLOC ||
+	      page_private(page) == L_PTE_MT_DEV_CACHED))
+		return;
+
 	dma_cache_maint_page(page, off, size, dir, dmac_map_area);
 
 	paddr = page_to_phys(page) + off;
@@ -500,7 +506,15 @@ EXPORT_SYMBOL(___dma_page_cpu_to_dev);
 void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
 	size_t size, enum dma_data_direction dir)
 {
-	unsigned long paddr = page_to_phys(page) + off;
+	unsigned long paddr;
+
+	if (PageUncached(page) &&
+	    !(page_private(page) == L_PTE_MT_WRITEBACK ||
+	      page_private(page) == L_PTE_MT_WRITEALLOC ||
+	      page_private(page) == L_PTE_MT_DEV_CACHED))
+		return;
+
+	paddr = page_to_phys(page) + off;
 
 	/* FIXME: non-speculating: not required */
 	/* don't bother invalidating if DMA to device */
diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c
index c6844cb..33e900a 100644
--- a/arch/arm/mm/flush.c
+++ b/arch/arm/mm/flush.c
@@ -153,6 +153,15 @@ void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
 
 void __flush_dcache_page(struct address_space *mapping, struct page *page)
 {
+	/* PageUncached is used to indicate that cache attribute flags
+	 * are stored with the page; cache maintenance may still be
+	 * required if the page is mapped inner-cacheable, outer
+	 * non-cacheable */
+	if (PageUncached(page) &&
+	    !(page_private(page) == L_PTE_MT_WRITEBACK ||
+	      page_private(page) == L_PTE_MT_WRITEALLOC ||
+	      page_private(page) == L_PTE_MT_DEV_CACHED))
+		return;
 	/*
 	 * Writeback any data associated with the kernel mapping of this
 	 * page.  This ensures that data in the physical page is mutually
diff --git a/arch/arm/mm/highmem.c b/arch/arm/mm/highmem.c
index 6ab2440..b25ebee 100644
--- a/arch/arm/mm/highmem.c
+++ b/arch/arm/mm/highmem.c
@@ -109,6 +109,7 @@ void *kmap_atomic_pfn(unsigned long pfn, enum km_type type)
 {
 	unsigned int idx;
 	unsigned long vaddr;
+	struct page *page = pfn_to_page(pfn);
 
 	pagefault_disable();
 
@@ -117,7 +118,7 @@ void *kmap_atomic_pfn(unsigned long pfn, enum km_type type)
 #ifdef CONFIG_DEBUG_HIGHMEM
 	BUG_ON(!pte_none(*(TOP_PTE(vaddr))));
 #endif
-	set_pte_ext(TOP_PTE(vaddr), pfn_pte(pfn, kmap_prot), 0);
+	set_pte_ext(TOP_PTE(vaddr), mk_pte(page, kmap_prot), 0);
 	local_flush_tlb_kernel_page(vaddr);
 
 	return (void *)vaddr;
-- 
1.7.0.4

[PATCH 3/4] [ARM] mm: add memory type for inner-writeback

[Gary King]

For streaming-style operations (e.g., software rendering of graphics
surfaces shared with non-coherent DMA devices), the cost of performing
L2 cache maintenance can exceed the benefit of having the larger cache
(this is particularly true for OUTER_CACHE configurations like the ARM
PL2x0).

This change uses the currently-unused mapping 5 (TEX[0]=1, C=0, B=1)
in the tex remapping tables as an inner-writeback-write-allocate, outer
non-cacheable memory type, so that this mapping will be available to
clients which will benefit from the reduced L2 maintenance.

Signed-off-by: Gary King <gking@nvidia.com>
---
 arch/arm/include/asm/pgtable.h |    3 +++
 arch/arm/mm/proc-macros.S      |    2 +-
 arch/arm/mm/proc-v7.S          |    4 ++--
 arch/arm/mm/proc-xsc3.S        |    2 +-
 arch/arm/mm/proc-xscale.S      |    2 +-
 5 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h
index ab68cf1..4739ffe 100644
--- a/arch/arm/include/asm/pgtable.h
+++ b/arch/arm/include/asm/pgtable.h
@@ -184,6 +184,7 @@ extern void __pgd_error(const char *file, int line, unsigned long val);
 #define L_PTE_MT_DEV_NONSHARED	(0x0c << 2)	/* 1100 */
 #define L_PTE_MT_DEV_WC		(0x09 << 2)	/* 1001 */
 #define L_PTE_MT_DEV_CACHED	(0x0b << 2)	/* 1011 */
+#define L_PTE_MT_INNER_WB	(0x05 << 2)	/* 0101 (armv6, armv7) */
 #define L_PTE_MT_MASK		(0x0f << 2)
 
 #ifndef __ASSEMBLY__
@@ -321,6 +322,8 @@ static inline pte_t pte_mkspecial(pte_t pte) { return pte; }
 #define pgprot_dmacoherent(prot) \
 	__pgprot_modify(prot, L_PTE_MT_MASK|L_PTE_EXEC, L_PTE_MT_UNCACHED)
 #endif
+#define pgprot_inner_writeback(prot) \
+	__pgprot_modify(prot, L_PTE_MT_MASK, L_PTE_MT_INNER_WB)
 
 #define pmd_none(pmd)		(!pmd_val(pmd))
 #define pmd_present(pmd)	(pmd_val(pmd))
diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S
index 7d63bea..6da85e3 100644
--- a/arch/arm/mm/proc-macros.S
+++ b/arch/arm/mm/proc-macros.S
@@ -107,7 +107,7 @@
 	.long	PTE_CACHEABLE					@ L_PTE_MT_WRITETHROUGH
 	.long	PTE_CACHEABLE | PTE_BUFFERABLE			@ L_PTE_MT_WRITEBACK
 	.long	PTE_BUFFERABLE					@ L_PTE_MT_DEV_SHARED
-	.long	0x00						@ unused
+	.long	PTE_EXT_TEX(4) | PTE_BUFFERABLE			@ L_PTE_MT_INNER_WB
 	.long	0x00						@ L_PTE_MT_MINICACHE (not present)
 	.long	PTE_EXT_TEX(1) | PTE_CACHEABLE | PTE_BUFFERABLE	@ L_PTE_MT_WRITEALLOC
 	.long	0x00						@ unused
diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S
index 7aaf88a..fa21329 100644
--- a/arch/arm/mm/proc-v7.S
+++ b/arch/arm/mm/proc-v7.S
@@ -270,8 +270,8 @@ __v7_setup:
 	 *   NS1 = PRRR[19] = 1		- normal shareable property
 	 *   NOS = PRRR[24+n] = 1	- not outer shareable
 	 */
-	ldr	r5, =0xff0a81a8			@ PRRR
-	ldr	r6, =0x40e040e0			@ NMRR
+	ldr	r5, =0xff0a89a8
+	ldr	r6, =0x40e044e0
 	mcr	p15, 0, r5, c10, c2, 0		@ write PRRR
 	mcr	p15, 0, r6, c10, c2, 1		@ write NMRR
 #endif
diff --git a/arch/arm/mm/proc-xsc3.S b/arch/arm/mm/proc-xsc3.S
index e5797f1..47756dc 100644
--- a/arch/arm/mm/proc-xsc3.S
+++ b/arch/arm/mm/proc-xsc3.S
@@ -377,7 +377,7 @@ cpu_xsc3_mt_table:
 	.long	PTE_EXT_TEX(5) | PTE_CACHEABLE			@ L_PTE_MT_WRITETHROUGH
 	.long	PTE_CACHEABLE | PTE_BUFFERABLE			@ L_PTE_MT_WRITEBACK
 	.long	PTE_EXT_TEX(1) | PTE_BUFFERABLE			@ L_PTE_MT_DEV_SHARED
-	.long	0x00						@ unused
+	.long	PTE_EXT_TEX(4) | PTE_BUFFERABLE			@ L_PTE_MT_INNER_WB (not present?)
 	.long	0x00						@ L_PTE_MT_MINICACHE (not present)
 	.long	PTE_EXT_TEX(5) | PTE_CACHEABLE | PTE_BUFFERABLE	@ L_PTE_MT_WRITEALLOC (not present?)
 	.long	0x00						@ unused
diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S
index 63037e2..647296b 100644
--- a/arch/arm/mm/proc-xscale.S
+++ b/arch/arm/mm/proc-xscale.S
@@ -473,7 +473,7 @@ cpu_xscale_mt_table:
 	.long	PTE_CACHEABLE					@ L_PTE_MT_WRITETHROUGH
 	.long	PTE_CACHEABLE | PTE_BUFFERABLE			@ L_PTE_MT_WRITEBACK
 	.long	PTE_EXT_TEX(1) | PTE_BUFFERABLE			@ L_PTE_MT_DEV_SHARED
-	.long	0x00						@ unused
+	.long	PTE_EXT_TEX(1) | PTE_CACHEABLE | PTE_BUFFERABLE	@ L_PTE_MT_INNER_WB
 	.long	PTE_EXT_TEX(1) | PTE_CACHEABLE			@ L_PTE_MT_MINICACHE
 	.long	PTE_EXT_TEX(1) | PTE_CACHEABLE | PTE_BUFFERABLE	@ L_PTE_MT_WRITEALLOC
 	.long	0x00						@ unused
-- 
1.7.0.4

[PATCH 4/4] [ARM] dma-mapping: add support for inner-writeback pages

[Gary King]

add support for pages allocated with the inner-writeback
allocation attribute to the cache flush and DMA cache maintenance
code.

pages allocated from the attribute page allocator with the inner-
writeback attribute selected can bypass outer cache maintenance,
since outer cachelines will not be allocated for these pages.
this results in a significant reduction in the time spent performing
cache maintenance for these pages.

Signed-off-by: Gary King <gking@nvidia.com>
---
 arch/arm/mm/dma-mapping.c |   23 +++++++++++++++--------
 arch/arm/mm/flush.c       |    3 ++-
 2 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index fa94be5..5de376a 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -488,16 +488,20 @@ void ___dma_page_cpu_to_dev(struct page *page, unsigned long off,
 	if (PageUncached(page) &&
 	    !(page_private(page) == L_PTE_MT_WRITEBACK ||
 	      page_private(page) == L_PTE_MT_WRITEALLOC ||
-	      page_private(page) == L_PTE_MT_DEV_CACHED))
+	      page_private(page) == L_PTE_MT_DEV_CACHED ||
+	      page_private(page) == L_PTE_MT_INNER_WB))
 		return;
 
 	dma_cache_maint_page(page, off, size, dir, dmac_map_area);
 
 	paddr = page_to_phys(page) + off;
-	if (dir == DMA_FROM_DEVICE) {
-		outer_inv_range(paddr, paddr + size);
-	} else {
-		outer_clean_range(paddr, paddr + size);
+
+	if (!PageUncached(page) || (page_private(page) != L_PTE_MT_INNER_WB)) {
+		if (dir == DMA_FROM_DEVICE) {
+			outer_inv_range(paddr, paddr + size);
+		} else {
+			outer_clean_range(paddr, paddr + size);
+		}
 	}
 	/* FIXME: non-speculating: flush on bidirectional mappings? */
 }
@@ -511,15 +515,18 @@ void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
 	if (PageUncached(page) &&
 	    !(page_private(page) == L_PTE_MT_WRITEBACK ||
 	      page_private(page) == L_PTE_MT_WRITEALLOC ||
-	      page_private(page) == L_PTE_MT_DEV_CACHED))
+	      page_private(page) == L_PTE_MT_DEV_CACHED ||
+	      page_private(page) == L_PTE_MT_INNER_WB))
 		return;
 
 	paddr = page_to_phys(page) + off;
 
 	/* FIXME: non-speculating: not required */
 	/* don't bother invalidating if DMA to device */
-	if (dir != DMA_TO_DEVICE)
-		outer_inv_range(paddr, paddr + size);
+	if (!PageUncached(page) || (page_private(page) != L_PTE_MT_INNER_WB)) {
+		if (dir != DMA_TO_DEVICE)
+			outer_inv_range(paddr, paddr + size);
+	}
 
 	dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
 }
diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c
index 33e900a..4f6f115 100644
--- a/arch/arm/mm/flush.c
+++ b/arch/arm/mm/flush.c
@@ -160,7 +160,8 @@ void __flush_dcache_page(struct address_space *mapping, struct page *page)
 	if (PageUncached(page) &&
 	    !(page_private(page) == L_PTE_MT_WRITEBACK ||
 	      page_private(page) == L_PTE_MT_WRITEALLOC ||
-	      page_private(page) == L_PTE_MT_DEV_CACHED))
+	      page_private(page) == L_PTE_MT_DEV_CACHED ||
+	      page_private(page) == L_PTE_MT_INNER_WB))
 		return;
 	/*
 	 * Writeback any data associated with the kernel mapping of this
-- 
1.7.0.4

[PATCH 3/4] [ARM] mm: add memory type for inner-writeback

[Russell King - ARM Linux]

On Mon, Aug 02, 2010 at 07:42:48PM -0700, Gary King wrote:
> This change uses the currently-unused mapping 5 (TEX[0]=1, C=0, B=1)
> in the tex remapping tables as an inner-writeback-write-allocate, outer
> non-cacheable memory type, so that this mapping will be available to
> clients which will benefit from the reduced L2 maintenance.

No.  This is not "free for use".  Mapping 5 is unused because it's not
architecturally defined - CPU implementations may not implement it.
This is what the ARM ARM says:

  For seven of the eight possible combinations of the TEX[0], C and B bits,
  a field in the PRRR defines the corresponding memory region as being
  Normal, Device or Strongly-ordered memory a field in the NMRR defines
  the Inner cache attributes that apply if the PRRR field identifies the
  region as Normal memory a second field in the NMRR defines the Outer
  cache attributes that apply if the PRRR field identifies the region as
  Normal memory.

  The meaning of the eighth combination for the TEX[0], C and B bits is
  IMPLEMENTATION DEFINED

So we can't be sure that the PRRR and NMRR bits which correspond with
mapping 5 even exist.

[PATCH 3/4] [ARM] mm: add memory type for inner-writeback

[Gary King]

Russell,

 > No.  This is not "free for use".  Mapping 5 is unused because it's not
 > architecturally defined - CPU implementations may not implement it.
 > This is what the ARM ARM says:

According to the ARMv7-A ARM, n = 6 is implementation defined; n = 5 is
available:

"The meaning of the field with n = 6 is IMPLEMENTATION DEFINED and might
   differ from the meaning given here. This is because the meaning of the
   attribute combination {TEX[0] = 1, C = 1, B = 0} is IMPLEMENTATION
   DEFINED."

- Gary


On 08/03/2010 12:41 AM, Russell King - ARM Linux wrote:
>
> On Mon, Aug 02, 2010 at 07:42:48PM -0700, Gary King wrote:
> > This change uses the currently-unused mapping 5 (TEX[0]=1, C=0, B=1)
> > in the tex remapping tables as an inner-writeback-write-allocate, outer
> > non-cacheable memory type, so that this mapping will be available to
> > clients which will benefit from the reduced L2 maintenance.
>
> No.  This is not "free for use".  Mapping 5 is unused because it's not
> architecturally defined - CPU implementations may not implement it.
> This is what the ARM ARM says:
>
>   For seven of the eight possible combinations of the TEX[0], C and B 
> bits,
>   a field in the PRRR defines the corresponding memory region as being
>   Normal, Device or Strongly-ordered memory a field in the NMRR defines
>   the Inner cache attributes that apply if the PRRR field identifies the
>   region as Normal memory a second field in the NMRR defines the Outer
>   cache attributes that apply if the PRRR field identifies the region as
>   Normal memory.
>
>   The meaning of the eighth combination for the TEX[0], C and B bits is
>   IMPLEMENTATION DEFINED
>
> So we can't be sure that the PRRR and NMRR bits which correspond with
> mapping 5 even exist.
>