* [PATCH 1/4] [ARM] mmu: add option to map lowmem with page mappings
2010-08-03 2:42 [PATCH 0/4] Add support for page alloc w/ custom cache attributes Gary King
@ 2010-08-03 2:42 ` Gary King
2010-08-03 2:42 ` [PATCH 2/4] [ARM] mm: add page allocator for customizing cache attributes Gary King
` (2 subsequent siblings)
3 siblings, 0 replies; 7+ messages in thread
From: Gary King @ 2010-08-03 2:42 UTC (permalink / raw)
To: linux-arm-kernel
add a kernel configuration to map the kernel's lowmem pages using PTE
mappings, rather than the default behavior of 1MiB section mappings.
on ARMv7 processors, to support allocating pages with DMA-coherent
cache attributes, the cache attributes specified in the kernel's
mapping must match cache attributes specified for other mappings;
to ensure that this is the case, the kernel's attributes must be
specified on a per-page basis.
to avoid problems caused by the init_mm page table allocations exceeding
the available initial memory, when this config is enabled lowmem is
initially mapped using sections (matches current behavior), then remapped
using pages after bootmem is initialized
Signed-off-by: Gary King <gking@nvidia.com>
---
arch/arm/mm/Kconfig | 9 +++++++
arch/arm/mm/mmu.c | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 72 insertions(+), 0 deletions(-)
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index 101105e..176e815 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -837,3 +837,12 @@ config ARCH_HAS_BARRIERS
help
This option allows the use of custom mandatory barriers
included via the mach/barriers.h file.
+
+config ARCH_LOWMEM_IN_PTES
+ bool
+ help
+ This option will map the kernel direct-mapped lowmem region
+ using page table mappings rather than section mappings.
+
+config ARCH_USES_PG_UNCACHED
+ bool
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index 2858941..5818e83 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -247,7 +247,10 @@ static struct mem_type mem_types[] = {
.domain = DOMAIN_USER,
},
[MT_MEMORY] = {
+ .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
+ L_PTE_EXEC,
.prot_sect = PMD_TYPE_SECT | PMD_SECT_AP_WRITE,
+ .prot_l1 = PMD_TYPE_TABLE,
.domain = DOMAIN_KERNEL,
},
[MT_ROM] = {
@@ -463,6 +466,7 @@ static void __init build_mem_type_table(void)
mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask;
mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask;
mem_types[MT_MEMORY].prot_sect |= ecc_mask | cp->pmd;
+ mem_types[MT_MEMORY].prot_pte |= pgprot_kernel;
mem_types[MT_ROM].prot_sect |= cp->pmd;
switch (cp->pmd) {
@@ -506,6 +510,30 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
} while (pte++, addr += PAGE_SIZE, addr != end);
}
+#ifdef CONFIG_ARCH_LOWMEM_IN_PTES
+static void __init realloc_init_pte(pmd_t *pmd, unsigned long addr,
+ unsigned long end, unsigned long pfn,
+ const struct mem_type *type)
+{
+ pte_t *pte, *ptep;
+
+ if ((pmd_val(*pmd) & PMD_TYPE_MASK) != PMD_TYPE_SECT)
+ return;
+
+ pte = alloc_bootmem_low_pages(2 * PTRS_PER_PTE * sizeof(pte_t));
+ if (WARN_ON(!pte))
+ return;
+
+ ptep = pte + PTRS_PER_PTE + __pte_index(addr);
+ do {
+ set_pte_ext(ptep, pfn_pte(pfn, __pgprot(type->prot_pte)), 0);
+ pfn++;
+ } while (ptep++, addr += PAGE_SIZE, addr != end);
+
+ __pmd_populate(pmd, __pa(pte) | type->prot_l1);
+}
+#endif
+
static void __init alloc_init_section(pgd_t *pgd, unsigned long addr,
unsigned long end, unsigned long phys,
const struct mem_type *type)
@@ -1045,6 +1073,40 @@ static void __init map_lowmem(void)
}
}
+static void __init remap_lowmem(void)
+{
+#ifdef CONFIG_ARCH_LOWMEM_IN_PTES
+ struct meminfo *mi = &meminfo;
+ const struct mem_type *type = &mem_types[MT_MEMORY];
+ int i;
+
+ for (i = 0; i < mi->nr_banks; i++) {
+ pgd_t *pgd;
+ unsigned long phys, addr, end;
+ struct membank *bank = &mi->bank[i];
+
+ if (bank->highmem)
+ continue;
+
+ phys = __pfn_to_phys(bank_pfn_start(bank));
+ addr = __phys_to_virt(bank_phys_start(bank));
+ end = addr + bank_phys_size(bank);
+
+ pgd = pgd_offset_k(addr);
+ do {
+ unsigned long next = pgd_addr_end(addr, end);
+ pmd_t *pmd = pmd_offset(pgd, addr);
+
+ realloc_init_pte(pmd, addr, next,
+ __phys_to_pfn(phys), type);
+
+ phys += next - addr;
+ addr = next;
+ } while (pgd++, addr != end);
+ }
+#endif
+}
+
static int __init meminfo_cmp(const void *_a, const void *_b)
{
const struct membank *a = _a, *b = _b;
@@ -1067,6 +1129,7 @@ void __init paging_init(struct machine_desc *mdesc)
prepare_page_table();
map_lowmem();
bootmem_init();
+ remap_lowmem();
devicemaps_init(mdesc);
kmap_init();
--
1.7.0.4
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 2/4] [ARM] mm: add page allocator for customizing cache attributes
2010-08-03 2:42 [PATCH 0/4] Add support for page alloc w/ custom cache attributes Gary King
2010-08-03 2:42 ` [PATCH 1/4] [ARM] mmu: add option to map lowmem with page mappings Gary King
@ 2010-08-03 2:42 ` Gary King
2010-08-03 2:42 ` [PATCH 3/4] [ARM] mm: add memory type for inner-writeback Gary King
2010-08-03 2:42 ` [PATCH 4/4] [ARM] dma-mapping: add support for inner-writeback pages Gary King
3 siblings, 0 replies; 7+ messages in thread
From: Gary King @ 2010-08-03 2:42 UTC (permalink / raw)
To: linux-arm-kernel
ARM CPUs with speculative prefetching have undefined behaviors when the
same physical page is mapped to two different virtual addresses with
conflicting cache attributes.
since many recent systems include IOMMU functionality (i.e., remapping
of discontiguous physical pages into a virtually-contiguous address
range for I/O devices), it is desirable to support allocating any
available OS memory for use by the I/O devices. however, since many
systems do not support cache coherency between the CPU and DMA devices,
these devices are left with using DMA-coherent allocations from the OS
(which severely limits the benefit of an IOMMU) or performing cache
maintenance (which can be a severe performance loss, particularly on
systems with outer caches, compared to using DMA-coherent memory).
this change adds an API for allocating pages from the OS with specific
cache maintenance properties and ensures that the kernel's mapping
of the page reflects the desired cache attributes, in line with the
ARMv7 architectural requirements
since the kmap page properties are now page-dependent, to ensure that
highmem pages are always mapped with the desired attributes, kmap_prot
is implemented as a static inline function which expects that "page"
is declared as a struct page * variable in any function which uses
kmap_prot, so that the correct properties will be returned.
Signed-off-by: Gary King <gking@nvidia.com>
---
arch/arm/include/asm/attrib_alloc.h | 57 ++++++++++++++++
arch/arm/include/asm/highmem.h | 12 ++++
arch/arm/mm/Kconfig | 24 +++++++
arch/arm/mm/Makefile | 2 +
arch/arm/mm/attrib_alloc.c | 122 +++++++++++++++++++++++++++++++++++
arch/arm/mm/dma-mapping.c | 16 ++++-
arch/arm/mm/flush.c | 9 +++
arch/arm/mm/highmem.c | 3 +-
8 files changed, 243 insertions(+), 2 deletions(-)
create mode 100644 arch/arm/include/asm/attrib_alloc.h
create mode 100644 arch/arm/mm/attrib_alloc.c
diff --git a/arch/arm/include/asm/attrib_alloc.h b/arch/arm/include/asm/attrib_alloc.h
new file mode 100644
index 0000000..609939d
--- /dev/null
+++ b/arch/arm/include/asm/attrib_alloc.h
@@ -0,0 +1,57 @@
+/*
+ * arch/arm/include/asm/attrib_alloc.h
+ *
+ * Page allocator with custom cache attributes
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __ARCH_ARM_ATTRIB_ALLOC_H
+#define __ARCH_ARM_ATTRIB_ALLOC_H
+
+#include <linux/types.h>
+#include <asm/page.h>
+
+#ifdef CONFIG_ARM_ATTRIB_ALLOCATOR
+struct page *arm_attrib_alloc_pages_node(int nid, gfp_t gfp,
+ unsigned int order, pgprot_t prot);
+
+void arm_attrib_free_pages(struct page *page, unsigned int order);
+#else
+static inline struct page *arm_attrib_alloc_pages_node(int, gfp_t,
+ unsigned int, pgprot_t)
+{
+ return NULL;
+}
+
+static inline arm_attrib_free_pages(struct page *, unsigned int)
+{
+}
+#endif
+
+static inline struct page *arm_attrib_alloc_pages(gfp_t gfp,
+ unsigned int order, pgprot_t prot)
+{
+ return arm_attrib_alloc_pages_node(-1, gfp, order, prot);
+}
+
+#define arm_attrib_alloc_page(gfp, prot) \
+ arm_attrib_alloc_pages((gfp), 0, (prot))
+
+#define arm_attrib_free_page(page) arm_attrib_free_pages((page), 0)
+
+#endif /* __ARCH_ARM_ATTRIB_ALLOC_H */
diff --git a/arch/arm/include/asm/highmem.h b/arch/arm/include/asm/highmem.h
index feb988a..d8c464c 100644
--- a/arch/arm/include/asm/highmem.h
+++ b/arch/arm/include/asm/highmem.h
@@ -9,7 +9,19 @@
#define PKMAP_NR(virt) (((virt) - PKMAP_BASE) >> PAGE_SHIFT)
#define PKMAP_ADDR(nr) (PKMAP_BASE + ((nr) << PAGE_SHIFT))
+#ifndef CONFIG_ARM_ATTRIB_ALLOCATOR
#define kmap_prot PAGE_KERNEL
+#else
+static inline pgprot_t kmap_prot_of(struct page *page)
+{
+ if (PageUncached(page)) {
+ return __pgprot_modify(PAGE_KERNEL, L_PTE_MT_MASK,
+ page->private);
+ } else
+ return PAGE_KERNEL;
+}
+#define kmap_prot kmap_prot_of(page)
+#endif
#define flush_cache_kmaps() \
do { \
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index 176e815..4abe9ee 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -811,6 +811,30 @@ config ARM_L1_CACHE_SHIFT
default 6 if ARM_L1_CACHE_SHIFT_6
default 5
+config ARM_ATTRIB_ALLOCATOR
+ bool "Support custom cache attribute allocations in low memory"
+ select ARCH_LOWMEM_IN_PTES
+ select ARCH_USES_PG_UNCACHED
+ depends on MMU && !CPU_CACHE_VIVT
+ help
+ Historically, the kernel has only reserved a small region
+ of physical memory for uncached access, and relied on
+ explicit cache maintenance for ensuring coherency between
+ the CPU and DMA.
+
+ However, many recent systems support mapping discontiguous
+ physical pages into contiguous DMA addresses (so-called
+ system MMUs). For some DMA clients (notably graphics and
+ multimedia engines), performing explict cache maintenance
+ between CPU and DMA mappings can be prohibitively expensive,
+ and since ARMv7, mapping the same physical page with different
+ cache attributes is disallowed and has unpredictable behavior.
+
+ Say 'Y' here to include page allocation support with explicit
+ cache attributes; on ARMv7 systems this will also force the
+ kernel's page tables to be mapped using page tables rather
+ than sections.
+
config ARM_DMA_MEM_BUFFERABLE
bool "Use non-cacheable memory for DMA" if CPU_V6 && !CPU_V7
depends on !(MACH_REALVIEW_PB1176 || REALVIEW_EB_ARM11MP || \
diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile
index e8d34a8..ce803db 100644
--- a/arch/arm/mm/Makefile
+++ b/arch/arm/mm/Makefile
@@ -12,6 +12,8 @@ ifneq ($(CONFIG_MMU),y)
obj-y += nommu.o
endif
+obj-$(CONFIG_ARM_ATTRIB_ALLOCATOR) += attrib_alloc.o
+
obj-$(CONFIG_MODULES) += proc-syms.o
obj-$(CONFIG_ALIGNMENT_TRAP) += alignment.o
diff --git a/arch/arm/mm/attrib_alloc.c b/arch/arm/mm/attrib_alloc.c
new file mode 100644
index 0000000..c188772
--- /dev/null
+++ b/arch/arm/mm/attrib_alloc.c
@@ -0,0 +1,122 @@
+/*
+ * arch/arm/mm/attrib_alloc.c
+ *
+ * Page allocator with custom cache attributes
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include <linux/kernel.h>
+#include <linux/highmem.h>
+#include <linux/gfp.h>
+#include <linux/page-flags.h>
+#include <asm/tlbflush.h>
+#include <asm/cacheflush.h>
+#include <asm/pgtable.h>
+#include <asm/pgalloc.h>
+#include <asm/fixmap.h>
+#include <asm/outercache.h>
+#include <asm/attrib_alloc.h>
+#include "mm.h"
+
+#define pgprot_type(pte) (pgprot_val(pte) & ~L_PTE_MT_MASK)
+
+static void update_kmap_pte(struct page *page, pgprot_t prot)
+{
+ unsigned long addr;
+ pte_t *pte;
+
+ addr = (unsigned long)kmap_high_get(page);
+ BUG_ON(!PageHighMem(page) || addr >= FIXADDR_START);
+ if (!addr)
+ return;
+
+ pte = &pkmap_page_table[PKMAP_NR(addr)];
+ set_pte_at(&init_mm, addr, pte, mk_pte(page, __pgprot(prot)));
+ flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
+ kunmap_high(page);
+}
+
+static void update_pte(struct page *page, pgprot_t prot)
+{
+ unsigned long addr = (unsigned long)page_address(page);
+ pgd_t *pgd = pgd_offset_k(addr);
+ pmd_t *pmd = pmd_offset(pgd, addr);
+ pte_t *pte;
+
+ BUG_ON(pmd_none(*pmd));
+ pte = pte_offset_kernel(pmd, addr);
+ set_pte_at(&init_mm, addr, pte, mk_pte(page, __pgprot(prot)));
+ flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
+}
+
+void arm_attrib_free_pages(struct page *page, unsigned int order)
+{
+ /* reset the page's mappings back to the standard kernel mappings
+ * before returning it to the page allocator */
+ if (PageUncached(page)) {
+ struct page *loop;
+ unsigned int i;
+
+ for (i = 0, loop = page; i < (1 << order); i++, loop++) {
+
+ if (PageHighMem(loop))
+ update_kmap_pte(loop, pgprot_kernel);
+ else
+ update_pte(loop, pgprot_kernel);
+
+ ClearPageUncached(page);
+ set_page_private(page, 0);
+ }
+ }
+ __free_pages(page, order);
+}
+
+struct page *arm_attrib_alloc_pages_node(int nid, gfp_t gfp,
+ unsigned int order, pgprot_t prot)
+{
+ struct page *page, *loop;
+ unsigned int i;
+ unsigned int type = pgprot_type(prot);
+
+ page = alloc_pages_node(nid, gfp, order);
+ /* if the requested cache attributes match the default value,
+ * just return because no special handling will be needed for
+ * this page */
+ if (!page || (type == pgprot_type(pgprot_kernel)))
+ return page;
+
+ for (i = 0, loop = page; i < (1 << order); i++, loop++) {
+ unsigned long phys = page_to_phys(page);
+ __flush_dcache_page(page_mapping(page), page);
+ outer_flush_range(phys, phys + PAGE_SIZE);
+
+ SetPageUncached(loop);
+ set_page_private(page, type);
+
+ /* even though a freshly-allocated highmem page shouldn't
+ * be mapped, because the kmaps are flushed lazily, it
+ * is possible that a mapping from an old kmap_high call
+ * is still present, and its cache attributes need to
+ * be updated to match the new expectations */
+ if (PageHighMem(loop))
+ update_kmap_pte(loop, prot);
+ else
+ update_pte(loop, prot);
+ }
+ return page;
+}
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index 9e7742f..fa94be5 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -485,6 +485,12 @@ void ___dma_page_cpu_to_dev(struct page *page, unsigned long off,
{
unsigned long paddr;
+ if (PageUncached(page) &&
+ !(page_private(page) == L_PTE_MT_WRITEBACK ||
+ page_private(page) == L_PTE_MT_WRITEALLOC ||
+ page_private(page) == L_PTE_MT_DEV_CACHED))
+ return;
+
dma_cache_maint_page(page, off, size, dir, dmac_map_area);
paddr = page_to_phys(page) + off;
@@ -500,7 +506,15 @@ EXPORT_SYMBOL(___dma_page_cpu_to_dev);
void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
size_t size, enum dma_data_direction dir)
{
- unsigned long paddr = page_to_phys(page) + off;
+ unsigned long paddr;
+
+ if (PageUncached(page) &&
+ !(page_private(page) == L_PTE_MT_WRITEBACK ||
+ page_private(page) == L_PTE_MT_WRITEALLOC ||
+ page_private(page) == L_PTE_MT_DEV_CACHED))
+ return;
+
+ paddr = page_to_phys(page) + off;
/* FIXME: non-speculating: not required */
/* don't bother invalidating if DMA to device */
diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c
index c6844cb..33e900a 100644
--- a/arch/arm/mm/flush.c
+++ b/arch/arm/mm/flush.c
@@ -153,6 +153,15 @@ void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
void __flush_dcache_page(struct address_space *mapping, struct page *page)
{
+ /* PageUncached is used to indicate that cache attribute flags
+ * are stored with the page; cache maintenance may still be
+ * required if the page is mapped inner-cacheable, outer
+ * non-cacheable */
+ if (PageUncached(page) &&
+ !(page_private(page) == L_PTE_MT_WRITEBACK ||
+ page_private(page) == L_PTE_MT_WRITEALLOC ||
+ page_private(page) == L_PTE_MT_DEV_CACHED))
+ return;
/*
* Writeback any data associated with the kernel mapping of this
* page. This ensures that data in the physical page is mutually
diff --git a/arch/arm/mm/highmem.c b/arch/arm/mm/highmem.c
index 6ab2440..b25ebee 100644
--- a/arch/arm/mm/highmem.c
+++ b/arch/arm/mm/highmem.c
@@ -109,6 +109,7 @@ void *kmap_atomic_pfn(unsigned long pfn, enum km_type type)
{
unsigned int idx;
unsigned long vaddr;
+ struct page *page = pfn_to_page(pfn);
pagefault_disable();
@@ -117,7 +118,7 @@ void *kmap_atomic_pfn(unsigned long pfn, enum km_type type)
#ifdef CONFIG_DEBUG_HIGHMEM
BUG_ON(!pte_none(*(TOP_PTE(vaddr))));
#endif
- set_pte_ext(TOP_PTE(vaddr), pfn_pte(pfn, kmap_prot), 0);
+ set_pte_ext(TOP_PTE(vaddr), mk_pte(page, kmap_prot), 0);
local_flush_tlb_kernel_page(vaddr);
return (void *)vaddr;
--
1.7.0.4
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 3/4] [ARM] mm: add memory type for inner-writeback
2010-08-03 2:42 [PATCH 0/4] Add support for page alloc w/ custom cache attributes Gary King
2010-08-03 2:42 ` [PATCH 1/4] [ARM] mmu: add option to map lowmem with page mappings Gary King
2010-08-03 2:42 ` [PATCH 2/4] [ARM] mm: add page allocator for customizing cache attributes Gary King
@ 2010-08-03 2:42 ` Gary King
2010-08-03 7:41 ` Russell King - ARM Linux
2010-08-03 2:42 ` [PATCH 4/4] [ARM] dma-mapping: add support for inner-writeback pages Gary King
3 siblings, 1 reply; 7+ messages in thread
From: Gary King @ 2010-08-03 2:42 UTC (permalink / raw)
To: linux-arm-kernel
For streaming-style operations (e.g., software rendering of graphics
surfaces shared with non-coherent DMA devices), the cost of performing
L2 cache maintenance can exceed the benefit of having the larger cache
(this is particularly true for OUTER_CACHE configurations like the ARM
PL2x0).
This change uses the currently-unused mapping 5 (TEX[0]=1, C=0, B=1)
in the tex remapping tables as an inner-writeback-write-allocate, outer
non-cacheable memory type, so that this mapping will be available to
clients which will benefit from the reduced L2 maintenance.
Signed-off-by: Gary King <gking@nvidia.com>
---
arch/arm/include/asm/pgtable.h | 3 +++
arch/arm/mm/proc-macros.S | 2 +-
arch/arm/mm/proc-v7.S | 4 ++--
arch/arm/mm/proc-xsc3.S | 2 +-
arch/arm/mm/proc-xscale.S | 2 +-
5 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h
index ab68cf1..4739ffe 100644
--- a/arch/arm/include/asm/pgtable.h
+++ b/arch/arm/include/asm/pgtable.h
@@ -184,6 +184,7 @@ extern void __pgd_error(const char *file, int line, unsigned long val);
#define L_PTE_MT_DEV_NONSHARED (0x0c << 2) /* 1100 */
#define L_PTE_MT_DEV_WC (0x09 << 2) /* 1001 */
#define L_PTE_MT_DEV_CACHED (0x0b << 2) /* 1011 */
+#define L_PTE_MT_INNER_WB (0x05 << 2) /* 0101 (armv6, armv7) */
#define L_PTE_MT_MASK (0x0f << 2)
#ifndef __ASSEMBLY__
@@ -321,6 +322,8 @@ static inline pte_t pte_mkspecial(pte_t pte) { return pte; }
#define pgprot_dmacoherent(prot) \
__pgprot_modify(prot, L_PTE_MT_MASK|L_PTE_EXEC, L_PTE_MT_UNCACHED)
#endif
+#define pgprot_inner_writeback(prot) \
+ __pgprot_modify(prot, L_PTE_MT_MASK, L_PTE_MT_INNER_WB)
#define pmd_none(pmd) (!pmd_val(pmd))
#define pmd_present(pmd) (pmd_val(pmd))
diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S
index 7d63bea..6da85e3 100644
--- a/arch/arm/mm/proc-macros.S
+++ b/arch/arm/mm/proc-macros.S
@@ -107,7 +107,7 @@
.long PTE_CACHEABLE @ L_PTE_MT_WRITETHROUGH
.long PTE_CACHEABLE | PTE_BUFFERABLE @ L_PTE_MT_WRITEBACK
.long PTE_BUFFERABLE @ L_PTE_MT_DEV_SHARED
- .long 0x00 @ unused
+ .long PTE_EXT_TEX(4) | PTE_BUFFERABLE @ L_PTE_MT_INNER_WB
.long 0x00 @ L_PTE_MT_MINICACHE (not present)
.long PTE_EXT_TEX(1) | PTE_CACHEABLE | PTE_BUFFERABLE @ L_PTE_MT_WRITEALLOC
.long 0x00 @ unused
diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S
index 7aaf88a..fa21329 100644
--- a/arch/arm/mm/proc-v7.S
+++ b/arch/arm/mm/proc-v7.S
@@ -270,8 +270,8 @@ __v7_setup:
* NS1 = PRRR[19] = 1 - normal shareable property
* NOS = PRRR[24+n] = 1 - not outer shareable
*/
- ldr r5, =0xff0a81a8 @ PRRR
- ldr r6, =0x40e040e0 @ NMRR
+ ldr r5, =0xff0a89a8
+ ldr r6, =0x40e044e0
mcr p15, 0, r5, c10, c2, 0 @ write PRRR
mcr p15, 0, r6, c10, c2, 1 @ write NMRR
#endif
diff --git a/arch/arm/mm/proc-xsc3.S b/arch/arm/mm/proc-xsc3.S
index e5797f1..47756dc 100644
--- a/arch/arm/mm/proc-xsc3.S
+++ b/arch/arm/mm/proc-xsc3.S
@@ -377,7 +377,7 @@ cpu_xsc3_mt_table:
.long PTE_EXT_TEX(5) | PTE_CACHEABLE @ L_PTE_MT_WRITETHROUGH
.long PTE_CACHEABLE | PTE_BUFFERABLE @ L_PTE_MT_WRITEBACK
.long PTE_EXT_TEX(1) | PTE_BUFFERABLE @ L_PTE_MT_DEV_SHARED
- .long 0x00 @ unused
+ .long PTE_EXT_TEX(4) | PTE_BUFFERABLE @ L_PTE_MT_INNER_WB (not present?)
.long 0x00 @ L_PTE_MT_MINICACHE (not present)
.long PTE_EXT_TEX(5) | PTE_CACHEABLE | PTE_BUFFERABLE @ L_PTE_MT_WRITEALLOC (not present?)
.long 0x00 @ unused
diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S
index 63037e2..647296b 100644
--- a/arch/arm/mm/proc-xscale.S
+++ b/arch/arm/mm/proc-xscale.S
@@ -473,7 +473,7 @@ cpu_xscale_mt_table:
.long PTE_CACHEABLE @ L_PTE_MT_WRITETHROUGH
.long PTE_CACHEABLE | PTE_BUFFERABLE @ L_PTE_MT_WRITEBACK
.long PTE_EXT_TEX(1) | PTE_BUFFERABLE @ L_PTE_MT_DEV_SHARED
- .long 0x00 @ unused
+ .long PTE_EXT_TEX(1) | PTE_CACHEABLE | PTE_BUFFERABLE @ L_PTE_MT_INNER_WB
.long PTE_EXT_TEX(1) | PTE_CACHEABLE @ L_PTE_MT_MINICACHE
.long PTE_EXT_TEX(1) | PTE_CACHEABLE | PTE_BUFFERABLE @ L_PTE_MT_WRITEALLOC
.long 0x00 @ unused
--
1.7.0.4
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 4/4] [ARM] dma-mapping: add support for inner-writeback pages
2010-08-03 2:42 [PATCH 0/4] Add support for page alloc w/ custom cache attributes Gary King
` (2 preceding siblings ...)
2010-08-03 2:42 ` [PATCH 3/4] [ARM] mm: add memory type for inner-writeback Gary King
@ 2010-08-03 2:42 ` Gary King
3 siblings, 0 replies; 7+ messages in thread
From: Gary King @ 2010-08-03 2:42 UTC (permalink / raw)
To: linux-arm-kernel
add support for pages allocated with the inner-writeback
allocation attribute to the cache flush and DMA cache maintenance
code.
pages allocated from the attribute page allocator with the inner-
writeback attribute selected can bypass outer cache maintenance,
since outer cachelines will not be allocated for these pages.
this results in a significant reduction in the time spent performing
cache maintenance for these pages.
Signed-off-by: Gary King <gking@nvidia.com>
---
arch/arm/mm/dma-mapping.c | 23 +++++++++++++++--------
arch/arm/mm/flush.c | 3 ++-
2 files changed, 17 insertions(+), 9 deletions(-)
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index fa94be5..5de376a 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -488,16 +488,20 @@ void ___dma_page_cpu_to_dev(struct page *page, unsigned long off,
if (PageUncached(page) &&
!(page_private(page) == L_PTE_MT_WRITEBACK ||
page_private(page) == L_PTE_MT_WRITEALLOC ||
- page_private(page) == L_PTE_MT_DEV_CACHED))
+ page_private(page) == L_PTE_MT_DEV_CACHED ||
+ page_private(page) == L_PTE_MT_INNER_WB))
return;
dma_cache_maint_page(page, off, size, dir, dmac_map_area);
paddr = page_to_phys(page) + off;
- if (dir == DMA_FROM_DEVICE) {
- outer_inv_range(paddr, paddr + size);
- } else {
- outer_clean_range(paddr, paddr + size);
+
+ if (!PageUncached(page) || (page_private(page) != L_PTE_MT_INNER_WB)) {
+ if (dir == DMA_FROM_DEVICE) {
+ outer_inv_range(paddr, paddr + size);
+ } else {
+ outer_clean_range(paddr, paddr + size);
+ }
}
/* FIXME: non-speculating: flush on bidirectional mappings? */
}
@@ -511,15 +515,18 @@ void ___dma_page_dev_to_cpu(struct page *page, unsigned long off,
if (PageUncached(page) &&
!(page_private(page) == L_PTE_MT_WRITEBACK ||
page_private(page) == L_PTE_MT_WRITEALLOC ||
- page_private(page) == L_PTE_MT_DEV_CACHED))
+ page_private(page) == L_PTE_MT_DEV_CACHED ||
+ page_private(page) == L_PTE_MT_INNER_WB))
return;
paddr = page_to_phys(page) + off;
/* FIXME: non-speculating: not required */
/* don't bother invalidating if DMA to device */
- if (dir != DMA_TO_DEVICE)
- outer_inv_range(paddr, paddr + size);
+ if (!PageUncached(page) || (page_private(page) != L_PTE_MT_INNER_WB)) {
+ if (dir != DMA_TO_DEVICE)
+ outer_inv_range(paddr, paddr + size);
+ }
dma_cache_maint_page(page, off, size, dir, dmac_unmap_area);
}
diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c
index 33e900a..4f6f115 100644
--- a/arch/arm/mm/flush.c
+++ b/arch/arm/mm/flush.c
@@ -160,7 +160,8 @@ void __flush_dcache_page(struct address_space *mapping, struct page *page)
if (PageUncached(page) &&
!(page_private(page) == L_PTE_MT_WRITEBACK ||
page_private(page) == L_PTE_MT_WRITEALLOC ||
- page_private(page) == L_PTE_MT_DEV_CACHED))
+ page_private(page) == L_PTE_MT_DEV_CACHED ||
+ page_private(page) == L_PTE_MT_INNER_WB))
return;
/*
* Writeback any data associated with the kernel mapping of this
--
1.7.0.4
^ permalink raw reply related [flat|nested] 7+ messages in thread