* [PATCH 11/17] powerpc/nohash32: set GUARDED attribute in the PMD directly
From: Christophe Leroy @ 2018-05-04 12:34 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
aneesh.kumar
Cc: linux-kernel, linuxppc-dev
In-Reply-To: <cover.1525435203.git.christophe.leroy@c-s.fr>
On the 8xx, the GUARDED attribute of the pages is managed in the
L1 entry, therefore to avoid having to copy it into L1 entry
at each TLB miss, we set it in the PMD.
For this, we split the VM alloc space in two parts, one
for VM alloc and non Guarded IO, and one for Guarded IO.
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
arch/powerpc/include/asm/nohash/32/pgalloc.h | 10 ++++++++++
arch/powerpc/include/asm/nohash/32/pgtable.h | 18 ++++++++++++++++--
arch/powerpc/include/asm/nohash/32/pte-8xx.h | 3 ++-
arch/powerpc/kernel/head_8xx.S | 18 +++++++-----------
arch/powerpc/mm/dump_linuxpagetables.c | 26 ++++++++++++++++++++++++--
arch/powerpc/mm/ioremap.c | 11 ++++++++---
arch/powerpc/mm/mem.c | 9 +++++++++
arch/powerpc/mm/pgtable_32.c | 28 +++++++++++++++++++++++++++-
arch/powerpc/platforms/Kconfig.cputype | 3 +++
9 files changed, 106 insertions(+), 20 deletions(-)
diff --git a/arch/powerpc/include/asm/nohash/32/pgalloc.h b/arch/powerpc/include/asm/nohash/32/pgalloc.h
index 29d37bd1f3b3..1c6461e7c6aa 100644
--- a/arch/powerpc/include/asm/nohash/32/pgalloc.h
+++ b/arch/powerpc/include/asm/nohash/32/pgalloc.h
@@ -58,6 +58,12 @@ static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp,
*pmdp = __pmd(__pa(pte) | _PMD_PRESENT);
}
+static inline void pmd_populate_kernel_g(struct mm_struct *mm, pmd_t *pmdp,
+ pte_t *pte)
+{
+ *pmdp = __pmd(__pa(pte) | _PMD_PRESENT | _PMD_GUARDED);
+}
+
static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp,
pgtable_t pte_page)
{
@@ -83,6 +89,10 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp,
#define pmd_pgtable(pmd) pmd_page(pmd)
#endif
+#define pte_alloc_kernel_g(pmd, address) \
+ ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel_g(pmd, address))? \
+ NULL: pte_offset_kernel(pmd, address))
+
extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr);
extern pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addr);
diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h
index 93dc22dbe964..009a5b3d3192 100644
--- a/arch/powerpc/include/asm/nohash/32/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/32/pgtable.h
@@ -69,9 +69,14 @@ extern int icache_44x_need_flush;
* virtual space that goes below PKMAP and FIXMAP
*/
#ifdef CONFIG_HIGHMEM
-#define KVIRT_TOP PKMAP_BASE
+#define _KVIRT_TOP PKMAP_BASE
#else
-#define KVIRT_TOP (0xfe000000UL) /* for now, could be FIXMAP_BASE ? */
+#define _KVIRT_TOP (0xfe000000UL) /* for now, could be FIXMAP_BASE ? */
+#endif
+#ifdef CONFIG_PPC_GUARDED_PAGE_IN_PMD
+#define KVIRT_TOP _ALIGN_DOWN(_KVIRT_TOP, PGDIR_SIZE)
+#else
+#define KVIRT_TOP _KVIRT_TOP
#endif
/*
@@ -84,7 +89,11 @@ extern int icache_44x_need_flush;
#else
#define IOREMAP_END KVIRT_TOP
#endif
+#ifdef CONFIG_PPC_GUARDED_PAGE_IN_PMD
+#define IOREMAP_BASE _ALIGN_UP(VMALLOC_BASE + (IOREMAP_END - VMALLOC_BASE) / 2, PGDIR_SIZE)
+#else
#define IOREMAP_BASE VMALLOC_BASE
+#endif
/*
* Just any arbitrary offset to the start of the vmalloc VM area: the
@@ -103,8 +112,13 @@ extern int icache_44x_need_flush;
#else
#define VMALLOC_BASE _ALIGN_DOWN((long)high_memory + VMALLOC_OFFSET, VMALLOC_OFFSET)
#endif
+#ifdef CONFIG_PPC_GUARDED_PAGE_IN_PMD
+#define VMALLOC_START VMALLOC_BASE
+#define VMALLOC_END IOREMAP_BASE
+#else
#define VMALLOC_START ioremap_bot
#define VMALLOC_END IOREMAP_END
+#endif
/*
* Bits in a linux-style PTE. These match the bits in the
diff --git a/arch/powerpc/include/asm/nohash/32/pte-8xx.h b/arch/powerpc/include/asm/nohash/32/pte-8xx.h
index f04cb46ae8a1..a9a2919251e0 100644
--- a/arch/powerpc/include/asm/nohash/32/pte-8xx.h
+++ b/arch/powerpc/include/asm/nohash/32/pte-8xx.h
@@ -47,10 +47,11 @@
#define _PAGE_RO 0x0600 /* Supervisor RO, User no access */
#define _PMD_PRESENT 0x0001
-#define _PMD_BAD 0x0fd0
+#define _PMD_BAD 0x0fc0
#define _PMD_PAGE_MASK 0x000c
#define _PMD_PAGE_8M 0x000c
#define _PMD_PAGE_512K 0x0004
+#define _PMD_GUARDED 0x0010
#define _PMD_USER 0x0020 /* APG 1 */
/* Until my rework is finished, 8xx still needs atomic PTE updates */
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index c3b831bb8bad..85b017c67e11 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -345,6 +345,10 @@ _ENTRY(ITLBMiss_cmp)
rlwinm r10, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
#ifdef CONFIG_HUGETLB_PAGE
mtcr r11
+#endif
+ /* Load the MI_TWC with the attributes for this "segment." */
+ mtspr SPRN_MI_TWC, r11 /* Set segment attributes */
+#ifdef CONFIG_HUGETLB_PAGE
bt- 28, 10f /* bit 28 = Large page (8M) */
bt- 29, 20f /* bit 29 = Large page (8M or 512k) */
#endif
@@ -354,8 +358,6 @@ _ENTRY(ITLBMiss_cmp)
#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
mtcr r12
#endif
- /* Load the MI_TWC with the attributes for this "segment." */
- mtspr SPRN_MI_TWC, r11 /* Set segment attributes */
#ifdef CONFIG_SWAP
rlwinm r11, r10, 32-5, _PAGE_PRESENT
@@ -457,6 +459,9 @@ _ENTRY(DTLBMiss_jmp)
rlwinm r10, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
#ifdef CONFIG_HUGETLB_PAGE
mtcr r11
+#endif
+ mtspr SPRN_MD_TWC, r11
+#ifdef CONFIG_HUGETLB_PAGE
bt- 28, 10f /* bit 28 = Large page (8M) */
bt- 29, 20f /* bit 29 = Large page (8M or 512k) */
#endif
@@ -465,15 +470,6 @@ _ENTRY(DTLBMiss_jmp)
4:
mtcr r12
- /* Insert the Guarded flag into the TWC from the Linux PTE.
- * It is bit 27 of both the Linux PTE and the TWC (at least
- * I got that right :-). It will be better when we can put
- * this into the Linux pgd/pmd and load it in the operation
- * above.
- */
- rlwimi r11, r10, 0, _PAGE_GUARDED
- mtspr SPRN_MD_TWC, r11
-
/* Both _PAGE_ACCESSED and _PAGE_PRESENT has to be set.
* We also need to know if the insn is a load/store, so:
* Clear _PAGE_PRESENT and load that which will
diff --git a/arch/powerpc/mm/dump_linuxpagetables.c b/arch/powerpc/mm/dump_linuxpagetables.c
index 6022adb899b7..cd3797be5e05 100644
--- a/arch/powerpc/mm/dump_linuxpagetables.c
+++ b/arch/powerpc/mm/dump_linuxpagetables.c
@@ -74,9 +74,9 @@ struct addr_marker {
static struct addr_marker address_markers[] = {
{ 0, "Start of kernel VM" },
+#ifdef CONFIG_PPC64
{ 0, "vmalloc() Area" },
{ 0, "vmalloc() End" },
-#ifdef CONFIG_PPC64
{ 0, "isa I/O start" },
{ 0, "isa I/O end" },
{ 0, "phb I/O start" },
@@ -85,8 +85,19 @@ static struct addr_marker address_markers[] = {
{ 0, "I/O remap end" },
{ 0, "vmemmap start" },
#else
+#ifdef CONFIG_PPC_GUARDED_PAGE_IN_PMD
+ { 0, "vmalloc() Area" },
+ { 0, "vmalloc() End" },
{ 0, "Early I/O remap start" },
{ 0, "Early I/O remap end" },
+ { 0, "I/O remap start" },
+ { 0, "I/O remap end" },
+#else
+ { 0, "Early I/O remap start" },
+ { 0, "Early I/O remap end" },
+ { 0, "vmalloc() I/O remap start" },
+ { 0, "vmalloc() I/O remap end" },
+#endif
#ifdef CONFIG_NOT_COHERENT_CACHE
{ 0, "Consistent mem start" },
{ 0, "Consistent mem end" },
@@ -437,9 +448,9 @@ static void populate_markers(void)
int i = 0;
address_markers[i++].start_address = PAGE_OFFSET;
+#ifdef CONFIG_PPC64
address_markers[i++].start_address = VMALLOC_START;
address_markers[i++].start_address = VMALLOC_END;
-#ifdef CONFIG_PPC64
address_markers[i++].start_address = ISA_IO_BASE;
address_markers[i++].start_address = ISA_IO_END;
address_markers[i++].start_address = PHB_IO_BASE;
@@ -452,8 +463,19 @@ static void populate_markers(void)
address_markers[i++].start_address = VMEMMAP_BASE;
#endif
#else /* !CONFIG_PPC64 */
+#ifdef CONFIG_PPC_GUARDED_PAGE_IN_PMD
+ address_markers[i++].start_address = VMALLOC_START;
+ address_markers[i++].start_address = VMALLOC_END;
address_markers[i++].start_address = IOREMAP_BASE;
address_markers[i++].start_address = ioremap_bot;
+ address_markers[i++].start_address = ioremap_bot;
+ address_markers[i++].start_address = IOREMAP_END;
+#else
+ address_markers[i++].start_address = IOREMAP_BASE;
+ address_markers[i++].start_address = ioremap_bot;
+ address_markers[i++].start_address = ioremap_bot;
+ address_markers[i++].start_address = IOREMAP_END;
+#endif
#ifdef CONFIG_NOT_COHERENT_CACHE
address_markers[i++].start_address = IOREMAP_END;
address_markers[i++].start_address = IOREMAP_END +
diff --git a/arch/powerpc/mm/ioremap.c b/arch/powerpc/mm/ioremap.c
index 59be5dfcb3e9..b8c347077e02 100644
--- a/arch/powerpc/mm/ioremap.c
+++ b/arch/powerpc/mm/ioremap.c
@@ -132,9 +132,14 @@ void __iomem * __ioremap_caller(phys_addr_t addr, unsigned long size,
if (slab_is_available()) {
struct vm_struct *area;
- area = __get_vm_area_caller(size, VM_IOREMAP,
- ioremap_bot, IOREMAP_END,
- caller);
+ if (flags & _PAGE_GUARDED)
+ area = __get_vm_area_caller(size, VM_IOREMAP,
+ ioremap_bot, IOREMAP_END,
+ caller);
+ else
+ area = __get_vm_area_caller(size, VM_IOREMAP,
+ VMALLOC_START, VMALLOC_END,
+ caller);
if (area == NULL)
return NULL;
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index b680aa78a4ac..fd7af7af5b58 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -386,10 +386,19 @@ void __init mem_init(void)
pr_info(" * 0x%08lx..0x%08lx : consistent mem\n",
IOREMAP_END, IOREMAP_END + CONFIG_CONSISTENT_SIZE);
#endif /* CONFIG_NOT_COHERENT_CACHE */
+#ifdef CONFIG_PPC_GUARDED_PAGE_IN_PMD
+ pr_info(" * 0x%08lx..0x%08lx : ioremap\n",
+ ioremap_bot, IOREMAP_END);
pr_info(" * 0x%08lx..0x%08lx : early ioremap\n",
IOREMAP_BASE, ioremap_bot);
+ pr_info(" * 0x%08lx..0x%08lx : vmalloc\n",
+ VMALLOC_START, VMALLOC_END);
+#else
pr_info(" * 0x%08lx..0x%08lx : vmalloc & ioremap\n",
VMALLOC_START, VMALLOC_END);
+ pr_info(" * 0x%08lx..0x%08lx : early ioremap\n",
+ IOREMAP_BASE, ioremap_bot);
+#endif
#endif /* CONFIG_PPC32 */
}
diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c
index 54a5bc0767a9..3aa0c78db95d 100644
--- a/arch/powerpc/mm/pgtable_32.c
+++ b/arch/powerpc/mm/pgtable_32.c
@@ -70,6 +70,27 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
return ptepage;
}
+#ifdef CONFIG_PPC_GUARDED_PAGE_IN_PMD
+int __pte_alloc_kernel_g(pmd_t *pmd, unsigned long address)
+{
+ pte_t *new = pte_alloc_one_kernel(&init_mm, address);
+ if (!new)
+ return -ENOMEM;
+
+ smp_wmb(); /* See comment in __pte_alloc */
+
+ spin_lock(&init_mm.page_table_lock);
+ if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
+ pmd_populate_kernel_g(&init_mm, pmd, new);
+ new = NULL;
+ }
+ spin_unlock(&init_mm.page_table_lock);
+ if (new)
+ pte_free_kernel(&init_mm, new);
+ return 0;
+}
+#endif
+
int map_kernel_page(unsigned long va, phys_addr_t pa, int flags)
{
pmd_t *pd;
@@ -79,7 +100,12 @@ int map_kernel_page(unsigned long va, phys_addr_t pa, int flags)
/* Use upper 10 bits of VA to index the first level map */
pd = pmd_offset(pud_offset(pgd_offset_k(va), va), va);
/* Use middle 10 bits of VA to index the second-level map */
- pg = pte_alloc_kernel(pd, va);
+#ifdef CONFIG_PPC_GUARDED_PAGE_IN_PMD
+ if (flags & _PAGE_GUARDED)
+ pg = pte_alloc_kernel_g(pd, va);
+ else
+#endif
+ pg = pte_alloc_kernel(pd, va);
if (pg != 0) {
err = 0;
/* The PTE should never be already set nor present in the
diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype
index 67d3125d0610..f860f0326c78 100644
--- a/arch/powerpc/platforms/Kconfig.cputype
+++ b/arch/powerpc/platforms/Kconfig.cputype
@@ -319,6 +319,9 @@ config ARCH_ENABLE_HUGEPAGE_MIGRATION
def_bool y
depends on PPC_BOOK3S_64 && HUGETLB_PAGE && MIGRATION
+config PPC_GUARDED_PAGE_IN_PMD
+ def_bool y
+ depends on PPC_8xx
config PPC_MMU_NOHASH
def_bool y
--
2.13.3
^ permalink raw reply related
* [PATCH 12/17] powerpc/8xx: Remove PTE_ATOMIC_UPDATES
From: Christophe Leroy @ 2018-05-04 12:34 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
aneesh.kumar
Cc: linux-kernel, linuxppc-dev
In-Reply-To: <cover.1525435203.git.christophe.leroy@c-s.fr>
commit 1bc54c03117b9 ("powerpc: rework 4xx PTE access and TLB miss")
introduced non atomic PTE updates and started the work of removing
PTE updates in TLB miss handlers, but kept PTE_ATOMIC_UPDATES for the
8xx with the following comment:
/* Until my rework is finished, 8xx still needs atomic PTE updates */
commit fe11dc3f9628e ("powerpc/8xx: Update TLB asm so it behaves as linux
mm expects") removed all PTE updates done in TLB miss handlers
Therefore, atomic PTE updates are not needed anymore for the 8xx
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
arch/powerpc/include/asm/nohash/32/pte-8xx.h | 3 ---
1 file changed, 3 deletions(-)
diff --git a/arch/powerpc/include/asm/nohash/32/pte-8xx.h b/arch/powerpc/include/asm/nohash/32/pte-8xx.h
index a9a2919251e0..31401320c1a5 100644
--- a/arch/powerpc/include/asm/nohash/32/pte-8xx.h
+++ b/arch/powerpc/include/asm/nohash/32/pte-8xx.h
@@ -54,9 +54,6 @@
#define _PMD_GUARDED 0x0010
#define _PMD_USER 0x0020 /* APG 1 */
-/* Until my rework is finished, 8xx still needs atomic PTE updates */
-#define PTE_ATOMIC_UPDATES 1
-
#ifdef CONFIG_PPC_16K_PAGES
#define _PAGE_PSIZE _PAGE_HUGE
#endif
--
2.13.3
^ permalink raw reply related
* [PATCH 13/17] powerpc/mm: Use hardware assistance in TLB handlers on the 8xx
From: Christophe Leroy @ 2018-05-04 12:34 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
aneesh.kumar
Cc: linux-kernel, linuxppc-dev
In-Reply-To: <cover.1525435203.git.christophe.leroy@c-s.fr>
Today, on the 8xx the TLB handlers do SW tablewalk by doing all
the calculation in ASM, in order to match with the Linux page
table structure.
The 8xx offers hardware assistance which allows significant size
reduction of the TLB handlers, hence also reduces the time spent
in the handlers.
However, using this HW assistance implies some constraints on the
page table structure:
- Regardless of the main page size used (4k or 16k), the
level 1 table (PGD) contains 1024 entries and each PGD entry covers
a 4Mbytes area which is managed by a level 2 table (PTE) containing
also 1024 entries each describing a 4k page.
- 16k pages require 4 identifical entries in the L2 table
- 512k pages PTE have to be spread every 128 bytes in the L2 table
- 8M pages PTE are at the address pointed by the L1 entry and each
8M page require 2 identical entries in the PGD.
In order to use hardware assistance, this patch does the following
modifications:
- Make PGD size independant of the main page size
- In 16k pages mode, redefine pte_t as a struct with 4 elements,
and populate those 4 elements in __set_pte_at() and pte_update()
- Modify the TLB handlers to use HW assistance
- Adapt the size of the hugepage tables.
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
arch/powerpc/include/asm/hugetlb.h | 4 +-
arch/powerpc/include/asm/nohash/32/pgtable.h | 16 +-
arch/powerpc/include/asm/nohash/pgtable.h | 4 +
arch/powerpc/include/asm/pgtable-types.h | 4 +
arch/powerpc/kernel/head_8xx.S | 227 +++++++++------------------
arch/powerpc/mm/8xx_mmu.c | 10 +-
arch/powerpc/mm/hugetlbpage.c | 12 ++
7 files changed, 112 insertions(+), 165 deletions(-)
diff --git a/arch/powerpc/include/asm/hugetlb.h b/arch/powerpc/include/asm/hugetlb.h
index 78540c074d70..6d29be6bac74 100644
--- a/arch/powerpc/include/asm/hugetlb.h
+++ b/arch/powerpc/include/asm/hugetlb.h
@@ -77,7 +77,9 @@ static inline pte_t *hugepte_offset(hugepd_t hpd, unsigned long addr,
unsigned long idx = 0;
pte_t *dir = hugepd_page(hpd);
-#ifndef CONFIG_PPC_FSL_BOOK3E
+#ifdef CONFIG_PPC_8xx
+ idx = (addr & ((1UL << pdshift) - 1)) >> PAGE_SHIFT;
+#elif !defined(CONFIG_PPC_FSL_BOOK3E)
idx = (addr & ((1UL << pdshift) - 1)) >> hugepd_shift(hpd);
#endif
diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h
index 009a5b3d3192..3efd616bbc80 100644
--- a/arch/powerpc/include/asm/nohash/32/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/32/pgtable.h
@@ -18,7 +18,11 @@ extern int icache_44x_need_flush;
#endif /* __ASSEMBLY__ */
+#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PPC_16K_PAGES)
+#define PTE_INDEX_SIZE (PTE_SHIFT - 2)
+#else
#define PTE_INDEX_SIZE PTE_SHIFT
+#endif
#define PMD_INDEX_SIZE 0
#define PUD_INDEX_SIZE 0
#define PGD_INDEX_SIZE (32 - PGDIR_SHIFT)
@@ -47,7 +51,11 @@ extern int icache_44x_need_flush;
* -Matt
*/
/* PGDIR_SHIFT determines what a top-level page table entry can map */
+#ifdef CONFIG_PPC_8xx
+#define PGDIR_SHIFT 22
+#else
#define PGDIR_SHIFT (PAGE_SHIFT + PTE_INDEX_SIZE)
+#endif
#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
#define PGDIR_MASK (~(PGDIR_SIZE-1))
@@ -190,7 +198,13 @@ static inline unsigned long pte_update(pte_t *p,
: "cc" );
#else /* PTE_ATOMIC_UPDATES */
unsigned long old = pte_val(*p);
- *p = __pte((old & ~clr) | set);
+ unsigned long new = (old & ~clr) | set;
+
+#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PPC_16K_PAGES)
+ p->pte = p->pte1 = p->pte2 = p->pte3 = new;
+#else
+ *p = __pte(new);
+#endif
#endif /* !PTE_ATOMIC_UPDATES */
#ifdef CONFIG_44x
diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h
index 077472640b35..e4b6c084be5c 100644
--- a/arch/powerpc/include/asm/nohash/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/pgtable.h
@@ -165,7 +165,11 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
/* Anything else just stores the PTE normally. That covers all 64-bit
* cases, and 32-bit non-hash with 32-bit PTEs.
*/
+#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PPC_16K_PAGES)
+ ptep->pte = ptep->pte1 = ptep->pte2 = ptep->pte3 = pte_val(pte);
+#else
*ptep = pte;
+#endif
/*
* With hardware tablewalk, a sync is needed to ensure that
diff --git a/arch/powerpc/include/asm/pgtable-types.h b/arch/powerpc/include/asm/pgtable-types.h
index eccb30b38b47..3b0edf041b2e 100644
--- a/arch/powerpc/include/asm/pgtable-types.h
+++ b/arch/powerpc/include/asm/pgtable-types.h
@@ -3,7 +3,11 @@
#define _ASM_POWERPC_PGTABLE_TYPES_H
/* PTE level */
+#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PPC_16K_PAGES)
+typedef struct { pte_basic_t pte, pte1, pte2, pte3; } pte_t;
+#else
typedef struct { pte_basic_t pte; } pte_t;
+#endif
#define __pte(x) ((pte_t) { (x) })
static inline pte_basic_t pte_val(pte_t x)
{
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 85b017c67e11..4855d5a36f70 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -275,7 +275,7 @@ SystemCall:
. = 0x1100
/*
* For the MPC8xx, this is a software tablewalk to load the instruction
- * TLB. The task switch loads the M_TW register with the pointer to the first
+ * TLB. The task switch loads the M_TWB register with the pointer to the first
* level table.
* If we discover there is no second level table (value is zero) or if there
* is an invalid pte, we load that into the TLB, which causes another fault
@@ -285,106 +285,100 @@ SystemCall:
*/
#ifdef CONFIG_8xx_CPU15
-#define INVALIDATE_ADJACENT_PAGES_CPU15(tmp, addr) \
- addi tmp, addr, PAGE_SIZE; \
- tlbie tmp; \
- addi tmp, addr, -PAGE_SIZE; \
- tlbie tmp
+#define INVALIDATE_ADJACENT_PAGES_CPU15(addr) \
+ addi addr, addr, PAGE_SIZE; \
+ tlbie addr; \
+ addi addr, addr, -(PAGE_SIZE << 1); \
+ tlbie addr; \
+ addi addr, addr, PAGE_SIZE
#else
-#define INVALIDATE_ADJACENT_PAGES_CPU15(tmp, addr)
+#define INVALIDATE_ADJACENT_PAGES_CPU15(addr)
#endif
InstructionTLBMiss:
mtspr SPRN_SPRG_SCRATCH0, r10
+#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_SWAP)
mtspr SPRN_SPRG_SCRATCH1, r11
-#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
- mtspr SPRN_SPRG_SCRATCH2, r12
+#ifdef ITLB_MISS_KERNEL
+ mfcr r11
+#endif
#endif
/* If we are faulting a kernel address, we have to use the
* kernel page tables.
*/
mfspr r10, SPRN_SRR0 /* Get effective address of fault */
- INVALIDATE_ADJACENT_PAGES_CPU15(r11, r10)
+ INVALIDATE_ADJACENT_PAGES_CPU15(r10)
/* Only modules will cause ITLB Misses as we always
* pin the first 8MB of kernel memory */
-#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
- mfcr r12
-#endif
+ mtspr SPRN_MD_EPN, r10
#ifdef ITLB_MISS_KERNEL
#if defined(SIMPLE_KERNEL_ADDRESS) && defined(CONFIG_PIN_TLB_TEXT)
- andis. r11, r10, 0x8000 /* Address >= 0x80000000 */
+ cmpi cr0, r10, 0 /* Address >= 0x80000000 */
#else
- rlwinm r11, r10, 16, 0xfff8
- cmpli cr0, r11, PAGE_OFFSET@h
+ rlwinm r10, r10, 16, 0xfff8
+ cmpli cr0, r10, PAGE_OFFSET@h
#ifndef CONFIG_PIN_TLB_TEXT
/* It is assumed that kernel code fits into the first 8M page */
_ENTRY(ITLBMiss_cmp)
- cmpli cr7, r11, (PAGE_OFFSET + 0x0800000)@h
+ cmpli cr7, r10, (PAGE_OFFSET + 0x0800000)@h
#endif
#endif
#endif
- mfspr r11, SPRN_M_TW /* Get level 1 table */
+ mfspr r10, SPRN_M_TWB /* Get level 1 table */
#ifdef ITLB_MISS_KERNEL
#if defined(SIMPLE_KERNEL_ADDRESS) && defined(CONFIG_PIN_TLB_TEXT)
- beq+ 3f
+ bge+ 3f
#else
blt+ 3f
#endif
#ifndef CONFIG_PIN_TLB_TEXT
blt cr7, ITLBMissLinear
#endif
- lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
+ rlwinm r10, r10, 0, 20, 31
+ oris r10, r10, (swapper_pg_dir - PAGE_OFFSET)@h
+ ori r10, r10, (swapper_pg_dir - PAGE_OFFSET)@l
3:
#endif
+#ifdef ITLB_MISS_KERNEL
+ mfcr r11
+#endif
/* Insert level 1 index */
- rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
- lwz r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11) /* Get the level 1 entry */
+ lwz r10, 0(r10) /* Get the level 1 entry */
+ mtspr SPRN_MI_TWC, r10 /* Set segment attributes */
+ mtspr SPRN_MD_TWC, r10
- /* Extract level 2 index */
- rlwinm r10, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
-#ifdef CONFIG_HUGETLB_PAGE
- mtcr r11
-#endif
- /* Load the MI_TWC with the attributes for this "segment." */
- mtspr SPRN_MI_TWC, r11 /* Set segment attributes */
-#ifdef CONFIG_HUGETLB_PAGE
- bt- 28, 10f /* bit 28 = Large page (8M) */
- bt- 29, 20f /* bit 29 = Large page (8M or 512k) */
-#endif
- rlwimi r10, r11, 0, 0, 32 - PAGE_SHIFT - 1 /* Add level 2 base */
+ mfspr r10, SPRN_MD_TWC
lwz r10, 0(r10) /* Get the pte */
-4:
-#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
- mtcr r12
-#endif
#ifdef CONFIG_SWAP
rlwinm r11, r10, 32-5, _PAGE_PRESENT
and r11, r11, r10
rlwimi r10, r11, 0, _PAGE_PRESENT
#endif
- li r11, RPN_PATTERN | 0x200
/* The Linux PTE won't go exactly into the MMU TLB.
* Software indicator bits 20 and 23 must be clear.
* Software indicator bits 22, 24, 25, 26, and 27 must be
* set. All other Linux PTE bits control the behavior
* of the MMU.
*/
- rlwimi r11, r10, 4, 0x0400 /* Copy _PAGE_EXEC into bit 21 */
- rlwimi r10, r11, 0, 0x0ff0 /* Set 22, 24-27, clear 20,23 */
+ rlwimi r10, r10, 0, 0x0f00 /* Clear bits 20-23 */
+ rlwimi r10, r10, 4, 0x0400 /* Copy _PAGE_EXEC into bit 21 */
+ ori r10, r10, RPN_PATTERN | 0x200 /* Set 22 and 24-27 */
mtspr SPRN_MI_RPN, r10 /* Update TLB entry */
/* Restore registers */
_ENTRY(itlb_miss_exit_1)
mfspr r10, SPRN_SPRG_SCRATCH0
+#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_SWAP)
mfspr r11, SPRN_SPRG_SCRATCH1
-#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
- mfspr r12, SPRN_SPRG_SCRATCH2
#endif
rfi
#ifdef CONFIG_PERF_EVENTS
_ENTRY(itlb_miss_perf)
+#if !defined(ITLB_MISS_KERNEL) && !defined(CONFIG_SWAP)
+ mtspr SPRN_SPRG_SCRATCH1, r11
+#endif
lis r10, (itlb_miss_counter - PAGE_OFFSET)@ha
lwz r11, (itlb_miss_counter - PAGE_OFFSET)@l(r10)
addi r11, r11, 1
@@ -392,83 +386,42 @@ _ENTRY(itlb_miss_perf)
#endif
mfspr r10, SPRN_SPRG_SCRATCH0
mfspr r11, SPRN_SPRG_SCRATCH1
-#if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
- mfspr r12, SPRN_SPRG_SCRATCH2
-#endif
rfi
-#ifdef CONFIG_HUGETLB_PAGE
-10: /* 8M pages */
-#ifdef CONFIG_PPC_16K_PAGES
- /* Extract level 2 index */
- rlwinm r10, r10, 32 - (PAGE_SHIFT_8M - PAGE_SHIFT), 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1), 29
- /* Add level 2 base */
- rlwimi r10, r11, 0, 0, 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1) - 1
-#else
- /* Level 2 base */
- rlwinm r10, r11, 0, ~HUGEPD_SHIFT_MASK
-#endif
- lwz r10, 0(r10) /* Get the pte */
- b 4b
-
-20: /* 512k pages */
- /* Extract level 2 index */
- rlwinm r10, r10, 32 - (PAGE_SHIFT_512K - PAGE_SHIFT), 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1), 29
- /* Add level 2 base */
- rlwimi r10, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1
- lwz r10, 0(r10) /* Get the pte */
- b 4b
-#endif
-
. = 0x1200
DataStoreTLBMiss:
mtspr SPRN_SPRG_SCRATCH0, r10
mtspr SPRN_SPRG_SCRATCH1, r11
- mtspr SPRN_SPRG_SCRATCH2, r12
- mfcr r12
+ mfcr r11
/* If we are faulting a kernel address, we have to use the
* kernel page tables.
*/
mfspr r10, SPRN_MD_EPN
- rlwinm r11, r10, 16, 0xfff8
- cmpli cr0, r11, PAGE_OFFSET@h
- mfspr r11, SPRN_M_TW /* Get level 1 table */
- blt+ 3f
- rlwinm r11, r10, 16, 0xfff8
+ rlwinm r10, r10, 16, 0xfff8
+ cmpli cr0, r10, PAGE_OFFSET@h
#ifndef CONFIG_PIN_TLB_IMMR
- cmpli cr0, r11, VIRT_IMMR_BASE@h
+ cmpli cr6, r10, VIRT_IMMR_BASE@h
#endif
_ENTRY(DTLBMiss_cmp)
- cmpli cr7, r11, (PAGE_OFFSET + 0x1800000)@h
+ cmpli cr7, r10, (PAGE_OFFSET + 0x1800000)@h
+ mfspr r10, SPRN_M_TWB /* Get level 1 table */
+ blt+ 3f
#ifndef CONFIG_PIN_TLB_IMMR
_ENTRY(DTLBMiss_jmp)
- beq- DTLBMissIMMR
+ beq- cr6, DTLBMissIMMR
#endif
blt cr7, DTLBMissLinear
- lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
+ rlwinm r10, r10, 0, 20, 31
+ oris r10, r10, (swapper_pg_dir - PAGE_OFFSET)@h
+ ori r10, r10, (swapper_pg_dir - PAGE_OFFSET)@l
3:
-
- /* Insert level 1 index */
- rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
- lwz r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11) /* Get the level 1 entry */
-
- /* We have a pte table, so load fetch the pte from the table.
- */
- /* Extract level 2 index */
- rlwinm r10, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
-#ifdef CONFIG_HUGETLB_PAGE
+ lwz r10, 0(r10) /* Get the level 1 entry */
mtcr r11
-#endif
- mtspr SPRN_MD_TWC, r11
-#ifdef CONFIG_HUGETLB_PAGE
- bt- 28, 10f /* bit 28 = Large page (8M) */
- bt- 29, 20f /* bit 29 = Large page (8M or 512k) */
-#endif
- rlwimi r10, r11, 0, 0, 32 - PAGE_SHIFT - 1 /* Add level 2 base */
+
+ mtspr SPRN_MD_TWC, r10
+ mfspr r10, SPRN_MD_TWC
lwz r10, 0(r10) /* Get the pte */
-4:
- mtcr r12
/* Both _PAGE_ACCESSED and _PAGE_PRESENT has to be set.
* We also need to know if the insn is a load/store, so:
@@ -498,7 +451,6 @@ _ENTRY(DTLBMiss_jmp)
_ENTRY(dtlb_miss_exit_1)
mfspr r10, SPRN_SPRG_SCRATCH0
mfspr r11, SPRN_SPRG_SCRATCH1
- mfspr r12, SPRN_SPRG_SCRATCH2
rfi
#ifdef CONFIG_PERF_EVENTS
_ENTRY(dtlb_miss_perf)
@@ -509,32 +461,8 @@ _ENTRY(dtlb_miss_perf)
#endif
mfspr r10, SPRN_SPRG_SCRATCH0
mfspr r11, SPRN_SPRG_SCRATCH1
- mfspr r12, SPRN_SPRG_SCRATCH2
rfi
-#ifdef CONFIG_HUGETLB_PAGE
-10: /* 8M pages */
- /* Extract level 2 index */
-#ifdef CONFIG_PPC_16K_PAGES
- rlwinm r10, r10, 32 - (PAGE_SHIFT_8M - PAGE_SHIFT), 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1), 29
- /* Add level 2 base */
- rlwimi r10, r11, 0, 0, 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1) - 1
-#else
- /* Level 2 base */
- rlwinm r10, r11, 0, ~HUGEPD_SHIFT_MASK
-#endif
- lwz r10, 0(r10) /* Get the pte */
- b 4b
-
-20: /* 512k pages */
- /* Extract level 2 index */
- rlwinm r10, r10, 32 - (PAGE_SHIFT_512K - PAGE_SHIFT), 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1), 29
- /* Add level 2 base */
- rlwimi r10, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1
- lwz r10, 0(r10) /* Get the pte */
- b 4b
-#endif
-
/* This is an instruction TLB error on the MPC8xx. This could be due
* to many reasons, such as executing guarded memory or illegal instruction
* addresses. There is nothing to do but handle a big time error fault.
@@ -642,7 +570,7 @@ InstructionBreakpoint:
* not enough space in the DataStoreTLBMiss area.
*/
DTLBMissIMMR:
- mtcr r12
+ mtcr r11
/* Set 512k byte guarded page and mark it valid */
li r10, MD_PS512K | MD_GUARDED | MD_SVALID
mtspr SPRN_MD_TWC, r10
@@ -657,15 +585,14 @@ DTLBMissIMMR:
_ENTRY(dtlb_miss_exit_2)
mfspr r10, SPRN_SPRG_SCRATCH0
mfspr r11, SPRN_SPRG_SCRATCH1
- mfspr r12, SPRN_SPRG_SCRATCH2
rfi
DTLBMissLinear:
- mtcr r12
+ mtcr r11
/* Set 8M byte page and mark it valid */
li r11, MD_PS8MEG | MD_SVALID
mtspr SPRN_MD_TWC, r11
- rlwinm r10, r10, 0, 0x0f800000 /* 8xx supports max 256Mb RAM */
+ rlwinm r10, r10, 20, 0x0f800000 /* 8xx supports max 256Mb RAM */
ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \
_PAGE_PRESENT
mtspr SPRN_MD_RPN, r10 /* Update TLB entry */
@@ -675,16 +602,15 @@ DTLBMissLinear:
_ENTRY(dtlb_miss_exit_3)
mfspr r10, SPRN_SPRG_SCRATCH0
mfspr r11, SPRN_SPRG_SCRATCH1
- mfspr r12, SPRN_SPRG_SCRATCH2
rfi
#ifndef CONFIG_PIN_TLB_TEXT
ITLBMissLinear:
- mtcr r12
+ mtcr r11
/* Set 8M byte page and mark it valid */
li r11, MI_PS8MEG | MI_SVALID
mtspr SPRN_MI_TWC, r11
- rlwinm r10, r10, 0, 0x0f800000 /* 8xx supports max 256Mb RAM */
+ rlwinm r10, r10, 20, 0x0f800000 /* 8xx supports max 256Mb RAM */
ori r10, r10, 0xf0 | MI_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \
_PAGE_PRESENT
mtspr SPRN_MI_RPN, r10 /* Update TLB entry */
@@ -692,7 +618,6 @@ ITLBMissLinear:
_ENTRY(itlb_miss_exit_2)
mfspr r10, SPRN_SPRG_SCRATCH0
mfspr r11, SPRN_SPRG_SCRATCH1
- mfspr r12, SPRN_SPRG_SCRATCH2
rfi
#endif
@@ -706,9 +631,10 @@ FixupDAR:/* Entry point for dcbx workaround. */
mtspr SPRN_SPRG_SCRATCH2, r10
/* fetch instruction from memory. */
mfspr r10, SPRN_SRR0
+ mtspr SPRN_MD_EPN, r10
rlwinm r11, r10, 16, 0xfff8
cmpli cr0, r11, PAGE_OFFSET@h
- mfspr r11, SPRN_M_TW /* Get level 1 table */
+ mfspr r11, SPRN_M_TWB /* Get level 1 table */
blt+ 3f
rlwinm r11, r10, 16, 0xfff8
_ENTRY(FixupDAR_cmp)
@@ -716,19 +642,20 @@ _ENTRY(FixupDAR_cmp)
/* create physical page address from effective address */
tophys(r11, r10)
blt- cr7, 201f
- lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
- /* Insert level 1 index */
-3: rlwimi r11, r10, 32 - ((PAGE_SHIFT - 2) << 1), (PAGE_SHIFT - 2) << 1, 29
- lwz r11, (swapper_pg_dir-PAGE_OFFSET)@l(r11) /* Get the level 1 entry */
+ mfspr r11, SPRN_M_TWB /* Get level 1 table */
+ rlwinm r11, r11, 0, 20, 31
+ oris r11, r11, (swapper_pg_dir - PAGE_OFFSET)@h
+ ori r11, r11, (swapper_pg_dir - PAGE_OFFSET)@l
+3:
+ lwz r11, 0(r11) /* Get the level 1 entry */
+ mtspr SPRN_MD_TWC, r11
mtcr r11
+ mfspr r11, SPRN_MD_TWC
+ lwz r11, 0(r11) /* Get the pte */
bt 28,200f /* bit 28 = Large page (8M) */
bt 29,202f /* bit 29 = Large page (8M or 512K) */
- rlwinm r11, r11,0,0,19 /* Extract page descriptor page address */
- /* Insert level 2 index */
- rlwimi r11, r10, 32 - (PAGE_SHIFT - 2), 32 - PAGE_SHIFT, 29
- lwz r11, 0(r11) /* Get the pte */
/* concat physical page address(r11) and page offset(r10) */
- rlwimi r11, r10, 0, 32 - PAGE_SHIFT, 31
+ rlwimi r11, r10, 0, 20, 31
201: lwz r11,0(r11)
/* Check if it really is a dcbx instruction. */
/* dcbt and dcbtst does not generate DTLB Misses/Errors,
@@ -748,23 +675,12 @@ _ENTRY(FixupDAR_cmp)
141: mfspr r10,SPRN_SPRG_SCRATCH2
b DARFixed /* Nope, go back to normal TLB processing */
- /* concat physical page address(r11) and page offset(r10) */
200:
-#ifdef CONFIG_PPC_16K_PAGES
- rlwinm r11, r11, 0, 0, 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1) - 1
- rlwimi r11, r10, 32 - (PAGE_SHIFT_8M - 2), 32 + PAGE_SHIFT_8M - (PAGE_SHIFT << 1), 29
-#else
- rlwinm r11, r10, 0, ~HUGEPD_SHIFT_MASK
-#endif
- lwz r11, 0(r11) /* Get the pte */
/* concat physical page address(r11) and page offset(r10) */
rlwimi r11, r10, 0, 32 - PAGE_SHIFT_8M, 31
b 201b
202:
- rlwinm r11, r11, 0, 0, 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1) - 1
- rlwimi r11, r10, 32 - (PAGE_SHIFT_512K - 2), 32 + PAGE_SHIFT_512K - (PAGE_SHIFT << 1), 29
- lwz r11, 0(r11) /* Get the pte */
/* concat physical page address(r11) and page offset(r10) */
rlwimi r11, r10, 0, 32 - PAGE_SHIFT_512K, 31
b 201b
@@ -898,9 +814,10 @@ start_here:
* init's THREAD like the context switch code does, but this is
* easier......until someone changes init's static structures.
*/
- lis r6, swapper_pg_dir@ha
+ lis r6, swapper_pg_dir@h
+ ori r6, r6, swapper_pg_dir@l
tophys(r6,r6)
- mtspr SPRN_M_TW, r6
+ mtspr SPRN_M_TWB, r6
lis r4,2f@h
ori r4,r4,2f@l
tophys(r4,r4)
diff --git a/arch/powerpc/mm/8xx_mmu.c b/arch/powerpc/mm/8xx_mmu.c
index 5d53684c2ebd..54a02b8e21ec 100644
--- a/arch/powerpc/mm/8xx_mmu.c
+++ b/arch/powerpc/mm/8xx_mmu.c
@@ -173,8 +173,6 @@ void __init setup_initial_memory_limit(phys_addr_t first_memblock_base,
*/
void set_context(unsigned long id, pgd_t *pgd)
{
- s16 offset = (s16)(__pa(swapper_pg_dir));
-
#ifdef CONFIG_BDI_SWITCH
pgd_t **ptr = *(pgd_t ***)(KERNELBASE + 0xf0);
@@ -184,12 +182,8 @@ void set_context(unsigned long id, pgd_t *pgd)
*(ptr + 1) = pgd;
#endif
- /* Register M_TW will contain base address of level 1 table minus the
- * lower part of the kernel PGDIR base address, so that all accesses to
- * level 1 table are done relative to lower part of kernel PGDIR base
- * address.
- */
- mtspr(SPRN_M_TW, __pa(pgd) - offset);
+ /* Register M_TWB will contain base address of level 1 table */
+ mtspr(SPRN_M_TWB, __pa(pgd));
/* Update context */
mtspr(SPRN_M_CASID, id - 1);
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index f1153f8254e3..6c07d40eed0f 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -61,7 +61,11 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp,
cachep = hugepte_cache;
num_hugepd = 1 << (pshift - pdshift);
} else {
+#ifdef CONFIG_PPC_8xx
+ cachep = PGT_CACHE(PTE_SHIFT);
+#else
cachep = PGT_CACHE(pdshift - pshift);
+#endif
num_hugepd = 1;
}
@@ -326,7 +330,11 @@ static void free_hugepd_range(struct mmu_gather *tlb, hugepd_t *hpdp, int pdshif
if (shift >= pdshift)
hugepd_free(tlb, hugepte);
else
+#ifdef CONFIG_PPC_8xx
+ pgtable_free_tlb(tlb, hugepte, PTE_SHIFT);
+#else
pgtable_free_tlb(tlb, hugepte, pdshift - shift);
+#endif
}
static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
@@ -689,7 +697,11 @@ static int __init hugetlbpage_init(void)
* use pgt cache for hugepd.
*/
if (pdshift > shift)
+#ifdef CONFIG_PPC_8xx
+ pgtable_cache_add(PTE_SHIFT, NULL);
+#else
pgtable_cache_add(pdshift - shift, NULL);
+#endif
#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx)
else if (!hugepte_cache) {
/*
--
2.13.3
^ permalink raw reply related
* [PATCH 14/17] powerpc/8xx: reunify TLB handler routines
From: Christophe Leroy @ 2018-05-04 12:34 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
aneesh.kumar
Cc: linux-kernel, linuxppc-dev
In-Reply-To: <cover.1525435203.git.christophe.leroy@c-s.fr>
Each handler must not exceed 64 instructions to fit into the main
exception area.
Following the significant size reduction of TLB handler routines,
the side handlers can be brought back close to the main part.
In the worst case:
Main part of ITLB handler is 45 insn, side part is 9 insn ==> total 54
Main part of DTLB handler is 37 insn, side part is 23 insn ==> total 60
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
arch/powerpc/kernel/head_8xx.S | 108 ++++++++++++++++++++---------------------
1 file changed, 52 insertions(+), 56 deletions(-)
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index 4855d5a36f70..c98a4ebb5a4d 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -388,6 +388,23 @@ _ENTRY(itlb_miss_perf)
mfspr r11, SPRN_SPRG_SCRATCH1
rfi
+#ifndef CONFIG_PIN_TLB_TEXT
+ITLBMissLinear:
+ mtcr r11
+ /* Set 8M byte page and mark it valid */
+ li r11, MI_PS8MEG | MI_SVALID
+ mtspr SPRN_MI_TWC, r11
+ rlwinm r10, r10, 20, 0x0f800000 /* 8xx supports max 256Mb RAM */
+ ori r10, r10, 0xf0 | MI_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \
+ _PAGE_PRESENT
+ mtspr SPRN_MI_RPN, r10 /* Update TLB entry */
+
+_ENTRY(itlb_miss_exit_2)
+ mfspr r10, SPRN_SPRG_SCRATCH0
+ mfspr r11, SPRN_SPRG_SCRATCH1
+ rfi
+#endif
+
. = 0x1200
DataStoreTLBMiss:
mtspr SPRN_SPRG_SCRATCH0, r10
@@ -463,6 +480,41 @@ _ENTRY(dtlb_miss_perf)
mfspr r11, SPRN_SPRG_SCRATCH1
rfi
+DTLBMissIMMR:
+ mtcr r11
+ /* Set 512k byte guarded page and mark it valid */
+ li r10, MD_PS512K | MD_GUARDED | MD_SVALID
+ mtspr SPRN_MD_TWC, r10
+ mfspr r10, SPRN_IMMR /* Get current IMMR */
+ rlwinm r10, r10, 0, 0xfff80000 /* Get 512 kbytes boundary */
+ ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \
+ _PAGE_PRESENT | _PAGE_NO_CACHE
+ mtspr SPRN_MD_RPN, r10 /* Update TLB entry */
+
+ li r11, RPN_PATTERN
+ mtspr SPRN_DAR, r11 /* Tag DAR */
+_ENTRY(dtlb_miss_exit_2)
+ mfspr r10, SPRN_SPRG_SCRATCH0
+ mfspr r11, SPRN_SPRG_SCRATCH1
+ rfi
+
+DTLBMissLinear:
+ mtcr r11
+ /* Set 8M byte page and mark it valid */
+ li r11, MD_PS8MEG | MD_SVALID
+ mtspr SPRN_MD_TWC, r11
+ rlwinm r10, r10, 20, 0x0f800000 /* 8xx supports max 256Mb RAM */
+ ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \
+ _PAGE_PRESENT
+ mtspr SPRN_MD_RPN, r10 /* Update TLB entry */
+
+ li r11, RPN_PATTERN
+ mtspr SPRN_DAR, r11 /* Tag DAR */
+_ENTRY(dtlb_miss_exit_3)
+ mfspr r10, SPRN_SPRG_SCRATCH0
+ mfspr r11, SPRN_SPRG_SCRATCH1
+ rfi
+
/* This is an instruction TLB error on the MPC8xx. This could be due
* to many reasons, such as executing guarded memory or illegal instruction
* addresses. There is nothing to do but handle a big time error fault.
@@ -565,62 +617,6 @@ InstructionBreakpoint:
. = 0x2000
-/*
- * Bottom part of DataStoreTLBMiss handlers for IMMR area and linear RAM.
- * not enough space in the DataStoreTLBMiss area.
- */
-DTLBMissIMMR:
- mtcr r11
- /* Set 512k byte guarded page and mark it valid */
- li r10, MD_PS512K | MD_GUARDED | MD_SVALID
- mtspr SPRN_MD_TWC, r10
- mfspr r10, SPRN_IMMR /* Get current IMMR */
- rlwinm r10, r10, 0, 0xfff80000 /* Get 512 kbytes boundary */
- ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \
- _PAGE_PRESENT | _PAGE_NO_CACHE
- mtspr SPRN_MD_RPN, r10 /* Update TLB entry */
-
- li r11, RPN_PATTERN
- mtspr SPRN_DAR, r11 /* Tag DAR */
-_ENTRY(dtlb_miss_exit_2)
- mfspr r10, SPRN_SPRG_SCRATCH0
- mfspr r11, SPRN_SPRG_SCRATCH1
- rfi
-
-DTLBMissLinear:
- mtcr r11
- /* Set 8M byte page and mark it valid */
- li r11, MD_PS8MEG | MD_SVALID
- mtspr SPRN_MD_TWC, r11
- rlwinm r10, r10, 20, 0x0f800000 /* 8xx supports max 256Mb RAM */
- ori r10, r10, 0xf0 | MD_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \
- _PAGE_PRESENT
- mtspr SPRN_MD_RPN, r10 /* Update TLB entry */
-
- li r11, RPN_PATTERN
- mtspr SPRN_DAR, r11 /* Tag DAR */
-_ENTRY(dtlb_miss_exit_3)
- mfspr r10, SPRN_SPRG_SCRATCH0
- mfspr r11, SPRN_SPRG_SCRATCH1
- rfi
-
-#ifndef CONFIG_PIN_TLB_TEXT
-ITLBMissLinear:
- mtcr r11
- /* Set 8M byte page and mark it valid */
- li r11, MI_PS8MEG | MI_SVALID
- mtspr SPRN_MI_TWC, r11
- rlwinm r10, r10, 20, 0x0f800000 /* 8xx supports max 256Mb RAM */
- ori r10, r10, 0xf0 | MI_SPS16K | _PAGE_PRIVILEGED | _PAGE_DIRTY | \
- _PAGE_PRESENT
- mtspr SPRN_MI_RPN, r10 /* Update TLB entry */
-
-_ENTRY(itlb_miss_exit_2)
- mfspr r10, SPRN_SPRG_SCRATCH0
- mfspr r11, SPRN_SPRG_SCRATCH1
- rfi
-#endif
-
/* This is the procedure to calculate the data EA for buggy dcbx,dcbi instructions
* by decoding the registers used by the dcbx instruction and adding them.
* DAR is set to the calculated address.
--
2.13.3
^ permalink raw reply related
* [PATCH 15/17] powerpc/8xx: Free up SPRN_SPRG_SCRATCH2
From: Christophe Leroy @ 2018-05-04 12:34 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
aneesh.kumar
Cc: linux-kernel, linuxppc-dev
In-Reply-To: <cover.1525435203.git.christophe.leroy@c-s.fr>
We can now use SPRN_M_TW in the DAR Fixup code, freeing
SPRN_SPRG_SCRATCH2
Then SPRN_SPRG_SCRATCH2 may be used for something else in the future.
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
arch/powerpc/kernel/head_8xx.S | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
index c98a4ebb5a4d..8e96b526f109 100644
--- a/arch/powerpc/kernel/head_8xx.S
+++ b/arch/powerpc/kernel/head_8xx.S
@@ -624,7 +624,7 @@ InstructionBreakpoint:
/* define if you don't want to use self modifying code */
#define NO_SELF_MODIFYING_CODE
FixupDAR:/* Entry point for dcbx workaround. */
- mtspr SPRN_SPRG_SCRATCH2, r10
+ mtspr SPRN_M_TW, r10
/* fetch instruction from memory. */
mfspr r10, SPRN_SRR0
mtspr SPRN_MD_EPN, r10
@@ -668,7 +668,7 @@ _ENTRY(FixupDAR_cmp)
beq+ 142f
cmpwi cr0, r10, 1964 /* Is icbi? */
beq+ 142f
-141: mfspr r10,SPRN_SPRG_SCRATCH2
+141: mfspr r10,SPRN_M_TW
b DARFixed /* Nope, go back to normal TLB processing */
200:
@@ -703,7 +703,7 @@ modified_instr:
bne+ 143f
subf r10,r0,r10 /* r10=r10-r0, only if reg RA is r0 */
143: mtdar r10 /* store faulting EA in DAR */
- mfspr r10,SPRN_SPRG_SCRATCH2
+ mfspr r10,SPRN_M_TW
b DARFixed /* Go back to normal TLB handling */
#else
mfctr r10
@@ -757,7 +757,7 @@ modified_instr:
mfdar r11
mtctr r11 /* restore ctr reg from DAR */
mtdar r10 /* save fault EA to DAR */
- mfspr r10,SPRN_SPRG_SCRATCH2
+ mfspr r10,SPRN_M_TW
b DARFixed /* Go back to normal TLB handling */
/* special handling for r10,r11 since these are modified already */
--
2.13.3
^ permalink raw reply related
* [PATCH 16/17] powerpc/mm: Make pte_fragment_alloc() common to PPC32 and PPC64
From: Christophe Leroy @ 2018-05-04 12:34 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
aneesh.kumar
Cc: linux-kernel, linuxppc-dev
In-Reply-To: <cover.1525435203.git.christophe.leroy@c-s.fr>
In order to allow the 8xx to handle pte_fragments, this patch
makes in common to PPC32 and PPC64
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
arch/powerpc/include/asm/mmu_context.h | 28 ++++++++++++++
arch/powerpc/mm/mmu_context_book3s64.c | 28 --------------
arch/powerpc/mm/pgtable.c | 67 ++++++++++++++++++++++++++++++++++
arch/powerpc/mm/pgtable_64.c | 67 ----------------------------------
arch/powerpc/platforms/Kconfig.cputype | 5 +++
5 files changed, 100 insertions(+), 95 deletions(-)
diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h
index 1835ca1505d6..252988f7e219 100644
--- a/arch/powerpc/include/asm/mmu_context.h
+++ b/arch/powerpc/include/asm/mmu_context.h
@@ -262,5 +262,33 @@ static inline u64 pte_to_hpte_pkey_bits(u64 pteflags)
#endif /* CONFIG_PPC_MEM_KEYS */
+#ifdef CONFIG_NEED_PTE_FRAG
+static inline void destroy_pagetable_page(struct mm_struct *mm)
+{
+ int count;
+ void *pte_frag;
+ struct page *page;
+
+ pte_frag = mm->context.pte_frag;
+ if (!pte_frag)
+ return;
+
+ page = virt_to_page(pte_frag);
+ /* drop all the pending references */
+ count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT;
+ /* We allow PTE_FRAG_NR fragments from a PTE page */
+ if (page_ref_sub_and_test(page, PTE_FRAG_NR - count)) {
+ pgtable_page_dtor(page);
+ free_unref_page(page);
+ }
+}
+
+#else
+static inline void destroy_pagetable_page(struct mm_struct *mm)
+{
+ return;
+}
+#endif
+
#endif /* __KERNEL__ */
#endif /* __ASM_POWERPC_MMU_CONTEXT_H */
diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c
index b75194dff64c..2f55a4e3c09a 100644
--- a/arch/powerpc/mm/mmu_context_book3s64.c
+++ b/arch/powerpc/mm/mmu_context_book3s64.c
@@ -192,34 +192,6 @@ static void destroy_contexts(mm_context_t *ctx)
spin_unlock(&mmu_context_lock);
}
-#ifdef CONFIG_PPC_64K_PAGES
-static void destroy_pagetable_page(struct mm_struct *mm)
-{
- int count;
- void *pte_frag;
- struct page *page;
-
- pte_frag = mm->context.pte_frag;
- if (!pte_frag)
- return;
-
- page = virt_to_page(pte_frag);
- /* drop all the pending references */
- count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT;
- /* We allow PTE_FRAG_NR fragments from a PTE page */
- if (page_ref_sub_and_test(page, PTE_FRAG_NR - count)) {
- pgtable_page_dtor(page);
- free_unref_page(page);
- }
-}
-
-#else
-static inline void destroy_pagetable_page(struct mm_struct *mm)
-{
- return;
-}
-#endif
-
void destroy_context(struct mm_struct *mm)
{
#ifdef CONFIG_SPAPR_TCE_IOMMU
diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c
index 9f361ae571e9..2d34755ed727 100644
--- a/arch/powerpc/mm/pgtable.c
+++ b/arch/powerpc/mm/pgtable.c
@@ -264,3 +264,70 @@ unsigned long vmalloc_to_phys(void *va)
return __pa(pfn_to_kaddr(pfn)) + offset_in_page(va);
}
EXPORT_SYMBOL_GPL(vmalloc_to_phys);
+
+#ifdef CONFIG_NEED_PTE_FRAG
+static pte_t *get_from_cache(struct mm_struct *mm)
+{
+ void *pte_frag, *ret;
+
+ spin_lock(&mm->page_table_lock);
+ ret = mm->context.pte_frag;
+ if (ret) {
+ pte_frag = ret + PTE_FRAG_SIZE;
+ /*
+ * If we have taken up all the fragments mark PTE page NULL
+ */
+ if (((unsigned long)pte_frag & ~PAGE_MASK) == 0)
+ pte_frag = NULL;
+ mm->context.pte_frag = pte_frag;
+ }
+ spin_unlock(&mm->page_table_lock);
+ return (pte_t *)ret;
+}
+
+static pte_t *__alloc_for_cache(struct mm_struct *mm, int kernel)
+{
+ void *ret = NULL;
+ struct page *page;
+
+ if (!kernel) {
+ page = alloc_page(PGALLOC_GFP | __GFP_ACCOUNT);
+ if (!page)
+ return NULL;
+ if (!pgtable_page_ctor(page)) {
+ __free_page(page);
+ return NULL;
+ }
+ } else {
+ page = alloc_page(PGALLOC_GFP);
+ if (!page)
+ return NULL;
+ }
+
+ ret = page_address(page);
+ spin_lock(&mm->page_table_lock);
+ /*
+ * If we find pgtable_page set, we return
+ * the allocated page with single fragement
+ * count.
+ */
+ if (likely(!mm->context.pte_frag)) {
+ set_page_count(page, PTE_FRAG_NR);
+ mm->context.pte_frag = ret + PTE_FRAG_SIZE;
+ }
+ spin_unlock(&mm->page_table_lock);
+
+ return (pte_t *)ret;
+}
+
+pte_t *pte_fragment_alloc(struct mm_struct *mm, unsigned long vmaddr, int kernel)
+{
+ pte_t *pte;
+
+ pte = get_from_cache(mm);
+ if (pte)
+ return pte;
+
+ return __alloc_for_cache(mm, kernel);
+}
+#endif
diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
index dd1102a246e4..1d8dc37d98a7 100644
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -139,73 +139,6 @@ struct page *pmd_page(pmd_t pmd)
return virt_to_page(pmd_page_vaddr(pmd));
}
-#ifdef CONFIG_PPC_64K_PAGES
-static pte_t *get_from_cache(struct mm_struct *mm)
-{
- void *pte_frag, *ret;
-
- spin_lock(&mm->page_table_lock);
- ret = mm->context.pte_frag;
- if (ret) {
- pte_frag = ret + PTE_FRAG_SIZE;
- /*
- * If we have taken up all the fragments mark PTE page NULL
- */
- if (((unsigned long)pte_frag & ~PAGE_MASK) == 0)
- pte_frag = NULL;
- mm->context.pte_frag = pte_frag;
- }
- spin_unlock(&mm->page_table_lock);
- return (pte_t *)ret;
-}
-
-static pte_t *__alloc_for_cache(struct mm_struct *mm, int kernel)
-{
- void *ret = NULL;
- struct page *page;
-
- if (!kernel) {
- page = alloc_page(PGALLOC_GFP | __GFP_ACCOUNT);
- if (!page)
- return NULL;
- if (!pgtable_page_ctor(page)) {
- __free_page(page);
- return NULL;
- }
- } else {
- page = alloc_page(PGALLOC_GFP);
- if (!page)
- return NULL;
- }
-
- ret = page_address(page);
- spin_lock(&mm->page_table_lock);
- /*
- * If we find pgtable_page set, we return
- * the allocated page with single fragement
- * count.
- */
- if (likely(!mm->context.pte_frag)) {
- set_page_count(page, PTE_FRAG_NR);
- mm->context.pte_frag = ret + PTE_FRAG_SIZE;
- }
- spin_unlock(&mm->page_table_lock);
-
- return (pte_t *)ret;
-}
-
-pte_t *pte_fragment_alloc(struct mm_struct *mm, unsigned long vmaddr, int kernel)
-{
- pte_t *pte;
-
- pte = get_from_cache(mm);
- if (pte)
- return pte;
-
- return __alloc_for_cache(mm, kernel);
-}
-#endif /* CONFIG_PPC_64K_PAGES */
-
void pte_fragment_free(unsigned long *table, int kernel)
{
struct page *page = virt_to_page(table);
diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype
index f860f0326c78..7172b04c91b5 100644
--- a/arch/powerpc/platforms/Kconfig.cputype
+++ b/arch/powerpc/platforms/Kconfig.cputype
@@ -337,6 +337,11 @@ config PPC_MM_SLICES
default y if PPC_8xx && HUGETLB_PAGE
default n
+config NEED_PTE_FRAG
+ bool
+ default y if PPC_BOOK3S_64 && PPC_64K_PAGES
+ default n
+
config PPC_HAVE_PMU_SUPPORT
bool
--
2.13.3
^ permalink raw reply related
* [PATCH BAD 17/17] powerpc/mm: Use pte_fragment_alloc() on 8xx
From: Christophe Leroy @ 2018-05-04 12:34 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
aneesh.kumar
Cc: linux-kernel, linuxppc-dev
In-Reply-To: <cover.1525435203.git.christophe.leroy@c-s.fr>
DO NOT APPLY THAT ONE, IT BUGS. But comments are welcome.
In 16k pages mode, the 8xx still need only 4k for the page table.
This patch makes use of the pte_fragment functions in order
to avoid wasting memory space
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
arch/powerpc/include/asm/mmu-8xx.h | 4 ++++
arch/powerpc/include/asm/nohash/32/pgalloc.h | 29 +++++++++++++++++++++++++++-
arch/powerpc/include/asm/nohash/32/pgtable.h | 5 ++++-
arch/powerpc/mm/mmu_context_nohash.c | 4 ++++
arch/powerpc/mm/pgtable.c | 10 +++++++++-
arch/powerpc/mm/pgtable_32.c | 12 ++++++++++++
arch/powerpc/platforms/Kconfig.cputype | 1 +
7 files changed, 62 insertions(+), 3 deletions(-)
diff --git a/arch/powerpc/include/asm/mmu-8xx.h b/arch/powerpc/include/asm/mmu-8xx.h
index 193f53116c7a..4f4cb754afd8 100644
--- a/arch/powerpc/include/asm/mmu-8xx.h
+++ b/arch/powerpc/include/asm/mmu-8xx.h
@@ -190,6 +190,10 @@ typedef struct {
struct slice_mask mask_8m;
# endif
#endif
+#ifdef CONFIG_NEED_PTE_FRAG
+ /* for 4K PTE fragment support */
+ void *pte_frag;
+#endif
} mm_context_t;
#define PHYS_IMMR_BASE (mfspr(SPRN_IMMR) & 0xfff80000)
diff --git a/arch/powerpc/include/asm/nohash/32/pgalloc.h b/arch/powerpc/include/asm/nohash/32/pgalloc.h
index 1c6461e7c6aa..1e3b8f580499 100644
--- a/arch/powerpc/include/asm/nohash/32/pgalloc.h
+++ b/arch/powerpc/include/asm/nohash/32/pgalloc.h
@@ -93,6 +93,32 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmdp,
((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel_g(pmd, address))? \
NULL: pte_offset_kernel(pmd, address))
+#ifdef CONFIG_NEED_PTE_FRAG
+extern pte_t *pte_fragment_alloc(struct mm_struct *, unsigned long, int);
+extern void pte_fragment_free(unsigned long *, int);
+
+static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
+ unsigned long address)
+{
+ return (pte_t *)pte_fragment_alloc(mm, address, 1);
+}
+
+static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
+ unsigned long address)
+{
+ return (pgtable_t)pte_fragment_alloc(mm, address, 0);
+}
+
+static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
+{
+ pte_fragment_free((unsigned long *)pte, 1);
+}
+
+static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
+{
+ pte_fragment_free((unsigned long *)ptepage, 0);
+}
+#else
extern pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr);
extern pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addr);
@@ -106,11 +132,12 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
pgtable_page_dtor(ptepage);
__free_page(ptepage);
}
+#endif
static inline void pgtable_free(void *table, unsigned index_size)
{
if (!index_size) {
- free_page((unsigned long)table);
+ pte_free_kernel(NULL, table);
} else {
BUG_ON(index_size > MAX_PGTABLE_INDEX_SIZE);
kmem_cache_free(PGT_CACHE(index_size), table);
diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h
index 3efd616bbc80..e2a22c8dc7f6 100644
--- a/arch/powerpc/include/asm/nohash/32/pgtable.h
+++ b/arch/powerpc/include/asm/nohash/32/pgtable.h
@@ -20,6 +20,9 @@ extern int icache_44x_need_flush;
#if defined(CONFIG_PPC_8xx) && defined(CONFIG_PPC_16K_PAGES)
#define PTE_INDEX_SIZE (PTE_SHIFT - 2)
+#define PTE_FRAG_NR 4
+#define PTE_FRAG_SIZE_SHIFT 12
+#define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT)
#else
#define PTE_INDEX_SIZE PTE_SHIFT
#endif
@@ -303,7 +306,7 @@ static inline void __ptep_set_access_flags(struct mm_struct *mm,
*/
#ifndef CONFIG_BOOKE
#define pmd_page_vaddr(pmd) \
- ((unsigned long) __va(pmd_val(pmd) & PAGE_MASK))
+ ((unsigned long) __va(pmd_val(pmd) & ~(PTE_TABLE_SIZE - 1)))
#define pmd_page(pmd) \
pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)
#else
diff --git a/arch/powerpc/mm/mmu_context_nohash.c b/arch/powerpc/mm/mmu_context_nohash.c
index e09228a9ad00..8b0ab33673e5 100644
--- a/arch/powerpc/mm/mmu_context_nohash.c
+++ b/arch/powerpc/mm/mmu_context_nohash.c
@@ -390,6 +390,9 @@ int init_new_context(struct task_struct *t, struct mm_struct *mm)
#endif
mm->context.id = MMU_NO_CONTEXT;
mm->context.active = 0;
+#ifdef CONFIG_NEED_PTE_FRAG
+ mm->context.pte_frag = NULL;
+#endif
return 0;
}
@@ -418,6 +421,7 @@ void destroy_context(struct mm_struct *mm)
nr_free_contexts++;
}
raw_spin_unlock_irqrestore(&context_lock, flags);
+ destroy_pagetable_page(mm);
}
#ifdef CONFIG_SMP
diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c
index 2d34755ed727..96cc5aa73331 100644
--- a/arch/powerpc/mm/pgtable.c
+++ b/arch/powerpc/mm/pgtable.c
@@ -23,6 +23,7 @@
#include <linux/kernel.h>
#include <linux/gfp.h>
+#include <linux/memblock.h>
#include <linux/mm.h>
#include <linux/percpu.h>
#include <linux/hardirq.h>
@@ -320,10 +321,17 @@ static pte_t *__alloc_for_cache(struct mm_struct *mm, int kernel)
return (pte_t *)ret;
}
-pte_t *pte_fragment_alloc(struct mm_struct *mm, unsigned long vmaddr, int kernel)
+__ref pte_t *pte_fragment_alloc(struct mm_struct *mm, unsigned long vmaddr, int kernel)
{
pte_t *pte;
+ if (kernel && !slab_is_available()) {
+ pte = __va(memblock_alloc(PTE_FRAG_SIZE, PTE_FRAG_SIZE));
+ if (pte)
+ memset(pte, 0, PTE_FRAG_SIZE);
+
+ return pte;
+ }
pte = get_from_cache(mm);
if (pte)
return pte;
diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c
index 3aa0c78db95d..5c8737cf2945 100644
--- a/arch/powerpc/mm/pgtable_32.c
+++ b/arch/powerpc/mm/pgtable_32.c
@@ -40,6 +40,17 @@
extern char etext[], _stext[], _sinittext[], _einittext[];
+#ifdef CONFIG_NEED_PTE_FRAG
+void pte_fragment_free(unsigned long *table, int kernel)
+{
+ struct page *page = virt_to_page(table);
+ if (put_page_testzero(page)) {
+ if (!kernel)
+ pgtable_page_dtor(page);
+ free_unref_page(page);
+ }
+}
+#else
__ref pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
{
pte_t *pte;
@@ -69,6 +80,7 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
}
return ptepage;
}
+#endif
#ifdef CONFIG_PPC_GUARDED_PAGE_IN_PMD
int __pte_alloc_kernel_g(pmd_t *pmd, unsigned long address)
diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype
index 7172b04c91b5..eff6210ad3c0 100644
--- a/arch/powerpc/platforms/Kconfig.cputype
+++ b/arch/powerpc/platforms/Kconfig.cputype
@@ -340,6 +340,7 @@ config PPC_MM_SLICES
config NEED_PTE_FRAG
bool
default y if PPC_BOOK3S_64 && PPC_64K_PAGES
+ default y if PPC_8xx && PPC_16K_PAGES
default n
config PPC_HAVE_PMU_SUPPORT
--
2.13.3
^ permalink raw reply related
* Re: [PATCH 1/3] powerpc/nohash: remove hash related code from nohash headers.
From: Christophe LEROY @ 2018-05-04 12:35 UTC (permalink / raw)
To: Michael Ellerman, kbuild test robot
Cc: kbuild-all, Benjamin Herrenschmidt, Paul Mackerras, linux-kernel,
linuxppc-dev
In-Reply-To: <878t8z8yah.fsf@concordia.ellerman.id.au>
Le 04/05/2018 à 13:17, Michael Ellerman a écrit :
> kbuild test robot <lkp@intel.com> writes:
>
>> Hi Christophe,
>>
>> Thank you for the patch! Yet something to improve:
>>
>> [auto build test ERROR on powerpc/next]
>> [also build test ERROR on v4.17-rc2 next-20180426]
>> [if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
>>
>> url: https://github.com/0day-ci/linux/commits/Christophe-Leroy/powerpc-nohash-remove-hash-related-code-from-nohash-headers/20180425-182026
>> base: https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
>> config: powerpc-ppc64e_defconfig (attached as .config)
>> compiler: powerpc64-linux-gnu-gcc (Debian 7.2.0-11) 7.2.0
>> reproduce:
>> wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
>> chmod +x ~/bin/make.cross
>> # save the attached .config to linux build tree
>> make.cross ARCH=powerpc
>>
>> All errors (new ones prefixed by >>):
>>
>> In file included from arch/powerpc/include/asm/nohash/pgtable.h:6:0,
>> from arch/powerpc/include/asm/pgtable.h:19,
>> from include/linux/memremap.h:8,
>> from include/linux/mm.h:27,
>> from include/linux/mman.h:5,
>> from arch/powerpc/kernel/asm-offsets.c:22:
>> arch/powerpc/include/asm/nohash/64/pgtable.h: In function '__ptep_test_and_clear_young':
>>>> arch/powerpc/include/asm/nohash/64/pgtable.h:214:6: error: implicit declaration of function 'pte_young'; did you mean 'pte_pud'? [-Werror=implicit-function-declaration]
>> if (pte_young(*ptep))
>> ^~~~~~~~~
>> pte_pud
>
> Urk.
>
> There's a circular dependency here.
>
> I fixed it with the patch below, which seems to be the least worst
> solution. Possibly we can clean things up further in future.
Oops, I just sent you a new version of the patch not using pte_young()
anymore. I'll let you decide what to do.
Christophe
>
> cheers
>
> diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h
> index 140f8e74b478..987a658b18e1 100644
> --- a/arch/powerpc/include/asm/nohash/32/pgtable.h
> +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h
> @@ -267,6 +267,11 @@ static inline void __ptep_set_access_flags(struct mm_struct *mm,
> pte_update(ptep, clr, set);
> }
>
> +static inline int pte_young(pte_t pte)
> +{
> + return pte_val(pte) & _PAGE_ACCESSED;
> +}
> +
> #define __HAVE_ARCH_PTE_SAME
> #define pte_same(A,B) ((pte_val(A) ^ pte_val(B)) == 0)
>
> diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h
> index e8de7cb4d3fb..6ac8381f4c7a 100644
> --- a/arch/powerpc/include/asm/nohash/64/pgtable.h
> +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h
> @@ -204,6 +204,11 @@ static inline unsigned long pte_update(struct mm_struct *mm,
> return old;
> }
>
> +static inline int pte_young(pte_t pte)
> +{
> + return pte_val(pte) & _PAGE_ACCESSED;
> +}
> +
> static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
> unsigned long addr, pte_t *ptep)
> {
> diff --git a/arch/powerpc/include/asm/nohash/pgtable.h b/arch/powerpc/include/asm/nohash/pgtable.h
> index 077472640b35..2160be2e4339 100644
> --- a/arch/powerpc/include/asm/nohash/pgtable.h
> +++ b/arch/powerpc/include/asm/nohash/pgtable.h
> @@ -17,7 +17,6 @@ static inline int pte_write(pte_t pte)
> }
> static inline int pte_read(pte_t pte) { return 1; }
> static inline int pte_dirty(pte_t pte) { return pte_val(pte) & _PAGE_DIRTY; }
> -static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; }
> static inline int pte_special(pte_t pte) { return pte_val(pte) & _PAGE_SPECIAL; }
> static inline int pte_none(pte_t pte) { return (pte_val(pte) & ~_PTE_NONE_MASK) == 0; }
> static inline pgprot_t pte_pgprot(pte_t pte) { return __pgprot(pte_val(pte) & PAGE_PROT_BITS); }
>
^ permalink raw reply
* [PATCH v3] On ppc64le we HAVE_RELIABLE_STACKTRACE
From: Torsten Duwe @ 2018-05-04 12:38 UTC (permalink / raw)
To: Josh Poimboeuf
Cc: Michael Ellerman, Jiri Kosina, linuxppc-dev, linux-kernel,
Nicholas Piggin, live-patching
In-Reply-To: <20180312153536.7avozx64ku4lvd3e@treble>
The "Power Architecture 64-Bit ELF V2 ABI" says in section 2.3.2.3:
[...] There are several rules that must be adhered to in order to ensure
reliable and consistent call chain backtracing:
* Before a function calls any other function, it shall establish its
own stack frame, whose size shall be a multiple of 16 bytes.
– In instances where a function’s prologue creates a stack frame, the
back-chain word of the stack frame shall be updated atomically with
the value of the stack pointer (r1) when a back chain is implemented.
(This must be supported as default by all ELF V2 ABI-compliant
environments.)
[...]
– The function shall save the link register that contains its return
address in the LR save doubleword of its caller’s stack frame before
calling another function.
To me this sounds like the equivalent of HAVE_RELIABLE_STACKTRACE.
This patch may be unneccessarily limited to ppc64le, but OTOH the only
user of this flag so far is livepatching, which is only implemented on
PPCs with 64-LE, a.k.a. ELF ABI v2.
Feel free to add other ppc variants, but so far only ppc64le got tested.
This change also implements save_stack_trace_tsk_reliable() for ppc64le
that checks for the above conditions, where possible.
Signed-off-by: Torsten Duwe <duwe@suse.de>
Signed-off-by: Nicolai Stange <nstange@suse.de>
---
v3:
* big bunch of fixes, credits go to Nicolai Stange:
- get the correct return address from the graph tracer,
should it be active.
IMO this should be moved into to generic code, but I'll
leave it like this for now, to get things going.
- bail out on a kretprobe
- also stop at an exception frame
- accomodate for the different stack layout of the idle task
- use an even more beautiful test: __kernel_text_address()
v2:
* implemented save_stack_trace_tsk_reliable(), with a bunch of sanity
checks. The test for a kernel code pointer is much nicer now, and
the exit condition is exact (when compared to last week's follow-up)
---
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index c32a181a7cbb..54f1daf4f9e5 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -220,6 +220,7 @@ config PPC
select HAVE_PERF_USER_STACK_DUMP
select HAVE_RCU_TABLE_FREE if SMP
select HAVE_REGS_AND_STACK_ACCESS_API
+ select HAVE_RELIABLE_STACKTRACE if PPC64 && CPU_LITTLE_ENDIAN
select HAVE_SYSCALL_TRACEPOINTS
select HAVE_VIRT_CPU_ACCOUNTING
select HAVE_IRQ_TIME_ACCOUNTING
diff --git a/arch/powerpc/kernel/stacktrace.c b/arch/powerpc/kernel/stacktrace.c
index d534ed901538..3d62ecb2587b 100644
--- a/arch/powerpc/kernel/stacktrace.c
+++ b/arch/powerpc/kernel/stacktrace.c
@@ -2,7 +2,7 @@
* Stack trace utility
*
* Copyright 2008 Christoph Hellwig, IBM Corp.
- *
+ * Copyright 2018 SUSE Linux GmbH
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
@@ -11,11 +11,16 @@
*/
#include <linux/export.h>
+#include <linux/kallsyms.h>
+#include <linux/module.h>
#include <linux/sched.h>
#include <linux/sched/debug.h>
+#include <linux/sched/task_stack.h>
#include <linux/stacktrace.h>
#include <asm/ptrace.h>
#include <asm/processor.h>
+#include <linux/ftrace.h>
+#include <asm/kprobes.h>
/*
* Save stack-backtrace addresses into a stack_trace buffer.
@@ -76,3 +81,114 @@ save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace)
save_context_stack(trace, regs->gpr[1], current, 0);
}
EXPORT_SYMBOL_GPL(save_stack_trace_regs);
+
+#ifdef CONFIG_HAVE_RELIABLE_STACKTRACE
+int
+save_stack_trace_tsk_reliable(struct task_struct *tsk,
+ struct stack_trace *trace)
+{
+ unsigned long sp;
+ unsigned long stack_page = (unsigned long)task_stack_page(tsk);
+ unsigned long stack_end;
+ int graph_idx = 0;
+
+ /* The last frame (unwinding first) may not yet have saved
+ * its LR onto the stack.
+ */
+ int firstframe = 1;
+
+ if (tsk == current)
+ sp = current_stack_pointer();
+ else
+ sp = tsk->thread.ksp;
+
+ stack_end = stack_page + THREAD_SIZE;
+ if (!is_idle_task(tsk)) {
+ /*
+ * For user tasks, this is the SP value loaded on
+ * kernel entry, see "PACAKSAVE(r13)" in _switch() and
+ * system_call_common()/EXCEPTION_PROLOG_COMMON().
+ *
+ * Likewise for non-swapper kernel threads,
+ * this also happens to be the top of the stack
+ * as setup by copy_thread().
+ *
+ * Note that stack backlinks are not properly setup by
+ * copy_thread() and thus, a forked task() will have
+ * an unreliable stack trace until it's been
+ * _switch()'ed to for the first time.
+ */
+ stack_end -= STACK_FRAME_OVERHEAD + sizeof(struct pt_regs);
+ } else {
+ /*
+ * idle tasks have a custom stack layout,
+ * c.f. cpu_idle_thread_init().
+ */
+ stack_end -= STACK_FRAME_OVERHEAD;
+ }
+
+ if (sp < stack_page + sizeof(struct thread_struct) ||
+ sp > stack_end - STACK_FRAME_MIN_SIZE) {
+ return 1;
+ }
+
+ for (;;) {
+ unsigned long *stack = (unsigned long *) sp;
+ unsigned long newsp, ip;
+
+ /* sanity check: ABI requires SP to be aligned 16 bytes. */
+ if (sp & 0xF)
+ return 1;
+
+ /* Mark stacktraces with exception frames as unreliable. */
+ if (sp <= stack_end - STACK_INT_FRAME_SIZE &&
+ stack[STACK_FRAME_MARKER] == STACK_FRAME_REGS_MARKER) {
+ return 1;
+ }
+
+ newsp = stack[0];
+ /* Stack grows downwards; unwinder may only go up. */
+ if (newsp <= sp)
+ return 1;
+
+ if (newsp != stack_end &&
+ newsp > stack_end - STACK_FRAME_MIN_SIZE) {
+ return 1; /* invalid backlink, too far up. */
+ }
+
+ /* Examine the saved LR: it must point into kernel code. */
+ ip = stack[STACK_FRAME_LR_SAVE];
+ if (!firstframe && !__kernel_text_address(ip))
+ return 1;
+ firstframe = 0;
+
+ /*
+ * FIXME: IMHO these tests do not belong in
+ * arch-dependent code, they are generic.
+ */
+ ip = ftrace_graph_ret_addr(tsk, &graph_idx, ip, NULL);
+
+ /*
+ * Mark stacktraces with kretprobed functions on them
+ * as unreliable.
+ */
+ if (ip == (unsigned long)kretprobe_trampoline)
+ return 1;
+
+ if (!trace->skip)
+ trace->entries[trace->nr_entries++] = ip;
+ else
+ trace->skip--;
+
+ if (newsp == stack_end)
+ break;
+
+ if (trace->nr_entries >= trace->max_entries)
+ return -E2BIG;
+
+ sp = newsp;
+ }
+ return 0;
+}
+EXPORT_SYMBOL_GPL(save_stack_trace_tsk_reliable);
+#endif /* CONFIG_HAVE_RELIABLE_STACKTRACE */
^ permalink raw reply related
* [PATCH 1/2] powerpc/trace elfv1: Update syscall name matching logic
From: Naveen N. Rao @ 2018-05-04 13:14 UTC (permalink / raw)
To: Michael Ellerman; +Cc: linuxppc-dev, Dominik Brodowski
On powerpc64 ABIv1, we are enabling syscall tracing for only ~20
syscalls. This is due to commit e145242ea0df6 ("syscalls/core,
syscalls/x86: Clean up syscall stub naming convention") which has
changed the syscall entry wrapper prefix from "SyS" to "__se_sys".
Update the logic for ABIv1 to not just skip the initial dot, but also
the "__se_sys" prefix.
Fixes: commit e145242ea0df6 ("syscalls/core, syscalls/x86: Clean up syscall stub naming convention")
Reported-by: Michael Ellerman <mpe@ellerman.id.au>
Signed-off-by: Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/ftrace.h | 10 +++-------
1 file changed, 3 insertions(+), 7 deletions(-)
diff --git a/arch/powerpc/include/asm/ftrace.h b/arch/powerpc/include/asm/ftrace.h
index f7a23c2dce74..731cb4314a42 100644
--- a/arch/powerpc/include/asm/ftrace.h
+++ b/arch/powerpc/include/asm/ftrace.h
@@ -71,13 +71,9 @@ struct dyn_arch_ftrace {
#define ARCH_HAS_SYSCALL_MATCH_SYM_NAME
static inline bool arch_syscall_match_sym_name(const char *sym, const char *name)
{
- /*
- * Compare the symbol name with the system call name. Skip the .sys or .SyS
- * prefix from the symbol name and the sys prefix from the system call name and
- * just match the rest. This is only needed on ppc64 since symbol names on
- * 32bit do not start with a period so the generic function will work.
- */
- return !strcmp(sym + 4, name + 3);
+ /* We need to skip past the initial dot, and the __se_sys alias */
+ return !strcmp(sym + 1, name) ||
+ (!strncmp(sym, ".__se_sys", 9) && !strcmp(sym + 6, name));
}
#endif
#endif /* CONFIG_FTRACE_SYSCALLS && !__ASSEMBLY__ */
--
2.17.0
^ permalink raw reply related
* [PATCH 2/2] powerpc/trace: Update syscall name matching logic to account for ppc_ prefix
From: Naveen N. Rao @ 2018-05-04 13:14 UTC (permalink / raw)
To: Michael Ellerman; +Cc: linuxppc-dev, Dominik Brodowski
In-Reply-To: <c50df60c9ca8309df56e0eb9a00072c154897764.1525439237.git.naveen.n.rao@linux.vnet.ibm.com>
Some syscall entry functions on powerpc are prefixed with
ppc_/ppc32_/ppc64_ rather than the usual sys_/__se_sys prefix. fork(),
clone(), swapcontext() are some examples of syscalls with such entry
points. We need to match against these names when initializing ftrace
syscall tracing.
Signed-off-by: Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
---
Tracing fork() and clone() was working so far since those had "ppc_"
prefix and we were jumping past the initial 3 characters to account for
"SyS_" prefix. We were not enabling tracing for ppc32_/ppc64_ prefixed
syscall entry points so far.
- Naveen
arch/powerpc/include/asm/ftrace.h | 21 +++++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/arch/powerpc/include/asm/ftrace.h b/arch/powerpc/include/asm/ftrace.h
index 731cb4314a42..331b17b2e46e 100644
--- a/arch/powerpc/include/asm/ftrace.h
+++ b/arch/powerpc/include/asm/ftrace.h
@@ -67,13 +67,30 @@ struct dyn_arch_ftrace {
#endif
#if defined(CONFIG_FTRACE_SYSCALLS) && !defined(__ASSEMBLY__)
-#ifdef PPC64_ELF_ABI_v1
+/*
+ * Some syscall entry functions on powerpc start with "ppc_" (fork and clone,
+ * for instance) or ppc32_/ppc64_. We should also match the sys_ variant with
+ * those.
+ */
#define ARCH_HAS_SYSCALL_MATCH_SYM_NAME
+#ifdef PPC64_ELF_ABI_v1
static inline bool arch_syscall_match_sym_name(const char *sym, const char *name)
{
/* We need to skip past the initial dot, and the __se_sys alias */
return !strcmp(sym + 1, name) ||
- (!strncmp(sym, ".__se_sys", 9) && !strcmp(sym + 6, name));
+ (!strncmp(sym, ".__se_sys", 9) && !strcmp(sym + 6, name)) ||
+ (!strncmp(sym, ".ppc_", 5) && !strcmp(sym + 5, name + 4)) ||
+ (!strncmp(sym, ".ppc32_", 7) && !strcmp(sym + 7, name + 4)) ||
+ (!strncmp(sym, ".ppc64_", 7) && !strcmp(sym + 7, name + 4));
+}
+#else
+static inline bool arch_syscall_match_sym_name(const char *sym, const char *name)
+{
+ return !strcmp(sym, name) ||
+ (!strncmp(sym, "__se_sys", 8) && !strcmp(sym + 5, name)) ||
+ (!strncmp(sym, "ppc_", 4) && !strcmp(sym + 4, name + 4)) ||
+ (!strncmp(sym, "ppc32_", 6) && !strcmp(sym + 6, name + 4)) ||
+ (!strncmp(sym, "ppc64_", 6) && !strcmp(sym + 6, name + 4));
}
#endif
#endif /* CONFIG_FTRACE_SYSCALLS && !__ASSEMBLY__ */
--
2.17.0
^ permalink raw reply related
* Re: [PATCH 12/17] powerpc/8xx: Remove PTE_ATOMIC_UPDATES
From: Joakim Tjernlund @ 2018-05-04 13:16 UTC (permalink / raw)
To: christophe.leroy@c-s.fr, paulus@samba.org, mpe@ellerman.id.au,
benh@kernel.crashing.org, aneesh.kumar@linux.vnet.ibm.com
Cc: linuxppc-dev@lists.ozlabs.org, linux-kernel@vger.kernel.org
In-Reply-To: <3ece8e0d69f3016111cdb9d643dfc4a158849ae0.1525435203.git.christophe.leroy@c-s.fr>
T24gRnJpLCAyMDE4LTA1LTA0IGF0IDE0OjM0ICswMjAwLCBDaHJpc3RvcGhlIExlcm95IHdyb3Rl
Og0KPiBDQVVUSU9OOiBUaGlzIGVtYWlsIG9yaWdpbmF0ZWQgZnJvbSBvdXRzaWRlIG9mIHRoZSBv
cmdhbml6YXRpb24uIERvIG5vdCBjbGljayBsaW5rcyBvciBvcGVuIGF0dGFjaG1lbnRzIHVubGVz
cyB5b3UgcmVjb2duaXplIHRoZSBzZW5kZXIgYW5kIGtub3cgdGhlIGNvbnRlbnQgaXMgc2FmZS4N
Cj4gDQo+IA0KPiBjb21taXQgMWJjNTRjMDMxMTdiOSAoInBvd2VycGM6IHJld29yayA0eHggUFRF
IGFjY2VzcyBhbmQgVExCIG1pc3MiKQ0KPiBpbnRyb2R1Y2VkIG5vbiBhdG9taWMgUFRFIHVwZGF0
ZXMgYW5kIHN0YXJ0ZWQgdGhlIHdvcmsgb2YgcmVtb3ZpbmcNCj4gUFRFIHVwZGF0ZXMgaW4gVExC
IG1pc3MgaGFuZGxlcnMsIGJ1dCBrZXB0IFBURV9BVE9NSUNfVVBEQVRFUyBmb3IgdGhlDQo+IDh4
eCB3aXRoIHRoZSBmb2xsb3dpbmcgY29tbWVudDoNCj4gLyogVW50aWwgbXkgcmV3b3JrIGlzIGZp
bmlzaGVkLCA4eHggc3RpbGwgbmVlZHMgYXRvbWljIFBURSB1cGRhdGVzICovDQo+IA0KPiBjb21t
aXQgZmUxMWRjM2Y5NjI4ZSAoInBvd2VycGMvOHh4OiBVcGRhdGUgVExCIGFzbSBzbyBpdCBiZWhh
dmVzIGFzIGxpbnV4DQo+IG1tIGV4cGVjdHMiKSByZW1vdmVkIGFsbCBQVEUgdXBkYXRlcyBkb25l
IGluIFRMQiBtaXNzIGhhbmRsZXJzDQoNCklzIHRoYXQgbXkgNyB5ZWFyIG9sZCBjb21taXQgPw0K
DQo+IA0KPiBUaGVyZWZvcmUsIGF0b21pYyBQVEUgdXBkYXRlcyBhcmUgbm90IG5lZWRlZCBhbnlt
b3JlIGZvciB0aGUgOHh4DQoNCkFib3V0IHRpbWUgcmVtb3ZpbmcgYXRvbWljIHVwZGF0ZXMgdGhl
biA6KQ0KDQo+IA0KPiBTaWduZWQtb2ZmLWJ5OiBDaHJpc3RvcGhlIExlcm95IDxjaHJpc3RvcGhl
Lmxlcm95QGMtcy5mcj4NCj4gDQo=
^ permalink raw reply
* Re: [PATCH 1/6] powerpc/syscalls: Switch trivial cases to SYSCALL_DEFINE
From: Naveen N. Rao @ 2018-05-04 15:03 UTC (permalink / raw)
To: linuxppc-dev, Michael Ellerman; +Cc: linux-kernel, viro
In-Reply-To: <20180502132051.28861-1-mpe@ellerman.id.au>
Michael Ellerman wrote:
> From: Al Viro <viro@zeniv.linux.org.uk>
>=20
> Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
> ---
> arch/powerpc/kernel/pci_32.c | 6 +++---
> arch/powerpc/kernel/pci_64.c | 4 ++--
> arch/powerpc/mm/subpage-prot.c | 4 +++-
> arch/powerpc/platforms/cell/spu_syscalls.c | 3 ++-
> 4 files changed, 10 insertions(+), 7 deletions(-)
>=20
I suppose we can also do this for switch_endian?
diff --git a/arch/powerpc/kernel/syscalls.c b/arch/powerpc/kernel/syscalls.=
c
index 466216506eb2..290265f2700c 100644
--- a/arch/powerpc/kernel/syscalls.c
+++ b/arch/powerpc/kernel/syscalls.c
@@ -123,7 +123,7 @@ long ppc_fadvise64_64(int fd, int advice, u32 offset_hi=
gh, u32 offset_low,
(u64)len_high << 32 | len_low, advice);
}
=20
-long sys_switch_endian(void)
+SYSCALL_DEFINE0(switch_endian)
{
struct thread_info *ti;
- Naveen
=
^ permalink raw reply related
* Re: [PATCH v10 24/25] x86/mm: add speculative pagefault handling
From: Punit Agrawal @ 2018-05-04 15:55 UTC (permalink / raw)
To: Laurent Dufour
Cc: Punit Agrawal, akpm, mhocko, peterz, kirill, ak, dave, jack,
Matthew Wilcox, benh, mpe, paulus, Thomas Gleixner, Ingo Molnar,
hpa, Will Deacon, Sergey Senozhatsky, Andrea Arcangeli,
Alexei Starovoitov, kemi.wang, sergey.senozhatsky.work,
Daniel Jordan, David Rientjes, Jerome Glisse, Ganesh Mahendran,
linux-kernel, linux-mm, haren, khandual, npiggin, bsingharora,
paulmck, Tim Chen <tim.c.ch
In-Reply-To: <fb143123-d54e-b08d-1bd8-07767c86c7d0@linux.vnet.ibm.com>
Laurent Dufour <ldufour@linux.vnet.ibm.com> writes:
> On 30/04/2018 20:43, Punit Agrawal wrote:
>> Hi Laurent,
>>
>> I am looking to add support for speculative page fault handling to
>> arm64 (effectively porting this patch) and had a few questions.
>> Apologies if I've missed an obvious explanation for my queries. I'm
>> jumping in bit late to the discussion.
>
> Hi Punit,
>
> Thanks for giving this series a review.
> I don't have arm64 hardware to play with, but I'll be happy to add arm64
> patches to my series and to try to maintain them.
I'll be happy to try them on arm64 platforms I have access to and
provide feedback.
>
>>
>> On Tue, Apr 17, 2018 at 3:33 PM, Laurent Dufour
>> <ldufour@linux.vnet.ibm.com> wrote:
>>> From: Peter Zijlstra <peterz@infradead.org>
>>>
[...]
>>>
>>> - vma = find_vma(mm, address);
>>> + if (!vma || !can_reuse_spf_vma(vma, address))
>>> + vma = find_vma(mm, address);
>>
>> Is there a measurable benefit from reusing the vma?
>>
>> Dropping the vma reference unconditionally after speculative page
>> fault handling gets rid of the implicit state when "vma != NULL"
>> (increased ref-count). I found it a bit confusing to follow.
>
> I do agree, this is quite confusing. My initial goal was to be able to reuse
> the VMA in the case a protection key error was detected, but it's not really
> necessary on x86 since we know at the beginning of the fault operation that
> protection key are in the loop. This is not the case on ppc64 but I couldn't
> find a way to easily rely on the speculatively fetched VMA neither, so for
> protection keys, this didn't help.
>
> Regarding the measurable benefit of reusing the fetched vma, I did further
> tests using will-it-scale/page_fault2_threads test, and I'm no more really
> convince that this worth the added complexity. I think I'll drop the patch "mm:
> speculative page fault handler return VMA" of the series, and thus remove the
> call to can_reuse_spf_vma().
Makes sense. Thanks for giving this a go.
Punit
[...]
^ permalink raw reply
* [PATCH 00/11] assortment of timer, watchdog, interrupt
From: Nicholas Piggin @ 2018-05-04 17:19 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin, Benjamin Herrenschmidt
These are a bunch of small things I've built up from looking
through code trying to track down some rare irq latency issues.
None of them actually fix any long irq latencies, but they
hopefully make the code a bit neater, get rid of some small
glitches, increase watchdog coverage etc.
Ben spotted a bug with the first patch last time I posted,
that's fixed.
Thanks,
Nick
Nicholas Piggin (11):
powerpc/64: irq_work avoid interrupt when called with hardware irqs
enabled
powerpc/pseries: put cede MSR[EE] check under IRQ_SOFT_MASK_DEBUG
powerpc/64s: make PACA_IRQ_HARD_DIS track MSR[EE] closely
powerpc/64s: micro-optimise __hard_irq_enable() for mtmsrd L=1 support
powerpc/64: remove start_tb and accum_tb from thread_struct
powerpc/pseries: lparcfg calculate PURR on demand
powerpc: generic clockevents broadcast receiver call
tick_receive_broadcast
powerpc: allow soft-NMI watchdog to cover timer interrupts with large
decrementers
powerpc: move timer broadcast code under GENERIC_CLOCKEVENTS_BROADCAST
ifdef
powerpc: move a stray NMI IPI case under NMI_IPI ifdef
powerpc/time: account broadcast timer event interrupts separately
arch/powerpc/include/asm/hardirq.h | 1 +
arch/powerpc/include/asm/hw_irq.h | 15 ++-
arch/powerpc/include/asm/plpar_wrappers.h | 8 +-
arch/powerpc/include/asm/processor.h | 4 -
arch/powerpc/include/asm/time.h | 9 --
arch/powerpc/kernel/entry_64.S | 8 ++
arch/powerpc/kernel/exceptions-64s.S | 5 +-
arch/powerpc/kernel/irq.c | 34 +++--
arch/powerpc/kernel/process.c | 18 ---
arch/powerpc/kernel/smp.c | 14 ++-
arch/powerpc/kernel/time.c | 143 +++++++++++++---------
arch/powerpc/platforms/pseries/lparcfg.c | 18 +--
12 files changed, 155 insertions(+), 122 deletions(-)
--
2.17.0
^ permalink raw reply
* [PATCH 01/11] powerpc/64: irq_work avoid interrupt when called with hardware irqs enabled
From: Nicholas Piggin @ 2018-05-04 17:19 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin, Benjamin Herrenschmidt
In-Reply-To: <20180504171935.25410-1-npiggin@gmail.com>
irq_work_raise should not cause a decrementer exception unless it is
called from NMI context. Doing so often just results in an immediate
masked decrementer interrupt:
<...>-550 90d... 4us : update_curr_rt <-dequeue_task_rt
<...>-550 90d... 5us : dbs_update_util_handler <-update_curr_rt
<...>-550 90d... 6us : arch_irq_work_raise <-irq_work_queue
<...>-550 90d... 7us : soft_nmi_interrupt <-soft_nmi_common
<...>-550 90d... 7us : printk_nmi_enter <-soft_nmi_interrupt
<...>-550 90d.Z. 8us : rcu_nmi_enter <-soft_nmi_interrupt
<...>-550 90d.Z. 9us : rcu_nmi_exit <-soft_nmi_interrupt
<...>-550 90d... 9us : printk_nmi_exit <-soft_nmi_interrupt
<...>-550 90d... 10us : cpuacct_charge <-update_curr_rt
The soft_nmi_interrupt here is the call into the watchdog, due to the
decrementer interrupt firing with irqs soft-disabled. This is
harmless, but sub-optimal.
When it's not called from NMI context or with interrupts enabled, mark
the decrementer pending in the irq_happened mask directly, rather than
having the masked decrementer interupt handler do it. This will be
replayed at the next local_irq_enable. See the comment for details.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/kernel/time.c | 33 +++++++++++++++++++++++++++++++--
1 file changed, 31 insertions(+), 2 deletions(-)
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index 360e71d455cc..e7e8611e8863 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -513,6 +513,35 @@ static inline void clear_irq_work_pending(void)
"i" (offsetof(struct paca_struct, irq_work_pending)));
}
+void arch_irq_work_raise(void)
+{
+ preempt_disable();
+ set_irq_work_pending_flag();
+ /*
+ * Non-nmi code running with interrupts disabled will replay
+ * irq_happened before it re-enables interrupts, so setthe
+ * decrementer there instead of causing a hardware exception
+ * which would immediately hit the masked interrupt handler
+ * and have the net effect of setting the decrementer in
+ * irq_happened.
+ *
+ * NMI interrupts can not check this when they return, so the
+ * decrementer hardware exception is raised, which will fire
+ * when interrupts are next enabled.
+ *
+ * BookE does not support this yet, it must audit all NMI
+ * interrupt handlers to ensure they call nmi_enter() so this
+ * check would be correct.
+ */
+ if (IS_ENABLED(CONFIG_BOOKE) || !irqs_disabled() || in_nmi()) {
+ set_dec(1);
+ } else {
+ hard_irq_disable();
+ local_paca->irq_happened |= PACA_IRQ_DEC;
+ }
+ preempt_enable();
+}
+
#else /* 32-bit */
DEFINE_PER_CPU(u8, irq_work_pending);
@@ -521,8 +550,6 @@ DEFINE_PER_CPU(u8, irq_work_pending);
#define test_irq_work_pending() __this_cpu_read(irq_work_pending)
#define clear_irq_work_pending() __this_cpu_write(irq_work_pending, 0)
-#endif /* 32 vs 64 bit */
-
void arch_irq_work_raise(void)
{
preempt_disable();
@@ -531,6 +558,8 @@ void arch_irq_work_raise(void)
preempt_enable();
}
+#endif /* 32 vs 64 bit */
+
#else /* CONFIG_IRQ_WORK */
#define test_irq_work_pending() 0
--
2.17.0
^ permalink raw reply related
* [PATCH 02/11] powerpc/pseries: put cede MSR[EE] check under IRQ_SOFT_MASK_DEBUG
From: Nicholas Piggin @ 2018-05-04 17:19 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin, Benjamin Herrenschmidt
In-Reply-To: <20180504171935.25410-1-npiggin@gmail.com>
This check does not catch IRQ soft mask bugs, but this option is
slightly more suitable than TRACE_IRQFLAGS.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/plpar_wrappers.h | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/arch/powerpc/include/asm/plpar_wrappers.h b/arch/powerpc/include/asm/plpar_wrappers.h
index 96c1a46acbd0..cff5a411e595 100644
--- a/arch/powerpc/include/asm/plpar_wrappers.h
+++ b/arch/powerpc/include/asm/plpar_wrappers.h
@@ -39,10 +39,10 @@ static inline long extended_cede_processor(unsigned long latency_hint)
set_cede_latency_hint(latency_hint);
rc = cede_processor();
-#ifdef CONFIG_TRACE_IRQFLAGS
- /* Ensure that H_CEDE returns with IRQs on */
- if (WARN_ON(!(mfmsr() & MSR_EE)))
- __hard_irq_enable();
+#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
+ /* Ensure that H_CEDE returns with IRQs on */
+ if (WARN_ON(!(mfmsr() & MSR_EE)))
+ __hard_irq_enable();
#endif
set_cede_latency_hint(old_latency_hint);
--
2.17.0
^ permalink raw reply related
* [PATCH 03/11] powerpc/64s: make PACA_IRQ_HARD_DIS track MSR[EE] closely
From: Nicholas Piggin @ 2018-05-04 17:19 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin, Benjamin Herrenschmidt
In-Reply-To: <20180504171935.25410-1-npiggin@gmail.com>
When the masked interrupt handler clears MSR[EE] for an interrupt in
the PACA_IRQ_MUST_HARD_MASK set, it does not set PACA_IRQ_HARD_DIS.
This makes them get out of synch.
With that taken into account, it's only low level irq manipulation
(and interrupt entry before reconcile) where they can be out of synch.
This makes the code less surprising.
It also allows the IRQ replay code to rely on the IRQ_HARD_DIS value
and not have to mtmsrd again in this case (e.g., for an external
interrupt that has been masked). The bigger benefit might just be
that there is not such an element of surprise in these two bits of
state.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/hw_irq.h | 10 ++++++----
arch/powerpc/kernel/entry_64.S | 8 ++++++++
arch/powerpc/kernel/exceptions-64s.S | 5 ++++-
arch/powerpc/kernel/irq.c | 28 +++++++++++++++++++---------
4 files changed, 37 insertions(+), 14 deletions(-)
diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h
index 855e17d158b1..8004d7887ff6 100644
--- a/arch/powerpc/include/asm/hw_irq.h
+++ b/arch/powerpc/include/asm/hw_irq.h
@@ -248,14 +248,16 @@ static inline bool lazy_irq_pending(void)
/*
* This is called by asynchronous interrupts to conditionally
- * re-enable hard interrupts when soft-disabled after having
- * cleared the source of the interrupt
+ * re-enable hard interrupts after having cleared the source
+ * of the interrupt. They are kept disabled if there is a different
+ * soft-masked interrupt pending that requires hard masking.
*/
static inline void may_hard_irq_enable(void)
{
- get_paca()->irq_happened &= ~PACA_IRQ_HARD_DIS;
- if (!(get_paca()->irq_happened & PACA_IRQ_MUST_HARD_MASK))
+ if (!(get_paca()->irq_happened & PACA_IRQ_MUST_HARD_MASK)) {
+ get_paca()->irq_happened &= ~PACA_IRQ_HARD_DIS;
__hard_irq_enable();
+ }
}
static inline bool arch_irq_disabled_regs(struct pt_regs *regs)
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index 51695608c68b..db4df061c33a 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -973,6 +973,14 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
or r4,r4,r3
std r4,_TRAP(r1)
+ /*
+ * PACA_IRQ_HARD_DIS won't always be set here, so set it now
+ * to reconcile the IRQ state. Tracing is already accounted for.
+ */
+ ld r4,PACAIRQHAPPENED(r13)
+ ori r4,r4,PACA_IRQ_HARD_DIS
+ stb r4,PACAIRQHAPPENED(r13)
+
/*
* Then find the right handler and call it. Interrupts are
* still soft-disabled and we keep them that way.
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index ae6a849db60b..69172dd41b11 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -1498,7 +1498,10 @@ masked_##_H##interrupt: \
mfspr r10,SPRN_##_H##SRR1; \
xori r10,r10,MSR_EE; /* clear MSR_EE */ \
mtspr SPRN_##_H##SRR1,r10; \
-2: mtcrf 0x80,r9; \
+ ori r11,r11,PACA_IRQ_HARD_DIS; \
+ stb r11,PACAIRQHAPPENED(r13); \
+2: /* done */ \
+ mtcrf 0x80,r9; \
ld r9,PACA_EXGEN+EX_R9(r13); \
ld r10,PACA_EXGEN+EX_R10(r13); \
ld r11,PACA_EXGEN+EX_R11(r13); \
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index 061aa0f47bb1..6569b5ffff93 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -145,8 +145,20 @@ notrace unsigned int __check_irq_replay(void)
trace_hardirqs_on();
trace_hardirqs_off();
+ /*
+ * We are always hard disabled here, but PACA_IRQ_HARD_DIS may
+ * not be set, which means interrupts have only just been hard
+ * disabled as part of the local_irq_restore or interrupt return
+ * code. In that case, skip the decrementr check becaus it's
+ * expensive to read the TB.
+ *
+ * HARD_DIS then gets cleared here, but it's reconciled later.
+ * Either local_irq_disable will replay the interrupt and that
+ * will reconcile state like other hard interrupts. Or interrupt
+ * retur will replay the interrupt and in that case it sets
+ * PACA_IRQ_HARD_DIS by hand (see comments in entry_64.S).
+ */
if (happened & PACA_IRQ_HARD_DIS) {
- /* Clear bit 0 which we wouldn't clear otherwise */
local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
/*
@@ -256,16 +268,14 @@ notrace void arch_local_irq_restore(unsigned long mask)
* __check_irq_replay(). We also need to soft-disable
* again to avoid warnings in there due to the use of
* per-cpu variables.
- *
- * We know that if the value in irq_happened is exactly 0x01
- * then we are already hard disabled (there are other less
- * common cases that we'll ignore for now), so we skip the
- * (expensive) mtmsrd.
*/
- if (unlikely(irq_happened != PACA_IRQ_HARD_DIS))
+ if (!(irq_happened & PACA_IRQ_HARD_DIS)) {
+#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
+ WARN_ON(!(mfmsr() & MSR_EE));
+#endif
__hard_irq_disable();
#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
- else {
+ } else {
/*
* We should already be hard disabled here. We had bugs
* where that wasn't the case so let's dbl check it and
@@ -274,8 +284,8 @@ notrace void arch_local_irq_restore(unsigned long mask)
*/
if (WARN_ON(mfmsr() & MSR_EE))
__hard_irq_disable();
- }
#endif
+ }
irq_soft_mask_set(IRQS_ALL_DISABLED);
trace_hardirqs_off();
--
2.17.0
^ permalink raw reply related
* [PATCH 04/11] powerpc/64s: micro-optimise __hard_irq_enable() for mtmsrd L=1 support
From: Nicholas Piggin @ 2018-05-04 17:19 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin, Benjamin Herrenschmidt
In-Reply-To: <20180504171935.25410-1-npiggin@gmail.com>
Book3S minimum supported ISA version now requires mtmsrd L=1. This
instruction does not require bits other than RI and EE to be supplied,
so __hard_irq_enable() and __hard_irq_disable() does not have to read
the kernel_msr from paca.
Interrupt entry code already relies on L=1 support.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/hw_irq.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h
index 8004d7887ff6..fbc2d83808aa 100644
--- a/arch/powerpc/include/asm/hw_irq.h
+++ b/arch/powerpc/include/asm/hw_irq.h
@@ -228,8 +228,8 @@ static inline bool arch_irqs_disabled(void)
#define __hard_irq_enable() asm volatile("wrteei 1" : : : "memory")
#define __hard_irq_disable() asm volatile("wrteei 0" : : : "memory")
#else
-#define __hard_irq_enable() __mtmsrd(local_paca->kernel_msr | MSR_EE, 1)
-#define __hard_irq_disable() __mtmsrd(local_paca->kernel_msr, 1)
+#define __hard_irq_enable() __mtmsrd(MSR_EE|MSR_RI, 1)
+#define __hard_irq_disable() __mtmsrd(MSR_RI, 1)
#endif
#define hard_irq_disable() do { \
--
2.17.0
^ permalink raw reply related
* [PATCH 05/11] powerpc/64: remove start_tb and accum_tb from thread_struct
From: Nicholas Piggin @ 2018-05-04 17:19 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin, Benjamin Herrenschmidt
In-Reply-To: <20180504171935.25410-1-npiggin@gmail.com>
These fields are only written to.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/processor.h | 4 ----
arch/powerpc/kernel/process.c | 6 +-----
2 files changed, 1 insertion(+), 9 deletions(-)
diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index c4b36a494a63..eff269adfa71 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -264,10 +264,6 @@ struct thread_struct {
struct thread_fp_state *fp_save_area;
int fpexc_mode; /* floating-point exception mode */
unsigned int align_ctl; /* alignment handling control */
-#ifdef CONFIG_PPC64
- unsigned long start_tb; /* Start purr when proc switched in */
- unsigned long accum_tb; /* Total accumulated purr for process */
-#endif
#ifdef CONFIG_HAVE_HW_BREAKPOINT
struct perf_event *ptrace_bps[HBP_NUM];
/*
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 1237f13fed51..ff7344d996e3 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -1187,11 +1187,7 @@ struct task_struct *__switch_to(struct task_struct *prev,
*/
if (firmware_has_feature(FW_FEATURE_SPLPAR)) {
struct cpu_usage *cu = this_cpu_ptr(&cpu_usage_array);
- long unsigned start_tb, current_tb;
- start_tb = old_thread->start_tb;
- cu->current_tb = current_tb = mfspr(SPRN_PURR);
- old_thread->accum_tb += (current_tb - start_tb);
- new_thread->start_tb = current_tb;
+ cu->current_tb = mfspr(SPRN_PURR);
}
#endif /* CONFIG_PPC64 */
--
2.17.0
^ permalink raw reply related
* [PATCH 06/11] powerpc/pseries: lparcfg calculate PURR on demand
From: Nicholas Piggin @ 2018-05-04 17:19 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin, Benjamin Herrenschmidt
In-Reply-To: <20180504171935.25410-1-npiggin@gmail.com>
For SPLPAR, lparcfg provides a sum of PURR registers for all CPUs.
Currently this is done by reading PURR in context switch and timer
interrupt, and storing that into a per-CPU variable. These are summed
to provide the value.
This does not work with all timer schemes (e.g., NO_HZ_FULL), and it
is sub-optimal for performance because it reads the PURR register on
every context switch, although that's been difficult to distinguish
from noise in the contxt_switch microbenchmark.
This patch implements the sum by calling a function on each CPU, to
read and add PURR values of each CPU.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/time.h | 8 --------
arch/powerpc/kernel/process.c | 14 --------------
arch/powerpc/kernel/time.c | 8 --------
arch/powerpc/platforms/pseries/lparcfg.c | 18 ++++++++++--------
4 files changed, 10 insertions(+), 38 deletions(-)
diff --git a/arch/powerpc/include/asm/time.h b/arch/powerpc/include/asm/time.h
index db546c034905..c965c79765c4 100644
--- a/arch/powerpc/include/asm/time.h
+++ b/arch/powerpc/include/asm/time.h
@@ -196,14 +196,6 @@ extern u64 mulhdu(u64, u64);
extern void div128_by_32(u64 dividend_high, u64 dividend_low,
unsigned divisor, struct div_result *dr);
-/* Used to store Processor Utilization register (purr) values */
-
-struct cpu_usage {
- u64 current_tb; /* Holds the current purr register values */
-};
-
-DECLARE_PER_CPU(struct cpu_usage, cpu_usage_array);
-
extern void secondary_cpu_time_init(void);
extern void __init time_init(void);
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index ff7344d996e3..e6ff36923d84 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -845,10 +845,6 @@ bool ppc_breakpoint_available(void)
}
EXPORT_SYMBOL_GPL(ppc_breakpoint_available);
-#ifdef CONFIG_PPC64
-DEFINE_PER_CPU(struct cpu_usage, cpu_usage_array);
-#endif
-
static inline bool hw_brk_match(struct arch_hw_breakpoint *a,
struct arch_hw_breakpoint *b)
{
@@ -1181,16 +1177,6 @@ struct task_struct *__switch_to(struct task_struct *prev,
WARN_ON(!irqs_disabled());
-#ifdef CONFIG_PPC64
- /*
- * Collect processor utilization data per process
- */
- if (firmware_has_feature(FW_FEATURE_SPLPAR)) {
- struct cpu_usage *cu = this_cpu_ptr(&cpu_usage_array);
- cu->current_tb = mfspr(SPRN_PURR);
- }
-#endif /* CONFIG_PPC64 */
-
#ifdef CONFIG_PPC_BOOK3S_64
batch = this_cpu_ptr(&ppc64_tlb_batch);
if (batch->active) {
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index e7e8611e8863..1fe6a24357e7 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -597,14 +597,6 @@ static void __timer_interrupt(void)
__this_cpu_inc(irq_stat.timer_irqs_others);
}
-#ifdef CONFIG_PPC64
- /* collect purr register values often, for accurate calculations */
- if (firmware_has_feature(FW_FEATURE_SPLPAR)) {
- struct cpu_usage *cu = this_cpu_ptr(&cpu_usage_array);
- cu->current_tb = mfspr(SPRN_PURR);
- }
-#endif
-
trace_timer_interrupt_exit(regs);
}
diff --git a/arch/powerpc/platforms/pseries/lparcfg.c b/arch/powerpc/platforms/pseries/lparcfg.c
index c508c938dc71..7c872dc01bdb 100644
--- a/arch/powerpc/platforms/pseries/lparcfg.c
+++ b/arch/powerpc/platforms/pseries/lparcfg.c
@@ -52,18 +52,20 @@
* Track sum of all purrs across all processors. This is used to further
* calculate usage values by different applications
*/
+static void cpu_get_purr(void *arg)
+{
+ atomic64_t *sum = arg;
+
+ atomic64_add(mfspr(SPRN_PURR), sum);
+}
+
static unsigned long get_purr(void)
{
- unsigned long sum_purr = 0;
- int cpu;
+ atomic64_t purr = ATOMIC64_INIT(0);
- for_each_possible_cpu(cpu) {
- struct cpu_usage *cu;
+ on_each_cpu(cpu_get_purr, &purr, 1);
- cu = &per_cpu(cpu_usage_array, cpu);
- sum_purr += cu->current_tb;
- }
- return sum_purr;
+ return atomic64_read(&purr);
}
/*
--
2.17.0
^ permalink raw reply related
* [PATCH 07/11] powerpc: generic clockevents broadcast receiver call tick_receive_broadcast
From: Nicholas Piggin @ 2018-05-04 17:19 UTC (permalink / raw)
To: linuxppc-dev
Cc: Nicholas Piggin, Benjamin Herrenschmidt, Srivatsa S . Bhat,
Preeti U Murthy
In-Reply-To: <20180504171935.25410-1-npiggin@gmail.com>
The broadcast tick recipient can call tick_receive_broadcast rather
than re-running the full timer interrupt.
It does not have to check for the next event time, because the sender
already determined the timer has expired. It does not have to test
irq_work_pending, because that's a direct decrementer interrupt and
does not go through the clock events subsystem. And it does not have
to read PURR because that was removed with the previous patch.
This results in no code size change, but both the decrementer and
broadcast path lengths are reduced.
Cc: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Cc: Preeti U Murthy <preeti@linux.vnet.ibm.com>
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/hw_irq.h | 1 +
arch/powerpc/include/asm/time.h | 1 -
arch/powerpc/kernel/smp.c | 4 +-
arch/powerpc/kernel/time.c | 84 ++++++++++++++-----------------
4 files changed, 42 insertions(+), 48 deletions(-)
diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h
index fbc2d83808aa..46fe8307fc43 100644
--- a/arch/powerpc/include/asm/hw_irq.h
+++ b/arch/powerpc/include/asm/hw_irq.h
@@ -55,6 +55,7 @@ extern void replay_system_reset(void);
extern void __replay_interrupt(unsigned int vector);
extern void timer_interrupt(struct pt_regs *);
+extern void timer_broadcast_interrupt(void);
extern void performance_monitor_exception(struct pt_regs *regs);
extern void WatchdogException(struct pt_regs *regs);
extern void unknown_exception(struct pt_regs *regs);
diff --git a/arch/powerpc/include/asm/time.h b/arch/powerpc/include/asm/time.h
index c965c79765c4..69b89f941252 100644
--- a/arch/powerpc/include/asm/time.h
+++ b/arch/powerpc/include/asm/time.h
@@ -28,7 +28,6 @@ extern struct clock_event_device decrementer_clockevent;
struct rtc_time;
extern void to_tm(int tim, struct rtc_time * tm);
-extern void tick_broadcast_ipi_handler(void);
extern void generic_calibrate_decr(void);
extern void hdec_interrupt(struct pt_regs *regs);
diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index 9ca7148b5881..5441a47701b1 100644
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -157,7 +157,7 @@ static irqreturn_t reschedule_action(int irq, void *data)
static irqreturn_t tick_broadcast_ipi_action(int irq, void *data)
{
- tick_broadcast_ipi_handler();
+ timer_broadcast_interrupt();
return IRQ_HANDLED;
}
@@ -278,7 +278,7 @@ irqreturn_t smp_ipi_demux_relaxed(void)
if (all & IPI_MESSAGE(PPC_MSG_RESCHEDULE))
scheduler_ipi();
if (all & IPI_MESSAGE(PPC_MSG_TICK_BROADCAST))
- tick_broadcast_ipi_handler();
+ timer_broadcast_interrupt();
#ifdef CONFIG_NMI_IPI
if (all & IPI_MESSAGE(PPC_MSG_NMI_IPI))
nmi_ipi_action(0, NULL);
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index 1fe6a24357e7..ad876906f847 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -567,47 +567,16 @@ void arch_irq_work_raise(void)
#endif /* CONFIG_IRQ_WORK */
-static void __timer_interrupt(void)
-{
- struct pt_regs *regs = get_irq_regs();
- u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
- struct clock_event_device *evt = this_cpu_ptr(&decrementers);
- u64 now;
-
- trace_timer_interrupt_entry(regs);
-
- if (test_irq_work_pending()) {
- clear_irq_work_pending();
- irq_work_run();
- }
-
- now = get_tb_or_rtc();
- if (now >= *next_tb) {
- *next_tb = ~(u64)0;
- if (evt->event_handler)
- evt->event_handler(evt);
- __this_cpu_inc(irq_stat.timer_irqs_event);
- } else {
- now = *next_tb - now;
- if (now <= decrementer_max)
- set_dec(now);
- /* We may have raced with new irq work */
- if (test_irq_work_pending())
- set_dec(1);
- __this_cpu_inc(irq_stat.timer_irqs_others);
- }
-
- trace_timer_interrupt_exit(regs);
-}
-
/*
* timer_interrupt - gets called when the decrementer overflows,
* with interrupts disabled.
*/
-void timer_interrupt(struct pt_regs * regs)
+void timer_interrupt(struct pt_regs *regs)
{
- struct pt_regs *old_regs;
+ struct clock_event_device *evt = this_cpu_ptr(&decrementers);
u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
+ struct pt_regs *old_regs;
+ u64 now;
/* Ensure a positive value is written to the decrementer, or else
* some CPUs will continue to take decrementer exceptions.
@@ -638,13 +607,47 @@ void timer_interrupt(struct pt_regs * regs)
old_regs = set_irq_regs(regs);
irq_enter();
+ trace_timer_interrupt_entry(regs);
+
+ if (test_irq_work_pending()) {
+ clear_irq_work_pending();
+ irq_work_run();
+ }
+
+ now = get_tb_or_rtc();
+ if (now >= *next_tb) {
+ *next_tb = ~(u64)0;
+ if (evt->event_handler)
+ evt->event_handler(evt);
+ __this_cpu_inc(irq_stat.timer_irqs_event);
+ } else {
+ now = *next_tb - now;
+ if (now <= decrementer_max)
+ set_dec(now);
+ /* We may have raced with new irq work */
+ if (test_irq_work_pending())
+ set_dec(1);
+ __this_cpu_inc(irq_stat.timer_irqs_others);
+ }
- __timer_interrupt();
+ trace_timer_interrupt_exit(regs);
irq_exit();
set_irq_regs(old_regs);
}
EXPORT_SYMBOL(timer_interrupt);
+void timer_broadcast_interrupt(void)
+{
+ u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
+ struct pt_regs *regs = get_irq_regs();
+
+ trace_timer_interrupt_entry(regs);
+ *next_tb = ~(u64)0;
+ tick_receive_broadcast();
+ __this_cpu_inc(irq_stat.timer_irqs_event);
+ trace_timer_interrupt_exit(regs);
+}
+
/*
* Hypervisor decrementer interrupts shouldn't occur but are sometimes
* left pending on exit from a KVM guest. We don't need to do anything
@@ -992,15 +995,6 @@ static int decrementer_shutdown(struct clock_event_device *dev)
return 0;
}
-/* Interrupt handler for the timer broadcast IPI */
-void tick_broadcast_ipi_handler(void)
-{
- u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
-
- *next_tb = get_tb_or_rtc();
- __timer_interrupt();
-}
-
static void register_decrementer_clockevent(int cpu)
{
struct clock_event_device *dec = &per_cpu(decrementers, cpu);
--
2.17.0
^ permalink raw reply related
* [PATCH 08/11] powerpc: allow soft-NMI watchdog to cover timer interrupts with large decrementers
From: Nicholas Piggin @ 2018-05-04 17:19 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin, Benjamin Herrenschmidt
In-Reply-To: <20180504171935.25410-1-npiggin@gmail.com>
Large decrementers (e.g., POWER9) can take a very long time to wrap,
so when the timer iterrupt handler sets the decrementer to max so as
to avoid taking another decrementer interrupt when hard enabling
interrupts before running timers, it effectively disables the soft
NMI coverage for timer interrupts.
Fix this by using the traditional 31-bit value instead, which wraps
after a few seconds. masked interrupt code does the same thing, and
in normal operation neither of these paths would ever wrap even the
31 bit value.
Note: the SMP watchdog should catch timer interrupt lockups, but it
is preferable for the local soft-NMI to catch them, mainly to avoid
the IPI.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/kernel/time.c | 19 +++++++++++++------
1 file changed, 13 insertions(+), 6 deletions(-)
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index ad876906f847..5862a3611795 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -578,22 +578,29 @@ void timer_interrupt(struct pt_regs *regs)
struct pt_regs *old_regs;
u64 now;
- /* Ensure a positive value is written to the decrementer, or else
- * some CPUs will continue to take decrementer exceptions.
- */
- set_dec(decrementer_max);
-
/* Some implementations of hotplug will get timer interrupts while
* offline, just ignore these and we also need to set
* decrementers_next_tb as MAX to make sure __check_irq_replay
* don't replay timer interrupt when return, otherwise we'll trap
* here infinitely :(
*/
- if (!cpu_online(smp_processor_id())) {
+ if (unlikely(!cpu_online(smp_processor_id()))) {
*next_tb = ~(u64)0;
+ set_dec(decrementer_max);
return;
}
+ /* Ensure a positive value is written to the decrementer, or else
+ * some CPUs will continue to take decrementer exceptions. When the
+ * PPC_WATCHDOG (decrementer based) is configured, keep this at most
+ * 31 bits, which is about 4 seconds on most systems, which gives
+ * the watchdog a chance of catching timer interrupt hard lockups.
+ */
+ if (IS_ENABLED(CONFIG_PPC_WATCHDOG))
+ set_dec(0x7fffffff);
+ else
+ set_dec(decrementer_max);
+
/* Conditionally hard-enable interrupts now that the DEC has been
* bumped to its maximum value
*/
--
2.17.0
^ permalink raw reply related
* [PATCH 09/11] powerpc: move timer broadcast code under GENERIC_CLOCKEVENTS_BROADCAST ifdef
From: Nicholas Piggin @ 2018-05-04 17:19 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin, Benjamin Herrenschmidt
In-Reply-To: <20180504171935.25410-1-npiggin@gmail.com>
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/kernel/smp.c | 8 ++++++++
arch/powerpc/kernel/time.c | 2 ++
2 files changed, 10 insertions(+)
diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index 5441a47701b1..914708eeb43f 100644
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -155,11 +155,13 @@ static irqreturn_t reschedule_action(int irq, void *data)
return IRQ_HANDLED;
}
+#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
static irqreturn_t tick_broadcast_ipi_action(int irq, void *data)
{
timer_broadcast_interrupt();
return IRQ_HANDLED;
}
+#endif
#ifdef CONFIG_NMI_IPI
static irqreturn_t nmi_ipi_action(int irq, void *data)
@@ -172,7 +174,9 @@ static irqreturn_t nmi_ipi_action(int irq, void *data)
static irq_handler_t smp_ipi_action[] = {
[PPC_MSG_CALL_FUNCTION] = call_function_action,
[PPC_MSG_RESCHEDULE] = reschedule_action,
+#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
[PPC_MSG_TICK_BROADCAST] = tick_broadcast_ipi_action,
+#endif
#ifdef CONFIG_NMI_IPI
[PPC_MSG_NMI_IPI] = nmi_ipi_action,
#endif
@@ -186,7 +190,9 @@ static irq_handler_t smp_ipi_action[] = {
const char *smp_ipi_name[] = {
[PPC_MSG_CALL_FUNCTION] = "ipi call function",
[PPC_MSG_RESCHEDULE] = "ipi reschedule",
+#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
[PPC_MSG_TICK_BROADCAST] = "ipi tick-broadcast",
+#endif
[PPC_MSG_NMI_IPI] = "nmi ipi",
};
@@ -277,8 +283,10 @@ irqreturn_t smp_ipi_demux_relaxed(void)
generic_smp_call_function_interrupt();
if (all & IPI_MESSAGE(PPC_MSG_RESCHEDULE))
scheduler_ipi();
+#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
if (all & IPI_MESSAGE(PPC_MSG_TICK_BROADCAST))
timer_broadcast_interrupt();
+#endif
#ifdef CONFIG_NMI_IPI
if (all & IPI_MESSAGE(PPC_MSG_NMI_IPI))
nmi_ipi_action(0, NULL);
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index 5862a3611795..23921f7b6e67 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -643,6 +643,7 @@ void timer_interrupt(struct pt_regs *regs)
}
EXPORT_SYMBOL(timer_interrupt);
+#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
void timer_broadcast_interrupt(void)
{
u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
@@ -654,6 +655,7 @@ void timer_broadcast_interrupt(void)
__this_cpu_inc(irq_stat.timer_irqs_event);
trace_timer_interrupt_exit(regs);
}
+#endif
/*
* Hypervisor decrementer interrupts shouldn't occur but are sometimes
--
2.17.0
^ permalink raw reply related
* [PATCH 10/11] powerpc: move a stray NMI IPI case under NMI_IPI ifdef
From: Nicholas Piggin @ 2018-05-04 17:19 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin, Benjamin Herrenschmidt
In-Reply-To: <20180504171935.25410-1-npiggin@gmail.com>
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/kernel/smp.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index 914708eeb43f..28ec1638a540 100644
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -193,7 +193,9 @@ const char *smp_ipi_name[] = {
#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
[PPC_MSG_TICK_BROADCAST] = "ipi tick-broadcast",
#endif
+#ifdef CONFIG_NMI_IPI
[PPC_MSG_NMI_IPI] = "nmi ipi",
+#endif
};
/* optional function to request ipi, for controllers with >= 4 ipis */
--
2.17.0
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox