* [PATCH v2 1/9] Revert "powerpc/64s/hash: remove user SLB data from the paca"
2018-10-02 14:27 [PATCH v2 0/9] Fixes for SLB to C series Nicholas Piggin
@ 2018-10-02 14:27 ` Nicholas Piggin
2018-10-02 14:27 ` [PATCH v2 2/9] powerpc/64: add struct int_regs to save additional registers on stack Nicholas Piggin
` (7 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Nicholas Piggin @ 2018-10-02 14:27 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Aneesh Kumar K . V, Nicholas Piggin
This reverts commit 8fed04d0f6aedf99b3d811ba58d38bb7f938a47a.
There are a number of problems with this patch, there are minor bugs
like vmalloc_sllp update no longer being picked up into pacas, but
more fundamentally the SLB flush can not be broadcast out to other
CPUs because it must be done in places where interrupts can be
disabled, which would deadlock.
Rather than try put some bandaids on this, the best fix is to revert
it. I think it's a worthwhile approach, but it should be solved by
demotig things eagerly at setup or map time, rather than in low level
hash faults. Most cases are rarely used by software or apply to very
old hardware.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/book3s/64/mmu-hash.h | 1 -
arch/powerpc/include/asm/paca.h | 13 ++++++
arch/powerpc/kernel/asm-offsets.c | 9 ++++
arch/powerpc/kernel/paca.c | 22 +++++++++
arch/powerpc/mm/hash_utils_64.c | 46 ++++++++++++++-----
arch/powerpc/mm/mmu_context.c | 3 +-
arch/powerpc/mm/slb.c | 20 +-------
arch/powerpc/mm/slice.c | 29 ++++++++----
8 files changed, 103 insertions(+), 40 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
index bbeaf6adf93c..14e552ea5e52 100644
--- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
@@ -503,7 +503,6 @@ struct slb_entry {
};
extern void slb_initialize(void);
-extern void core_flush_all_slbs(struct mm_struct *mm);
extern void slb_flush_and_rebolt(void);
void slb_flush_all_realmode(void);
void __slb_restore_bolted_realmode(void);
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 6d6b3706232c..4838149ee07b 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -146,6 +146,18 @@ struct paca_struct {
struct tlb_core_data tcd;
#endif /* CONFIG_PPC_BOOK3E */
+#ifdef CONFIG_PPC_BOOK3S
+ mm_context_id_t mm_ctx_id;
+#ifdef CONFIG_PPC_MM_SLICES
+ unsigned char mm_ctx_low_slices_psize[BITS_PER_LONG / BITS_PER_BYTE];
+ unsigned char mm_ctx_high_slices_psize[SLICE_ARRAY_SIZE];
+ unsigned long mm_ctx_slb_addr_limit;
+#else
+ u16 mm_ctx_user_psize;
+ u16 mm_ctx_sllp;
+#endif
+#endif
+
/*
* then miscellaneous read-write fields
*/
@@ -248,6 +260,7 @@ struct paca_struct {
#endif /* CONFIG_PPC_BOOK3S_64 */
} ____cacheline_aligned;
+extern void copy_mm_to_paca(struct mm_struct *mm);
extern struct paca_struct **paca_ptrs;
extern void initialise_paca(struct paca_struct *new_paca, int cpu);
extern void setup_paca(struct paca_struct *new_paca);
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index ba9d0fc98730..27c78b6c9909 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -180,6 +180,15 @@ int main(void)
OFFSET(PACAIRQSOFTMASK, paca_struct, irq_soft_mask);
OFFSET(PACAIRQHAPPENED, paca_struct, irq_happened);
OFFSET(PACA_FTRACE_ENABLED, paca_struct, ftrace_enabled);
+#ifdef CONFIG_PPC_BOOK3S
+ OFFSET(PACACONTEXTID, paca_struct, mm_ctx_id);
+#ifdef CONFIG_PPC_MM_SLICES
+ OFFSET(PACALOWSLICESPSIZE, paca_struct, mm_ctx_low_slices_psize);
+ OFFSET(PACAHIGHSLICEPSIZE, paca_struct, mm_ctx_high_slices_psize);
+ OFFSET(PACA_SLB_ADDR_LIMIT, paca_struct, mm_ctx_slb_addr_limit);
+ DEFINE(MMUPSIZEDEFSIZE, sizeof(struct mmu_psize_def));
+#endif /* CONFIG_PPC_MM_SLICES */
+#endif
#ifdef CONFIG_PPC_BOOK3E
OFFSET(PACAPGD, paca_struct, pgd);
diff --git a/arch/powerpc/kernel/paca.c b/arch/powerpc/kernel/paca.c
index 0cf84e30d1cd..0ee3e6d50f28 100644
--- a/arch/powerpc/kernel/paca.c
+++ b/arch/powerpc/kernel/paca.c
@@ -258,3 +258,25 @@ void __init free_unused_pacas(void)
printk(KERN_DEBUG "Allocated %u bytes for %u pacas\n",
paca_ptrs_size + paca_struct_size, nr_cpu_ids);
}
+
+void copy_mm_to_paca(struct mm_struct *mm)
+{
+#ifdef CONFIG_PPC_BOOK3S
+ mm_context_t *context = &mm->context;
+
+ get_paca()->mm_ctx_id = context->id;
+#ifdef CONFIG_PPC_MM_SLICES
+ VM_BUG_ON(!mm->context.slb_addr_limit);
+ get_paca()->mm_ctx_slb_addr_limit = mm->context.slb_addr_limit;
+ memcpy(&get_paca()->mm_ctx_low_slices_psize,
+ &context->low_slices_psize, sizeof(context->low_slices_psize));
+ memcpy(&get_paca()->mm_ctx_high_slices_psize,
+ &context->high_slices_psize, TASK_SLICE_ARRAY_SZ(mm));
+#else /* CONFIG_PPC_MM_SLICES */
+ get_paca()->mm_ctx_user_psize = context->user_psize;
+ get_paca()->mm_ctx_sllp = context->sllp;
+#endif
+#else /* !CONFIG_PPC_BOOK3S */
+ return;
+#endif
+}
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 88c95dc8b141..f23a89d8e4ce 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -1088,16 +1088,16 @@ unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap)
}
#ifdef CONFIG_PPC_MM_SLICES
-static unsigned int get_psize(struct mm_struct *mm, unsigned long addr)
+static unsigned int get_paca_psize(unsigned long addr)
{
unsigned char *psizes;
unsigned long index, mask_index;
if (addr < SLICE_LOW_TOP) {
- psizes = mm->context.low_slices_psize;
+ psizes = get_paca()->mm_ctx_low_slices_psize;
index = GET_LOW_SLICE_INDEX(addr);
} else {
- psizes = mm->context.high_slices_psize;
+ psizes = get_paca()->mm_ctx_high_slices_psize;
index = GET_HIGH_SLICE_INDEX(addr);
}
mask_index = index & 0x1;
@@ -1105,9 +1105,9 @@ static unsigned int get_psize(struct mm_struct *mm, unsigned long addr)
}
#else
-unsigned int get_psize(struct mm_struct *mm, unsigned long addr)
+unsigned int get_paca_psize(unsigned long addr)
{
- return mm->context.user_psize;
+ return get_paca()->mm_ctx_user_psize;
}
#endif
@@ -1118,11 +1118,15 @@ unsigned int get_psize(struct mm_struct *mm, unsigned long addr)
#ifdef CONFIG_PPC_64K_PAGES
void demote_segment_4k(struct mm_struct *mm, unsigned long addr)
{
- if (get_psize(mm, addr) == MMU_PAGE_4K)
+ if (get_slice_psize(mm, addr) == MMU_PAGE_4K)
return;
slice_set_range_psize(mm, addr, 1, MMU_PAGE_4K);
copro_flush_all_slbs(mm);
- core_flush_all_slbs(mm);
+ if ((get_paca_psize(addr) != MMU_PAGE_4K) && (current->mm == mm)) {
+
+ copy_mm_to_paca(mm);
+ slb_flush_and_rebolt();
+ }
}
#endif /* CONFIG_PPC_64K_PAGES */
@@ -1187,6 +1191,22 @@ void hash_failure_debug(unsigned long ea, unsigned long access,
trap, vsid, ssize, psize, lpsize, pte);
}
+static void check_paca_psize(unsigned long ea, struct mm_struct *mm,
+ int psize, bool user_region)
+{
+ if (user_region) {
+ if (psize != get_paca_psize(ea)) {
+ copy_mm_to_paca(mm);
+ slb_flush_and_rebolt();
+ }
+ } else if (get_paca()->vmalloc_sllp !=
+ mmu_psize_defs[mmu_vmalloc_psize].sllp) {
+ get_paca()->vmalloc_sllp =
+ mmu_psize_defs[mmu_vmalloc_psize].sllp;
+ slb_vmalloc_update();
+ }
+}
+
/* Result code is:
* 0 - handled
* 1 - normal page fault
@@ -1219,7 +1239,7 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea,
rc = 1;
goto bail;
}
- psize = get_psize(mm, ea);
+ psize = get_slice_psize(mm, ea);
ssize = user_segment_size(ea);
vsid = get_user_vsid(&mm->context, ea, ssize);
break;
@@ -1307,6 +1327,9 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea,
WARN_ON(1);
}
#endif
+ if (current->mm == mm)
+ check_paca_psize(ea, mm, psize, user_region);
+
goto bail;
}
@@ -1341,14 +1364,15 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea,
"to 4kB pages because of "
"non-cacheable mapping\n");
psize = mmu_vmalloc_psize = MMU_PAGE_4K;
- slb_vmalloc_update();
copro_flush_all_slbs(mm);
- core_flush_all_slbs(mm);
}
}
#endif /* CONFIG_PPC_64K_PAGES */
+ if (current->mm == mm)
+ check_paca_psize(ea, mm, psize, user_region);
+
#ifdef CONFIG_PPC_64K_PAGES
if (psize == MMU_PAGE_64K)
rc = __hash_page_64K(ea, access, vsid, ptep, trap,
@@ -1436,7 +1460,7 @@ int __hash_page(unsigned long ea, unsigned long msr, unsigned long trap,
#ifdef CONFIG_PPC_MM_SLICES
static bool should_hash_preload(struct mm_struct *mm, unsigned long ea)
{
- int psize = get_psize(mm, ea);
+ int psize = get_slice_psize(mm, ea);
/* We only prefault standard pages for now */
if (unlikely(psize != mm->context.user_psize))
diff --git a/arch/powerpc/mm/mmu_context.c b/arch/powerpc/mm/mmu_context.c
index 28ae2835db3d..f84e14f23e50 100644
--- a/arch/powerpc/mm/mmu_context.c
+++ b/arch/powerpc/mm/mmu_context.c
@@ -54,7 +54,8 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
* MMU context id, which is then moved to SPRN_PID.
*
* For the hash MMU it is either the first load from slb_cache
- * in switch_slb(), and/or load of MMU context id.
+ * in switch_slb(), and/or the store of paca->mm_ctx_id in
+ * copy_mm_to_paca().
*
* On the other side, the barrier is in mm/tlb-radix.c for
* radix which orders earlier stores to clear the PTEs vs
diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
index b438220c4336..74c3b6f8e9b7 100644
--- a/arch/powerpc/mm/slb.c
+++ b/arch/powerpc/mm/slb.c
@@ -430,6 +430,8 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
}
get_paca()->slb_used_bitmap = get_paca()->slb_kern_bitmap;
+ copy_mm_to_paca(mm);
+
/*
* We gradually age out SLBs after a number of context switches to
* reduce reload overhead of unused entries (like we do with FP/VEC
@@ -460,24 +462,6 @@ void slb_set_size(u16 size)
mmu_slb_size = size;
}
-static void cpu_flush_slb(void *parm)
-{
- struct mm_struct *mm = parm;
- unsigned long flags;
-
- if (mm != current->active_mm)
- return;
-
- local_irq_save(flags);
- slb_flush_and_rebolt();
- local_irq_restore(flags);
-}
-
-void core_flush_all_slbs(struct mm_struct *mm)
-{
- on_each_cpu(cpu_flush_slb, mm, 1);
-}
-
void slb_initialize(void)
{
unsigned long linear_llp, vmalloc_llp, io_llp;
diff --git a/arch/powerpc/mm/slice.c b/arch/powerpc/mm/slice.c
index fc5b3a1ec666..546dd07c8083 100644
--- a/arch/powerpc/mm/slice.c
+++ b/arch/powerpc/mm/slice.c
@@ -207,6 +207,23 @@ static bool slice_check_range_fits(struct mm_struct *mm,
return true;
}
+static void slice_flush_segments(void *parm)
+{
+#ifdef CONFIG_PPC64
+ struct mm_struct *mm = parm;
+ unsigned long flags;
+
+ if (mm != current->active_mm)
+ return;
+
+ copy_mm_to_paca(current->active_mm);
+
+ local_irq_save(flags);
+ slb_flush_and_rebolt();
+ local_irq_restore(flags);
+#endif
+}
+
static void slice_convert(struct mm_struct *mm,
const struct slice_mask *mask, int psize)
{
@@ -272,9 +289,6 @@ static void slice_convert(struct mm_struct *mm,
spin_unlock_irqrestore(&slice_convert_lock, flags);
copro_flush_all_slbs(mm);
-#ifdef CONFIG_PPC64
- core_flush_all_slbs(mm);
-#endif
}
/*
@@ -488,9 +502,8 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
* be already initialised beyond the old address limit.
*/
mm->context.slb_addr_limit = high_limit;
-#ifdef CONFIG_PPC64
- core_flush_all_slbs(mm);
-#endif
+
+ on_each_cpu(slice_flush_segments, mm, 1);
}
/* Sanity checks */
@@ -652,10 +665,8 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
(SLICE_NUM_HIGH &&
!bitmap_empty(potential_mask.high_slices, SLICE_NUM_HIGH))) {
slice_convert(mm, &potential_mask, psize);
-#ifdef CONFIG_PPC64
if (psize > MMU_PAGE_BASE)
- core_flush_all_slbs(mm);
-#endif
+ on_each_cpu(slice_flush_segments, mm, 1);
}
return newaddr;
--
2.18.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 2/9] powerpc/64: add struct int_regs to save additional registers on stack
2018-10-02 14:27 [PATCH v2 0/9] Fixes for SLB to C series Nicholas Piggin
2018-10-02 14:27 ` [PATCH v2 1/9] Revert "powerpc/64s/hash: remove user SLB data from the paca" Nicholas Piggin
@ 2018-10-02 14:27 ` Nicholas Piggin
2018-10-02 14:27 ` [PATCH v2 3/9] powerpc/64: interrupts save PPR on stack rather than thread_struct Nicholas Piggin
` (6 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Nicholas Piggin @ 2018-10-02 14:27 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Aneesh Kumar K . V, Nicholas Piggin
struct pt_regs is part of the user ABI and also the fundametal
structure for saving registers at interrupt.
The generic kernel code makes it difficult to completely decouple
these, but it's easy enough to add additional space required to save
more registers. Create a struct int_stack with struct pt_regs at
offset 0.
This is required for a following fix to save the PPR SPR on stack.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/ptrace.h | 17 +++++++---
arch/powerpc/kernel/asm-offsets.c | 21 ++++++++-----
arch/powerpc/kernel/process.c | 52 ++++++++++++++++---------------
arch/powerpc/kernel/stacktrace.c | 2 +-
4 files changed, 53 insertions(+), 39 deletions(-)
diff --git a/arch/powerpc/include/asm/ptrace.h b/arch/powerpc/include/asm/ptrace.h
index 447cbd1bee99..1a98cd8c49f6 100644
--- a/arch/powerpc/include/asm/ptrace.h
+++ b/arch/powerpc/include/asm/ptrace.h
@@ -26,7 +26,6 @@
#include <uapi/asm/ptrace.h>
#include <asm/asm-const.h>
-
#ifdef __powerpc64__
/*
@@ -44,7 +43,7 @@
#define STACK_FRAME_OVERHEAD 112 /* size of minimum stack frame */
#define STACK_FRAME_LR_SAVE 2 /* Location of LR in stack frame */
#define STACK_FRAME_REGS_MARKER ASM_CONST(0x7265677368657265)
-#define STACK_INT_FRAME_SIZE (sizeof(struct pt_regs) + \
+#define STACK_INT_FRAME_SIZE (sizeof(struct int_regs) + \
STACK_FRAME_OVERHEAD + KERNEL_REDZONE_SIZE)
#define STACK_FRAME_MARKER 12
@@ -76,6 +75,11 @@
#ifndef __ASSEMBLY__
+struct int_regs {
+ /* pt_regs must be offset 0 so r1 + STACK_FRAME_OVERHEAD points to it */
+ struct pt_regs pt_regs;
+};
+
#define GET_IP(regs) ((regs)->nip)
#define GET_USP(regs) ((regs)->gpr[1])
#define GET_FP(regs) (0)
@@ -119,8 +123,11 @@ extern int ptrace_get_reg(struct task_struct *task, int regno,
extern int ptrace_put_reg(struct task_struct *task, int regno,
unsigned long data);
-#define current_pt_regs() \
- ((struct pt_regs *)((unsigned long)current_thread_info() + THREAD_SIZE) - 1)
+#define current_int_regs() \
+ ((struct int_regs *)((unsigned long)current_thread_info() + THREAD_SIZE) - 1)
+
+#define current_pt_regs() (¤t_int_regs()->pt_regs)
+
/*
* We use the least-significant bit of the trap field to indicate
* whether we have saved the full set of registers, or only a
@@ -137,7 +144,7 @@ extern int ptrace_put_reg(struct task_struct *task, int regno,
#define TRAP(regs) ((regs)->trap & ~0xF)
#ifdef __powerpc64__
#define NV_REG_POISON 0xdeadbeefdeadbeefUL
-#define CHECK_FULL_REGS(regs) BUG_ON(regs->trap & 1)
+#define CHECK_FULL_REGS(regs) BUG_ON((regs)->trap & 1)
#else
#define NV_REG_POISON 0xdeadbeef
#define CHECK_FULL_REGS(regs) \
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 27c78b6c9909..509a113f97dc 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -72,8 +72,13 @@
#include <asm/fixmap.h>
#endif
-#define STACK_PT_REGS_OFFSET(sym, val) \
- DEFINE(sym, STACK_FRAME_OVERHEAD + offsetof(struct pt_regs, val))
+#define STACK_INT_REGS_OFFSET(sym, val) \
+ DEFINE(sym, STACK_FRAME_OVERHEAD + offsetof(struct int_regs, val))
+
+#define STACK_PT_REGS_OFFSET(sym, val) \
+ DEFINE(sym, STACK_FRAME_OVERHEAD + \
+ offsetof(struct int_regs, pt_regs) + \
+ offsetof(struct pt_regs, val))
int main(void)
{
@@ -150,7 +155,7 @@ int main(void)
OFFSET(THREAD_CKFPSTATE, thread_struct, ckfp_state.fpr);
/* Local pt_regs on stack for Transactional Memory funcs. */
DEFINE(TM_FRAME_SIZE, STACK_FRAME_OVERHEAD +
- sizeof(struct pt_regs) + 16);
+ sizeof(struct int_regs) + 16);
#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
OFFSET(TI_FLAGS, thread_info, flags);
@@ -273,11 +278,11 @@ int main(void)
/* Interrupt register frame */
DEFINE(INT_FRAME_SIZE, STACK_INT_FRAME_SIZE);
- DEFINE(SWITCH_FRAME_SIZE, STACK_FRAME_OVERHEAD + sizeof(struct pt_regs));
+ DEFINE(SWITCH_FRAME_SIZE, STACK_FRAME_OVERHEAD + sizeof(struct int_regs));
#ifdef CONFIG_PPC64
/* Create extra stack space for SRR0 and SRR1 when calling prom/rtas. */
- DEFINE(PROM_FRAME_SIZE, STACK_FRAME_OVERHEAD + sizeof(struct pt_regs) + 16);
- DEFINE(RTAS_FRAME_SIZE, STACK_FRAME_OVERHEAD + sizeof(struct pt_regs) + 16);
+ DEFINE(PROM_FRAME_SIZE, STACK_FRAME_OVERHEAD + sizeof(struct int_regs) + 16);
+ DEFINE(RTAS_FRAME_SIZE, STACK_FRAME_OVERHEAD + sizeof(struct int_regs) + 16);
#endif /* CONFIG_PPC64 */
STACK_PT_REGS_OFFSET(GPR0, gpr[0]);
STACK_PT_REGS_OFFSET(GPR1, gpr[1]);
@@ -324,8 +329,8 @@ int main(void)
STACK_PT_REGS_OFFSET(SOFTE, softe);
/* These _only_ to be used with {PROM,RTAS}_FRAME_SIZE!!! */
- DEFINE(_SRR0, STACK_FRAME_OVERHEAD+sizeof(struct pt_regs));
- DEFINE(_SRR1, STACK_FRAME_OVERHEAD+sizeof(struct pt_regs)+8);
+ DEFINE(_SRR0, STACK_FRAME_OVERHEAD+sizeof(struct int_regs));
+ DEFINE(_SRR1, STACK_FRAME_OVERHEAD+sizeof(struct int_regs)+8);
#endif /* CONFIG_PPC64 */
#if defined(CONFIG_PPC32)
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 03c2e1f134bc..532e9a83e526 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -1627,7 +1627,7 @@ static void setup_ksp_vsid(struct task_struct *p, unsigned long sp)
int copy_thread(unsigned long clone_flags, unsigned long usp,
unsigned long kthread_arg, struct task_struct *p)
{
- struct pt_regs *childregs, *kregs;
+ struct int_regs *childregs, *kregs;
extern void ret_from_fork(void);
extern void ret_from_kernel_thread(void);
void (*f)(void);
@@ -1637,44 +1637,44 @@ int copy_thread(unsigned long clone_flags, unsigned long usp,
klp_init_thread_info(ti);
/* Copy registers */
- sp -= sizeof(struct pt_regs);
- childregs = (struct pt_regs *) sp;
+ sp -= sizeof(struct int_regs);
+ childregs = (struct int_regs *) sp;
if (unlikely(p->flags & PF_KTHREAD)) {
/* kernel thread */
- memset(childregs, 0, sizeof(struct pt_regs));
- childregs->gpr[1] = sp + sizeof(struct pt_regs);
+ memset(childregs, 0, sizeof(struct int_regs));
+ childregs->pt_regs.gpr[1] = sp + sizeof(struct int_regs);
/* function */
if (usp)
- childregs->gpr[14] = ppc_function_entry((void *)usp);
+ childregs->pt_regs.gpr[14] = ppc_function_entry((void *)usp);
#ifdef CONFIG_PPC64
clear_tsk_thread_flag(p, TIF_32BIT);
- childregs->softe = IRQS_ENABLED;
+ childregs->pt_regs.softe = IRQS_ENABLED;
#endif
- childregs->gpr[15] = kthread_arg;
+ childregs->pt_regs.gpr[15] = kthread_arg;
p->thread.regs = NULL; /* no user register state */
ti->flags |= _TIF_RESTOREALL;
f = ret_from_kernel_thread;
} else {
/* user thread */
- struct pt_regs *regs = current_pt_regs();
- CHECK_FULL_REGS(regs);
+ struct int_regs *regs = current_int_regs();
+ CHECK_FULL_REGS(®s->pt_regs);
*childregs = *regs;
if (usp)
- childregs->gpr[1] = usp;
- p->thread.regs = childregs;
- childregs->gpr[3] = 0; /* Result from fork() */
+ childregs->pt_regs.gpr[1] = usp;
+ p->thread.regs = &childregs->pt_regs;
+ childregs->pt_regs.gpr[3] = 0; /* Result from fork() */
if (clone_flags & CLONE_SETTLS) {
#ifdef CONFIG_PPC64
if (!is_32bit_task())
- childregs->gpr[13] = childregs->gpr[6];
+ childregs->pt_regs.gpr[13] = childregs->pt_regs.gpr[6];
else
#endif
- childregs->gpr[2] = childregs->gpr[6];
+ childregs->pt_regs.gpr[2] = childregs->pt_regs.gpr[6];
}
f = ret_from_fork;
}
- childregs->msr &= ~(MSR_FP|MSR_VEC|MSR_VSX);
+ childregs->pt_regs.msr &= ~(MSR_FP|MSR_VEC|MSR_VSX);
sp -= STACK_FRAME_OVERHEAD;
/*
@@ -1686,8 +1686,8 @@ int copy_thread(unsigned long clone_flags, unsigned long usp,
* system call, using the stack frame created above.
*/
((unsigned long *)sp)[0] = 0;
- sp -= sizeof(struct pt_regs);
- kregs = (struct pt_regs *) sp;
+ sp -= sizeof(struct int_regs);
+ kregs = (struct int_regs *) sp;
sp -= STACK_FRAME_OVERHEAD;
p->thread.ksp = sp;
#ifdef CONFIG_PPC32
@@ -1715,7 +1715,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp,
p->thread.tidr = 0;
#endif
- kregs->nip = ppc_function_entry(f);
+ kregs->pt_regs.nip = ppc_function_entry(f);
return 0;
}
@@ -1739,8 +1739,8 @@ void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp)
* set. Do it now.
*/
if (!current->thread.regs) {
- struct pt_regs *regs = task_stack_page(current) + THREAD_SIZE;
- current->thread.regs = regs - 1;
+ struct int_regs *iregs = task_stack_page(current) + THREAD_SIZE;
+ current->thread.regs = &(iregs - 1)->pt_regs;
}
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -2106,11 +2106,13 @@ void show_stack(struct task_struct *tsk, unsigned long *stack)
*/
if (validate_sp(sp, tsk, STACK_INT_FRAME_SIZE)
&& stack[STACK_FRAME_MARKER] == STACK_FRAME_REGS_MARKER) {
- struct pt_regs *regs = (struct pt_regs *)
- (sp + STACK_FRAME_OVERHEAD);
- lr = regs->link;
+ struct int_regs *regs;
+ regs = (struct int_regs *)(sp + STACK_FRAME_OVERHEAD);
+ lr = regs->pt_regs.link;
printk("--- interrupt: %lx at %pS\n LR = %pS\n",
- regs->trap, (void *)regs->nip, (void *)lr);
+ regs->pt_regs.trap,
+ (void *)regs->pt_regs.nip,
+ (void *)lr);
firstframe = 1;
}
diff --git a/arch/powerpc/kernel/stacktrace.c b/arch/powerpc/kernel/stacktrace.c
index e2c50b55138f..463dedc4d664 100644
--- a/arch/powerpc/kernel/stacktrace.c
+++ b/arch/powerpc/kernel/stacktrace.c
@@ -120,7 +120,7 @@ save_stack_trace_tsk_reliable(struct task_struct *tsk,
* an unreliable stack trace until it's been
* _switch()'ed to for the first time.
*/
- stack_end -= STACK_FRAME_OVERHEAD + sizeof(struct pt_regs);
+ stack_end -= STACK_FRAME_OVERHEAD + sizeof(struct int_regs);
} else {
/*
* idle tasks have a custom stack layout,
--
2.18.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 3/9] powerpc/64: interrupts save PPR on stack rather than thread_struct
2018-10-02 14:27 [PATCH v2 0/9] Fixes for SLB to C series Nicholas Piggin
2018-10-02 14:27 ` [PATCH v2 1/9] Revert "powerpc/64s/hash: remove user SLB data from the paca" Nicholas Piggin
2018-10-02 14:27 ` [PATCH v2 2/9] powerpc/64: add struct int_regs to save additional registers on stack Nicholas Piggin
@ 2018-10-02 14:27 ` Nicholas Piggin
2018-10-02 14:27 ` [PATCH v2 4/9] powerpc/64s/hash: Fix preloading of SLB entries Nicholas Piggin
` (5 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Nicholas Piggin @ 2018-10-02 14:27 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Aneesh Kumar K . V, Nicholas Piggin
PPR is the odd register out when it comes to interrupt handling,
it is saved in current->thread.ppr while all others are saved on
the stack.
The difficulty with this is that accessing thread.ppr can cause a
SLB fault, but the SLB fault handler implementation in C change had
assumed the normal exception entry handlers would not cause an SLB
fault.
Fix this by allocating room in the interrupt stack to save PPR.
Fixes: 5e46e29e6a97 ("powerpc/64s/hash: convert SLB miss handlers to C")
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/exception-64s.h | 9 ++++-----
arch/powerpc/include/asm/processor.h | 12 +++++++-----
arch/powerpc/include/asm/ptrace.h | 1 +
arch/powerpc/kernel/asm-offsets.c | 2 +-
arch/powerpc/kernel/entry_64.S | 15 +++++----------
arch/powerpc/kernel/process.c | 2 +-
arch/powerpc/kernel/ptrace.c | 4 ++--
7 files changed, 21 insertions(+), 24 deletions(-)
diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h
index 47578b79f0fb..3b4767ed3ec5 100644
--- a/arch/powerpc/include/asm/exception-64s.h
+++ b/arch/powerpc/include/asm/exception-64s.h
@@ -228,11 +228,10 @@
* PPR save/restore macros used in exceptions_64s.S
* Used for P7 or later processors
*/
-#define SAVE_PPR(area, ra, rb) \
+#define SAVE_PPR(area, ra) \
BEGIN_FTR_SECTION_NESTED(940) \
- ld ra,PACACURRENT(r13); \
- ld rb,area+EX_PPR(r13); /* Read PPR from paca */ \
- std rb,TASKTHREADPPR(ra); \
+ ld ra,area+EX_PPR(r13); /* Read PPR from paca */ \
+ std ra,_PPR(r1); \
END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,940)
#define RESTORE_PPR_PACA(area, ra) \
@@ -500,7 +499,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
3: EXCEPTION_PROLOG_COMMON_1(); \
beq 4f; /* if from kernel mode */ \
ACCOUNT_CPU_USER_ENTRY(r13, r9, r10); \
- SAVE_PPR(area, r9, r10); \
+ SAVE_PPR(area, r9); \
4: EXCEPTION_PROLOG_COMMON_2(area) \
EXCEPTION_PROLOG_COMMON_3(n) \
ACCOUNT_STOLEN_TIME
diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index 350c584ca179..07251598056c 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -32,9 +32,9 @@
/* Default SMT priority is set to 3. Use 11- 13bits to save priority. */
#define PPR_PRIORITY 3
#ifdef __ASSEMBLY__
-#define INIT_PPR (PPR_PRIORITY << 50)
+#define DEFAULT_PPR (PPR_PRIORITY << 50)
#else
-#define INIT_PPR ((u64)PPR_PRIORITY << 50)
+#define DEFAULT_PPR ((u64)PPR_PRIORITY << 50)
#endif /* __ASSEMBLY__ */
#endif /* CONFIG_PPC64 */
@@ -247,7 +247,11 @@ struct thread_struct {
#ifdef CONFIG_PPC64
unsigned long ksp_vsid;
#endif
- struct pt_regs *regs; /* Pointer to saved register state */
+ union {
+ struct int_regs *iregs; /* Pointer to saved register state */
+ struct pt_regs *regs; /* Pointer to saved register state */
+ };
+
mm_segment_t addr_limit; /* for get_fs() validation */
#ifdef CONFIG_BOOKE
/* BookE base exception scratch space; align on cacheline */
@@ -342,7 +346,6 @@ struct thread_struct {
* onwards.
*/
int dscr_inherit;
- unsigned long ppr; /* used to save/restore SMT priority */
unsigned long tidr;
#endif
#ifdef CONFIG_PPC_BOOK3S_64
@@ -390,7 +393,6 @@ struct thread_struct {
.regs = (struct pt_regs *)INIT_SP - 1, /* XXX bogus, I think */ \
.addr_limit = KERNEL_DS, \
.fpexc_mode = 0, \
- .ppr = INIT_PPR, \
.fscr = FSCR_TAR | FSCR_EBB \
}
#endif
diff --git a/arch/powerpc/include/asm/ptrace.h b/arch/powerpc/include/asm/ptrace.h
index 1a98cd8c49f6..9a5a1cc85bd0 100644
--- a/arch/powerpc/include/asm/ptrace.h
+++ b/arch/powerpc/include/asm/ptrace.h
@@ -78,6 +78,7 @@
struct int_regs {
/* pt_regs must be offset 0 so r1 + STACK_FRAME_OVERHEAD points to it */
struct pt_regs pt_regs;
+ unsigned long ppr;
};
#define GET_IP(regs) ((regs)->nip)
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 509a113f97dc..56241174ea94 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -88,7 +88,6 @@ int main(void)
#ifdef CONFIG_PPC64
DEFINE(SIGSEGV, SIGSEGV);
DEFINE(NMI_MASK, NMI_MASK);
- OFFSET(TASKTHREADPPR, task_struct, thread.ppr);
#else
OFFSET(THREAD_INFO, task_struct, stack);
DEFINE(THREAD_INFO_GAP, _ALIGN_UP(sizeof(struct thread_info), 16));
@@ -283,6 +282,7 @@ int main(void)
/* Create extra stack space for SRR0 and SRR1 when calling prom/rtas. */
DEFINE(PROM_FRAME_SIZE, STACK_FRAME_OVERHEAD + sizeof(struct int_regs) + 16);
DEFINE(RTAS_FRAME_SIZE, STACK_FRAME_OVERHEAD + sizeof(struct int_regs) + 16);
+ STACK_INT_REGS_OFFSET(_PPR, ppr);
#endif /* CONFIG_PPC64 */
STACK_PT_REGS_OFFSET(GPR0, gpr[0]);
STACK_PT_REGS_OFFSET(GPR1, gpr[1]);
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index 77a888bfcb53..ce448f753d2e 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -386,10 +386,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
4: /* Anything else left to do? */
BEGIN_FTR_SECTION
- lis r3,INIT_PPR@highest /* Set thread.ppr = 3 */
- ld r10,PACACURRENT(r13)
+ lis r3,DEFAULT_PPR@highest /* Set default PPR */
sldi r3,r3,32 /* bits 11-13 are used for ppr */
- std r3,TASKTHREADPPR(r10)
+ std r3,_PPR(r1)
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
@@ -938,12 +937,6 @@ fast_exception_return:
andi. r0,r3,MSR_RI
beq- .Lunrecov_restore
- /* Load PPR from thread struct before we clear MSR:RI */
-BEGIN_FTR_SECTION
- ld r2,PACACURRENT(r13)
- ld r2,TASKTHREADPPR(r2)
-END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
-
/*
* Clear RI before restoring r13. If we are returning to
* userspace and we take an exception after restoring r13,
@@ -964,7 +957,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
andi. r0,r3,MSR_PR
beq 1f
BEGIN_FTR_SECTION
- mtspr SPRN_PPR,r2 /* Restore PPR */
+ /* Restore PPR */
+ ld r2,_PPR(r1)
+ mtspr SPRN_PPR,r2
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
ACCOUNT_CPU_USER_EXIT(r13, r2, r4)
REST_GPR(13, r1)
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 532e9a83e526..18a14fb8f759 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -1711,7 +1711,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp,
p->thread.dscr = mfspr(SPRN_DSCR);
}
if (cpu_has_feature(CPU_FTR_HAS_PPR))
- p->thread.ppr = INIT_PPR;
+ childregs->ppr = DEFAULT_PPR;
p->thread.tidr = 0;
#endif
diff --git a/arch/powerpc/kernel/ptrace.c b/arch/powerpc/kernel/ptrace.c
index 9667666eb18e..57c04176f686 100644
--- a/arch/powerpc/kernel/ptrace.c
+++ b/arch/powerpc/kernel/ptrace.c
@@ -1609,7 +1609,7 @@ static int ppr_get(struct task_struct *target,
void *kbuf, void __user *ubuf)
{
return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.ppr, 0, sizeof(u64));
+ &target->thread.iregs->ppr, 0, sizeof(u64));
}
static int ppr_set(struct task_struct *target,
@@ -1618,7 +1618,7 @@ static int ppr_set(struct task_struct *target,
const void *kbuf, const void __user *ubuf)
{
return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
- &target->thread.ppr, 0, sizeof(u64));
+ &target->thread.iregs->ppr, 0, sizeof(u64));
}
static int dscr_get(struct task_struct *target,
--
2.18.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 4/9] powerpc/64s/hash: Fix preloading of SLB entries
2018-10-02 14:27 [PATCH v2 0/9] Fixes for SLB to C series Nicholas Piggin
` (2 preceding siblings ...)
2018-10-02 14:27 ` [PATCH v2 3/9] powerpc/64: interrupts save PPR on stack rather than thread_struct Nicholas Piggin
@ 2018-10-02 14:27 ` Nicholas Piggin
2018-10-02 14:27 ` [PATCH v2 5/9] powerpc/64s/hash: add more barriers for slb preloading Nicholas Piggin
` (4 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Nicholas Piggin @ 2018-10-02 14:27 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Aneesh Kumar K . V, Nicholas Piggin
slb_setup_new_exec and preload_new_slb_context assumed if an address
missed the preload cache, then it would not be in the SLB and could
be added. This is wrong if the preload cache has started to overflow.
This can cause SLB multi-hits on user addresses.
That assumption came from an earlier version of the patch which
cleared the preload cache when copying the task, but even that was
technically wrong because some user accesses occur before these
preloads, and the preloads themselves could overflow the cache
depending on the size.
Fixes: 89ca4e126a3f ("powerpc/64s/hash: Add a SLB preload cache")
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/mm/slb.c | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
index 74c3b6f8e9b7..a5bd3c02d432 100644
--- a/arch/powerpc/mm/slb.c
+++ b/arch/powerpc/mm/slb.c
@@ -311,6 +311,13 @@ void slb_setup_new_exec(void)
struct mm_struct *mm = current->mm;
unsigned long exec = 0x10000000;
+ /*
+ * preload cache can only be used to determine whether a SLB
+ * entry exists if it does not start to overflow.
+ */
+ if (ti->slb_preload_nr + 2 > SLB_PRELOAD_NR)
+ return;
+
/*
* We have no good place to clear the slb preload cache on exec,
* flush_thread is about the earliest arch hook but that happens
@@ -345,6 +352,10 @@ void preload_new_slb_context(unsigned long start, unsigned long sp)
struct mm_struct *mm = current->mm;
unsigned long heap = mm->start_brk;
+ /* see above */
+ if (ti->slb_preload_nr + 3 > SLB_PRELOAD_NR)
+ return;
+
/* Userspace entry address. */
if (!is_kernel_addr(start)) {
if (preload_add(ti, start))
--
2.18.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 5/9] powerpc/64s/hash: add more barriers for slb preloading
2018-10-02 14:27 [PATCH v2 0/9] Fixes for SLB to C series Nicholas Piggin
` (3 preceding siblings ...)
2018-10-02 14:27 ` [PATCH v2 4/9] powerpc/64s/hash: Fix preloading of SLB entries Nicholas Piggin
@ 2018-10-02 14:27 ` Nicholas Piggin
2018-10-02 14:27 ` [PATCH v2 6/9] powerpc/64s/hash: SLB fix preemption issues Nicholas Piggin
` (3 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Nicholas Piggin @ 2018-10-02 14:27 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Aneesh Kumar K . V, Nicholas Piggin
In several places, more care has to be taken to prevent compiler or
CPU re-ordering of memory accesses into critical sections that must
not take SLB faults. Barriers are explained in the comments.
Fixes: 5e46e29e6a97 ("powerpc/64s/hash: convert SLB miss handlers to C")
Fixes: 89ca4e126a3f ("powerpc/64s/hash: Add a SLB preload cache")
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/mm/slb.c | 37 +++++++++++++++++++++++++++++++------
1 file changed, 31 insertions(+), 6 deletions(-)
diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
index a5bd3c02d432..8c38659f1b6b 100644
--- a/arch/powerpc/mm/slb.c
+++ b/arch/powerpc/mm/slb.c
@@ -344,6 +344,9 @@ void slb_setup_new_exec(void)
if (preload_add(ti, mm->mmap_base))
slb_allocate_user(mm, mm->mmap_base);
}
+
+ /* see switch_slb */
+ asm volatile("isync" : : : "memory");
}
void preload_new_slb_context(unsigned long start, unsigned long sp)
@@ -373,6 +376,9 @@ void preload_new_slb_context(unsigned long start, unsigned long sp)
if (preload_add(ti, heap))
slb_allocate_user(mm, heap);
}
+
+ /* see switch_slb */
+ asm volatile("isync" : : : "memory");
}
@@ -389,6 +395,7 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
* which would update the slb_cache/slb_cache_ptr fields in the PACA.
*/
hard_irq_disable();
+ asm volatile("isync" : : : "memory");
if (cpu_has_feature(CPU_FTR_ARCH_300)) {
/*
* SLBIA IH=3 invalidates all Class=1 SLBEs and their
@@ -396,7 +403,7 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
* switch_slb wants. So ARCH_300 does not use the slb
* cache.
*/
- asm volatile("isync ; " PPC_SLBIA(3)" ; isync");
+ asm volatile(PPC_SLBIA(3));
} else {
unsigned long offset = get_paca()->slb_cache_ptr;
@@ -404,7 +411,6 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
offset <= SLB_CACHE_ENTRIES) {
unsigned long slbie_data = 0;
- asm volatile("isync" : : : "memory");
for (i = 0; i < offset; i++) {
/* EA */
slbie_data = (unsigned long)
@@ -419,7 +425,6 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
if (!cpu_has_feature(CPU_FTR_ARCH_207S) && offset == 1)
asm volatile("slbie %0" : : "r" (slbie_data));
- asm volatile("isync" : : : "memory");
} else {
struct slb_shadow *p = get_slb_shadow();
unsigned long ksp_esid_data =
@@ -427,8 +432,7 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
unsigned long ksp_vsid_data =
be64_to_cpu(p->save_area[KSTACK_INDEX].vsid);
- asm volatile("isync\n"
- PPC_SLBIA(1) "\n"
+ asm volatile(PPC_SLBIA(1) "\n"
"slbmte %0,%1\n"
"isync"
:: "r"(ksp_vsid_data),
@@ -466,6 +470,13 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
slb_allocate_user(mm, ea);
}
+
+ /*
+ * Synchronize slbmte preloads with possible subsequent user memory
+ * address accesses by the kernel (user mode won't happen until
+ * rfid, which is safe).
+ */
+ asm volatile("isync" : : : "memory");
}
void slb_set_size(u16 size)
@@ -609,6 +620,17 @@ static long slb_insert_entry(unsigned long ea, unsigned long context,
if (!vsid)
return -EFAULT;
+ /*
+ * There must not be a kernel SLB fault in alloc_slb_index or before
+ * slbmte here or the allocation bitmaps could get out of whack with
+ * the SLB.
+ *
+ * User SLB faults or preloads take this path which might get inlined
+ * into the caller, so add compiler barriers here to ensure unsafe
+ * memory accesses do not come between
+ */
+ barrier();
+
index = alloc_slb_index(kernel);
vsid_data = __mk_vsid_data(vsid, ssize, flags);
@@ -617,10 +639,13 @@ static long slb_insert_entry(unsigned long ea, unsigned long context,
/*
* No need for an isync before or after this slbmte. The exception
* we enter with and the rfid we exit with are context synchronizing.
- * Also we only handle user segments here.
+ * User preloads should add isync afterwards in case the kernel
+ * accesses user memory before it returns to userspace with rfid.
*/
asm volatile("slbmte %0, %1" : : "r" (vsid_data), "r" (esid_data));
+ barrier();
+
if (!kernel)
slb_cache_update(esid_data);
--
2.18.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 6/9] powerpc/64s/hash: SLB fix preemption issues
2018-10-02 14:27 [PATCH v2 0/9] Fixes for SLB to C series Nicholas Piggin
` (4 preceding siblings ...)
2018-10-02 14:27 ` [PATCH v2 5/9] powerpc/64s/hash: add more barriers for slb preloading Nicholas Piggin
@ 2018-10-02 14:27 ` Nicholas Piggin
2018-10-02 14:27 ` [PATCH v2 7/9] powerpc/64s/hash: Fix false positive preempt warnings Nicholas Piggin
` (2 subsequent siblings)
8 siblings, 0 replies; 10+ messages in thread
From: Nicholas Piggin @ 2018-10-02 14:27 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Aneesh Kumar K . V, Nicholas Piggin
slb_setup_new_exec and preload_new_slb_context run with interrupts
and preemption enabled, which can be corrupted by re-entrant interrupt
or process touching SLB preload cache or SLB allocator.
Hard disable interrupts over these regions.
Fixes: 5e46e29e6a97 ("powerpc/64s/hash: convert SLB miss handlers to C")
Fixes: 89ca4e126a3f ("powerpc/64s/hash: Add a SLB preload cache")
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/mm/slb.c | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
index 8c38659f1b6b..b5a33da875b5 100644
--- a/arch/powerpc/mm/slb.c
+++ b/arch/powerpc/mm/slb.c
@@ -311,6 +311,8 @@ void slb_setup_new_exec(void)
struct mm_struct *mm = current->mm;
unsigned long exec = 0x10000000;
+ WARN_ON(irqs_disabled());
+
/*
* preload cache can only be used to determine whether a SLB
* entry exists if it does not start to overflow.
@@ -318,6 +320,8 @@ void slb_setup_new_exec(void)
if (ti->slb_preload_nr + 2 > SLB_PRELOAD_NR)
return;
+ hard_irq_disable();
+
/*
* We have no good place to clear the slb preload cache on exec,
* flush_thread is about the earliest arch hook but that happens
@@ -347,6 +351,8 @@ void slb_setup_new_exec(void)
/* see switch_slb */
asm volatile("isync" : : : "memory");
+
+ local_irq_enable();
}
void preload_new_slb_context(unsigned long start, unsigned long sp)
@@ -355,10 +361,14 @@ void preload_new_slb_context(unsigned long start, unsigned long sp)
struct mm_struct *mm = current->mm;
unsigned long heap = mm->start_brk;
+ WARN_ON(irqs_disabled());
+
/* see above */
if (ti->slb_preload_nr + 3 > SLB_PRELOAD_NR)
return;
+ hard_irq_disable();
+
/* Userspace entry address. */
if (!is_kernel_addr(start)) {
if (preload_add(ti, start))
@@ -379,6 +389,8 @@ void preload_new_slb_context(unsigned long start, unsigned long sp)
/* see switch_slb */
asm volatile("isync" : : : "memory");
+
+ local_irq_enable();
}
--
2.18.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 7/9] powerpc/64s/hash: Fix false positive preempt warnings
2018-10-02 14:27 [PATCH v2 0/9] Fixes for SLB to C series Nicholas Piggin
` (5 preceding siblings ...)
2018-10-02 14:27 ` [PATCH v2 6/9] powerpc/64s/hash: SLB fix preemption issues Nicholas Piggin
@ 2018-10-02 14:27 ` Nicholas Piggin
2018-10-02 14:27 ` [PATCH v2 8/9] powerpc/64s/hash: slb_flush_and_rebolt simplify Nicholas Piggin
2018-10-02 14:27 ` [PATCH v2 9/9] powerpc/64s/hash: add some slb debugging tests Nicholas Piggin
8 siblings, 0 replies; 10+ messages in thread
From: Nicholas Piggin @ 2018-10-02 14:27 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Aneesh Kumar K . V, Nicholas Piggin
Fixes: 5e46e29e6a97 ("powerpc/64s/hash: convert SLB miss handlers to C")
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/mm/slb.c | 24 ++++++++++++------------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
index b5a33da875b5..c273d0baf4fc 100644
--- a/arch/powerpc/mm/slb.c
+++ b/arch/powerpc/mm/slb.c
@@ -558,21 +558,21 @@ static void slb_cache_update(unsigned long esid_data)
/*
* Now update slb cache entries
*/
- slb_cache_index = get_paca()->slb_cache_ptr;
+ slb_cache_index = local_paca->slb_cache_ptr;
if (slb_cache_index < SLB_CACHE_ENTRIES) {
/*
* We have space in slb cache for optimized switch_slb().
* Top 36 bits from esid_data as per ISA
*/
- get_paca()->slb_cache[slb_cache_index++] = esid_data >> 28;
- get_paca()->slb_cache_ptr++;
+ local_paca->slb_cache[slb_cache_index++] = esid_data >> 28;
+ local_paca->slb_cache_ptr++;
} else {
/*
* Our cache is full and the current cache content strictly
* doesn't indicate the active SLB conents. Bump the ptr
* so that switch_slb() will ignore the cache.
*/
- get_paca()->slb_cache_ptr = SLB_CACHE_ENTRIES + 1;
+ local_paca->slb_cache_ptr = SLB_CACHE_ENTRIES + 1;
}
}
@@ -596,24 +596,24 @@ static enum slb_index alloc_slb_index(bool kernel)
* POWER7/8/9 have 32 SLB entries, this could be expanded if a
* future CPU has more.
*/
- if (get_paca()->slb_used_bitmap != U32_MAX) {
- index = ffz(get_paca()->slb_used_bitmap);
- get_paca()->slb_used_bitmap |= 1U << index;
+ if (local_paca->slb_used_bitmap != U32_MAX) {
+ index = ffz(local_paca->slb_used_bitmap);
+ local_paca->slb_used_bitmap |= 1U << index;
if (kernel)
- get_paca()->slb_kern_bitmap |= 1U << index;
+ local_paca->slb_kern_bitmap |= 1U << index;
} else {
/* round-robin replacement of slb starting at SLB_NUM_BOLTED. */
- index = get_paca()->stab_rr;
+ index = local_paca->stab_rr;
if (index < (mmu_slb_size - 1))
index++;
else
index = SLB_NUM_BOLTED;
- get_paca()->stab_rr = index;
+ local_paca->stab_rr = index;
if (index < 32) {
if (kernel)
- get_paca()->slb_kern_bitmap |= 1U << index;
+ local_paca->slb_kern_bitmap |= 1U << index;
else
- get_paca()->slb_kern_bitmap &= ~(1U << index);
+ local_paca->slb_kern_bitmap &= ~(1U << index);
}
}
BUG_ON(index < SLB_NUM_BOLTED);
--
2.18.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 8/9] powerpc/64s/hash: slb_flush_and_rebolt simplify
2018-10-02 14:27 [PATCH v2 0/9] Fixes for SLB to C series Nicholas Piggin
` (6 preceding siblings ...)
2018-10-02 14:27 ` [PATCH v2 7/9] powerpc/64s/hash: Fix false positive preempt warnings Nicholas Piggin
@ 2018-10-02 14:27 ` Nicholas Piggin
2018-10-02 14:27 ` [PATCH v2 9/9] powerpc/64s/hash: add some slb debugging tests Nicholas Piggin
8 siblings, 0 replies; 10+ messages in thread
From: Nicholas Piggin @ 2018-10-02 14:27 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Aneesh Kumar K . V, Nicholas Piggin
slb_flush_and_rebolt is misleading, it is called in virtual mode, so
it can not possibly change the stack, so it should not be touching the
shadow area. And since vmalloc is no longer bolted, it should not
change any bolted mappings at all.
Change the name to slb_flush_and_restore_bolted, and have it just load
the kernel stack from what's currently in the shadow SLB area.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/book3s/64/mmu-hash.h | 2 +-
arch/powerpc/kernel/swsusp_asm64.S | 2 +-
arch/powerpc/mm/hash_utils_64.c | 4 +-
arch/powerpc/mm/slb.c | 46 +++++++------------
arch/powerpc/mm/slice.c | 2 +-
5 files changed, 21 insertions(+), 35 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
index 14e552ea5e52..60cda8fb0677 100644
--- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
@@ -503,7 +503,7 @@ struct slb_entry {
};
extern void slb_initialize(void);
-extern void slb_flush_and_rebolt(void);
+void slb_flush_and_restore_bolted(void);
void slb_flush_all_realmode(void);
void __slb_restore_bolted_realmode(void);
void slb_restore_bolted_realmode(void);
diff --git a/arch/powerpc/kernel/swsusp_asm64.S b/arch/powerpc/kernel/swsusp_asm64.S
index f83bf6f72cb0..185216becb8b 100644
--- a/arch/powerpc/kernel/swsusp_asm64.S
+++ b/arch/powerpc/kernel/swsusp_asm64.S
@@ -262,7 +262,7 @@ END_FW_FTR_SECTION_IFCLR(FW_FEATURE_LPAR)
addi r1,r1,-128
#ifdef CONFIG_PPC_BOOK3S_64
- bl slb_flush_and_rebolt
+ bl slb_flush_and_restore_bolted
#endif
bl do_after_copyback
addi r1,r1,128
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index f23a89d8e4ce..974d0332e1d8 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -1125,7 +1125,7 @@ void demote_segment_4k(struct mm_struct *mm, unsigned long addr)
if ((get_paca_psize(addr) != MMU_PAGE_4K) && (current->mm == mm)) {
copy_mm_to_paca(mm);
- slb_flush_and_rebolt();
+ slb_flush_and_restore_bolted();
}
}
#endif /* CONFIG_PPC_64K_PAGES */
@@ -1197,7 +1197,7 @@ static void check_paca_psize(unsigned long ea, struct mm_struct *mm,
if (user_region) {
if (psize != get_paca_psize(ea)) {
copy_mm_to_paca(mm);
- slb_flush_and_rebolt();
+ slb_flush_and_restore_bolted();
}
} else if (get_paca()->vmalloc_sllp !=
mmu_psize_defs[mmu_vmalloc_psize].sllp) {
diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
index c273d0baf4fc..e3c5280f43bd 100644
--- a/arch/powerpc/mm/slb.c
+++ b/arch/powerpc/mm/slb.c
@@ -115,8 +115,6 @@ void __slb_restore_bolted_realmode(void)
/*
* Insert the bolted entries into an empty SLB.
- * This is not the same as rebolt because the bolted segments are not
- * changed, just loaded from the shadow area.
*/
void slb_restore_bolted_realmode(void)
{
@@ -135,12 +133,15 @@ void slb_flush_all_realmode(void)
asm volatile("slbmte %0,%0; slbia" : : "r" (0));
}
-void slb_flush_and_rebolt(void)
+/*
+ * This flushes non-bolted entries, it can be run in virtual mode. Must
+ * be called with interrupts disabled.
+ */
+void slb_flush_and_restore_bolted(void)
{
- /* If you change this make sure you change SLB_NUM_BOLTED
- * and PR KVM appropriately too. */
- unsigned long linear_llp, lflags;
- unsigned long ksp_esid_data, ksp_vsid_data;
+ struct slb_shadow *p = get_slb_shadow();
+
+ BUILD_BUG_ON(SLB_NUM_BOLTED != 2);
WARN_ON(!irqs_disabled());
@@ -150,30 +151,12 @@ void slb_flush_and_rebolt(void)
*/
hard_irq_disable();
- linear_llp = mmu_psize_defs[mmu_linear_psize].sllp;
- lflags = SLB_VSID_KERNEL | linear_llp;
-
- ksp_esid_data = mk_esid_data(get_paca()->kstack, mmu_kernel_ssize, KSTACK_INDEX);
- if ((ksp_esid_data & ~0xfffffffUL) <= PAGE_OFFSET) {
- ksp_esid_data &= ~SLB_ESID_V;
- ksp_vsid_data = 0;
- slb_shadow_clear(KSTACK_INDEX);
- } else {
- /* Update stack entry; others don't change */
- slb_shadow_update(get_paca()->kstack, mmu_kernel_ssize, lflags, KSTACK_INDEX);
- ksp_vsid_data =
- be64_to_cpu(get_slb_shadow()->save_area[KSTACK_INDEX].vsid);
- }
-
- /* We need to do this all in asm, so we're sure we don't touch
- * the stack between the slbia and rebolting it. */
asm volatile("isync\n"
"slbia\n"
- /* Slot 1 - kernel stack */
- "slbmte %0,%1\n"
- "isync"
- :: "r"(ksp_vsid_data),
- "r"(ksp_esid_data)
+ "slbmte %0, %1\n"
+ "isync\n"
+ :: "r" (be64_to_cpu(p->save_area[KSTACK_INDEX].vsid)),
+ "r" (be64_to_cpu(p->save_area[KSTACK_INDEX].esid))
: "memory");
get_paca()->slb_cache_ptr = 0;
@@ -254,7 +237,10 @@ void slb_dump_contents(struct slb_entry *slb_ptr)
void slb_vmalloc_update(void)
{
- slb_flush_and_rebolt();
+ /*
+ * vmalloc is not bolted, so just have to flush non-bolted.
+ */
+ slb_flush_and_restore_bolted();
}
static bool preload_hit(struct thread_info *ti, unsigned long esid)
diff --git a/arch/powerpc/mm/slice.c b/arch/powerpc/mm/slice.c
index 546dd07c8083..50ba3d0456a5 100644
--- a/arch/powerpc/mm/slice.c
+++ b/arch/powerpc/mm/slice.c
@@ -219,7 +219,7 @@ static void slice_flush_segments(void *parm)
copy_mm_to_paca(current->active_mm);
local_irq_save(flags);
- slb_flush_and_rebolt();
+ slb_flush_and_restore_bolted();
local_irq_restore(flags);
#endif
}
--
2.18.0
^ permalink raw reply related [flat|nested] 10+ messages in thread* [PATCH v2 9/9] powerpc/64s/hash: add some slb debugging tests
2018-10-02 14:27 [PATCH v2 0/9] Fixes for SLB to C series Nicholas Piggin
` (7 preceding siblings ...)
2018-10-02 14:27 ` [PATCH v2 8/9] powerpc/64s/hash: slb_flush_and_rebolt simplify Nicholas Piggin
@ 2018-10-02 14:27 ` Nicholas Piggin
8 siblings, 0 replies; 10+ messages in thread
From: Nicholas Piggin @ 2018-10-02 14:27 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Aneesh Kumar K . V, Nicholas Piggin
This adds CONFIG_DEBUG_VM checks to ensure:
- The kernel stack is in the SLB after it's flushed and bolted.
- We don't insert an SLB for an address that is aleady in the SLB.
- The kernel SLB miss handler does not take an SLB miss.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/paca.h | 3 ++
arch/powerpc/mm/slb.c | 53 +++++++++++++++++++++++++++++++--
2 files changed, 53 insertions(+), 3 deletions(-)
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 4838149ee07b..2bfbd8811b72 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -115,6 +115,9 @@ struct paca_struct {
u16 vmalloc_sllp;
u8 slb_cache_ptr;
u8 stab_rr; /* stab/slb round-robin counter */
+#ifdef CONFIG_DEBUG_VM
+ u8 in_kernel_slb_handler;
+#endif
u32 slb_used_bitmap; /* Bitmaps for first 32 SLB entries. */
u32 slb_kern_bitmap;
u32 slb_cache[SLB_CACHE_ENTRIES];
diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
index e3c5280f43bd..703c344f6751 100644
--- a/arch/powerpc/mm/slb.c
+++ b/arch/powerpc/mm/slb.c
@@ -58,6 +58,30 @@ static inline unsigned long mk_vsid_data(unsigned long ea, int ssize,
return __mk_vsid_data(get_kernel_vsid(ea, ssize), ssize, flags);
}
+static void assert_slb_exists(unsigned long ea)
+{
+#ifdef CONFIG_DEBUG_VM
+ unsigned long tmp;
+
+ WARN_ON_ONCE(mfmsr() & MSR_EE);
+
+ asm volatile("slbfee. %0, %1" : "=r"(tmp) : "r"(ea) : "cr0");
+ WARN_ON(tmp == 0);
+#endif
+}
+
+static void assert_slb_notexists(unsigned long ea)
+{
+#ifdef CONFIG_DEBUG_VM
+ unsigned long tmp;
+
+ WARN_ON_ONCE(mfmsr() & MSR_EE);
+
+ asm volatile("slbfee. %0, %1" : "=r"(tmp) : "r"(ea) : "cr0");
+ WARN_ON(tmp != 0);
+#endif
+}
+
static inline void slb_shadow_update(unsigned long ea, int ssize,
unsigned long flags,
enum slb_index index)
@@ -90,6 +114,7 @@ static inline void create_shadowed_slbe(unsigned long ea, int ssize,
*/
slb_shadow_update(ea, ssize, flags, index);
+ assert_slb_notexists(ea);
asm volatile("slbmte %0,%1" :
: "r" (mk_vsid_data(ea, ssize, flags)),
"r" (mk_esid_data(ea, ssize, index))
@@ -111,6 +136,8 @@ void __slb_restore_bolted_realmode(void)
: "r" (be64_to_cpu(p->save_area[index].vsid)),
"r" (be64_to_cpu(p->save_area[index].esid)));
}
+
+ assert_slb_exists(local_paca->kstack);
}
/*
@@ -158,6 +185,7 @@ void slb_flush_and_restore_bolted(void)
:: "r" (be64_to_cpu(p->save_area[KSTACK_INDEX].vsid)),
"r" (be64_to_cpu(p->save_area[KSTACK_INDEX].esid))
: "memory");
+ assert_slb_exists(get_paca()->kstack);
get_paca()->slb_cache_ptr = 0;
@@ -410,9 +438,17 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
unsigned long slbie_data = 0;
for (i = 0; i < offset; i++) {
- /* EA */
- slbie_data = (unsigned long)
+ unsigned long ea;
+
+ ea = (unsigned long)
get_paca()->slb_cache[i] << SID_SHIFT;
+ /*
+ * Could assert_slb_exists here, but hypervisor
+ * or machine check could have come in and
+ * removed the entry at this point.
+ */
+
+ slbie_data = ea;
slbie_data |= user_segment_size(slbie_data)
<< SLBIE_SSIZE_SHIFT;
slbie_data |= SLBIE_C; /* user slbs have C=1 */
@@ -640,6 +676,7 @@ static long slb_insert_entry(unsigned long ea, unsigned long context,
* User preloads should add isync afterwards in case the kernel
* accesses user memory before it returns to userspace with rfid.
*/
+ assert_slb_notexists(ea);
asm volatile("slbmte %0, %1" : : "r" (vsid_data), "r" (esid_data));
barrier();
@@ -740,7 +777,17 @@ long do_slb_fault(struct pt_regs *regs, unsigned long ea)
* if they go via fast_exception_return too.
*/
if (id >= KERNEL_REGION_ID) {
- return slb_allocate_kernel(ea, id);
+ long err;
+#ifdef CONFIG_DEBUG_VM
+ /* Catch recursive kernel SLB faults. */
+ BUG_ON(local_paca->in_kernel_slb_handler);
+ local_paca->in_kernel_slb_handler = 1;
+#endif
+ err = slb_allocate_kernel(ea, id);
+#ifdef CONFIG_DEBUG_VM
+ local_paca->in_kernel_slb_handler = 0;
+#endif
+ return err;
} else {
struct mm_struct *mm = current->mm;
long err;
--
2.18.0
^ permalink raw reply related [flat|nested] 10+ messages in thread