* [RFC PATCH 4/9] powerpc/64s: avoid reloading (H)SRR registers if they are still valid
From: Nicholas Piggin @ 2020-11-06 15:59 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
In-Reply-To: <20201106155929.2246055-1-npiggin@gmail.com>
When an interrupt is taken, the SRR registers are set to return to
where it left off. Unless they are modified in the meantime, or the
return address or MSR are modified, there is no need to reload these
registers when returning from interrupt.
Introduce per-CPU flags that track the validity of SRR and HSRR
registers, clear them when returning from interrupt, using the registers
for something else (e.g., OPAL calls), or adjusting return address or MSR.
This improves the performance of interrupt returns.
XXX: 64e build breaks
XXX: may not need to invalidate both hsrr and srr all the time
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/Kconfig.debug | 5 ++
arch/powerpc/include/asm/paca.h | 2 +
arch/powerpc/include/asm/ptrace.h | 15 +++++
arch/powerpc/kernel/asm-offsets.c | 2 +
arch/powerpc/kernel/entry_64.S | 75 +++++++++++++++++++---
arch/powerpc/kernel/exceptions-64s.S | 27 ++++++++
arch/powerpc/kernel/fpu.S | 2 +
arch/powerpc/kernel/kgdb.c | 2 +-
arch/powerpc/kernel/kprobes-ftrace.c | 2 +-
arch/powerpc/kernel/kprobes.c | 10 +--
arch/powerpc/kernel/process.c | 21 +++++-
arch/powerpc/kernel/rtas.c | 13 +++-
arch/powerpc/kernel/signal.c | 2 +-
arch/powerpc/kernel/signal_64.c | 14 ++++
arch/powerpc/kernel/syscalls.c | 2 +
arch/powerpc/kernel/traps.c | 18 +++---
arch/powerpc/kernel/vector.S | 4 ++
arch/powerpc/lib/sstep.c | 5 +-
arch/powerpc/math-emu/math.c | 2 +-
arch/powerpc/mm/fault.c | 2 +-
arch/powerpc/platforms/powernv/opal-call.c | 3 +
arch/powerpc/sysdev/fsl_pci.c | 2 +-
22 files changed, 197 insertions(+), 33 deletions(-)
diff --git a/arch/powerpc/Kconfig.debug b/arch/powerpc/Kconfig.debug
index b88900f4832f..ad1f5bf6ab3d 100644
--- a/arch/powerpc/Kconfig.debug
+++ b/arch/powerpc/Kconfig.debug
@@ -84,6 +84,11 @@ config MSI_BITMAP_SELFTEST
config PPC_IRQ_SOFT_MASK_DEBUG
bool "Include extra checks for powerpc irq soft masking"
+ depends on PPC64
+
+config PPC_RFI_SRR_DEBUG
+ bool "Include extra checks for RFI SRR register validity"
+ depends on PPC_BOOK3S_64
config XMON
bool "Include xmon kernel debugger"
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 9454d29ff4b4..58e9995c3184 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -170,6 +170,8 @@ struct paca_struct {
#ifdef CONFIG_PPC_BOOK3E
u16 trap_save; /* Used when bad stack is encountered */
#endif
+ u8 hsrr_valid; /* HSRRs set for HRFID */
+ u8 srr_valid; /* SRRs set for RFID */
u8 irq_soft_mask; /* mask for irq soft masking */
u8 irq_happened; /* irq happened while soft-disabled */
u8 irq_work_pending; /* IRQ_WORK interrupt while soft-disable */
diff --git a/arch/powerpc/include/asm/ptrace.h b/arch/powerpc/include/asm/ptrace.h
index e2c778c176a3..2c3e773ce292 100644
--- a/arch/powerpc/include/asm/ptrace.h
+++ b/arch/powerpc/include/asm/ptrace.h
@@ -110,6 +110,7 @@ struct pt_regs
#endif /* __powerpc64__ */
#ifndef __ASSEMBLY__
+#include <asm/paca.h>
static inline unsigned long instruction_pointer(struct pt_regs *regs)
{
@@ -160,6 +161,20 @@ static inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc)
regs->gpr[3] = rc;
}
+static inline void regs_set_return_ip(struct pt_regs *regs, unsigned long ip)
+{
+ regs->nip = ip;
+#ifdef CONFIG_PPC_BOOK3S
+ local_paca->hsrr_valid = 0;
+ local_paca->srr_valid = 0;
+#endif
+}
+
+static inline void regs_add_return_ip(struct pt_regs *regs, long offset)
+{
+ regs_set_return_ip(regs, regs->nip + offset);
+}
+
#ifdef __powerpc64__
#define user_mode(regs) ((((regs)->msr) >> MSR_PR_LG) & 0x1)
#else
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index c2722ff36e98..ea13e35dd511 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -210,6 +210,8 @@ int main(void)
OFFSET(PACATOC, paca_struct, kernel_toc);
OFFSET(PACAKBASE, paca_struct, kernelbase);
OFFSET(PACAKMSR, paca_struct, kernel_msr);
+ OFFSET(PACAHSRR_VALID, paca_struct, hsrr_valid);
+ OFFSET(PACASRR_VALID, paca_struct, srr_valid);
OFFSET(PACAIRQSOFTMASK, paca_struct, irq_soft_mask);
OFFSET(PACAIRQHAPPENED, paca_struct, irq_happened);
OFFSET(PACA_FTRACE_ENABLED, paca_struct, ftrace_enabled);
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index 53027fc9cd31..6236a88a592f 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -65,6 +65,30 @@ exception_marker:
.align 7
#ifdef CONFIG_PPC_BOOK3S
+.macro DEBUG_SRR_VALID srr
+#ifdef CONFIG_PPC_RFI_SRR_DEBUG
+ .ifc \srr,srr
+ mfspr r11,SPRN_SRR0
+ ld r12,_NIP(r1)
+100: tdne r11,r12
+ EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
+ mfspr r11,SPRN_SRR1
+ ld r12,_MSR(r1)
+100: tdne r11,r12
+ EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
+ .else
+ mfspr r11,SPRN_HSRR0
+ ld r12,_NIP(r1)
+100: tdne r11,r12
+ EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
+ mfspr r11,SPRN_HSRR1
+ ld r12,_MSR(r1)
+100: tdne r11,r12
+ EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
+ .endif
+#endif
+.endm
+
.macro system_call_vectored name trapnr
.globl system_call_vectored_\name
system_call_vectored_\name:
@@ -289,6 +313,9 @@ END_BTB_FLUSH_SECTION
ld r11,exception_marker@toc(r2)
std r11,-16(r10) /* "regshere" marker */
+ li r11,1
+ stb r11,PACASRR_VALID(r13)
+
/*
* RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
* would clobber syscall parameters. Also we always enter with IRQs
@@ -310,18 +337,25 @@ END_BTB_FLUSH_SECTION
bl syscall_exit_prepare
ld r2,_CCR(r1)
+ ld r6,_LINK(r1)
+ mtlr r6
+
+ lbz r4,PACASRR_VALID(r13)
+ cmpdi r4,0
+ bne 1f
+ li r4,0
+ stb r4,PACASRR_VALID(r13)
ld r4,_NIP(r1)
ld r5,_MSR(r1)
- ld r6,_LINK(r1)
+ mtspr SPRN_SRR0,r4
+ mtspr SPRN_SRR1,r5
+1:
+ DEBUG_SRR_VALID srr
BEGIN_FTR_SECTION
stdcx. r0,0,r1 /* to clear the reservation */
END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
- mtspr SPRN_SRR0,r4
- mtspr SPRN_SRR1,r5
- mtlr r6
-
cmpdi r3,0
bne .Lsyscall_restore_regs
/* Zero volatile regs that may contain sensitive kernel data */
@@ -648,7 +682,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
blr
#ifdef CONFIG_PPC_BOOK3S
-
/*
* If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not
* touched, no exit work created, then this can be used.
@@ -683,19 +716,32 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\())
bne- .Lrestore_nvgprs_\srr
.Lfast_user_interrupt_return_\srr:
- ld r11,_NIP(r1)
- ld r12,_MSR(r1)
BEGIN_FTR_SECTION
ld r10,_PPR(r1)
mtspr SPRN_PPR,r10
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
+ .ifc \srr,srr
+ lbz r4,PACASRR_VALID(r13)
+ .else
+ lbz r4,PACAHSRR_VALID(r13)
+ .endif
+ cmpdi r4,0
+ li r4,0
+ bne 1f
+ ld r11,_NIP(r1)
+ ld r12,_MSR(r1)
.ifc \srr,srr
mtspr SPRN_SRR0,r11
mtspr SPRN_SRR1,r12
+1:
+ stb r4,PACASRR_VALID(r13)
.else
mtspr SPRN_HSRR0,r11
mtspr SPRN_HSRR1,r12
+1:
+ stb r4,PACAHSRR_VALID(r13)
.endif
+ DEBUG_SRR_VALID \srr
BEGIN_FTR_SECTION
stdcx. r0,0,r1 /* to clear the reservation */
@@ -740,15 +786,28 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
.Lfast_kernel_interrupt_return_\srr\():
cmpdi cr1,r3,0
+ .ifc \srr,srr
+ lbz r4,PACASRR_VALID(r13)
+ .else
+ lbz r4,PACAHSRR_VALID(r13)
+ .endif
+ cmpdi r4,0
+ li r4,0
+ bne 1f
ld r11,_NIP(r1)
ld r12,_MSR(r1)
.ifc \srr,srr
mtspr SPRN_SRR0,r11
mtspr SPRN_SRR1,r12
+1:
+ stb r4,PACASRR_VALID(r13)
.else
mtspr SPRN_HSRR0,r11
mtspr SPRN_HSRR1,r12
+1:
+ stb r4,PACAHSRR_VALID(r13)
.endif
+ DEBUG_SRR_VALID \srr
BEGIN_FTR_SECTION
stdcx. r0,0,r1 /* to clear the reservation */
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 1f725a3ac2f3..b370a5f334fc 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -567,6 +567,20 @@ DEFINE_FIXED_SYMBOL(\name\()_common_real)
std r0,GPR0(r1) /* save r0 in stackframe */
std r10,GPR1(r1) /* save r1 in stackframe */
+ /* Mark our [H]SRRs valid for return */
+ li r10,1
+ .if IHSRR_IF_HVMODE
+ BEGIN_FTR_SECTION
+ stb r10,PACAHSRR_VALID(r13)
+ FTR_SECTION_ELSE
+ stb r10,PACASRR_VALID(r13)
+ ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
+ .elseif IHSRR
+ stb r10,PACAHSRR_VALID(r13)
+ .else
+ stb r10,PACASRR_VALID(r13)
+ .endif
+
.if ISET_RI
li r10,MSR_RI
mtmsrd r10,1 /* Set MSR_RI */
@@ -666,10 +680,13 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
.macro EXCEPTION_RESTORE_REGS hsrr=0
/* Move original SRR0 and SRR1 into the respective regs */
ld r9,_MSR(r1)
+ li r10,0
.if \hsrr
mtspr SPRN_HSRR1,r9
+ stb r10,PACAHSRR_VALID(r13)
.else
mtspr SPRN_SRR1,r9
+ stb r10,PACASRR_VALID(r13)
.endif
ld r9,_NIP(r1)
.if \hsrr
@@ -1781,6 +1798,8 @@ EXC_COMMON_BEGIN(hdecrementer_common)
*
* Be careful to avoid touching the kernel stack.
*/
+ li r10,0
+ stb r10,PACAHSRR_VALID(r13)
ld r10,PACA_EXGEN+EX_CTR(r13)
mtctr r10
mtcrf 0x80,r9
@@ -2611,6 +2630,8 @@ BEGIN_FTR_SECTION
ld r10,PACA_EXGEN+EX_CFAR(r13)
mtspr SPRN_CFAR,r10
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
+ li r10,0
+ stb r10,PACAHSRR_VALID(r13)
ld r10,PACA_EXGEN+EX_R10(r13)
ld r11,PACA_EXGEN+EX_R11(r13)
ld r12,PACA_EXGEN+EX_R12(r13)
@@ -2783,6 +2804,12 @@ masked_interrupt:
ori r11,r11,PACA_IRQ_HARD_DIS
stb r11,PACAIRQHAPPENED(r13)
2: /* done */
+ li r10,0
+ .if \hsrr
+ stb r10,PACAHSRR_VALID(r13)
+ .else
+ stb r10,PACASRR_VALID(r13)
+ .endif
ld r10,PACA_EXGEN+EX_CTR(r13)
mtctr r10
mtcrf 0x80,r9
diff --git a/arch/powerpc/kernel/fpu.S b/arch/powerpc/kernel/fpu.S
index 3ff9a8fafa46..ae80b4a3da48 100644
--- a/arch/powerpc/kernel/fpu.S
+++ b/arch/powerpc/kernel/fpu.S
@@ -105,6 +105,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_VSX)
ori r12,r12,MSR_FP
or r12,r12,r4
std r12,_MSR(r1)
+ li r4,0
+ stb r4,PACASRR_VALID(r13)
#endif
li r4,1
stb r4,THREAD_LOAD_FP(r5)
diff --git a/arch/powerpc/kernel/kgdb.c b/arch/powerpc/kernel/kgdb.c
index 409080208a6c..dcac6c74a93c 100644
--- a/arch/powerpc/kernel/kgdb.c
+++ b/arch/powerpc/kernel/kgdb.c
@@ -147,7 +147,7 @@ static int kgdb_handle_breakpoint(struct pt_regs *regs)
return 0;
if (*(u32 *)regs->nip == BREAK_INSTR)
- regs->nip += BREAK_INSTR_SIZE;
+ regs_add_return_ip(regs, BREAK_INSTR_SIZE);
return 1;
}
diff --git a/arch/powerpc/kernel/kprobes-ftrace.c b/arch/powerpc/kernel/kprobes-ftrace.c
index 972cb28174b2..dc35d4f5564f 100644
--- a/arch/powerpc/kernel/kprobes-ftrace.c
+++ b/arch/powerpc/kernel/kprobes-ftrace.c
@@ -40,7 +40,7 @@ void kprobe_ftrace_handler(unsigned long nip, unsigned long parent_nip,
* Emulate singlestep (and also recover regs->nip)
* as if there is a nop
*/
- regs->nip += MCOUNT_INSN_SIZE;
+ regs_add_return_ip(regs, MCOUNT_INSN_SIZE);
if (unlikely(p->post_handler)) {
kcb->kprobe_status = KPROBE_HIT_SSDONE;
p->post_handler(p, regs, 0);
diff --git a/arch/powerpc/kernel/kprobes.c b/arch/powerpc/kernel/kprobes.c
index 01ab2163659e..8165ed71ab51 100644
--- a/arch/powerpc/kernel/kprobes.c
+++ b/arch/powerpc/kernel/kprobes.c
@@ -178,7 +178,7 @@ static nokprobe_inline void prepare_singlestep(struct kprobe *p, struct pt_regs
* variant as values in regs could play a part in
* if the trap is taken or not
*/
- regs->nip = (unsigned long)p->ainsn.insn;
+ regs_set_return_ip(regs, (unsigned long)p->ainsn.insn);
}
static nokprobe_inline void save_previous_kprobe(struct kprobe_ctlblk *kcb)
@@ -415,7 +415,7 @@ static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
* we end up emulating it in kprobe_handler(), which increments the nip
* again.
*/
- regs->nip = orig_ret_address - 4;
+ regs_set_return_ip(regs, orig_ret_address - 4);
regs->link = orig_ret_address;
return 0;
@@ -450,7 +450,7 @@ int kprobe_post_handler(struct pt_regs *regs)
}
/* Adjust nip to after the single-stepped instruction */
- regs->nip = (unsigned long)cur->addr + len;
+ regs_set_return_ip(regs, (unsigned long)cur->addr + len);
regs->msr |= kcb->kprobe_saved_msr;
/*Restore back the original saved kprobes variables and continue. */
@@ -490,7 +490,7 @@ int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
* and allow the page fault handler to continue as a
* normal page fault.
*/
- regs->nip = (unsigned long)cur->addr;
+ regs_set_return_ip(regs, (unsigned long)cur->addr);
regs->msr &= ~MSR_SINGLESTEP; /* Turn off 'trace' bits */
regs->msr |= kcb->kprobe_saved_msr;
if (kcb->kprobe_status == KPROBE_REENTER)
@@ -523,7 +523,7 @@ int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
* zero, try to fix up.
*/
if ((entry = search_exception_tables(regs->nip)) != NULL) {
- regs->nip = extable_fixup(entry);
+ regs_set_return_ip(regs, extable_fixup(entry));
return 1;
}
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 0bdd3ed653df..ea36a29c8b01 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -97,6 +97,8 @@ static void check_if_tm_restore_required(struct task_struct *tsk)
!test_thread_flag(TIF_RESTORE_TM)) {
tsk->thread.ckpt_regs.msr = tsk->thread.regs->msr;
set_thread_flag(TIF_RESTORE_TM);
+ local_paca->hsrr_valid = 0;
+ local_paca->srr_valid = 0;
}
}
@@ -161,6 +163,8 @@ static void __giveup_fpu(struct task_struct *tsk)
if (cpu_has_feature(CPU_FTR_VSX))
msr &= ~MSR_VSX;
tsk->thread.regs->msr = msr;
+ local_paca->hsrr_valid = 0;
+ local_paca->srr_valid = 0;
}
void giveup_fpu(struct task_struct *tsk)
@@ -244,6 +248,8 @@ static void __giveup_altivec(struct task_struct *tsk)
if (cpu_has_feature(CPU_FTR_VSX))
msr &= ~MSR_VSX;
tsk->thread.regs->msr = msr;
+ local_paca->hsrr_valid = 0;
+ local_paca->srr_valid = 0;
}
void giveup_altivec(struct task_struct *tsk)
@@ -559,6 +565,8 @@ void notrace restore_math(struct pt_regs *regs)
msr_check_and_clear(new_msr);
regs->msr |= new_msr | fpexc_mode;
+ local_paca->hsrr_valid = 0;
+ local_paca->srr_valid = 0;
}
}
#endif /* CONFIG_PPC_BOOK3S_64 */
@@ -1295,6 +1303,8 @@ struct task_struct *__switch_to(struct task_struct *prev,
atomic_read(¤t->mm->context.vas_windows)))
asm volatile(PPC_CP_ABORT);
}
+ local_paca->hsrr_valid = 0;
+ local_paca->srr_valid = 0;
#endif /* CONFIG_PPC_BOOK3S_64 */
return last;
@@ -1878,6 +1888,9 @@ void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp)
#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
thread_pkey_regs_init(¤t->thread);
+
+ local_paca->hsrr_valid = 0;
+ local_paca->srr_valid = 0;
}
EXPORT_SYMBOL(start_thread);
@@ -1925,9 +1938,12 @@ int set_fpexc_mode(struct task_struct *tsk, unsigned int val)
if (val > PR_FP_EXC_PRECISE)
return -EINVAL;
tsk->thread.fpexc_mode = __pack_fe01(val);
- if (regs != NULL && (regs->msr & MSR_FP) != 0)
+ if (regs != NULL && (regs->msr & MSR_FP) != 0) {
regs->msr = (regs->msr & ~(MSR_FE0|MSR_FE1))
| tsk->thread.fpexc_mode;
+ local_paca->hsrr_valid = 0;
+ local_paca->srr_valid = 0;
+ }
return 0;
}
@@ -1979,6 +1995,9 @@ int set_endian(struct task_struct *tsk, unsigned int val)
else
return -EINVAL;
+ local_paca->hsrr_valid = 0;
+ local_paca->srr_valid = 0;
+
return 0;
}
diff --git a/arch/powerpc/kernel/rtas.c b/arch/powerpc/kernel/rtas.c
index 954f41676f69..85b5541a24f7 100644
--- a/arch/powerpc/kernel/rtas.c
+++ b/arch/powerpc/kernel/rtas.c
@@ -46,6 +46,13 @@
/* This is here deliberately so it's only used in this file */
void enter_rtas(unsigned long);
+static inline void do_enter_rtas(unsigned long args)
+{
+ enter_rtas(args);
+ local_paca->hsrr_valid = 0;
+ local_paca->srr_valid = 0;
+}
+
struct rtas_t rtas = {
.lock = __ARCH_SPIN_LOCK_UNLOCKED
};
@@ -384,7 +391,7 @@ static char *__fetch_rtas_last_error(char *altbuf)
save_args = rtas.args;
rtas.args = err_args;
- enter_rtas(__pa(&rtas.args));
+ do_enter_rtas(__pa(&rtas.args));
err_args = rtas.args;
rtas.args = save_args;
@@ -430,7 +437,7 @@ va_rtas_call_unlocked(struct rtas_args *args, int token, int nargs, int nret,
for (i = 0; i < nret; ++i)
args->rets[i] = 0;
- enter_rtas(__pa(args));
+ do_enter_rtas(__pa(args));
}
void rtas_call_unlocked(struct rtas_args *args, int token, int nargs, int nret, ...)
@@ -1198,7 +1205,7 @@ SYSCALL_DEFINE1(rtas, struct rtas_args __user *, uargs)
flags = lock_rtas();
rtas.args = args;
- enter_rtas(__pa(&rtas.args));
+ do_enter_rtas(__pa(&rtas.args));
args = rtas.args;
/* A -1 return code indicates that the last command couldn't
diff --git a/arch/powerpc/kernel/signal.c b/arch/powerpc/kernel/signal.c
index 44ec7b34b27e..ca1a504d442d 100644
--- a/arch/powerpc/kernel/signal.c
+++ b/arch/powerpc/kernel/signal.c
@@ -242,7 +242,7 @@ static void check_syscall_restart(struct pt_regs *regs, struct k_sigaction *ka,
regs->gpr[0] = __NR_restart_syscall;
else
regs->gpr[3] = regs->orig_gpr3;
- regs->nip -= 4;
+ regs_add_return_ip(regs, - 4);
regs->result = 0;
} else {
if (trap_is_scv(regs)) {
diff --git a/arch/powerpc/kernel/signal_64.c b/arch/powerpc/kernel/signal_64.c
index bfc939360bad..8e60c89dcc87 100644
--- a/arch/powerpc/kernel/signal_64.c
+++ b/arch/powerpc/kernel/signal_64.c
@@ -693,6 +693,10 @@ SYSCALL_DEFINE3(swapcontext, struct ucontext __user *, old_ctx,
/* This returns like rt_sigreturn */
set_thread_flag(TIF_RESTOREALL);
+
+ local_paca->hsrr_valid = 0;
+ local_paca->srr_valid = 0;
+
return 0;
}
@@ -798,6 +802,10 @@ SYSCALL_DEFINE0(rt_sigreturn)
goto badframe;
set_thread_flag(TIF_RESTOREALL);
+
+ local_paca->hsrr_valid = 0;
+ local_paca->srr_valid = 0;
+
return 0;
badframe:
@@ -878,6 +886,7 @@ int handle_rt_signal64(struct ksignal *ksig, sigset_t *set,
err |= put_user(regs->gpr[1], (unsigned long __user *)newsp);
/* Set up "regs" so we "return" to the signal handler. */
+ /* XXX: use set return IP */
if (is_elf2_task()) {
regs->ctr = (unsigned long) ksig->ka.sa.sa_handler;
regs->gpr[12] = regs->ctr;
@@ -910,6 +919,9 @@ int handle_rt_signal64(struct ksignal *ksig, sigset_t *set,
if (err)
goto badframe;
+ local_paca->hsrr_valid = 0;
+ local_paca->srr_valid = 0;
+
return 0;
badframe:
@@ -917,6 +929,8 @@ int handle_rt_signal64(struct ksignal *ksig, sigset_t *set,
printk_ratelimited(regs->msr & MSR_64BIT ? fmt64 : fmt32,
tsk->comm, tsk->pid, "setup_rt_frame",
(long)frame, regs->nip, regs->link);
+ local_paca->hsrr_valid = 0;
+ local_paca->srr_valid = 0;
return 1;
}
diff --git a/arch/powerpc/kernel/syscalls.c b/arch/powerpc/kernel/syscalls.c
index 078608ec2e92..13bf62ccfa18 100644
--- a/arch/powerpc/kernel/syscalls.c
+++ b/arch/powerpc/kernel/syscalls.c
@@ -123,6 +123,8 @@ SYSCALL_DEFINE0(switch_endian)
struct thread_info *ti;
current->thread.regs->msr ^= MSR_LE;
+ local_paca->hsrr_valid = 0;
+ local_paca->srr_valid = 0;
/*
* Set TIF_RESTOREALL so that r3 isn't clobbered on return to
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index 95f84c542523..0796391edd16 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -1049,7 +1049,7 @@ static void p9_hmi_special_emu(struct pt_regs *regs)
#endif /* !__LITTLE_ENDIAN__ */
/* Go to next instruction */
- regs->nip += 4;
+ regs_add_return_ip(regs, 4);
}
#endif /* CONFIG_VSX */
@@ -1487,7 +1487,7 @@ DEFINE_INTERRUPT_HANDLER(program_check_exception)
if (!(regs->msr & MSR_PR) && /* not user-mode */
report_bug(bugaddr, regs) == BUG_TRAP_TYPE_WARN) {
- regs->nip += 4;
+ regs_add_return_ip(regs, 4);
return;
}
_exception(SIGTRAP, regs, TRAP_BRKPT, regs->nip);
@@ -1549,7 +1549,7 @@ DEFINE_INTERRUPT_HANDLER(program_check_exception)
if (reason & (REASON_ILLEGAL | REASON_PRIVILEGED)) {
switch (emulate_instruction(regs)) {
case 0:
- regs->nip += 4;
+ regs_add_return_ip(regs, 4);
emulate_single_step(regs);
return;
case -EFAULT:
@@ -1600,7 +1600,7 @@ DEFINE_INTERRUPT_HANDLER(alignment_exception)
if (fixed == 1) {
/* skip over emulated instruction */
- regs->nip += inst_length(reason);
+ regs_add_return_ip(regs, inst_length(reason));
emulate_single_step(regs);
return;
}
@@ -1767,7 +1767,7 @@ DEFINE_INTERRUPT_HANDLER(facility_unavailable_exception)
pr_err("DSCR based mfspr emulation failed\n");
return;
}
- regs->nip += 4;
+ regs_add_return_ip(regs, 4);
emulate_single_step(regs);
}
return;
@@ -2030,7 +2030,7 @@ DEFINE_INTERRUPT_HANDLER(altivec_assist_exception)
PPC_WARN_EMULATED(altivec, regs);
err = emulate_altivec(regs);
if (err == 0) {
- regs->nip += 4; /* skip emulated instruction */
+ regs_add_return_ip(regs, 4); /* skip emulated instruction */
emulate_single_step(regs);
return;
}
@@ -2094,7 +2094,7 @@ void SPEFloatingPointException(struct pt_regs *regs)
err = do_spe_mathemu(regs);
if (err == 0) {
- regs->nip += 4; /* skip emulated instruction */
+ regs_add_return_ip(regs, 4); /* skip emulated instruction */
emulate_single_step(regs);
return;
}
@@ -2125,10 +2125,10 @@ void SPEFloatingPointRoundException(struct pt_regs *regs)
giveup_spe(current);
preempt_enable();
- regs->nip -= 4;
+ regs_add_return_ip(regs, - 4);
err = speround_handler(regs);
if (err == 0) {
- regs->nip += 4; /* skip emulated instruction */
+ regs_add_return_ip(regs, 4); /* skip emulated instruction */
emulate_single_step(regs);
return;
}
diff --git a/arch/powerpc/kernel/vector.S b/arch/powerpc/kernel/vector.S
index 2c948e7b0d00..609e3526a66f 100644
--- a/arch/powerpc/kernel/vector.S
+++ b/arch/powerpc/kernel/vector.S
@@ -75,6 +75,8 @@ _GLOBAL(load_up_altivec)
addi r5,r4,THREAD /* Get THREAD */
oris r12,r12,MSR_VEC@h
std r12,_MSR(r1)
+ li r4,0
+ stb r4,PACASRR_VALID(r13)
#endif
li r4,1
stb r4,THREAD_LOAD_VEC(r5)
@@ -133,6 +135,8 @@ _GLOBAL(load_up_vsx)
/* enable use of VSX after return */
oris r12,r12,MSR_VSX@h
std r12,_MSR(r1)
+ li r4,0
+ stb r4,PACASRR_VALID(r13)
b fast_interrupt_return_srr
#endif /* CONFIG_VSX */
diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c
index 855457ed09b5..6c2d6a3c9c8a 100644
--- a/arch/powerpc/lib/sstep.c
+++ b/arch/powerpc/lib/sstep.c
@@ -3033,7 +3033,7 @@ void emulate_update_regs(struct pt_regs *regs, struct instruction_op *op)
default:
WARN_ON_ONCE(1);
}
- regs->nip = next_pc;
+ regs_set_return_ip(regs, next_pc);
}
NOKPROBE_SYMBOL(emulate_update_regs);
@@ -3310,6 +3310,9 @@ int emulate_step(struct pt_regs *regs, struct ppc_inst instr)
unsigned long val;
unsigned long ea;
+ local_paca->hsrr_valid = 0;
+ local_paca->srr_valid = 0;
+
r = analyse_instr(&op, regs, instr);
if (r < 0)
return r;
diff --git a/arch/powerpc/math-emu/math.c b/arch/powerpc/math-emu/math.c
index 30b4b69c6941..d92416d78aee 100644
--- a/arch/powerpc/math-emu/math.c
+++ b/arch/powerpc/math-emu/math.c
@@ -453,7 +453,7 @@ do_mathemu(struct pt_regs *regs)
break;
}
- regs->nip += 4;
+ regs_add_return_ip(regs, 4);
return 0;
illegal:
diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
index 1999982200ab..1d82dc53386f 100644
--- a/arch/powerpc/mm/fault.c
+++ b/arch/powerpc/mm/fault.c
@@ -615,7 +615,7 @@ void bad_page_fault(struct pt_regs *regs, int sig)
/* Are we prepared to handle this fault? */
if ((entry = search_exception_tables(regs->nip)) != NULL) {
- regs->nip = extable_fixup(entry);
+ regs_set_return_ip(regs, extable_fixup(entry));
return;
}
diff --git a/arch/powerpc/platforms/powernv/opal-call.c b/arch/powerpc/platforms/powernv/opal-call.c
index 5cd0f52d258f..1a7bc261d156 100644
--- a/arch/powerpc/platforms/powernv/opal-call.c
+++ b/arch/powerpc/platforms/powernv/opal-call.c
@@ -100,6 +100,9 @@ static int64_t opal_call(int64_t a0, int64_t a1, int64_t a2, int64_t a3,
bool mmu = (msr & (MSR_IR|MSR_DR));
int64_t ret;
+ local_paca->hsrr_valid = 0;
+ local_paca->srr_valid = 0;
+
msr &= ~MSR_EE;
if (unlikely(!mmu))
diff --git a/arch/powerpc/sysdev/fsl_pci.c b/arch/powerpc/sysdev/fsl_pci.c
index 040b9d01c079..af78e7c3108f 100644
--- a/arch/powerpc/sysdev/fsl_pci.c
+++ b/arch/powerpc/sysdev/fsl_pci.c
@@ -1072,7 +1072,7 @@ int fsl_pci_mcheck_exception(struct pt_regs *regs)
ret = get_kernel_nofault(inst, (void *)regs->nip);
if (!ret && mcheck_handle_load(regs, inst)) {
- regs->nip += 4;
+ regs_add_return_ip(regs, 4);
return 1;
}
}
--
2.23.0
^ permalink raw reply related
* [RFC PATCH 5/9] powerpc/64: move interrupt return asm to interrupt_64.S
From: Nicholas Piggin @ 2020-11-06 15:59 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
In-Reply-To: <20201106155929.2246055-1-npiggin@gmail.com>
The next patch would like to move interrupt return assembly code to a low
location before general text, so move it into its own file and include via
head_64.S
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/head-64.h | 2 +-
arch/powerpc/kernel/entry_64.S | 601 ----------------------------
arch/powerpc/kernel/head_64.S | 5 +-
arch/powerpc/kernel/interrupt_64.S | 608 +++++++++++++++++++++++++++++
4 files changed, 613 insertions(+), 603 deletions(-)
create mode 100644 arch/powerpc/kernel/interrupt_64.S
diff --git a/arch/powerpc/include/asm/head-64.h b/arch/powerpc/include/asm/head-64.h
index 4cb9efa2eb21..242204e12993 100644
--- a/arch/powerpc/include/asm/head-64.h
+++ b/arch/powerpc/include/asm/head-64.h
@@ -16,7 +16,7 @@
.section ".head.data.\name\()","a",@progbits
.endm
.macro use_ftsec name
- .section ".head.text.\name\()"
+ .section ".head.text.\name\()","ax",@progbits
.endm
/*
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index 6236a88a592f..65ddd159974b 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -32,7 +32,6 @@
#include <asm/irqflags.h>
#include <asm/hw_irq.h>
#include <asm/context_tracking.h>
-#include <asm/tm.h>
#include <asm/ppc-opcode.h>
#include <asm/barrier.h>
#include <asm/export.h>
@@ -48,411 +47,7 @@
/*
* System calls.
*/
- .section ".toc","aw"
-SYS_CALL_TABLE:
- .tc sys_call_table[TC],sys_call_table
-
-#ifdef CONFIG_COMPAT
-COMPAT_SYS_CALL_TABLE:
- .tc compat_sys_call_table[TC],compat_sys_call_table
-#endif
-
-/* This value is used to mark exception frames on the stack. */
-exception_marker:
- .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
-
.section ".text"
- .align 7
-
-#ifdef CONFIG_PPC_BOOK3S
-.macro DEBUG_SRR_VALID srr
-#ifdef CONFIG_PPC_RFI_SRR_DEBUG
- .ifc \srr,srr
- mfspr r11,SPRN_SRR0
- ld r12,_NIP(r1)
-100: tdne r11,r12
- EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
- mfspr r11,SPRN_SRR1
- ld r12,_MSR(r1)
-100: tdne r11,r12
- EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
- .else
- mfspr r11,SPRN_HSRR0
- ld r12,_NIP(r1)
-100: tdne r11,r12
- EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
- mfspr r11,SPRN_HSRR1
- ld r12,_MSR(r1)
-100: tdne r11,r12
- EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
- .endif
-#endif
-.endm
-
-.macro system_call_vectored name trapnr
- .globl system_call_vectored_\name
-system_call_vectored_\name:
-_ASM_NOKPROBE_SYMBOL(system_call_vectored_\name)
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
-BEGIN_FTR_SECTION
- extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
- bne .Ltabort_syscall
-END_FTR_SECTION_IFSET(CPU_FTR_TM)
-#endif
- INTERRUPT_TO_KERNEL
- mr r10,r1
- ld r1,PACAKSAVE(r13)
- std r10,0(r1)
- std r11,_NIP(r1)
- std r12,_MSR(r1)
- std r0,GPR0(r1)
- std r10,GPR1(r1)
- std r2,GPR2(r1)
- ld r2,PACATOC(r13)
- mfcr r12
- li r11,0
- /* Can we avoid saving r3-r8 in common case? */
- std r3,GPR3(r1)
- std r4,GPR4(r1)
- std r5,GPR5(r1)
- std r6,GPR6(r1)
- std r7,GPR7(r1)
- std r8,GPR8(r1)
- /* Zero r9-r12, this should only be required when restoring all GPRs */
- std r11,GPR9(r1)
- std r11,GPR10(r1)
- std r11,GPR11(r1)
- std r11,GPR12(r1)
- std r9,GPR13(r1)
- SAVE_NVGPRS(r1)
- std r11,_XER(r1)
- std r11,_LINK(r1)
- std r11,_CTR(r1)
-
- li r11,\trapnr
- std r11,_TRAP(r1)
- std r12,_CCR(r1)
- std r3,ORIG_GPR3(r1)
- addi r10,r1,STACK_FRAME_OVERHEAD
- ld r11,exception_marker@toc(r2)
- std r11,-16(r10) /* "regshere" marker */
-
-BEGIN_FTR_SECTION
- HMT_MEDIUM
-END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
-
- /*
- * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
- * would clobber syscall parameters. Also we always enter with IRQs
- * enabled and nothing pending. system_call_exception() will call
- * trace_hardirqs_off().
- *
- * scv enters with MSR[EE]=1, so don't set PACA_IRQ_HARD_DIS. The
- * entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED.
- */
-
- /* Calling convention has r9 = orig r0, r10 = regs */
- mr r9,r0
- bl system_call_exception
-
-.Lsyscall_vectored_\name\()_exit:
- addi r4,r1,STACK_FRAME_OVERHEAD
- li r5,1 /* scv */
- bl syscall_exit_prepare
-
- ld r2,_CCR(r1)
- ld r4,_NIP(r1)
- ld r5,_MSR(r1)
-
-BEGIN_FTR_SECTION
- stdcx. r0,0,r1 /* to clear the reservation */
-END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
-
-BEGIN_FTR_SECTION
- HMT_MEDIUM_LOW
-END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
-
- cmpdi r3,0
- bne .Lsyscall_vectored_\name\()_restore_regs
-
- /* rfscv returns with LR->NIA and CTR->MSR */
- mtlr r4
- mtctr r5
-
- /* Could zero these as per ABI, but we may consider a stricter ABI
- * which preserves these if libc implementations can benefit, so
- * restore them for now until further measurement is done. */
- ld r0,GPR0(r1)
- ld r4,GPR4(r1)
- ld r5,GPR5(r1)
- ld r6,GPR6(r1)
- ld r7,GPR7(r1)
- ld r8,GPR8(r1)
- /* Zero volatile regs that may contain sensitive kernel data */
- li r9,0
- li r10,0
- li r11,0
- li r12,0
- mtspr SPRN_XER,r0
-
- /*
- * We don't need to restore AMR on the way back to userspace for KUAP.
- * The value of AMR only matters while we're in the kernel.
- */
- mtcr r2
- ld r2,GPR2(r1)
- ld r3,GPR3(r1)
- ld r13,GPR13(r1)
- ld r1,GPR1(r1)
- RFSCV_TO_USER
- b . /* prevent speculative execution */
-
-.Lsyscall_vectored_\name\()_restore_regs:
- li r3,0
- mtmsrd r3,1
- mtspr SPRN_SRR0,r4
- mtspr SPRN_SRR1,r5
-
- ld r3,_CTR(r1)
- ld r4,_LINK(r1)
- ld r5,_XER(r1)
-
- REST_NVGPRS(r1)
- ld r0,GPR0(r1)
- mtcr r2
- mtctr r3
- mtlr r4
- mtspr SPRN_XER,r5
- REST_10GPRS(2, r1)
- REST_2GPRS(12, r1)
- ld r1,GPR1(r1)
- RFI_TO_USER
-.endm
-
-system_call_vectored common 0x3000
-/*
- * We instantiate another entry copy for the SIGILL variant, with TRAP=0x7ff0
- * which is tested by system_call_exception when r0 is -1 (as set by vector
- * entry code).
- */
-system_call_vectored sigill 0x7ff0
-
-
-/*
- * Entered via kernel return set up by kernel/sstep.c, must match entry regs
- */
- .globl system_call_vectored_emulate
-system_call_vectored_emulate:
-_ASM_NOKPROBE_SYMBOL(system_call_vectored_emulate)
- li r10,IRQS_ALL_DISABLED
- stb r10,PACAIRQSOFTMASK(r13)
- b system_call_vectored_common
-#endif
-
- .balign IFETCH_ALIGN_BYTES
- .globl system_call_common_real
-system_call_common_real:
- ld r10,PACAKMSR(r13) /* get MSR value for kernel */
- mtmsrd r10
-
- .balign IFETCH_ALIGN_BYTES
- .globl system_call_common
-system_call_common:
-_ASM_NOKPROBE_SYMBOL(system_call_common)
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
-BEGIN_FTR_SECTION
- extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
- bne .Ltabort_syscall
-END_FTR_SECTION_IFSET(CPU_FTR_TM)
-#endif
- mr r10,r1
- ld r1,PACAKSAVE(r13)
- std r10,0(r1)
- std r11,_NIP(r1)
- std r12,_MSR(r1)
- std r0,GPR0(r1)
- std r10,GPR1(r1)
- std r2,GPR2(r1)
-#ifdef CONFIG_PPC_FSL_BOOK3E
-START_BTB_FLUSH_SECTION
- BTB_FLUSH(r10)
-END_BTB_FLUSH_SECTION
-#endif
- ld r2,PACATOC(r13)
- mfcr r12
- li r11,0
- /* Can we avoid saving r3-r8 in common case? */
- std r3,GPR3(r1)
- std r4,GPR4(r1)
- std r5,GPR5(r1)
- std r6,GPR6(r1)
- std r7,GPR7(r1)
- std r8,GPR8(r1)
- /* Zero r9-r12, this should only be required when restoring all GPRs */
- std r11,GPR9(r1)
- std r11,GPR10(r1)
- std r11,GPR11(r1)
- std r11,GPR12(r1)
- std r9,GPR13(r1)
- SAVE_NVGPRS(r1)
- std r11,_XER(r1)
- std r11,_CTR(r1)
- mflr r10
-
- /*
- * This clears CR0.SO (bit 28), which is the error indication on
- * return from this system call.
- */
- rldimi r12,r11,28,(63-28)
- li r11,0xc00
- std r10,_LINK(r1)
- std r11,_TRAP(r1)
- std r12,_CCR(r1)
- std r3,ORIG_GPR3(r1)
- addi r10,r1,STACK_FRAME_OVERHEAD
- ld r11,exception_marker@toc(r2)
- std r11,-16(r10) /* "regshere" marker */
-
- li r11,1
- stb r11,PACASRR_VALID(r13)
-
- /*
- * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
- * would clobber syscall parameters. Also we always enter with IRQs
- * enabled and nothing pending. system_call_exception() will call
- * trace_hardirqs_off().
- */
- li r11,IRQS_ALL_DISABLED
- li r12,PACA_IRQ_HARD_DIS
- stb r11,PACAIRQSOFTMASK(r13)
- stb r12,PACAIRQHAPPENED(r13)
-
- /* Calling convention has r9 = orig r0, r10 = regs */
- mr r9,r0
- bl system_call_exception
-
-.Lsyscall_exit:
- addi r4,r1,STACK_FRAME_OVERHEAD
- li r5,0 /* !scv */
- bl syscall_exit_prepare
-
- ld r2,_CCR(r1)
- ld r6,_LINK(r1)
- mtlr r6
-
- lbz r4,PACASRR_VALID(r13)
- cmpdi r4,0
- bne 1f
- li r4,0
- stb r4,PACASRR_VALID(r13)
- ld r4,_NIP(r1)
- ld r5,_MSR(r1)
- mtspr SPRN_SRR0,r4
- mtspr SPRN_SRR1,r5
-1:
- DEBUG_SRR_VALID srr
-
-BEGIN_FTR_SECTION
- stdcx. r0,0,r1 /* to clear the reservation */
-END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
-
- cmpdi r3,0
- bne .Lsyscall_restore_regs
- /* Zero volatile regs that may contain sensitive kernel data */
- li r0,0
- li r4,0
- li r5,0
- li r6,0
- li r7,0
- li r8,0
- li r9,0
- li r10,0
- li r11,0
- li r12,0
- mtctr r0
- mtspr SPRN_XER,r0
-.Lsyscall_restore_regs_cont:
-
-BEGIN_FTR_SECTION
- HMT_MEDIUM_LOW
-END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
-
- /*
- * We don't need to restore AMR on the way back to userspace for KUAP.
- * The value of AMR only matters while we're in the kernel.
- */
- mtcr r2
- ld r2,GPR2(r1)
- ld r3,GPR3(r1)
- ld r13,GPR13(r1)
- ld r1,GPR1(r1)
- RFI_TO_USER
- b . /* prevent speculative execution */
-
-.Lsyscall_restore_regs:
- ld r3,_CTR(r1)
- ld r4,_XER(r1)
- REST_NVGPRS(r1)
- mtctr r3
- mtspr SPRN_XER,r4
- ld r0,GPR0(r1)
- REST_8GPRS(4, r1)
- ld r12,GPR12(r1)
- b .Lsyscall_restore_regs_cont
-
-#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
-.Ltabort_syscall:
- /* Firstly we need to enable TM in the kernel */
- mfmsr r10
- li r9, 1
- rldimi r10, r9, MSR_TM_LG, 63-MSR_TM_LG
- mtmsrd r10, 0
-
- /* tabort, this dooms the transaction, nothing else */
- li r9, (TM_CAUSE_SYSCALL|TM_CAUSE_PERSISTENT)
- TABORT(R9)
-
- /*
- * Return directly to userspace. We have corrupted user register state,
- * but userspace will never see that register state. Execution will
- * resume after the tbegin of the aborted transaction with the
- * checkpointed register state.
- */
- li r9, MSR_RI
- andc r10, r10, r9
- mtmsrd r10, 1
- mtspr SPRN_SRR0, r11
- mtspr SPRN_SRR1, r12
- RFI_TO_USER
- b . /* prevent speculative execution */
-#endif
-
-#ifdef CONFIG_PPC_BOOK3S
-_GLOBAL(ret_from_fork_scv)
- bl schedule_tail
- REST_NVGPRS(r1)
- li r3,0 /* fork() return value */
- b .Lsyscall_vectored_common_exit
-#endif
-
-_GLOBAL(ret_from_fork)
- bl schedule_tail
- REST_NVGPRS(r1)
- li r3,0 /* fork() return value */
- b .Lsyscall_exit
-
-_GLOBAL(ret_from_kernel_thread)
- bl schedule_tail
- REST_NVGPRS(r1)
- mtctr r14
- mr r3,r15
-#ifdef PPC64_ELF_ABI_v2
- mr r12,r14
-#endif
- bctrl
- li r3,0
- b .Lsyscall_exit
-
#ifdef CONFIG_PPC_BOOK3E
/* Save non-volatile GPRs, if not already saved. */
_GLOBAL(save_nvgprs)
@@ -681,202 +276,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
addi r1,r1,SWITCH_FRAME_SIZE
blr
-#ifdef CONFIG_PPC_BOOK3S
- /*
- * If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not
- * touched, no exit work created, then this can be used.
- */
- .balign IFETCH_ALIGN_BYTES
- .globl fast_interrupt_return_srr
-fast_interrupt_return_srr:
-_ASM_NOKPROBE_SYMBOL(fast_interrupt_return_srr)
- kuap_check_amr r3, r4
- ld r5,_MSR(r1)
- andi. r0,r5,MSR_PR
- bne .Lfast_user_interrupt_return_srr
- kuap_restore_amr r3, r4
- andi. r0,r5,MSR_RI
- li r3,0 /* 0 return value, no EMULATE_STACK_STORE */
- bne+ .Lfast_kernel_interrupt_return_srr
- addi r3,r1,STACK_FRAME_OVERHEAD
- bl unrecoverable_exception
- b . /* should not get here */
-
-.macro interrupt_return_macro srr
- .balign IFETCH_ALIGN_BYTES
- .globl interrupt_return_\srr
-interrupt_return_\srr\():
-_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\())
- ld r4,_MSR(r1)
- andi. r0,r4,MSR_PR
- beq .Lkernel_interrupt_return_\srr
- addi r3,r1,STACK_FRAME_OVERHEAD
- bl interrupt_exit_user_prepare
- cmpdi r3,0
- bne- .Lrestore_nvgprs_\srr
-
-.Lfast_user_interrupt_return_\srr:
-BEGIN_FTR_SECTION
- ld r10,_PPR(r1)
- mtspr SPRN_PPR,r10
-END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
- .ifc \srr,srr
- lbz r4,PACASRR_VALID(r13)
- .else
- lbz r4,PACAHSRR_VALID(r13)
- .endif
- cmpdi r4,0
- li r4,0
- bne 1f
- ld r11,_NIP(r1)
- ld r12,_MSR(r1)
- .ifc \srr,srr
- mtspr SPRN_SRR0,r11
- mtspr SPRN_SRR1,r12
-1:
- stb r4,PACASRR_VALID(r13)
- .else
- mtspr SPRN_HSRR0,r11
- mtspr SPRN_HSRR1,r12
-1:
- stb r4,PACAHSRR_VALID(r13)
- .endif
- DEBUG_SRR_VALID \srr
-
-BEGIN_FTR_SECTION
- stdcx. r0,0,r1 /* to clear the reservation */
-FTR_SECTION_ELSE
- ldarx r0,0,r1
-ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
-
- ld r3,_CCR(r1)
- ld r4,_LINK(r1)
- ld r5,_CTR(r1)
- ld r6,_XER(r1)
- li r0,0
-
- REST_4GPRS(7, r1)
- REST_2GPRS(11, r1)
- REST_GPR(13, r1)
-
- mtcr r3
- mtlr r4
- mtctr r5
- mtspr SPRN_XER,r6
-
- REST_4GPRS(2, r1)
- REST_GPR(6, r1)
- REST_GPR(0, r1)
- REST_GPR(1, r1)
- .ifc \srr,srr
- RFI_TO_USER
- .else
- HRFI_TO_USER
- .endif
- b . /* prevent speculative execution */
-
-.Lrestore_nvgprs_\srr\():
- REST_NVGPRS(r1)
- b .Lfast_user_interrupt_return_\srr
-
- .balign IFETCH_ALIGN_BYTES
-.Lkernel_interrupt_return_\srr\():
- addi r3,r1,STACK_FRAME_OVERHEAD
- bl interrupt_exit_kernel_prepare
-
-.Lfast_kernel_interrupt_return_\srr\():
- cmpdi cr1,r3,0
- .ifc \srr,srr
- lbz r4,PACASRR_VALID(r13)
- .else
- lbz r4,PACAHSRR_VALID(r13)
- .endif
- cmpdi r4,0
- li r4,0
- bne 1f
- ld r11,_NIP(r1)
- ld r12,_MSR(r1)
- .ifc \srr,srr
- mtspr SPRN_SRR0,r11
- mtspr SPRN_SRR1,r12
-1:
- stb r4,PACASRR_VALID(r13)
- .else
- mtspr SPRN_HSRR0,r11
- mtspr SPRN_HSRR1,r12
-1:
- stb r4,PACAHSRR_VALID(r13)
- .endif
- DEBUG_SRR_VALID \srr
-
-BEGIN_FTR_SECTION
- stdcx. r0,0,r1 /* to clear the reservation */
-FTR_SECTION_ELSE
- ldarx r0,0,r1
-ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
-
- ld r3,_LINK(r1)
- ld r4,_CTR(r1)
- ld r5,_XER(r1)
- ld r6,_CCR(r1)
- li r0,0
-
- REST_4GPRS(7, r1)
- REST_2GPRS(11, r1)
-
- mtlr r3
- mtctr r4
- mtspr SPRN_XER,r5
-
- /*
- * Leaving a stale exception_marker on the stack can confuse
- * the reliable stack unwinder later on. Clear it.
- */
- std r0,STACK_FRAME_OVERHEAD-16(r1)
-
- REST_4GPRS(2, r1)
-
- bne- cr1,1f /* emulate stack store */
- mtcr r6
- REST_GPR(6, r1)
- REST_GPR(0, r1)
- REST_GPR(1, r1)
- .ifc \srr,srr
- RFI_TO_KERNEL
- .else
- HRFI_TO_KERNEL
- .endif
- b . /* prevent speculative execution */
-
-1: /*
- * Emulate stack store with update. New r1 value was already calculated
- * and updated in our interrupt regs by emulate_loadstore, but we can't
- * store the previous value of r1 to the stack before re-loading our
- * registers from it, otherwise they could be clobbered. Use
- * PACA_EXGEN as temporary storage to hold the store data, as
- * interrupts are disabled here so it won't be clobbered.
- */
- mtcr r6
- std r9,PACA_EXGEN+0(r13)
- addi r9,r1,INT_FRAME_SIZE /* get original r1 */
- REST_GPR(6, r1)
- REST_GPR(0, r1)
- REST_GPR(1, r1)
- std r9,0(r1) /* perform store component of stdu */
- ld r9,PACA_EXGEN+0(r13)
-
- .ifc \srr,srr
- RFI_TO_KERNEL
- .else
- HRFI_TO_KERNEL
- .endif
- b . /* prevent speculative execution */
-.endm
-
-interrupt_return_macro srr
-interrupt_return_macro hsrr
-#endif /* CONFIG_PPC_BOOK3S */
-
#ifdef CONFIG_PPC_RTAS
/*
* On CHRP, the Run-Time Abstraction Services (RTAS) have to be
diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S
index 4e2591cb4bd1..b13a2d79dfb4 100644
--- a/arch/powerpc/kernel/head_64.S
+++ b/arch/powerpc/kernel/head_64.S
@@ -189,8 +189,9 @@ CLOSE_FIXED_SECTION(first_256B)
/* This value is used to mark exception frames on the stack. */
.section ".toc","aw"
+/* This value is used to mark exception frames on the stack. */
exception_marker:
- .tc ID_72656773_68657265[TC],0x7265677368657265
+ .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
.previous
/*
@@ -206,6 +207,8 @@ OPEN_TEXT_SECTION(0x100)
USE_TEXT_SECTION()
+#include "interrupt_64.S"
+
#ifdef CONFIG_PPC_BOOK3E
/*
* The booting_thread_hwid holds the thread id we want to boot in cpu
diff --git a/arch/powerpc/kernel/interrupt_64.S b/arch/powerpc/kernel/interrupt_64.S
new file mode 100644
index 000000000000..e121829ef717
--- /dev/null
+++ b/arch/powerpc/kernel/interrupt_64.S
@@ -0,0 +1,608 @@
+#include <asm/ppc_asm.h>
+#include <asm/head-64.h>
+#include <asm/bug.h>
+#include <asm/hw_irq.h>
+#include <asm/tm.h>
+#include <asm/mmu.h>
+#include <asm/asm-offsets.h>
+#include <asm/exception-64s.h>
+#include <asm/ptrace.h>
+#include <asm/head-64.h>
+#include <asm/feature-fixups.h>
+#include <asm/kup.h>
+
+ .section ".toc","aw"
+SYS_CALL_TABLE:
+ .tc sys_call_table[TC],sys_call_table
+
+#ifdef CONFIG_COMPAT
+COMPAT_SYS_CALL_TABLE:
+ .tc compat_sys_call_table[TC],compat_sys_call_table
+#endif
+ .previous
+
+#ifdef CONFIG_PPC_BOOK3S
+.macro DEBUG_SRR_VALID srr
+#ifdef CONFIG_PPC_RFI_SRR_DEBUG
+ .ifc \srr,srr
+ mfspr r11,SPRN_SRR0
+ ld r12,_NIP(r1)
+100: tdne r11,r12
+ EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
+ mfspr r11,SPRN_SRR1
+ ld r12,_MSR(r1)
+100: tdne r11,r12
+ EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
+ .else
+ mfspr r11,SPRN_HSRR0
+ ld r12,_NIP(r1)
+100: tdne r11,r12
+ EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
+ mfspr r11,SPRN_HSRR1
+ ld r12,_MSR(r1)
+100: tdne r11,r12
+ EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
+ .endif
+#endif
+.endm
+
+.macro system_call_vectored name trapnr
+ .globl system_call_vectored_\name
+system_call_vectored_\name:
+_ASM_NOKPROBE_SYMBOL(system_call_vectored_\name)
+#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
+BEGIN_FTR_SECTION
+ extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
+ bne .Ltabort_syscall
+END_FTR_SECTION_IFSET(CPU_FTR_TM)
+#endif
+ INTERRUPT_TO_KERNEL
+ mr r10,r1
+ ld r1,PACAKSAVE(r13)
+ std r10,0(r1)
+ std r11,_NIP(r1)
+ std r12,_MSR(r1)
+ std r0,GPR0(r1)
+ std r10,GPR1(r1)
+ std r2,GPR2(r1)
+ ld r2,PACATOC(r13)
+ mfcr r12
+ li r11,0
+ /* Can we avoid saving r3-r8 in common case? */
+ std r3,GPR3(r1)
+ std r4,GPR4(r1)
+ std r5,GPR5(r1)
+ std r6,GPR6(r1)
+ std r7,GPR7(r1)
+ std r8,GPR8(r1)
+ /* Zero r9-r12, this should only be required when restoring all GPRs */
+ std r11,GPR9(r1)
+ std r11,GPR10(r1)
+ std r11,GPR11(r1)
+ std r11,GPR12(r1)
+ std r9,GPR13(r1)
+ SAVE_NVGPRS(r1)
+ std r11,_XER(r1)
+ std r11,_LINK(r1)
+ std r11,_CTR(r1)
+
+ li r11,\trapnr
+ std r11,_TRAP(r1)
+ std r12,_CCR(r1)
+ std r3,ORIG_GPR3(r1)
+ addi r10,r1,STACK_FRAME_OVERHEAD
+ ld r11,exception_marker@toc(r2)
+ std r11,-16(r10) /* "regshere" marker */
+
+BEGIN_FTR_SECTION
+ HMT_MEDIUM
+END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
+
+ /*
+ * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
+ * would clobber syscall parameters. Also we always enter with IRQs
+ * enabled and nothing pending. system_call_exception() will call
+ * trace_hardirqs_off().
+ *
+ * scv enters with MSR[EE]=1, so don't set PACA_IRQ_HARD_DIS. The
+ * entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED.
+ */
+
+ /* Calling convention has r9 = orig r0, r10 = regs */
+ mr r9,r0
+ bl system_call_exception
+
+.Lsyscall_vectored_\name\()_exit:
+ addi r4,r1,STACK_FRAME_OVERHEAD
+ li r5,1 /* scv */
+ bl syscall_exit_prepare
+
+ ld r2,_CCR(r1)
+ ld r4,_NIP(r1)
+ ld r5,_MSR(r1)
+
+BEGIN_FTR_SECTION
+ stdcx. r0,0,r1 /* to clear the reservation */
+END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
+
+BEGIN_FTR_SECTION
+ HMT_MEDIUM_LOW
+END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
+
+ cmpdi r3,0
+ bne .Lsyscall_vectored_\name\()_restore_regs
+
+ /* rfscv returns with LR->NIA and CTR->MSR */
+ mtlr r4
+ mtctr r5
+
+ /* Could zero these as per ABI, but we may consider a stricter ABI
+ * which preserves these if libc implementations can benefit, so
+ * restore them for now until further measurement is done. */
+ ld r0,GPR0(r1)
+ ld r4,GPR4(r1)
+ ld r5,GPR5(r1)
+ ld r6,GPR6(r1)
+ ld r7,GPR7(r1)
+ ld r8,GPR8(r1)
+ /* Zero volatile regs that may contain sensitive kernel data */
+ li r9,0
+ li r10,0
+ li r11,0
+ li r12,0
+ mtspr SPRN_XER,r0
+
+ /*
+ * We don't need to restore AMR on the way back to userspace for KUAP.
+ * The value of AMR only matters while we're in the kernel.
+ */
+ mtcr r2
+ ld r2,GPR2(r1)
+ ld r3,GPR3(r1)
+ ld r13,GPR13(r1)
+ ld r1,GPR1(r1)
+ RFSCV_TO_USER
+ b . /* prevent speculative execution */
+
+.Lsyscall_vectored_\name\()_restore_regs:
+ li r3,0
+ mtmsrd r3,1
+ mtspr SPRN_SRR0,r4
+ mtspr SPRN_SRR1,r5
+
+ ld r3,_CTR(r1)
+ ld r4,_LINK(r1)
+ ld r5,_XER(r1)
+
+ REST_NVGPRS(r1)
+ ld r0,GPR0(r1)
+ mtcr r2
+ mtctr r3
+ mtlr r4
+ mtspr SPRN_XER,r5
+ REST_10GPRS(2, r1)
+ REST_2GPRS(12, r1)
+ ld r1,GPR1(r1)
+ RFI_TO_USER
+.endm
+
+system_call_vectored common 0x3000
+/*
+ * We instantiate another entry copy for the SIGILL variant, with TRAP=0x7ff0
+ * which is tested by system_call_exception when r0 is -1 (as set by vector
+ * entry code).
+ */
+system_call_vectored sigill 0x7ff0
+
+
+/*
+ * Entered via kernel return set up by kernel/sstep.c, must match entry regs
+ */
+ .globl system_call_vectored_emulate
+system_call_vectored_emulate:
+_ASM_NOKPROBE_SYMBOL(system_call_vectored_emulate)
+ li r10,IRQS_ALL_DISABLED
+ stb r10,PACAIRQSOFTMASK(r13)
+ b system_call_vectored_common
+#endif /* CONFIG_PPC_BOOK3S */
+
+ .balign IFETCH_ALIGN_BYTES
+ .globl system_call_common_real
+system_call_common_real:
+ ld r10,PACAKMSR(r13) /* get MSR value for kernel */
+ mtmsrd r10
+
+ .balign IFETCH_ALIGN_BYTES
+ .globl system_call_common
+system_call_common:
+_ASM_NOKPROBE_SYMBOL(system_call_common)
+#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
+BEGIN_FTR_SECTION
+ extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
+ bne .Ltabort_syscall
+END_FTR_SECTION_IFSET(CPU_FTR_TM)
+#endif
+ mr r10,r1
+ ld r1,PACAKSAVE(r13)
+ std r10,0(r1)
+ std r11,_NIP(r1)
+ std r12,_MSR(r1)
+ std r0,GPR0(r1)
+ std r10,GPR1(r1)
+ std r2,GPR2(r1)
+#ifdef CONFIG_PPC_FSL_BOOK3E
+START_BTB_FLUSH_SECTION
+ BTB_FLUSH(r10)
+END_BTB_FLUSH_SECTION
+#endif
+ ld r2,PACATOC(r13)
+ mfcr r12
+ li r11,0
+ /* Can we avoid saving r3-r8 in common case? */
+ std r3,GPR3(r1)
+ std r4,GPR4(r1)
+ std r5,GPR5(r1)
+ std r6,GPR6(r1)
+ std r7,GPR7(r1)
+ std r8,GPR8(r1)
+ /* Zero r9-r12, this should only be required when restoring all GPRs */
+ std r11,GPR9(r1)
+ std r11,GPR10(r1)
+ std r11,GPR11(r1)
+ std r11,GPR12(r1)
+ std r9,GPR13(r1)
+ SAVE_NVGPRS(r1)
+ std r11,_XER(r1)
+ std r11,_CTR(r1)
+ mflr r10
+
+ /*
+ * This clears CR0.SO (bit 28), which is the error indication on
+ * return from this system call.
+ */
+ rldimi r12,r11,28,(63-28)
+ li r11,0xc00
+ std r10,_LINK(r1)
+ std r11,_TRAP(r1)
+ std r12,_CCR(r1)
+ std r3,ORIG_GPR3(r1)
+ addi r10,r1,STACK_FRAME_OVERHEAD
+ ld r11,exception_marker@toc(r2)
+ std r11,-16(r10) /* "regshere" marker */
+
+ li r11,1
+ stb r11,PACASRR_VALID(r13)
+
+ /*
+ * RECONCILE_IRQ_STATE without calling trace_hardirqs_off(), which
+ * would clobber syscall parameters. Also we always enter with IRQs
+ * enabled and nothing pending. system_call_exception() will call
+ * trace_hardirqs_off().
+ */
+ li r11,IRQS_ALL_DISABLED
+ li r12,PACA_IRQ_HARD_DIS
+ stb r11,PACAIRQSOFTMASK(r13)
+ stb r12,PACAIRQHAPPENED(r13)
+
+ /* Calling convention has r9 = orig r0, r10 = regs */
+ mr r9,r0
+ bl system_call_exception
+
+.Lsyscall_exit:
+ addi r4,r1,STACK_FRAME_OVERHEAD
+ li r5,0 /* !scv */
+ bl syscall_exit_prepare
+
+ ld r2,_CCR(r1)
+ ld r6,_LINK(r1)
+ mtlr r6
+
+ lbz r4,PACASRR_VALID(r13)
+ cmpdi r4,0
+ bne 1f
+ li r4,0
+ stb r4,PACASRR_VALID(r13)
+ ld r4,_NIP(r1)
+ ld r5,_MSR(r1)
+ mtspr SPRN_SRR0,r4
+ mtspr SPRN_SRR1,r5
+1:
+ DEBUG_SRR_VALID srr
+
+BEGIN_FTR_SECTION
+ stdcx. r0,0,r1 /* to clear the reservation */
+END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
+
+ cmpdi r3,0
+ bne .Lsyscall_restore_regs
+ /* Zero volatile regs that may contain sensitive kernel data */
+ li r0,0
+ li r4,0
+ li r5,0
+ li r6,0
+ li r7,0
+ li r8,0
+ li r9,0
+ li r10,0
+ li r11,0
+ li r12,0
+ mtctr r0
+ mtspr SPRN_XER,r0
+.Lsyscall_restore_regs_cont:
+
+BEGIN_FTR_SECTION
+ HMT_MEDIUM_LOW
+END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
+
+ /*
+ * We don't need to restore AMR on the way back to userspace for KUAP.
+ * The value of AMR only matters while we're in the kernel.
+ */
+ mtcr r2
+ ld r2,GPR2(r1)
+ ld r3,GPR3(r1)
+ ld r13,GPR13(r1)
+ ld r1,GPR1(r1)
+ RFI_TO_USER
+ b . /* prevent speculative execution */
+
+.Lsyscall_restore_regs:
+ ld r3,_CTR(r1)
+ ld r4,_XER(r1)
+ REST_NVGPRS(r1)
+ mtctr r3
+ mtspr SPRN_XER,r4
+ ld r0,GPR0(r1)
+ REST_8GPRS(4, r1)
+ ld r12,GPR12(r1)
+ b .Lsyscall_restore_regs_cont
+
+#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
+.Ltabort_syscall:
+ /* Firstly we need to enable TM in the kernel */
+ mfmsr r10
+ li r9, 1
+ rldimi r10, r9, MSR_TM_LG, 63-MSR_TM_LG
+ mtmsrd r10, 0
+
+ /* tabort, this dooms the transaction, nothing else */
+ li r9, (TM_CAUSE_SYSCALL|TM_CAUSE_PERSISTENT)
+ TABORT(R9)
+
+ /*
+ * Return directly to userspace. We have corrupted user register state,
+ * but userspace will never see that register state. Execution will
+ * resume after the tbegin of the aborted transaction with the
+ * checkpointed register state.
+ */
+ li r9, MSR_RI
+ andc r10, r10, r9
+ mtmsrd r10, 1
+ mtspr SPRN_SRR0, r11
+ mtspr SPRN_SRR1, r12
+ RFI_TO_USER
+ b . /* prevent speculative execution */
+#endif
+
+#ifdef CONFIG_PPC_BOOK3S
+_GLOBAL(ret_from_fork_scv)
+ bl schedule_tail
+ REST_NVGPRS(r1)
+ li r3,0 /* fork() return value */
+ b .Lsyscall_vectored_common_exit
+#endif
+
+_GLOBAL(ret_from_fork)
+ bl schedule_tail
+ REST_NVGPRS(r1)
+ li r3,0 /* fork() return value */
+ b .Lsyscall_exit
+
+_GLOBAL(ret_from_kernel_thread)
+ bl schedule_tail
+ REST_NVGPRS(r1)
+ mtctr r14
+ mr r3,r15
+#ifdef PPC64_ELF_ABI_v2
+ mr r12,r14
+#endif
+ bctrl
+ li r3,0
+ b .Lsyscall_exit
+
+#ifdef CONFIG_PPC_BOOK3S
+ /*
+ * If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not
+ * touched, no exit work created, then this can be used.
+ */
+ .balign IFETCH_ALIGN_BYTES
+ .globl fast_interrupt_return_srr
+fast_interrupt_return_srr:
+_ASM_NOKPROBE_SYMBOL(fast_interrupt_return_srr)
+ kuap_check_amr r3, r4
+ ld r5,_MSR(r1)
+ andi. r0,r5,MSR_PR
+ bne .Lfast_user_interrupt_return_srr
+ kuap_restore_amr r3, r4
+ andi. r0,r5,MSR_RI
+ li r3,0 /* 0 return value, no EMULATE_STACK_STORE */
+ bne+ .Lfast_kernel_interrupt_return_srr
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl unrecoverable_exception
+ b . /* should not get here */
+
+.macro interrupt_return_macro srr
+ .balign IFETCH_ALIGN_BYTES
+ .globl interrupt_return_\srr
+interrupt_return_\srr\():
+_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\())
+ ld r4,_MSR(r1)
+ andi. r0,r4,MSR_PR
+ beq .Lkernel_interrupt_return_\srr
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl interrupt_exit_user_prepare
+ cmpdi r3,0
+ bne- .Lrestore_nvgprs_\srr
+
+.Lfast_user_interrupt_return_\srr:
+BEGIN_FTR_SECTION
+ ld r10,_PPR(r1)
+ mtspr SPRN_PPR,r10
+END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
+ .ifc \srr,srr
+ lbz r4,PACASRR_VALID(r13)
+ .else
+ lbz r4,PACAHSRR_VALID(r13)
+ .endif
+ cmpdi r4,0
+ li r4,0
+ bne 1f
+ ld r11,_NIP(r1)
+ ld r12,_MSR(r1)
+ .ifc \srr,srr
+ mtspr SPRN_SRR0,r11
+ mtspr SPRN_SRR1,r12
+1:
+ stb r4,PACASRR_VALID(r13)
+ .else
+ mtspr SPRN_HSRR0,r11
+ mtspr SPRN_HSRR1,r12
+1:
+ stb r4,PACAHSRR_VALID(r13)
+ .endif
+ DEBUG_SRR_VALID \srr
+
+BEGIN_FTR_SECTION
+ stdcx. r0,0,r1 /* to clear the reservation */
+FTR_SECTION_ELSE
+ ldarx r0,0,r1
+ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
+
+ ld r3,_CCR(r1)
+ ld r4,_LINK(r1)
+ ld r5,_CTR(r1)
+ ld r6,_XER(r1)
+ li r0,0
+
+ REST_4GPRS(7, r1)
+ REST_2GPRS(11, r1)
+ REST_GPR(13, r1)
+
+ mtcr r3
+ mtlr r4
+ mtctr r5
+ mtspr SPRN_XER,r6
+
+ REST_4GPRS(2, r1)
+ REST_GPR(6, r1)
+ REST_GPR(0, r1)
+ REST_GPR(1, r1)
+ .ifc \srr,srr
+ RFI_TO_USER
+ .else
+ HRFI_TO_USER
+ .endif
+ b . /* prevent speculative execution */
+
+.Lrestore_nvgprs_\srr\():
+ REST_NVGPRS(r1)
+ b .Lfast_user_interrupt_return_\srr
+
+ .balign IFETCH_ALIGN_BYTES
+.Lkernel_interrupt_return_\srr\():
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl interrupt_exit_kernel_prepare
+
+.Lfast_kernel_interrupt_return_\srr\():
+ cmpdi cr1,r3,0
+ .ifc \srr,srr
+ lbz r4,PACASRR_VALID(r13)
+ .else
+ lbz r4,PACAHSRR_VALID(r13)
+ .endif
+ cmpdi r4,0
+ li r4,0
+ bne 1f
+ ld r11,_NIP(r1)
+ ld r12,_MSR(r1)
+ .ifc \srr,srr
+ mtspr SPRN_SRR0,r11
+ mtspr SPRN_SRR1,r12
+1:
+ stb r4,PACASRR_VALID(r13)
+ .else
+ mtspr SPRN_HSRR0,r11
+ mtspr SPRN_HSRR1,r12
+1:
+ stb r4,PACAHSRR_VALID(r13)
+ .endif
+ DEBUG_SRR_VALID \srr
+
+BEGIN_FTR_SECTION
+ stdcx. r0,0,r1 /* to clear the reservation */
+FTR_SECTION_ELSE
+ ldarx r0,0,r1
+ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
+
+ ld r3,_LINK(r1)
+ ld r4,_CTR(r1)
+ ld r5,_XER(r1)
+ ld r6,_CCR(r1)
+ li r0,0
+
+ REST_4GPRS(7, r1)
+ REST_2GPRS(11, r1)
+
+ mtlr r3
+ mtctr r4
+ mtspr SPRN_XER,r5
+
+ /*
+ * Leaving a stale exception_marker on the stack can confuse
+ * the reliable stack unwinder later on. Clear it.
+ */
+ std r0,STACK_FRAME_OVERHEAD-16(r1)
+
+ REST_4GPRS(2, r1)
+
+ bne- cr1,1f /* emulate stack store */
+ mtcr r6
+ REST_GPR(6, r1)
+ REST_GPR(0, r1)
+ REST_GPR(1, r1)
+ .ifc \srr,srr
+ RFI_TO_KERNEL
+ .else
+ HRFI_TO_KERNEL
+ .endif
+ b . /* prevent speculative execution */
+
+1: /*
+ * Emulate stack store with update. New r1 value was already calculated
+ * and updated in our interrupt regs by emulate_loadstore, but we can't
+ * store the previous value of r1 to the stack before re-loading our
+ * registers from it, otherwise they could be clobbered. Use
+ * PACA_EXGEN as temporary storage to hold the store data, as
+ * interrupts are disabled here so it won't be clobbered.
+ */
+ mtcr r6
+ std r9,PACA_EXGEN+0(r13)
+ addi r9,r1,INT_FRAME_SIZE /* get original r1 */
+ REST_GPR(6, r1)
+ REST_GPR(0, r1)
+ REST_GPR(1, r1)
+ std r9,0(r1) /* perform store component of stdu */
+ ld r9,PACA_EXGEN+0(r13)
+
+ .ifc \srr,srr
+ RFI_TO_KERNEL
+ .else
+ HRFI_TO_KERNEL
+ .endif
+ b . /* prevent speculative execution */
+.endm
+
+interrupt_return_macro srr
+interrupt_return_macro hsrr
+
+#endif /* CONFIG_PPC_BOOK3S */
--
2.23.0
^ permalink raw reply related
* [RFC PATCH 6/9] powerpc/64s: save one more register in the masked interrupt handler
From: Nicholas Piggin @ 2020-11-06 15:59 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
In-Reply-To: <20201106155929.2246055-1-npiggin@gmail.com>
This frees up one more register (and takes advantage of that to
clean things up a little bit).
This register will be used in the following patch.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/kernel/exceptions-64s.S | 34 ++++++++++++++++------------
1 file changed, 20 insertions(+), 14 deletions(-)
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index b370a5f334fc..a01f69e774b5 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -2745,7 +2745,6 @@ INT_DEFINE_END(soft_nmi)
* and run it entirely with interrupts hard disabled.
*/
EXC_COMMON_BEGIN(soft_nmi_common)
- mfspr r11,SPRN_SRR0
mr r10,r1
ld r1,PACAEMERGSP(r13)
subi r1,r1,INT_FRAME_SIZE
@@ -2780,19 +2779,24 @@ masked_Hinterrupt:
.else
masked_interrupt:
.endif
- lbz r11,PACAIRQHAPPENED(r13)
- or r11,r11,r10
- stb r11,PACAIRQHAPPENED(r13)
+ stw r9,PACA_EXGEN+EX_CCR(r13)
+ lbz r9,PACAIRQHAPPENED(r13)
+ or r9,r9,r10
+ stb r9,PACAIRQHAPPENED(r13)
+
+ .if ! \hsrr
cmpwi r10,PACA_IRQ_DEC
bne 1f
- lis r10,0x7fff
- ori r10,r10,0xffff
- mtspr SPRN_DEC,r10
+ LOAD_REG_IMMEDIATE(r9, 0x7fffffff)
+ mtspr SPRN_DEC,r9
#ifdef CONFIG_PPC_WATCHDOG
+ lwz r9,PACA_EXGEN+EX_CCR(r13)
b soft_nmi_common
#else
b 2f
#endif
+ .endif
+
1: andi. r10,r10,PACA_IRQ_MUST_HARD_MASK
beq 2f
xori r12,r12,MSR_EE /* clear MSR_EE */
@@ -2801,17 +2805,19 @@ masked_interrupt:
.else
mtspr SPRN_SRR1,r12
.endif
- ori r11,r11,PACA_IRQ_HARD_DIS
- stb r11,PACAIRQHAPPENED(r13)
+ ori r9,r9,PACA_IRQ_HARD_DIS
+ stb r9,PACAIRQHAPPENED(r13)
2: /* done */
- li r10,0
+ li r9,0
.if \hsrr
- stb r10,PACAHSRR_VALID(r13)
+ stb r9,PACAHSRR_VALID(r13)
.else
- stb r10,PACASRR_VALID(r13)
+ stb r9,PACASRR_VALID(r13)
.endif
- ld r10,PACA_EXGEN+EX_CTR(r13)
- mtctr r10
+
+ ld r9,PACA_EXGEN+EX_CTR(r13)
+ mtctr r9
+ lwz r9,PACA_EXGEN+EX_CCR(r13)
mtcrf 0x80,r9
std r1,PACAR1(r13)
ld r9,PACA_EXGEN+EX_R9(r13)
--
2.23.0
^ permalink raw reply related
* [RFC PATCH 7/9] powerpc/64s: allow alternate return locations for soft-masked interrupts
From: Nicholas Piggin @ 2020-11-06 15:59 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
In-Reply-To: <20201106155929.2246055-1-npiggin@gmail.com>
This is a variation of the exception table code which adjusts a failed
page fault return location if it was taken at an address specified in
an exception table, to a corresponding fixup handler address.
This patch adds a similar masked interrupt restart table that is checked
when when an asynchronous interrupt is taken while soft-masked.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/interrupt.h | 18 ++++++++++++++
arch/powerpc/include/asm/ppc_asm.h | 8 +++++++
arch/powerpc/kernel/exceptions-64s.S | 36 +++++++++++++++++++++++++++-
arch/powerpc/kernel/interrupt_64.S | 3 +++
arch/powerpc/kernel/vmlinux.lds.S | 10 ++++++++
arch/powerpc/lib/Makefile | 2 +-
arch/powerpc/lib/restart_table.c | 26 ++++++++++++++++++++
arch/powerpc/perf/core-book3s.c | 19 ++++++++++++---
8 files changed, 117 insertions(+), 5 deletions(-)
create mode 100644 arch/powerpc/lib/restart_table.c
diff --git a/arch/powerpc/include/asm/interrupt.h b/arch/powerpc/include/asm/interrupt.h
index 1aeb9a841cc6..5d68c510e5e0 100644
--- a/arch/powerpc/include/asm/interrupt.h
+++ b/arch/powerpc/include/asm/interrupt.h
@@ -18,6 +18,11 @@ static inline void nap_adjust_return(struct pt_regs *regs)
#endif
}
+#ifdef CONFIG_PPC_BOOK3S_64
+extern char __end_soft_masked[];
+unsigned long search_kernel_restart_table(unsigned long addr);
+#endif
+
struct interrupt_state {
#ifdef CONFIG_PPC_BOOK3E_64
enum ctx_state ctx_state;
@@ -47,6 +52,9 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup
*/
if (TRAP(regs) != 0x700)
CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
+ BUG_ON(regs->nip < (unsigned long)__end_soft_masked));
+ if (arch_irq_disabled_regs(regs))
+ BUG_ON(search_kernel_restart_table(regs->nip));
}
#endif
@@ -126,6 +134,8 @@ static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inte
{
#ifdef CONFIG_PPC64
#ifdef CONFIG_PPC_BOOK3S_64
+ if (!(regs->msr & MSR_PR) && regs->nip < (unsigned long)__end_soft_masked)
+ regs->softe = IRQS_ALL_DISABLED;
state->irq_soft_mask = local_paca->irq_soft_mask;
state->irq_happened = local_paca->irq_happened;
@@ -163,6 +173,14 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter
nap_adjust_return(regs);
+#ifdef CONFIG_PPC_BOOK3S_64
+ if (arch_irq_disabled_regs(regs)) {
+ unsigned long rst = search_kernel_restart_table(regs->nip);
+ if (rst)
+ regs_set_return_ip(regs, rst);
+ }
+#endif
+
#ifdef CONFIG_PPC64
this_cpu_set_ftrace_enabled(state->ftrace_enabled);
diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h
index c1199f6c75a3..da084d695cfe 100644
--- a/arch/powerpc/include/asm/ppc_asm.h
+++ b/arch/powerpc/include/asm/ppc_asm.h
@@ -791,6 +791,14 @@ END_FTR_SECTION_NESTED(CPU_FTR_CELL_TB_BUG, CPU_FTR_CELL_TB_BUG, 96)
stringify_in_c(.long (_target) - . ;) \
stringify_in_c(.previous)
+#define RESTART_TABLE(_start, _end, _target) \
+ stringify_in_c(.section __restart_table,"a";)\
+ stringify_in_c(.balign 8;) \
+ stringify_in_c(.llong (_start);) \
+ stringify_in_c(.llong (_end);) \
+ stringify_in_c(.llong (_target);) \
+ stringify_in_c(.previous)
+
#ifdef CONFIG_PPC_FSL_BOOK3E
#define BTB_FLUSH(reg) \
lis reg,BUCSR_INIT@h; \
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index a01f69e774b5..0615c2e724ea 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -514,8 +514,9 @@ DEFINE_FIXED_SYMBOL(\name\()_common_real)
/* Kernel code running below __end_interrupts is implicitly
* soft-masked */
- LOAD_HANDLER(r10, __end_interrupts)
+ LOAD_HANDLER(r10, __end_soft_masked)
cmpld r11,r10
+
li r10,IMASK
blt- 1f
@@ -673,6 +674,28 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
__GEN_COMMON_BODY \name
.endm
+.macro SEARCH_RESTART_TABLE
+ LOAD_REG_IMMEDIATE_SYM(r9, r12, __start___restart_table)
+ LOAD_REG_IMMEDIATE_SYM(r10, r12, __stop___restart_table)
+300:
+ cmpd r9,r10
+ beq 302f
+ ld r12,0(r9)
+ cmpld r11,r12
+ blt 301f
+ ld r12,8(r9)
+ cmpld r11,r12
+ bge 301f
+ ld r12,16(r9)
+ b 303f
+301:
+ addi r9,r9,24
+ b 300b
+302:
+ li r12,0
+303:
+.endm
+
/*
* Restore all registers including H/SRR0/1 saved in a stack frame of a
* standard exception.
@@ -2758,6 +2781,7 @@ EXC_COMMON_BEGIN(soft_nmi_common)
mtmsrd r9,1
kuap_restore_amr r9, r10
+
EXCEPTION_RESTORE_REGS hsrr=0
RFI_TO_KERNEL
@@ -2815,6 +2839,16 @@ masked_interrupt:
stb r9,PACASRR_VALID(r13)
.endif
+ SEARCH_RESTART_TABLE
+ cmpdi r12,0
+ beq 3f
+ .if \hsrr
+ mtspr SPRN_HSRR0,r12
+ .else
+ mtspr SPRN_SRR0,r12
+ .endif
+3:
+
ld r9,PACA_EXGEN+EX_CTR(r13)
mtctr r9
lwz r9,PACA_EXGEN+EX_CCR(r13)
diff --git a/arch/powerpc/kernel/interrupt_64.S b/arch/powerpc/kernel/interrupt_64.S
index e121829ef717..9b44f6d3463b 100644
--- a/arch/powerpc/kernel/interrupt_64.S
+++ b/arch/powerpc/kernel/interrupt_64.S
@@ -605,4 +605,7 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
interrupt_return_macro srr
interrupt_return_macro hsrr
+ .globl __end_soft_masked
+__end_soft_masked:
+DEFINE_FIXED_SYMBOL(__end_soft_masked)
#endif /* CONFIG_PPC_BOOK3S */
diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S
index e0548b4950de..211c82e96420 100644
--- a/arch/powerpc/kernel/vmlinux.lds.S
+++ b/arch/powerpc/kernel/vmlinux.lds.S
@@ -9,6 +9,14 @@
#define EMITS_PT_NOTE
#define RO_EXCEPTION_TABLE_ALIGN 0
+#define RESTART_TABLE(align) \
+ . = ALIGN(align); \
+ __restart_table : AT(ADDR(__restart_table) - LOAD_OFFSET) { \
+ __start___restart_table = .; \
+ KEEP(*(__restart_table)) \
+ __stop___restart_table = .; \
+ }
+
#include <asm/page.h>
#include <asm-generic/vmlinux.lds.h>
#include <asm/cache.h>
@@ -124,6 +132,8 @@ SECTIONS
RO_DATA(PAGE_SIZE)
#ifdef CONFIG_PPC64
+ RESTART_TABLE(8)
+
. = ALIGN(8);
__stf_entry_barrier_fixup : AT(ADDR(__stf_entry_barrier_fixup) - LOAD_OFFSET) {
__start___stf_entry_barrier_fixup = .;
diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile
index 69a91b571845..5d90bebcf9cf 100644
--- a/arch/powerpc/lib/Makefile
+++ b/arch/powerpc/lib/Makefile
@@ -36,7 +36,7 @@ extra-$(CONFIG_PPC64) += crtsavres.o
endif
obj-$(CONFIG_PPC_BOOK3S_64) += copyuser_power7.o copypage_power7.o \
- memcpy_power7.o
+ memcpy_power7.o restart_table.o
obj64-y += copypage_64.o copyuser_64.o mem_64.o hweight_64.o \
memcpy_64.o copy_mc_64.o
diff --git a/arch/powerpc/lib/restart_table.c b/arch/powerpc/lib/restart_table.c
new file mode 100644
index 000000000000..f2df84c46f24
--- /dev/null
+++ b/arch/powerpc/lib/restart_table.c
@@ -0,0 +1,26 @@
+struct restart_table_entry {
+ unsigned long start;
+ unsigned long end;
+ unsigned long fixup;
+};
+
+extern struct restart_table_entry __start___restart_table[];
+extern struct restart_table_entry __stop___restart_table[];
+
+/* Given an address, look for it in the kernel exception table */
+unsigned long search_kernel_restart_table(unsigned long addr)
+{
+ struct restart_table_entry *rte = __start___restart_table;
+
+ while (rte < __stop___restart_table) {
+ unsigned long start = rte->start;
+ unsigned long end = rte->end;
+ unsigned long fixup = rte->fixup;
+
+ if (addr >= start && addr < end)
+ return fixup;
+
+ rte++;
+ }
+ return 0;
+}
diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c
index 08643cba1494..4afd292a7a5e 100644
--- a/arch/powerpc/perf/core-book3s.c
+++ b/arch/powerpc/perf/core-book3s.c
@@ -332,9 +332,15 @@ static inline void perf_read_regs(struct pt_regs *regs)
* If interrupts were soft-disabled when a PMU interrupt occurs, treat
* it as an NMI.
*/
+extern char __end_soft_masked[];
static inline int perf_intr_is_nmi(struct pt_regs *regs)
{
- return (regs->softe & IRQS_DISABLED);
+ if (regs->softe & IRQS_DISABLED)
+ return true;
+
+ if (!(regs->msr & MSR_PR) && regs->nip < (unsigned long)__end_soft_masked)
+ return true;
+ return false;
}
/*
@@ -2214,6 +2220,8 @@ static bool pmc_overflow(unsigned long val)
return false;
}
+unsigned long search_kernel_restart_table(unsigned long addr);
+
/*
* Performance monitor interrupt stuff
*/
@@ -2301,10 +2309,15 @@ static void __perf_event_interrupt(struct pt_regs *regs)
*/
write_mmcr0(cpuhw, cpuhw->mmcr.mmcr0);
- if (nmi)
+ if (nmi) {
+ unsigned long rst = search_kernel_restart_table(regs->nip);
+ if (rst)
+ regs_set_return_ip(regs, rst);
+
nmi_exit();
- else
+ } else {
irq_exit();
+ }
}
static void perf_event_interrupt(struct pt_regs *regs)
--
2.23.0
^ permalink raw reply related
* [RFC PATCH 8/9] powerpc/64s: interrupt soft-enable race fix
From: Nicholas Piggin @ 2020-11-06 15:59 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
In-Reply-To: <20201106155929.2246055-1-npiggin@gmail.com>
Prevent interrupt restore from allowing racing hard interrupts going
ahead of previous soft-pending ones, by using the soft-masked restart
handler to allow a store to clear the soft-mask while knowing nothing
is soft-pending.
This probably doesn't matter much in practice, but it's a simple
demonstrator / test case to exercise the restart table logic.
XXX: 64e can't do this unless it adds restart table support
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/kernel/irq.c | 79 +++++++++++++++++++++++++--------------
1 file changed, 50 insertions(+), 29 deletions(-)
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index c8185f709d26..9671468b2c51 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -317,52 +317,73 @@ notrace void arch_local_irq_restore(unsigned long mask)
unsigned char irq_happened;
/* Write the new soft-enabled value */
- irq_soft_mask_set(mask);
- if (mask)
+ if (mask) {
+ irq_soft_mask_set(mask);
return;
+ }
/*
- * From this point onward, we can take interrupts, preempt,
- * etc... unless we got hard-disabled. We check if an event
- * happened. If none happened, we know we can just return.
- *
- * We may have preempted before the check below, in which case
- * we are checking the "new" CPU instead of the old one. This
- * is only a problem if an event happened on the "old" CPU.
+ * After the stb, interrupts are unmasked and there are no interrupts
+ * pending replay. The restart sequence makes this atomic with
+ * respect to soft-masked interrupts. If this was just a simple code
+ * sequence, a soft-masked interrupt could become pending right after
+ * the comparison and before the stb.
*
- * External interrupt events will have caused interrupts to
- * be hard-disabled, so there is no problem, we
- * cannot have preempted.
+ * This allows interrupts to be unmasked without hard disabling, and
+ * also without new hard interrupts coming in ahead of pending ones.
*/
+ asm_volatile_goto(
+"1: \n"
+" lbz 9,%0(13) \n"
+" cmpwi 9,0 \n"
+" bne %l[happened] \n"
+" stb 9,%1(13) \n"
+"2: \n"
+ RESTART_TABLE(1b, 2b, 1b)
+ : : "i" (offsetof(struct paca_struct, irq_happened)),
+ "i" (offsetof(struct paca_struct, irq_soft_mask))
+ : "cr0", "r9"
+ : happened);
+
+ if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
+ WARN_ON_ONCE(!(mfmsr() & MSR_EE));
+
+ return;
+
+happened:
irq_happened = get_irq_happened();
- if (!irq_happened) {
+ if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
+ WARN_ON_ONCE(!irq_happened);
+
+ if (irq_happened == PACA_IRQ_HARD_DIS) {
if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
- WARN_ON_ONCE(!(mfmsr() & MSR_EE));
+ WARN_ON_ONCE(mfmsr() & MSR_EE);
+ irq_soft_mask_set(IRQS_ENABLED);
+ local_paca->irq_happened = 0;
+ __hard_irq_enable();
return;
}
- /* We need to hard disable to replay. */
+ /* Have interrupts to replay, need to hard disable first */
if (!(irq_happened & PACA_IRQ_HARD_DIS)) {
- if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
- WARN_ON_ONCE(!(mfmsr() & MSR_EE));
+ if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) {
+ if (!(mfmsr() & MSR_EE)) {
+ /*
+ * An interrupt could have come in and cleared
+ * MSR[EE] and set IRQ_HARD_DIS, so check
+ * IRQ_HARD_DIS again and warn if it is still
+ * clear.
+ */
+ irq_happened = get_irq_happened();
+ WARN_ON_ONCE(!(irq_happened & PACA_IRQ_HARD_DIS));
+ }
+ }
__hard_irq_disable();
} else {
- /*
- * We should already be hard disabled here. We had bugs
- * where that wasn't the case so let's dbl check it and
- * warn if we are wrong. Only do that when IRQ tracing
- * is enabled as mfmsr() can be costly.
- */
if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) {
if (WARN_ON_ONCE(mfmsr() & MSR_EE))
__hard_irq_disable();
}
-
- if (irq_happened == PACA_IRQ_HARD_DIS) {
- local_paca->irq_happened = 0;
- __hard_irq_enable();
- return;
- }
}
/*
--
2.23.0
^ permalink raw reply related
* [RFC PATCH 9/9] powerpc/64s: use interrupt restart table to speed up return from interrupt
From: Nicholas Piggin @ 2020-11-06 15:59 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
In-Reply-To: <20201106155929.2246055-1-npiggin@gmail.com>
The restart table facility is used to return from interrupt without
disabling MSR[EE] or MSR[RI].
Interrupt return code is put into the low soft-masked region, and
critical code that has return SRRs set, soft-masked state set with no
soft-pending interrupts to replay, and no exit work, is put into
alternate return sections. r1 is saved in the paca. If an interrupt hits
at this point, our fixup location re-loads r1 and from there we can get
to regs, critical things are re-loaded, and we go around and do the
exit sequence again.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/asm-prototypes.h | 4 +-
arch/powerpc/include/asm/paca.h | 1 +
arch/powerpc/include/asm/ptrace.h | 13 +-
arch/powerpc/kernel/asm-offsets.c | 3 +
arch/powerpc/kernel/interrupt_64.S | 119 ++++++++++-
arch/powerpc/kernel/syscall_64.c | 242 +++++++++++++---------
6 files changed, 277 insertions(+), 105 deletions(-)
diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h
index 939f3c94c8f3..e0b0fc2913c1 100644
--- a/arch/powerpc/include/asm/asm-prototypes.h
+++ b/arch/powerpc/include/asm/asm-prototypes.h
@@ -71,8 +71,8 @@ void __init machine_init(u64 dt_ptr);
#endif
long system_call_exception(long r3, long r4, long r5, long r6, long r7, long r8, unsigned long r0, struct pt_regs *regs);
notrace unsigned long syscall_exit_prepare(unsigned long r3, struct pt_regs *regs, long scv);
-notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr);
-notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsigned long msr);
+notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs);
+notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs);
long ppc_fadvise64_64(int fd, int advice, u32 offset_high, u32 offset_low,
u32 len_high, u32 len_low);
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 58e9995c3184..ab123dc85b81 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -167,6 +167,7 @@ struct paca_struct {
u64 kstack; /* Saved Kernel stack addr */
u64 saved_r1; /* r1 save for RTAS calls or PM or EE=0 */
u64 saved_msr; /* MSR saved here by enter_rtas */
+ u64 exit_save_r1; /* Syscall/interrupt R1 save */
#ifdef CONFIG_PPC_BOOK3E
u16 trap_save; /* Used when bad stack is encountered */
#endif
diff --git a/arch/powerpc/include/asm/ptrace.h b/arch/powerpc/include/asm/ptrace.h
index 2c3e773ce292..cd49769d4194 100644
--- a/arch/powerpc/include/asm/ptrace.h
+++ b/arch/powerpc/include/asm/ptrace.h
@@ -50,14 +50,23 @@ struct pt_regs
union {
struct {
-#ifdef CONFIG_PPC64
+#ifdef CONFIG_PPC_BOOK3S_64
unsigned long ppr;
+ unsigned long exit_result;
#endif
#ifdef CONFIG_PPC_KUAP
unsigned long kuap;
#endif
};
- unsigned long __pad[2]; /* Maintain 16 byte interrupt stack alignment */
+
+ /* Maintain 16 byte interrupt stack alignment */
+#ifdef CONFIG_PPC_KUAP
+#ifdef CONFIG_PPC_BOOK3S_64
+ unsigned long __pad[4];
+#else
+ unsigned long __pad[2];
+#endif
+#endif
};
};
#endif
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index ea13e35dd511..eec4e92bd938 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -290,6 +290,7 @@ int main(void)
OFFSET(ACCOUNT_STARTTIME_USER, paca_struct, accounting.starttime_user);
OFFSET(ACCOUNT_USER_TIME, paca_struct, accounting.utime);
OFFSET(ACCOUNT_SYSTEM_TIME, paca_struct, accounting.stime);
+ OFFSET(PACA_EXIT_SAVE_R1, paca_struct, exit_save_r1);
#ifdef CONFIG_PPC_BOOK3E
OFFSET(PACA_TRAP_SAVE, paca_struct, trap_save);
#endif
@@ -353,7 +354,9 @@ int main(void)
STACK_PT_REGS_OFFSET(_ESR, dsisr);
#else /* CONFIG_PPC64 */
STACK_PT_REGS_OFFSET(SOFTE, softe);
+#ifdef CONFIG_PPC_BOOK3S_64
STACK_PT_REGS_OFFSET(_PPR, ppr);
+#endif
#endif /* CONFIG_PPC64 */
#ifdef CONFIG_PPC_KUAP
diff --git a/arch/powerpc/kernel/interrupt_64.S b/arch/powerpc/kernel/interrupt_64.S
index 9b44f6d3463b..a686d625b8ee 100644
--- a/arch/powerpc/kernel/interrupt_64.S
+++ b/arch/powerpc/kernel/interrupt_64.S
@@ -113,9 +113,18 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
bl system_call_exception
.Lsyscall_vectored_\name\()_exit:
- addi r4,r1,STACK_FRAME_OVERHEAD
+ addi r4,r1,STACK_FRAME_OVERHEAD
li r5,1 /* scv */
bl syscall_exit_prepare
+ std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
+.Lsyscall_vectored_\name\()_rst_start:
+ lbz r11,PACAIRQHAPPENED(r13)
+ andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l
+ bne- .Lsyscall_vectored_\name\()_restart
+ li r11,IRQS_ENABLED
+ stb r11,PACAIRQSOFTMASK(r13)
+ li r11,0
+ stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
ld r2,_CCR(r1)
ld r4,_NIP(r1)
@@ -165,8 +174,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
b . /* prevent speculative execution */
.Lsyscall_vectored_\name\()_restore_regs:
- li r3,0
- mtmsrd r3,1
mtspr SPRN_SRR0,r4
mtspr SPRN_SRR1,r5
@@ -184,9 +191,26 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
REST_2GPRS(12, r1)
ld r1,GPR1(r1)
RFI_TO_USER
+.Lsyscall_vectored_\name\()_rst_end:
+
+.Lsyscall_vectored_\name\()_restart:
+ GET_PACA(r13)
+ ld r1,PACA_EXIT_SAVE_R1(r13)
+ ld r2,PACATOC(r13)
+ ld r3,RESULT(r1)
+ addi r4,r1,STACK_FRAME_OVERHEAD
+ li r11,IRQS_ALL_DISABLED
+ stb r11,PACAIRQSOFTMASK(r13)
+ bl syscall_exit_restart
+ std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
+ b .Lsyscall_vectored_\name\()_rst_start
+
+RESTART_TABLE(.Lsyscall_vectored_\name\()_rst_start, .Lsyscall_vectored_\name\()_rst_end, .Lsyscall_vectored_\name\()_restart)
+
.endm
system_call_vectored common 0x3000
+
/*
* We instantiate another entry copy for the SIGILL variant, with TRAP=0x7ff0
* which is tested by system_call_exception when r0 is -1 (as set by vector
@@ -289,9 +313,18 @@ END_BTB_FLUSH_SECTION
bl system_call_exception
.Lsyscall_exit:
- addi r4,r1,STACK_FRAME_OVERHEAD
+ addi r4,r1,STACK_FRAME_OVERHEAD
li r5,0 /* !scv */
bl syscall_exit_prepare
+ std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
+.Lsyscall_rst_start:
+ lbz r11,PACAIRQHAPPENED(r13)
+ andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l
+ bne- .Lsyscall_restart
+ li r11,IRQS_ENABLED
+ stb r11,PACAIRQSOFTMASK(r13)
+ li r11,0
+ stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
ld r2,_CCR(r1)
ld r6,_LINK(r1)
@@ -356,6 +389,21 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
REST_8GPRS(4, r1)
ld r12,GPR12(r1)
b .Lsyscall_restore_regs_cont
+.Lsyscall_rst_end:
+
+.Lsyscall_restart:
+ GET_PACA(r13)
+ ld r1,PACA_EXIT_SAVE_R1(r13)
+ ld r2,PACATOC(r13)
+ ld r3,RESULT(r1)
+ addi r4,r1,STACK_FRAME_OVERHEAD
+ li r11,IRQS_ALL_DISABLED
+ stb r11,PACAIRQSOFTMASK(r13)
+ bl syscall_exit_restart
+ std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
+ b .Lsyscall_rst_start
+
+RESTART_TABLE(.Lsyscall_rst_start, .Lsyscall_rst_end, .Lsyscall_restart)
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
.Ltabort_syscall:
@@ -439,16 +487,32 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\())
ld r4,_MSR(r1)
andi. r0,r4,MSR_PR
beq .Lkernel_interrupt_return_\srr
+.Linterrupt_return_\srr\()_user:
+ li r11,IRQS_ALL_DISABLED
+ stb r11,PACAIRQSOFTMASK(r13)
addi r3,r1,STACK_FRAME_OVERHEAD
bl interrupt_exit_user_prepare
cmpdi r3,0
bne- .Lrestore_nvgprs_\srr
+.Lrestore_nvgprs_\srr\()_cont:
+
+ std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
+.Linterrupt_return_\srr\()_user_rst_start:
+ lbz r11,PACAIRQHAPPENED(r13)
+ andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l
+ bne- .Linterrupt_return_\srr\()_user_restart
+ li r11,IRQS_ENABLED
+ stb r11,PACAIRQSOFTMASK(r13)
+ li r11,0
+ stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
.Lfast_user_interrupt_return_\srr:
BEGIN_FTR_SECTION
ld r10,_PPR(r1)
mtspr SPRN_PPR,r10
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
+ lbz r4,PACAIRQSOFTMASK(r13)
+ tdnei r4,IRQS_ENABLED
.ifc \srr,srr
lbz r4,PACASRR_VALID(r13)
.else
@@ -503,16 +567,46 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
HRFI_TO_USER
.endif
b . /* prevent speculative execution */
+.Linterrupt_return_\srr\()_user_rst_end:
.Lrestore_nvgprs_\srr\():
REST_NVGPRS(r1)
- b .Lfast_user_interrupt_return_\srr
+ b .Lrestore_nvgprs_\srr\()_cont
+
+.Linterrupt_return_\srr\()_user_restart:
+ GET_PACA(r13)
+ ld r1,PACA_EXIT_SAVE_R1(r13)
+ ld r2,PACATOC(r13)
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ li r11,IRQS_ALL_DISABLED
+ stb r11,PACAIRQSOFTMASK(r13)
+ bl interrupt_exit_user_restart
+ std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
+ b .Linterrupt_return_\srr\()_user_rst_start
+
+RESTART_TABLE(.Linterrupt_return_\srr\()_user_rst_start, .Linterrupt_return_\srr\()_user_rst_end, .Linterrupt_return_\srr\()_user_restart)
.balign IFETCH_ALIGN_BYTES
.Lkernel_interrupt_return_\srr\():
+.Linterrupt_return_\srr\()_kernel:
+ li r11,IRQS_ALL_DISABLED
+ stb r11,PACAIRQSOFTMASK(r13)
addi r3,r1,STACK_FRAME_OVERHEAD
bl interrupt_exit_kernel_prepare
+ std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
+.Linterrupt_return_\srr\()_kernel_rst_start:
+ lbz r11,SOFTE(r1)
+ cmpwi r11,IRQS_ENABLED
+ stb r11,PACAIRQSOFTMASK(r13)
+ bne 1f
+ lbz r11,PACAIRQHAPPENED(r13)
+ andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l
+ bne- .Linterrupt_return_\srr\()_kernel_restart
+ li r11,0
+ stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
+1:
+
.Lfast_kernel_interrupt_return_\srr\():
cmpdi cr1,r3,0
.ifc \srr,srr
@@ -600,6 +694,21 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
HRFI_TO_KERNEL
.endif
b . /* prevent speculative execution */
+.Linterrupt_return_\srr\()_kernel_rst_end:
+
+.Linterrupt_return_\srr\()_kernel_restart:
+ GET_PACA(r13)
+ ld r1,PACA_EXIT_SAVE_R1(r13)
+ ld r2,PACATOC(r13)
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ li r11,IRQS_ALL_DISABLED
+ stb r11,PACAIRQSOFTMASK(r13)
+ bl interrupt_exit_kernel_restart
+ std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
+ b .Linterrupt_return_\srr\()_kernel_rst_start
+
+RESTART_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, .Linterrupt_return_\srr\()_kernel_rst_end, .Linterrupt_return_\srr\()_kernel_restart)
+
.endm
interrupt_return_macro srr
diff --git a/arch/powerpc/kernel/syscall_64.c b/arch/powerpc/kernel/syscall_64.c
index 672f2a796487..5add073a21b7 100644
--- a/arch/powerpc/kernel/syscall_64.c
+++ b/arch/powerpc/kernel/syscall_64.c
@@ -109,87 +109,27 @@ notrace long system_call_exception(long r3, long r4, long r5,
return f(r3, r4, r5, r6, r7, r8);
}
-/*
- * local irqs must be disabled. Returns false if the caller must re-enable
- * them, check for new work, and try again.
- */
-static notrace inline bool prep_irq_for_enabled_exit(bool clear_ri)
+static notrace inline bool prep_irq_for_enabled_exit(void)
{
/* This must be done with RI=1 because tracing may touch vmaps */
trace_hardirqs_on();
- /* This pattern matches prep_irq_for_idle */
- if (clear_ri)
- __hard_EE_RI_disable();
- else
- __hard_irq_disable();
if (unlikely(lazy_irq_pending_nocheck())) {
- /* Took an interrupt, may have more exit work to do. */
- if (clear_ri)
- __hard_RI_enable();
trace_hardirqs_off();
- local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
-
return false;
}
- local_paca->irq_happened = 0;
- irq_soft_mask_set(IRQS_ENABLED);
return true;
}
-/*
- * This should be called after a syscall returns, with r3 the return value
- * from the syscall. If this function returns non-zero, the system call
- * exit assembly should additionally load all GPR registers and CTR and XER
- * from the interrupt frame.
- *
- * The function graph tracer can not trace the return side of this function,
- * because RI=0 and soft mask state is "unreconciled", so it is marked notrace.
- */
-notrace unsigned long syscall_exit_prepare(unsigned long r3,
- struct pt_regs *regs,
- long scv)
+notrace unsigned long syscall_exit_prepare_main(unsigned long r3,
+ struct pt_regs *regs)
{
unsigned long *ti_flagsp = ¤t_thread_info()->flags;
unsigned long ti_flags;
unsigned long ret = 0;
- CT_WARN_ON(ct_state() == CONTEXT_USER);
-
- kuap_check_amr();
-
- regs->result = r3;
-
- /* Check whether the syscall is issued inside a restartable sequence */
- rseq_syscall(regs);
-
- ti_flags = *ti_flagsp;
-
- if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && !scv) {
- if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) {
- r3 = -r3;
- regs->ccr |= 0x10000000; /* Set SO bit in CR */
- }
- }
-
- if (unlikely(ti_flags & _TIF_PERSYSCALL_MASK)) {
- if (ti_flags & _TIF_RESTOREALL)
- ret = _TIF_RESTOREALL;
- else
- regs->gpr[3] = r3;
- clear_bits(_TIF_PERSYSCALL_MASK, ti_flagsp);
- } else {
- regs->gpr[3] = r3;
- }
-
- if (unlikely(ti_flags & _TIF_SYSCALL_DOTRACE)) {
- do_syscall_trace_leave(regs);
- ret |= _TIF_RESTOREALL;
- }
-
again:
- local_irq_disable();
ti_flags = READ_ONCE(*ti_flagsp);
while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) {
local_irq_enable();
@@ -233,15 +173,16 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,
}
}
- user_enter_irqoff();
-
- /* scv need not set RI=0 because SRRs are not used */
- if (unlikely(!prep_irq_for_enabled_exit(!scv))) {
- user_exit_irqoff();
+ if (unlikely(lazy_irq_pending_nocheck())) {
local_irq_enable();
+ local_irq_disable();
goto again;
}
+ trace_hardirqs_on();
+
+ user_enter_irqoff();
+
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
local_paca->tm_scratch = regs->msr;
#endif
@@ -251,17 +192,95 @@ notrace unsigned long syscall_exit_prepare(unsigned long r3,
return ret;
}
+/*
+ * This should be called after a syscall returns, with r3 the return value
+ * from the syscall. If this function returns non-zero, the system call
+ * exit assembly should additionally load all GPR registers and CTR and XER
+ * from the interrupt frame.
+ *
+ * The function graph tracer can not trace the return side of this function,
+ * because RI=0 and soft mask state is "unreconciled", so it is marked notrace.
+ */
+notrace unsigned long syscall_exit_prepare(unsigned long r3,
+ struct pt_regs *regs,
+ long scv)
+{
+ unsigned long *ti_flagsp = ¤t_thread_info()->flags;
+ unsigned long ti_flags;
+ unsigned long ret = 0;
+
+ CT_WARN_ON(ct_state() == CONTEXT_USER);
+
+ kuap_check_amr();
+
+ regs->result = r3;
+
+ /* Check whether the syscall is issued inside a restartable sequence */
+ rseq_syscall(regs);
+
+ ti_flags = *ti_flagsp;
+
+ if (unlikely(r3 >= (unsigned long)-MAX_ERRNO) && !scv) {
+ if (likely(!(ti_flags & (_TIF_NOERROR | _TIF_RESTOREALL)))) {
+ r3 = -r3;
+ regs->ccr |= 0x10000000; /* Set SO bit in CR */
+ }
+ }
+
+ if (unlikely(ti_flags & _TIF_PERSYSCALL_MASK)) {
+ if (ti_flags & _TIF_RESTOREALL)
+ ret = _TIF_RESTOREALL;
+ else
+ regs->gpr[3] = r3;
+ clear_bits(_TIF_PERSYSCALL_MASK, ti_flagsp);
+ } else {
+ regs->gpr[3] = r3;
+ }
+
+ if (unlikely(ti_flags & _TIF_SYSCALL_DOTRACE)) {
+ do_syscall_trace_leave(regs);
+ ret |= _TIF_RESTOREALL;
+ }
+
+ local_irq_disable();
+ ret |= syscall_exit_prepare_main(r3, regs);
+
+ regs->exit_result = ret;
+
+ return ret;
+}
+
+notrace unsigned long syscall_exit_restart(unsigned long r3, struct pt_regs *regs)
+{
+ /* This is called in when detecting a soft-pending interrupt as well,
+ * so can't just have restart table returns clear SRR1[MSR] and set
+ * PACA_IRQ_HARD_DIS here (unless the soft-pending case were to clear
+ * MSR[EE] too).
+ */
+ hard_irq_disable();
+
+ trace_hardirqs_off();
+ user_exit_irqoff();
+ account_cpu_user_entry();
+
+ BUG_ON(!user_mode(regs));
+
+ regs->exit_result |= syscall_exit_prepare_main(r3, regs);
+
+ return regs->exit_result;
+}
+
#ifdef CONFIG_PPC_BOOK3S /* BOOK3E not yet using this */
-notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr)
+notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs)
{
#ifdef CONFIG_PPC_BOOK3E
struct thread_struct *ts = ¤t->thread;
#endif
unsigned long *ti_flagsp = ¤t_thread_info()->flags;
unsigned long ti_flags;
- unsigned long flags;
unsigned long ret = 0;
+ BUG_ON(!irqs_disabled());
if (IS_ENABLED(CONFIG_PPC_BOOK3S))
BUG_ON(!(regs->msr & MSR_RI));
BUG_ON(!(regs->msr & MSR_PR));
@@ -275,8 +294,6 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned
*/
kuap_check_amr();
- local_irq_save(flags);
-
again:
ti_flags = READ_ONCE(*ti_flagsp);
while (unlikely(ti_flags & (_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM))) {
@@ -310,14 +327,16 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned
}
}
- user_enter_irqoff();
- if (unlikely(!prep_irq_for_enabled_exit(true))) {
- user_exit_irqoff();
+ if (unlikely(lazy_irq_pending_nocheck())) {
local_irq_enable();
local_irq_disable();
goto again;
}
+ trace_hardirqs_on();
+
+ user_enter_irqoff();
+
#ifdef CONFIG_PPC_BOOK3E
if (unlikely(ts->debug.dbcr0 & DBCR0_IDM)) {
/*
@@ -336,19 +355,21 @@ notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned
account_cpu_user_exit();
+ regs->exit_result = ret;
+
return ret;
}
void unrecoverable_exception(struct pt_regs *regs);
void preempt_schedule_irq(void);
-notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsigned long msr)
+notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs)
{
unsigned long *ti_flagsp = ¤t_thread_info()->flags;
- unsigned long flags;
unsigned long ret = 0;
unsigned long amr;
+ BUG_ON(!irqs_disabled());
if (IS_ENABLED(CONFIG_PPC_BOOK3S) && unlikely(!(regs->msr & MSR_RI)))
unrecoverable_exception(regs);
BUG_ON(regs->msr & MSR_PR);
@@ -362,13 +383,6 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign
amr = kuap_get_and_check_amr();
- if (unlikely(*ti_flagsp & _TIF_EMULATE_STACK_STORE)) {
- clear_bits(_TIF_EMULATE_STACK_STORE, ti_flagsp);
- ret = 1;
- }
-
- local_irq_save(flags);
-
if (regs->softe == IRQS_ENABLED) {
/* Returning to a kernel context with local irqs enabled. */
WARN_ON_ONCE(!(regs->msr & MSR_EE));
@@ -381,27 +395,33 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign
}
}
- if (unlikely(!prep_irq_for_enabled_exit(true))) {
- /*
- * Can't local_irq_restore to replay if we were in
- * interrupt context. Must replay directly.
- */
- if (irqs_disabled_flags(flags)) {
- replay_soft_interrupts();
- } else {
- local_irq_restore(flags);
- local_irq_save(flags);
- }
+ if (unlikely(lazy_irq_pending_nocheck())) {
+ hard_irq_disable();
+ replay_soft_interrupts();
/* Took an interrupt, may have more exit work to do. */
goto again;
}
+ local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
+
+ trace_hardirqs_on();
} else {
/* Returning to a kernel context with local irqs disabled. */
- __hard_EE_RI_disable();
if (regs->msr & MSR_EE)
local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
}
+ if (unlikely(*ti_flagsp & _TIF_EMULATE_STACK_STORE)) {
+ /* Stack store can't be restarted (stack might have been clobbered) */
+ __hard_EE_RI_disable();
+ if (unlikely(lazy_irq_pending_nocheck()) && regs->softe == IRQS_ENABLED) {
+ local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
+ __hard_RI_enable();
+ goto again;
+ }
+
+ clear_bits(_TIF_EMULATE_STACK_STORE, ti_flagsp);
+ ret = 1;
+ }
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
local_paca->tm_scratch = regs->msr;
@@ -416,4 +436,34 @@ notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsign
return ret;
}
+
+notrace unsigned long interrupt_exit_user_restart(struct pt_regs *regs)
+{
+ hard_irq_disable();
+
+ trace_hardirqs_off();
+ user_exit_irqoff();
+ account_cpu_user_entry();
+
+ BUG_ON(!user_mode(regs));
+
+ regs->exit_result |= interrupt_exit_user_prepare(regs);
+
+ return regs->exit_result;
+}
+
+notrace unsigned long interrupt_exit_kernel_restart(struct pt_regs *regs)
+{
+ hard_irq_disable();
+
+ set_kuap(AMR_KUAP_BLOCKED);
+
+ if (regs->softe == IRQS_ENABLED)
+ trace_hardirqs_off();
+
+ BUG_ON(user_mode(regs));
+
+ return interrupt_exit_kernel_prepare(regs);
+}
+
#endif
--
2.23.0
^ permalink raw reply related
* Re: [PATCH 02/29] powerpc/rtas: prevent suspend-related sys_rtas use on LE
From: Nathan Lynch @ 2020-11-06 16:44 UTC (permalink / raw)
To: Andrew Donnellan, linuxppc-dev; +Cc: tyreld, brking, mmc, cforno12, drt
In-Reply-To: <95c8b599-f939-ad28-5671-cfbf7436508e@linux.ibm.com>
Andrew Donnellan <ajd@linux.ibm.com> writes:
> On 30/10/20 11:10 pm, Nathan Lynch wrote:
>>> And there's a zero chance that drmgr will ever be fixed on LE?
>>
>> It's always used the sysfs interface on LE, and the only way to provoke
>> it to attempt the syscalls is by doing something like this before
>> running the migration:
>>
>> # echo 0 > /tmp/fake_api_version
>> # mount -o bind,ro /tmp/fake_api_version /sys/kernel/mobility/api_version
>>
>> So I'm not aware of any circumstance that would actually motivate
>> someone to make it work on LE. I'd say it's more likely that drmgr will
>> drop its support for using the syscall altogether.
>>
>
> Okay. librtas should handle the error fine, so I don't have any huge
> objection to this, though perhaps a documentation patch to librtas to
> mention that these calls are BE-only would be in order.
Good suggestion, I've drafted a documentation change as well as an
attempt to mark the appropriate APIs deprecated for LE here:
https://github.com/nathanlynch/librtas/tree/topic/document-suspend-filter
which I'll send to Tyrel if/when the Linux change is staged.
^ permalink raw reply
* [PATCH net-next] Revert ibmvnic merge do_change_param_reset into do_reset
From: Dany Madden @ 2020-11-06 19:17 UTC (permalink / raw)
To: davem; +Cc: Dany Madden, netdev, linuxppc-dev
This reverts commit 16b5f5ce351f8709a6b518cc3cbf240c378305bf
where it restructures do_reset. There are patches being tested that
would require major rework if this is committed first.
We will resend this after the other patches have been applied.
Signed-off-by: Dany Madden <drt@linux.ibm.com>
---
drivers/net/ethernet/ibm/ibmvnic.c | 147 ++++++++++++++++++++---------
1 file changed, 104 insertions(+), 43 deletions(-)
diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c
index f4167de30461..af4dfbe28d56 100644
--- a/drivers/net/ethernet/ibm/ibmvnic.c
+++ b/drivers/net/ethernet/ibm/ibmvnic.c
@@ -1826,6 +1826,86 @@ static int ibmvnic_set_mac(struct net_device *netdev, void *p)
return rc;
}
+/**
+ * do_change_param_reset returns zero if we are able to keep processing reset
+ * events, or non-zero if we hit a fatal error and must halt.
+ */
+static int do_change_param_reset(struct ibmvnic_adapter *adapter,
+ struct ibmvnic_rwi *rwi,
+ u32 reset_state)
+{
+ struct net_device *netdev = adapter->netdev;
+ int i, rc;
+
+ netdev_dbg(adapter->netdev, "Change param resetting driver (%d)\n",
+ rwi->reset_reason);
+
+ netif_carrier_off(netdev);
+ adapter->reset_reason = rwi->reset_reason;
+
+ ibmvnic_cleanup(netdev);
+
+ if (reset_state == VNIC_OPEN) {
+ rc = __ibmvnic_close(netdev);
+ if (rc)
+ return rc;
+ }
+
+ release_resources(adapter);
+ release_sub_crqs(adapter, 1);
+ release_crq_queue(adapter);
+
+ adapter->state = VNIC_PROBED;
+
+ rc = init_crq_queue(adapter);
+
+ if (rc) {
+ netdev_err(adapter->netdev,
+ "Couldn't initialize crq. rc=%d\n", rc);
+ return rc;
+ }
+
+ rc = ibmvnic_reset_init(adapter, true);
+ if (rc)
+ return IBMVNIC_INIT_FAILED;
+
+ /* If the adapter was in PROBE state prior to the reset,
+ * exit here.
+ */
+ if (reset_state == VNIC_PROBED)
+ return 0;
+
+ rc = ibmvnic_login(netdev);
+ if (rc) {
+ adapter->state = reset_state;
+ return rc;
+ }
+
+ rc = init_resources(adapter);
+ if (rc)
+ return rc;
+
+ ibmvnic_disable_irqs(adapter);
+
+ adapter->state = VNIC_CLOSED;
+
+ if (reset_state == VNIC_CLOSED)
+ return 0;
+
+ rc = __ibmvnic_open(netdev);
+ if (rc)
+ return IBMVNIC_OPEN_FAILED;
+
+ /* refresh device's multicast list */
+ ibmvnic_set_multi(netdev);
+
+ /* kick napi */
+ for (i = 0; i < adapter->req_rx_queues; i++)
+ napi_schedule(&adapter->napi[i]);
+
+ return 0;
+}
+
/**
* do_reset returns zero if we are able to keep processing reset events, or
* non-zero if we hit a fatal error and must halt.
@@ -1841,12 +1921,10 @@ static int do_reset(struct ibmvnic_adapter *adapter,
netdev_dbg(adapter->netdev, "Re-setting driver (%d)\n",
rwi->reset_reason);
- adapter->reset_reason = rwi->reset_reason;
- /* requestor of VNIC_RESET_CHANGE_PARAM already has the rtnl lock */
- if (!(adapter->reset_reason == VNIC_RESET_CHANGE_PARAM))
- rtnl_lock();
+ rtnl_lock();
netif_carrier_off(netdev);
+ adapter->reset_reason = rwi->reset_reason;
old_num_rx_queues = adapter->req_rx_queues;
old_num_tx_queues = adapter->req_tx_queues;
@@ -1858,37 +1936,25 @@ static int do_reset(struct ibmvnic_adapter *adapter,
if (reset_state == VNIC_OPEN &&
adapter->reset_reason != VNIC_RESET_MOBILITY &&
adapter->reset_reason != VNIC_RESET_FAILOVER) {
- if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {
- rc = __ibmvnic_close(netdev);
- if (rc)
- goto out;
- } else {
- adapter->state = VNIC_CLOSING;
-
- /* Release the RTNL lock before link state change and
- * re-acquire after the link state change to allow
- * linkwatch_event to grab the RTNL lock and run during
- * a reset.
- */
- rtnl_unlock();
- rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);
- rtnl_lock();
- if (rc)
- goto out;
+ adapter->state = VNIC_CLOSING;
- if (adapter->state != VNIC_CLOSING) {
- rc = -1;
- goto out;
- }
+ /* Release the RTNL lock before link state change and
+ * re-acquire after the link state change to allow
+ * linkwatch_event to grab the RTNL lock and run during
+ * a reset.
+ */
+ rtnl_unlock();
+ rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);
+ rtnl_lock();
+ if (rc)
+ goto out;
- adapter->state = VNIC_CLOSED;
+ if (adapter->state != VNIC_CLOSING) {
+ rc = -1;
+ goto out;
}
- }
- if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {
- release_resources(adapter);
- release_sub_crqs(adapter, 1);
- release_crq_queue(adapter);
+ adapter->state = VNIC_CLOSED;
}
if (adapter->reset_reason != VNIC_RESET_NON_FATAL) {
@@ -1897,9 +1963,7 @@ static int do_reset(struct ibmvnic_adapter *adapter,
*/
adapter->state = VNIC_PROBED;
- if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {
- rc = init_crq_queue(adapter);
- } else if (adapter->reset_reason == VNIC_RESET_MOBILITY) {
+ if (adapter->reset_reason == VNIC_RESET_MOBILITY) {
rc = ibmvnic_reenable_crq_queue(adapter);
release_sub_crqs(adapter, 1);
} else {
@@ -1939,11 +2003,7 @@ static int do_reset(struct ibmvnic_adapter *adapter,
goto out;
}
- if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {
- rc = init_resources(adapter);
- if (rc)
- goto out;
- } else if (adapter->req_rx_queues != old_num_rx_queues ||
+ if (adapter->req_rx_queues != old_num_rx_queues ||
adapter->req_tx_queues != old_num_tx_queues ||
adapter->req_rx_add_entries_per_subcrq !=
old_num_rx_slots ||
@@ -2004,9 +2064,7 @@ static int do_reset(struct ibmvnic_adapter *adapter,
rc = 0;
out:
- /* requestor of VNIC_RESET_CHANGE_PARAM should still hold the rtnl lock */
- if (!(adapter->reset_reason == VNIC_RESET_CHANGE_PARAM))
- rtnl_unlock();
+ rtnl_unlock();
return rc;
}
@@ -2140,7 +2198,10 @@ static void __ibmvnic_reset(struct work_struct *work)
}
spin_unlock_irqrestore(&adapter->state_lock, flags);
- if (adapter->force_reset_recovery) {
+ if (rwi->reset_reason == VNIC_RESET_CHANGE_PARAM) {
+ /* CHANGE_PARAM requestor holds rtnl_lock */
+ rc = do_change_param_reset(adapter, rwi, reset_state);
+ } else if (adapter->force_reset_recovery) {
/* Transport event occurred during previous reset */
if (adapter->wait_for_reset) {
/* Previous was CHANGE_PARAM; caller locked */
--
2.18.2
^ permalink raw reply related
* Re: [PATCH net-next] Revert ibmvnic merge do_change_param_reset into do_reset
From: ljp @ 2020-11-06 19:30 UTC (permalink / raw)
To: Dany Madden, wvoigt; +Cc: netdev, linuxppc-dev, davem, Linuxppc-dev
In-Reply-To: <20201106191745.1679846-1-drt@linux.ibm.com>
On 2020-11-06 13:17, Dany Madden wrote:
> This reverts commit 16b5f5ce351f8709a6b518cc3cbf240c378305bf
> where it restructures do_reset. There are patches being tested that
> would require major rework if this is committed first.
>
> We will resend this after the other patches have been applied.
I discussed with my manager, and he has agreed not revert this one
since it is in the net-next tree and will not affect net tree for
current bug fix patches.
Sorry for the confusion.
Thanks,
Lijun
>
> Signed-off-by: Dany Madden <drt@linux.ibm.com>
> ---
> drivers/net/ethernet/ibm/ibmvnic.c | 147 ++++++++++++++++++++---------
> 1 file changed, 104 insertions(+), 43 deletions(-)
>
> diff --git a/drivers/net/ethernet/ibm/ibmvnic.c
> b/drivers/net/ethernet/ibm/ibmvnic.c
> index f4167de30461..af4dfbe28d56 100644
> --- a/drivers/net/ethernet/ibm/ibmvnic.c
> +++ b/drivers/net/ethernet/ibm/ibmvnic.c
> @@ -1826,6 +1826,86 @@ static int ibmvnic_set_mac(struct net_device
> *netdev, void *p)
> return rc;
> }
>
> +/**
> + * do_change_param_reset returns zero if we are able to keep
> processing reset
> + * events, or non-zero if we hit a fatal error and must halt.
> + */
> +static int do_change_param_reset(struct ibmvnic_adapter *adapter,
> + struct ibmvnic_rwi *rwi,
> + u32 reset_state)
> +{
> + struct net_device *netdev = adapter->netdev;
> + int i, rc;
> +
> + netdev_dbg(adapter->netdev, "Change param resetting driver (%d)\n",
> + rwi->reset_reason);
> +
> + netif_carrier_off(netdev);
> + adapter->reset_reason = rwi->reset_reason;
> +
> + ibmvnic_cleanup(netdev);
> +
> + if (reset_state == VNIC_OPEN) {
> + rc = __ibmvnic_close(netdev);
> + if (rc)
> + return rc;
> + }
> +
> + release_resources(adapter);
> + release_sub_crqs(adapter, 1);
> + release_crq_queue(adapter);
> +
> + adapter->state = VNIC_PROBED;
> +
> + rc = init_crq_queue(adapter);
> +
> + if (rc) {
> + netdev_err(adapter->netdev,
> + "Couldn't initialize crq. rc=%d\n", rc);
> + return rc;
> + }
> +
> + rc = ibmvnic_reset_init(adapter, true);
> + if (rc)
> + return IBMVNIC_INIT_FAILED;
> +
> + /* If the adapter was in PROBE state prior to the reset,
> + * exit here.
> + */
> + if (reset_state == VNIC_PROBED)
> + return 0;
> +
> + rc = ibmvnic_login(netdev);
> + if (rc) {
> + adapter->state = reset_state;
> + return rc;
> + }
> +
> + rc = init_resources(adapter);
> + if (rc)
> + return rc;
> +
> + ibmvnic_disable_irqs(adapter);
> +
> + adapter->state = VNIC_CLOSED;
> +
> + if (reset_state == VNIC_CLOSED)
> + return 0;
> +
> + rc = __ibmvnic_open(netdev);
> + if (rc)
> + return IBMVNIC_OPEN_FAILED;
> +
> + /* refresh device's multicast list */
> + ibmvnic_set_multi(netdev);
> +
> + /* kick napi */
> + for (i = 0; i < adapter->req_rx_queues; i++)
> + napi_schedule(&adapter->napi[i]);
> +
> + return 0;
> +}
> +
> /**
> * do_reset returns zero if we are able to keep processing reset
> events, or
> * non-zero if we hit a fatal error and must halt.
> @@ -1841,12 +1921,10 @@ static int do_reset(struct ibmvnic_adapter
> *adapter,
> netdev_dbg(adapter->netdev, "Re-setting driver (%d)\n",
> rwi->reset_reason);
>
> - adapter->reset_reason = rwi->reset_reason;
> - /* requestor of VNIC_RESET_CHANGE_PARAM already has the rtnl lock */
> - if (!(adapter->reset_reason == VNIC_RESET_CHANGE_PARAM))
> - rtnl_lock();
> + rtnl_lock();
>
> netif_carrier_off(netdev);
> + adapter->reset_reason = rwi->reset_reason;
>
> old_num_rx_queues = adapter->req_rx_queues;
> old_num_tx_queues = adapter->req_tx_queues;
> @@ -1858,37 +1936,25 @@ static int do_reset(struct ibmvnic_adapter
> *adapter,
> if (reset_state == VNIC_OPEN &&
> adapter->reset_reason != VNIC_RESET_MOBILITY &&
> adapter->reset_reason != VNIC_RESET_FAILOVER) {
> - if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {
> - rc = __ibmvnic_close(netdev);
> - if (rc)
> - goto out;
> - } else {
> - adapter->state = VNIC_CLOSING;
> -
> - /* Release the RTNL lock before link state change and
> - * re-acquire after the link state change to allow
> - * linkwatch_event to grab the RTNL lock and run during
> - * a reset.
> - */
> - rtnl_unlock();
> - rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);
> - rtnl_lock();
> - if (rc)
> - goto out;
> + adapter->state = VNIC_CLOSING;
>
> - if (adapter->state != VNIC_CLOSING) {
> - rc = -1;
> - goto out;
> - }
> + /* Release the RTNL lock before link state change and
> + * re-acquire after the link state change to allow
> + * linkwatch_event to grab the RTNL lock and run during
> + * a reset.
> + */
> + rtnl_unlock();
> + rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);
> + rtnl_lock();
> + if (rc)
> + goto out;
>
> - adapter->state = VNIC_CLOSED;
> + if (adapter->state != VNIC_CLOSING) {
> + rc = -1;
> + goto out;
> }
> - }
>
> - if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {
> - release_resources(adapter);
> - release_sub_crqs(adapter, 1);
> - release_crq_queue(adapter);
> + adapter->state = VNIC_CLOSED;
> }
>
> if (adapter->reset_reason != VNIC_RESET_NON_FATAL) {
> @@ -1897,9 +1963,7 @@ static int do_reset(struct ibmvnic_adapter
> *adapter,
> */
> adapter->state = VNIC_PROBED;
>
> - if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {
> - rc = init_crq_queue(adapter);
> - } else if (adapter->reset_reason == VNIC_RESET_MOBILITY) {
> + if (adapter->reset_reason == VNIC_RESET_MOBILITY) {
> rc = ibmvnic_reenable_crq_queue(adapter);
> release_sub_crqs(adapter, 1);
> } else {
> @@ -1939,11 +2003,7 @@ static int do_reset(struct ibmvnic_adapter
> *adapter,
> goto out;
> }
>
> - if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {
> - rc = init_resources(adapter);
> - if (rc)
> - goto out;
> - } else if (adapter->req_rx_queues != old_num_rx_queues ||
> + if (adapter->req_rx_queues != old_num_rx_queues ||
> adapter->req_tx_queues != old_num_tx_queues ||
> adapter->req_rx_add_entries_per_subcrq !=
> old_num_rx_slots ||
> @@ -2004,9 +2064,7 @@ static int do_reset(struct ibmvnic_adapter
> *adapter,
> rc = 0;
>
> out:
> - /* requestor of VNIC_RESET_CHANGE_PARAM should still hold the rtnl
> lock */
> - if (!(adapter->reset_reason == VNIC_RESET_CHANGE_PARAM))
> - rtnl_unlock();
> + rtnl_unlock();
>
> return rc;
> }
> @@ -2140,7 +2198,10 @@ static void __ibmvnic_reset(struct work_struct
> *work)
> }
> spin_unlock_irqrestore(&adapter->state_lock, flags);
>
> - if (adapter->force_reset_recovery) {
> + if (rwi->reset_reason == VNIC_RESET_CHANGE_PARAM) {
> + /* CHANGE_PARAM requestor holds rtnl_lock */
> + rc = do_change_param_reset(adapter, rwi, reset_state);
> + } else if (adapter->force_reset_recovery) {
> /* Transport event occurred during previous reset */
> if (adapter->wait_for_reset) {
> /* Previous was CHANGE_PARAM; caller locked */
^ permalink raw reply
* Re: [PATCH net-next] Revert ibmvnic merge do_change_param_reset into do_reset
From: Jakub Kicinski @ 2020-11-06 19:42 UTC (permalink / raw)
To: ljp; +Cc: wvoigt, netdev, Linuxppc-dev, Dany Madden, linuxppc-dev, davem
In-Reply-To: <0ff353cbada91b031d1bbae250a975d5@linux.vnet.ibm.com>
On Fri, 06 Nov 2020 13:30:25 -0600 ljp wrote:
> On 2020-11-06 13:17, Dany Madden wrote:
> > This reverts commit 16b5f5ce351f8709a6b518cc3cbf240c378305bf
> > where it restructures do_reset. There are patches being tested that
> > would require major rework if this is committed first.
> >
> > We will resend this after the other patches have been applied.
>
> I discussed with my manager, and he has agreed not revert this one
> since it is in the net-next tree and will not affect net tree for
> current bug fix patches.
We merge net into net-next periodically (~every week or so) so if you
keep making changes to both branches I will have to deal with the
fallout.
I'm assuming that the resolution for the current conflict which Stephen
Rothwell sent from linux-next is correct. Please confirm.
I will resolve it like he did when Linus pulls from net (hopefully
later today).
But if you know you have more fixes I'd rather revert this, get all the
relevant fixed into net, wait for net to be merged into net-next and
then redo the refactoring.
Hope that makes sense.
^ permalink raw reply
* Re: [PATCH net-next] Revert ibmvnic merge do_change_param_reset into do_reset
From: ljp @ 2020-11-06 20:15 UTC (permalink / raw)
To: Jakub Kicinski
Cc: wvoigt, netdev, Linuxppc-dev, Dany Madden, linuxppc-dev, davem
In-Reply-To: <20201106114208.4b0e8eec@kicinski-fedora-pc1c0hjn.dhcp.thefacebook.com>
On 2020-11-06 13:42, Jakub Kicinski wrote:
> On Fri, 06 Nov 2020 13:30:25 -0600 ljp wrote:
>> On 2020-11-06 13:17, Dany Madden wrote:
>> > This reverts commit 16b5f5ce351f8709a6b518cc3cbf240c378305bf
>> > where it restructures do_reset. There are patches being tested that
>> > would require major rework if this is committed first.
>> >
>> > We will resend this after the other patches have been applied.
>>
>> I discussed with my manager, and he has agreed not revert this one
>> since it is in the net-next tree and will not affect net tree for
>> current bug fix patches.
>
> We merge net into net-next periodically (~every week or so) so if you
> keep making changes to both branches I will have to deal with the
> fallout.
>
> I'm assuming that the resolution for the current conflict which Stephen
> Rothwell sent from linux-next is correct. Please confirm.
That fix is correct.
>
> I will resolve it like he did when Linus pulls from net (hopefully
> later today).
>
> But if you know you have more fixes I'd rather revert this, get all the
> relevant fixed into net, wait for net to be merged into net-next and
> then redo the refactoring.
>
> Hope that makes sense.
^ permalink raw reply
* Re: [PATCH net-next] Revert ibmvnic merge do_change_param_reset into do_reset
From: ljp @ 2020-11-06 21:02 UTC (permalink / raw)
To: Jakub Kicinski
Cc: wvoigt, netdev, Linuxppc-dev, Dany Madden, linuxppc-dev, davem
In-Reply-To: <20201106114208.4b0e8eec@kicinski-fedora-pc1c0hjn.dhcp.thefacebook.com>
On 2020-11-06 13:42, Jakub Kicinski wrote:
> On Fri, 06 Nov 2020 13:30:25 -0600 ljp wrote:
>> On 2020-11-06 13:17, Dany Madden wrote:
>> > This reverts commit 16b5f5ce351f8709a6b518cc3cbf240c378305bf
>> > where it restructures do_reset. There are patches being tested that
>> > would require major rework if this is committed first.
>> >
>> > We will resend this after the other patches have been applied.
>>
>> I discussed with my manager, and he has agreed not revert this one
>> since it is in the net-next tree and will not affect net tree for
>> current bug fix patches.
>
> We merge net into net-next periodically (~every week or so) so if you
> keep making changes to both branches I will have to deal with the
> fallout.
>
> I'm assuming that the resolution for the current conflict which Stephen
> Rothwell sent from linux-next is correct. Please confirm.
>
> I will resolve it like he did when Linus pulls from net (hopefully
> later today).
>
> But if you know you have more fixes I'd rather revert this, get all the
> relevant fixed into net, wait for net to be merged into net-next and
> then redo the refactoring.
>
> Hope that makes sense.
Jakub,
We had further discussion in the team based on your comments above.
You can revert it for now.
Thanks for your efforts.
Lijun
^ permalink raw reply
* [PATCH v2 00/23] Rid W=1 warnings in MTD
From: Lee Jones @ 2020-11-06 21:36 UTC (permalink / raw)
To: lee.jones
Cc: Boris BREZILLON, Vignesh Raghavendra, Sergey Lapin, dri-devel,
linux-mtd, Frieder Schrempf, Miquel Raynal, Choudary Kalluri,
Robert Jarzmik, Sumit Semwal, Paul Mackerras, Dan Brown,
linux-samsung-soc, Adrian Hunter, Kamal Dasu, Krzysztof Kozlowski,
Thomas Gleixner, Chen-Yu Tsai, Kukjin Kim,
bcm-kernel-feedback-list, linux-media, Tudor Ambarus,
Maxime Ripard, linaro-mm-sig, Dmitriy B, Thomas Gleixner,
Jochen Schäuble, Naga Sureshkumar Relli, Yoshio Furuyama,
Nicolas Pitre, linuxppc-dev, linux-kernel, Ben Dooks,
Kyungmin Park, Boris Brezillon, Qiang Yu, Philipp Zabel,
Richard Weinberger, Jian Zhang, Brian Norris, David Woodhouse,
Christian König
This set is part of a larger effort attempting to clean-up W=1
kernel builds, which are currently overwhelmingly riddled with
niggly little warnings.
v1 => v2:
- Added tags
- Satisfied Miquel's review comments
Lee Jones (23):
mtd: mtdpart: Fix misdocumented function parameter 'mtd'
mtd: devices: phram: File headers are not good candidates for
kernel-doc
mtd: nand: onenand: onenand_base: Fix expected kernel-doc formatting
mtd: devices: docg3: Fix kernel-doc 'bad line' and 'excessive doc'
issues
mtd: mtdcore: Fix misspelled function parameter 'section'
mtd: nand: onenand: onenand_bbt: Fix expected kernel-doc formatting
mtd: spi-nor: controllers: hisi-sfc: Demote non-conformant kernel-doc
mtd: ubi: build: Document 'ubi_num' in struct mtd_dev_param
mtd: nand: spi: toshiba: Demote non-conformant kernel-doc header
mtd: ubi: kapi: Correct documentation for 'ubi_leb_read_sg's 'sgl'
parameter
mtd: ubi: eba: Fix a couple of misdocumentation issues
mtd: ubi: wl: Fix a couple of kernel-doc issues
mtd: nand: raw: brcmnand: brcmnand: Demote non-conformant kernel-doc
headers
mtd: ubi: gluebi: Fix misnamed function parameter documentation
mtd: nand: raw: diskonchip: Marking unused variables as
__always_unused
mtd: nand: raw: cafe_nand: Remove superfluous param doc and add
another
mtd: nand: raw: s3c2410: Add documentation for 2 missing struct
members
mtd: nand: raw: omap_elm: Finish half populated function header,
demote empty ones
mtd: nand: raw: omap2: Fix a bunch of kernel-doc misdemeanours
mtd: nand: raw: sunxi_nand: Document 'sunxi_nfc's 'caps' member
mtd: nand: raw: arasan-nand-controller: Document 'anfc_op's 'buf'
member
mtd: nand: onenand: onenand_base: Fix some kernel-doc misdemeanours
mtd: devices: powernv_flash: Add function names to headers and fix
'dev'
drivers/mtd/devices/docg3.c | 5 +-
drivers/mtd/devices/phram.c | 2 +-
drivers/mtd/devices/powernv_flash.c | 5 +-
drivers/mtd/mtdcore.c | 2 +-
drivers/mtd/mtdpart.c | 2 +-
drivers/mtd/nand/onenand/onenand_base.c | 444 +++++++++---------
drivers/mtd/nand/onenand/onenand_bbt.c | 32 +-
drivers/mtd/nand/raw/arasan-nand-controller.c | 1 +
drivers/mtd/nand/raw/brcmnand/brcmnand.c | 6 +-
drivers/mtd/nand/raw/cafe_nand.c | 2 +-
drivers/mtd/nand/raw/diskonchip.c | 4 +-
drivers/mtd/nand/raw/omap2.c | 18 +-
drivers/mtd/nand/raw/omap_elm.c | 7 +-
drivers/mtd/nand/raw/s3c2410.c | 4 +-
drivers/mtd/nand/raw/sunxi_nand.c | 1 +
drivers/mtd/nand/spi/toshiba.c | 2 +-
drivers/mtd/spi-nor/controllers/hisi-sfc.c | 2 +-
drivers/mtd/ubi/build.c | 1 +
drivers/mtd/ubi/eba.c | 3 +-
drivers/mtd/ubi/gluebi.c | 2 +-
drivers/mtd/ubi/kapi.c | 2 +-
drivers/mtd/ubi/wl.c | 3 +-
22 files changed, 278 insertions(+), 272 deletions(-)
Cc: Adrian Hunter <ext-adrian.hunter@nokia.com>
Cc: bcm-kernel-feedback-list@broadcom.com
Cc: Ben Dooks <ben@simtec.co.uk>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Boris BREZILLON <b.brezillon.dev@gmail.com>
Cc: Boris Brezillon <boris.brezillon@collabora.com>
Cc: Brian Norris <computersforpeace@gmail.com>
Cc: Chen-Yu Tsai <wens@csie.org>
Cc: Choudary Kalluri <punnaia@xilinx.com>
Cc: "Christian König" <christian.koenig@amd.com>
Cc: Dan Brown <dan_brown@ieee.org>
Cc: David Woodhouse <dwmw2@infradead.org>
Cc: Dmitriy B <rzk333@gmail.com>
Cc: dri-devel@lists.freedesktop.org
Cc: Frieder Schrempf <frieder.schrempf@kontron.de>
Cc: Jian Zhang <jzhang@ti.com>
Cc: "Jochen Schäuble" <psionic@psionic.de>
Cc: Kamal Dasu <kdasu.kdev@gmail.com>
Cc: Krzysztof Kozlowski <krzk@kernel.org>
Cc: Kukjin Kim <kgene@kernel.org>
Cc: Kyungmin Park <kyungmin.park@samsung.com>
Cc: linaro-mm-sig@lists.linaro.org
Cc: linux-media@vger.kernel.org
Cc: linux-mtd@lists.infradead.org
Cc: linuxppc-dev@lists.ozlabs.org
Cc: linux-samsung-soc@vger.kernel.org
Cc: Maxime Ripard <mripard@kernel.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Miquel Raynal <miquel.raynal@bootlin.com>
Cc: Naga Sureshkumar Relli <nagasure@xilinx.com>
Cc: Nicolas Pitre <nico@fluxnic.net>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Philipp Zabel <p.zabel@pengutronix.de>
Cc: Qiang Yu <yuq825@gmail.com>
Cc: Richard Weinberger <richard@nod.at>
Cc: Robert Jarzmik <robert.jarzmik@free.fr>
Cc: Sergey Lapin <slapin@ossfans.org>
Cc: Sumit Semwal <sumit.semwal@linaro.org>
Cc: Thomas Gleixner <gleixner@linutronix.de>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Tudor Ambarus <tudor.ambarus@microchip.com>
Cc: Vignesh Raghavendra <vigneshr@ti.com>
Cc: Yoshio Furuyama <ytc-mb-yfuruyama7@kioxia.com>
--
2.25.1
^ permalink raw reply
* [PATCH v2 23/23] mtd: devices: powernv_flash: Add function names to headers and fix 'dev'
From: Lee Jones @ 2020-11-06 21:36 UTC (permalink / raw)
To: lee.jones
Cc: Vignesh Raghavendra, Richard Weinberger, Miquel Raynal,
linux-kernel, Paul Mackerras, linux-mtd, linuxppc-dev
In-Reply-To: <20201106213655.1838861-1-lee.jones@linaro.org>
Fixes the following W=1 kernel build warning(s):
drivers/mtd/devices/powernv_flash.c:129: warning: Cannot understand * @mtd: the device
drivers/mtd/devices/powernv_flash.c:145: warning: Cannot understand * @mtd: the device
drivers/mtd/devices/powernv_flash.c:161: warning: Cannot understand * @mtd: the device
drivers/mtd/devices/powernv_flash.c:184: warning: Function parameter or member 'dev' not described in 'powernv_flash_set_driver_info'
Cc: Miquel Raynal <miquel.raynal@bootlin.com>
Cc: Richard Weinberger <richard@nod.at>
Cc: Vignesh Raghavendra <vigneshr@ti.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: linux-mtd@lists.infradead.org
Cc: linuxppc-dev@lists.ozlabs.org
Signed-off-by: Lee Jones <lee.jones@linaro.org>
---
drivers/mtd/devices/powernv_flash.c | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/drivers/mtd/devices/powernv_flash.c b/drivers/mtd/devices/powernv_flash.c
index 0b757d9ba2f6b..6950a87648151 100644
--- a/drivers/mtd/devices/powernv_flash.c
+++ b/drivers/mtd/devices/powernv_flash.c
@@ -126,6 +126,7 @@ static int powernv_flash_async_op(struct mtd_info *mtd, enum flash_op op,
}
/**
+ * powernv_flash_read
* @mtd: the device
* @from: the offset to read from
* @len: the number of bytes to read
@@ -142,6 +143,7 @@ static int powernv_flash_read(struct mtd_info *mtd, loff_t from, size_t len,
}
/**
+ * powernv_flash_write
* @mtd: the device
* @to: the offset to write to
* @len: the number of bytes to write
@@ -158,6 +160,7 @@ static int powernv_flash_write(struct mtd_info *mtd, loff_t to, size_t len,
}
/**
+ * powernv_flash_erase
* @mtd: the device
* @erase: the erase info
* Returns 0 if erase successful or -ERRNO if an error occurred
@@ -176,7 +179,7 @@ static int powernv_flash_erase(struct mtd_info *mtd, struct erase_info *erase)
/**
* powernv_flash_set_driver_info - Fill the mtd_info structure and docg3
- * structure @pdev: The platform device
+ * @dev: The device structure
* @mtd: The structure to fill
*/
static int powernv_flash_set_driver_info(struct device *dev,
--
2.25.1
^ permalink raw reply related
* Re: [PATCH v3 1/2] ASoC: dt-bindings: fsl_aud2htx: Add binding doc for aud2htx module
From: Nicolin Chen @ 2020-11-06 21:27 UTC (permalink / raw)
To: Mark Brown
Cc: devicetree, alsa-devel, timur, Xiubo.Lee, lgirdwood,
Shengjiu Wang, linuxppc-dev, tiwai, perex, robh+dt, festevam,
linux-kernel
In-Reply-To: <20201106122013.GB49612@sirena.org.uk>
On Fri, Nov 06, 2020 at 12:20:13PM +0000, Mark Brown wrote:
> On Fri, Nov 06, 2020 at 11:54:23AM +0000, Mark Brown wrote:
> > On Mon, 2 Nov 2020 09:52:26 +0800, Shengjiu Wang wrote:
> > > AUD2HTX (Audio Subsystem TO HDMI TX Subsystem) is a new
> > > IP module found on i.MX8MP.
> >
> > Applied to
> >
> > https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git for-next
>
> Sorry, looks like me queueing this raced with the review comments coming
> in. I think the review commments are small enough that it'll be OK to
> fix incrementally?
Yes. I am okay if we move forward with this version.
Thanks
^ permalink raw reply
* Re: [PATCH v3 2/2] ASoC: fsl_aud2htx: Add aud2htx module driver
From: Nicolin Chen @ 2020-11-06 21:38 UTC (permalink / raw)
To: Shengjiu Wang
Cc: open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
alsa-devel, Timur Tabi, Xiubo Li, Fabio Estevam, Shengjiu Wang,
Liam Girdwood, Takashi Iwai, Rob Herring, Mark Brown,
linuxppc-dev, linux-kernel
In-Reply-To: <CAA+D8ANrkgXR+8JGp4fPLvkKJ05EqQyShWTO+xWpPOycPT9Tyw@mail.gmail.com>
On Fri, Nov 06, 2020 at 10:51:03AM +0800, Shengjiu Wang wrote:
> > > +static irqreturn_t fsl_aud2htx_isr(int irq, void *dev_id)
> > > +{
> > > + return IRQ_HANDLED;
> >
> > Empty isr? Perhaps can drop the request_irq() at all?
>
> I'd like to keep this for future enhancement, what do you think?
I believe that usually it will be a common practice that we add
when we use it -- exaggerating the situation, just like you will
not actually add an empty driver for future enhancement.
But I am not strongly against it, as it's small. Since Mark has
applied it, let's keep it then.
^ permalink raw reply
* Re: [PATCH net-next] Revert ibmvnic merge do_change_param_reset into do_reset
From: Jakub Kicinski @ 2020-11-07 1:30 UTC (permalink / raw)
To: Dany Madden; +Cc: netdev, linuxppc-dev, davem
In-Reply-To: <20201106191745.1679846-1-drt@linux.ibm.com>
On Fri, 6 Nov 2020 14:17:45 -0500 Dany Madden wrote:
> This reverts commit 16b5f5ce351f8709a6b518cc3cbf240c378305bf
> where it restructures do_reset. There are patches being tested that
> would require major rework if this is committed first.
>
> We will resend this after the other patches have been applied.
>
> Signed-off-by: Dany Madden <drt@linux.ibm.com>
Applied, thanks.
^ permalink raw reply
* Re: [PATCH net-next] Revert ibmvnic merge do_change_param_reset into do_reset
From: patchwork-bot+netdevbpf @ 2020-11-07 1:40 UTC (permalink / raw)
To: Dany Madden; +Cc: netdev, linuxppc-dev, davem
In-Reply-To: <20201106191745.1679846-1-drt@linux.ibm.com>
Hello:
This patch was applied to netdev/net-next.git (refs/heads/master):
On Fri, 6 Nov 2020 14:17:45 -0500 you wrote:
> This reverts commit 16b5f5ce351f8709a6b518cc3cbf240c378305bf
> where it restructures do_reset. There are patches being tested that
> would require major rework if this is committed first.
>
> We will resend this after the other patches have been applied.
>
> Signed-off-by: Dany Madden <drt@linux.ibm.com>
>
> [...]
Here is the summary with links:
- [net-next] Revert ibmvnic merge do_change_param_reset into do_reset
https://git.kernel.org/netdev/net-next/c/9f32c27eb4fc
You are awesome, thank you!
--
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html
^ permalink raw reply
* [PATCH v2] powerpc/64: irq replay remove decrementer overflow check
From: Nicholas Piggin @ 2020-11-07 1:43 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
This is way to catch some cases of decrementer overflow, when the
decrementer has underflowed an odd number of times, while MSR[EE] was
disabled.
With a typical small decrementer, a timer that fires when MSR[EE] is
disabled will be "lost" if MSR[EE] remains disabled for between 4.3 and
8.6 seconds after the timer expires. In any case, the decrementer
interrupt would be taken at 8.6 seconds and the timer would be found at
that point.
So this check is for catching extreme latency events, and it prevents
those latencies from being a further few seconds long. It's not obvious
this is a good tradeoff. This is already a watchdog magnitude event and
that situation is not improved a significantly with this check. For
large decrementers, it's useless.
Therefore remove this check, which avoids a mftb when enabling hard
disabled interrupts (e.g., when enabling after coming from hardware
interrupt handlers). Perhaps more importantly, it also removes the
clunky MSR[EE] vs PACA_IRQ_HARD_DIS incoherency in soft-interrupt replay
which simplifies the code.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
v2: Fixed 64e warnings
arch/powerpc/kernel/irq.c | 53 ++-------------------------
arch/powerpc/kernel/time.c | 9 ++---
arch/powerpc/platforms/powernv/opal.c | 2 +-
3 files changed, 8 insertions(+), 56 deletions(-)
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index 7d0f7682d01d..6b1eca53e36c 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -102,14 +102,6 @@ static inline notrace unsigned long get_irq_happened(void)
return happened;
}
-static inline notrace int decrementer_check_overflow(void)
-{
- u64 now = get_tb();
- u64 *next_tb = this_cpu_ptr(&decrementers_next_tb);
-
- return now >= *next_tb;
-}
-
#ifdef CONFIG_PPC_BOOK3E
/* This is called whenever we are re-enabling interrupts
@@ -142,35 +134,6 @@ notrace unsigned int __check_irq_replay(void)
trace_hardirqs_on();
trace_hardirqs_off();
- /*
- * We are always hard disabled here, but PACA_IRQ_HARD_DIS may
- * not be set, which means interrupts have only just been hard
- * disabled as part of the local_irq_restore or interrupt return
- * code. In that case, skip the decrementr check becaus it's
- * expensive to read the TB.
- *
- * HARD_DIS then gets cleared here, but it's reconciled later.
- * Either local_irq_disable will replay the interrupt and that
- * will reconcile state like other hard interrupts. Or interrupt
- * retur will replay the interrupt and in that case it sets
- * PACA_IRQ_HARD_DIS by hand (see comments in entry_64.S).
- */
- if (happened & PACA_IRQ_HARD_DIS) {
- local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
-
- /*
- * We may have missed a decrementer interrupt if hard disabled.
- * Check the decrementer register in case we had a rollover
- * while hard disabled.
- */
- if (!(happened & PACA_IRQ_DEC)) {
- if (decrementer_check_overflow()) {
- local_paca->irq_happened |= PACA_IRQ_DEC;
- happened |= PACA_IRQ_DEC;
- }
- }
- }
-
if (happened & PACA_IRQ_DEC) {
local_paca->irq_happened &= ~PACA_IRQ_DEC;
return 0x900;
@@ -186,6 +149,9 @@ notrace unsigned int __check_irq_replay(void)
return 0x280;
}
+ if (happened & PACA_IRQ_HARD_DIS)
+ local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
+
/* There should be nothing left ! */
BUG_ON(local_paca->irq_happened != 0);
@@ -229,18 +195,6 @@ void replay_soft_interrupts(void)
if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
WARN_ON_ONCE(mfmsr() & MSR_EE);
- if (happened & PACA_IRQ_HARD_DIS) {
- /*
- * We may have missed a decrementer interrupt if hard disabled.
- * Check the decrementer register in case we had a rollover
- * while hard disabled.
- */
- if (!(happened & PACA_IRQ_DEC)) {
- if (decrementer_check_overflow())
- happened |= PACA_IRQ_DEC;
- }
- }
-
/*
* Force the delivery of pending soft-disabled interrupts on PS3.
* Any HV call will have this side effect.
@@ -345,6 +299,7 @@ notrace void arch_local_irq_restore(unsigned long mask)
if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
WARN_ON_ONCE(!(mfmsr() & MSR_EE));
__hard_irq_disable();
+ local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
} else {
/*
* We should already be hard disabled here. We had bugs
diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c
index 74efe46f5532..7d372ff3504b 100644
--- a/arch/powerpc/kernel/time.c
+++ b/arch/powerpc/kernel/time.c
@@ -552,14 +552,11 @@ void timer_interrupt(struct pt_regs *regs)
struct pt_regs *old_regs;
u64 now;
- /* Some implementations of hotplug will get timer interrupts while
- * offline, just ignore these and we also need to set
- * decrementers_next_tb as MAX to make sure __check_irq_replay
- * don't replay timer interrupt when return, otherwise we'll trap
- * here infinitely :(
+ /*
+ * Some implementations of hotplug will get timer interrupts while
+ * offline, just ignore these.
*/
if (unlikely(!cpu_online(smp_processor_id()))) {
- *next_tb = ~(u64)0;
set_dec(decrementer_max);
return;
}
diff --git a/arch/powerpc/platforms/powernv/opal.c b/arch/powerpc/platforms/powernv/opal.c
index d95954ad4c0a..c61c3b62c8c6 100644
--- a/arch/powerpc/platforms/powernv/opal.c
+++ b/arch/powerpc/platforms/powernv/opal.c
@@ -731,7 +731,7 @@ int opal_hmi_exception_early2(struct pt_regs *regs)
return 1;
}
-/* HMI exception handler called in virtual mode during check_irq_replay. */
+/* HMI exception handler called in virtual mode when irqs are next enabled. */
int opal_handle_hmi_exception(struct pt_regs *regs)
{
/*
--
2.23.0
^ permalink raw reply related
* [PATCH] ASoC: fsl_aud2htx: Remove dev_err() usage after platform_get_irq()
From: Shengjiu Wang @ 2020-11-07 2:20 UTC (permalink / raw)
To: timur, nicoleotsuka, Xiubo.Lee, festevam, broonie, perex, tiwai,
alsa-devel
Cc: linuxppc-dev, linux-kernel
platform_get_irq() would print error message internally, so dev_err()
after platform_get_irq() is not needed
Signed-off-by: Shengjiu Wang <shengjiu.wang@nxp.com>
---
sound/soc/fsl/fsl_aud2htx.c | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/sound/soc/fsl/fsl_aud2htx.c b/sound/soc/fsl/fsl_aud2htx.c
index 124aeb70f24e..4091ccc7c3e9 100644
--- a/sound/soc/fsl/fsl_aud2htx.c
+++ b/sound/soc/fsl/fsl_aud2htx.c
@@ -211,11 +211,8 @@ static int fsl_aud2htx_probe(struct platform_device *pdev)
}
irq = platform_get_irq(pdev, 0);
- if (irq < 0) {
- dev_err(&pdev->dev, "no irq for node %s\n",
- dev_name(&pdev->dev));
+ if (irq < 0)
return irq;
- }
ret = devm_request_irq(&pdev->dev, irq, fsl_aud2htx_isr, 0,
dev_name(&pdev->dev), aud2htx);
--
2.27.0
^ permalink raw reply related
* [RFC PATCH] powerpc: show registers when unwinding interrupt frames
From: Nicholas Piggin @ 2020-11-07 2:33 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
It's often useful to know the register state for interrupts in
the stack frame. In the below example (with this patch applied),
the important information is the state of the page fault.
A blatant case like this probably rather should have the page
fault regs passed down to the warning, but quite often there are
less obvious cases where an interrupt shows up that might give
some more clues.
The downside is longer and more complex bug output.
Bug: Write fault blocked by AMR!
WARNING: CPU: 0 PID: 72 at arch/powerpc/include/asm/book3s/64/kup-radix.h:164 __do_page_fault+0x880/0xa90
Modules linked in:
CPU: 0 PID: 72 Comm: systemd-gpt-aut Not tainted
NIP: c00000000006e2f0 LR: c00000000006e2ec CTR: 0000000000000000
REGS: c00000000a4f3420 TRAP: 0700
MSR: 8000000000021033 <SF,ME,IR,DR,RI,LE> CR: 28002840 XER: 20040000
CFAR: c000000000128be0 IRQMASK: 3
GPR00: c00000000006e2ec c00000000a4f36c0 c0000000014f0700 0000000000000020
GPR04: 0000000000000001 c000000001290f50 0000000000000001 c000000001290f80
GPR08: c000000001612b08 0000000000000000 0000000000000000 00000000ffffe0f7
GPR12: 0000000048002840 c0000000016e0000 c00c000000021c80 c000000000fd6f60
GPR16: 0000000000000000 c00000000a104698 0000000000000003 c0000000087f0000
GPR20: 0000000000000100 c0000000070330b8 0000000000000000 0000000000000004
GPR24: 0000000002000000 0000000000000300 0000000002000000 c00000000a5b0c00
GPR28: 0000000000000000 000000000a000000 00007fffb2a90038 c00000000a4f3820
NIP [c00000000006e2f0] __do_page_fault+0x880/0xa90
LR [c00000000006e2ec] __do_page_fault+0x87c/0xa90
Call Trace:
[c00000000a4f36c0] [c00000000006e2ec] __do_page_fault+0x87c/0xa90 (unreliable)
[c00000000a4f3780] [c000000000e1c034] do_page_fault+0x34/0x90
[c00000000a4f37b0] [c000000000008908] data_access_common_virt+0x158/0x1b0
--- interrupt: 300 at __copy_tofrom_user_base+0x9c/0x5a4
NIP: c00000000009b028 LR: c000000000802978 CTR: 0000000000000800
REGS: c00000000a4f3820 TRAP: 0300
MSR: 800000000280b033 <SF,VEC,VSX,EE,FP,ME,IR,DR,RI,LE> CR: 24004840 XER: 00000000
CFAR: c00000000009aff4 DAR: 00007fffb2a90038 DSISR: 0a000000 IRQMASK: 0
GPR00: 0000000000000000 c00000000a4f3ac0 c0000000014f0700 00007fffb2a90028
GPR04: c000000008720010 0000000000010000 0000000000000000 0000000000000000
GPR08: 0000000000000000 0000000000000000 0000000000000000 0000000000000001
GPR12: 0000000000004000 c0000000016e0000 c00c000000021c80 c000000000fd6f60
GPR16: 0000000000000000 c00000000a104698 0000000000000003 c0000000087f0000
GPR20: 0000000000000100 c0000000070330b8 0000000000000000 0000000000000004
GPR24: c00000000a4f3c80 c000000008720000 0000000000010000 0000000000000000
GPR28: 0000000000010000 0000000008720000 0000000000010000 c000000001515b98
NIP [c00000000009b028] __copy_tofrom_user_base+0x9c/0x5a4
LR [c000000000802978] copyout+0x68/0xc0
--- interrupt: 300
[c00000000a4f3af0] [c0000000008074b8] copy_page_to_iter+0x188/0x540
[c00000000a4f3b50] [c00000000035c678] generic_file_buffered_read+0x358/0xd80
[c00000000a4f3c40] [c0000000004c1e90] blkdev_read_iter+0x50/0x80
[c00000000a4f3c60] [c00000000045733c] new_sync_read+0x12c/0x1c0
[c00000000a4f3d00] [c00000000045a1f0] vfs_read+0x1d0/0x240
[c00000000a4f3d50] [c00000000045a7f4] ksys_read+0x84/0x140
[c00000000a4f3da0] [c000000000033a60] system_call_exception+0x100/0x280
[c00000000a4f3e10] [c00000000000c508] system_call_common+0xf8/0x2f8
Instruction dump:
eae10078 3be0000b 4bfff890 60420000 792917e1 4182ff18 3c82ffab 3884a5e0
3c62ffab 3863a6e8 480ba891 60000000 <0fe00000> 3be0000b 4bfff860 e93c0938
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/kernel/process.c | 20 ++++++++++++++------
1 file changed, 14 insertions(+), 6 deletions(-)
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index ea36a29c8b01..799f00b32f74 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -1475,12 +1475,10 @@ static void print_msr_bits(unsigned long val)
#define LAST_VOLATILE 12
#endif
-void show_regs(struct pt_regs * regs)
+static void __show_regs(struct pt_regs *regs)
{
int i, trap;
- show_regs_print_info(KERN_DEFAULT);
-
printk("NIP: "REG" LR: "REG" CTR: "REG"\n",
regs->nip, regs->link, regs->ctr);
printk("REGS: %px TRAP: %04lx %s (%s)\n",
@@ -1522,6 +1520,12 @@ void show_regs(struct pt_regs * regs)
printk("NIP ["REG"] %pS\n", regs->nip, (void *)regs->nip);
printk("LR ["REG"] %pS\n", regs->link, (void *)regs->link);
}
+}
+
+void show_regs(struct pt_regs *regs)
+{
+ show_regs_print_info(KERN_DEFAULT);
+ __show_regs(regs);
show_stack(current, (unsigned long *) regs->gpr[1], KERN_DEFAULT);
if (!user_mode(regs))
show_instructions(regs);
@@ -2192,10 +2196,14 @@ void show_stack(struct task_struct *tsk, unsigned long *stack,
&& stack[STACK_FRAME_MARKER] == STACK_FRAME_REGS_MARKER) {
struct pt_regs *regs = (struct pt_regs *)
(sp + STACK_FRAME_OVERHEAD);
+
lr = regs->link;
- printk("%s--- interrupt: %lx at %pS\n LR = %pS\n",
- loglvl, regs->trap,
- (void *)regs->nip, (void *)lr);
+ printk("%s--- interrupt: %lx at %pS\n",
+ loglvl, regs->trap, (void *)regs->nip);
+ __show_regs(regs);
+ printk("%s--- interrupt: %lx\n",
+ loglvl, regs->trap);
+
firstframe = 1;
}
--
2.23.0
^ permalink raw reply related
* Re: [PATCH] ASoC: fsl_aud2htx: Remove dev_err() usage after platform_get_irq()
From: Nicolin Chen @ 2020-11-07 2:28 UTC (permalink / raw)
To: Shengjiu Wang
Cc: alsa-devel, timur, Xiubo.Lee, linuxppc-dev, tiwai, perex, broonie,
festevam, linux-kernel
In-Reply-To: <1604715643-29507-1-git-send-email-shengjiu.wang@nxp.com>
On Sat, Nov 07, 2020 at 10:20:43AM +0800, Shengjiu Wang wrote:
> platform_get_irq() would print error message internally, so dev_err()
> after platform_get_irq() is not needed
>
> Signed-off-by: Shengjiu Wang <shengjiu.wang@nxp.com>
Acked-by: Nicolin Chen <nicoleotsuka@gmail.com>
^ permalink raw reply
* [PATCH] powerpc: add compile-time support for lbarx, lwarx
From: Nicholas Piggin @ 2020-11-07 3:23 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
ISA v2.06 (POWER7 and up) as well as e6500 support lbarx and lwarx.
Add a compile option that allows code to use it, and add support in
cmpxchg and xchg 8 and 16 bit values.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/Kconfig | 3 +
arch/powerpc/include/asm/cmpxchg.h | 236 ++++++++++++++++++++++++-
arch/powerpc/platforms/Kconfig.cputype | 5 +
3 files changed, 243 insertions(+), 1 deletion(-)
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index e9f13fe08492..d231af06f75a 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -266,6 +266,9 @@ config PPC_BARRIER_NOSPEC
default y
depends on PPC_BOOK3S_64 || PPC_FSL_BOOK3E
+config PPC_LBARX_LWARX
+ bool
+
config EARLY_PRINTK
bool
default y
diff --git a/arch/powerpc/include/asm/cmpxchg.h b/arch/powerpc/include/asm/cmpxchg.h
index cf091c4c22e5..17fd996dc0d4 100644
--- a/arch/powerpc/include/asm/cmpxchg.h
+++ b/arch/powerpc/include/asm/cmpxchg.h
@@ -77,10 +77,76 @@ u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new) \
* the previous value stored there.
*/
+#ifndef CONFIG_PPC_LBARX_LWARX
XCHG_GEN(u8, _local, "memory");
XCHG_GEN(u8, _relaxed, "cc");
XCHG_GEN(u16, _local, "memory");
XCHG_GEN(u16, _relaxed, "cc");
+#else
+static __always_inline unsigned long
+__xchg_u8_local(volatile void *p, unsigned long val)
+{
+ unsigned long prev;
+
+ __asm__ __volatile__(
+"1: lbarx %0,0,%2 \n"
+" stbcx. %3,0,%2 \n\
+ bne- 1b"
+ : "=&r" (prev), "+m" (*(volatile unsigned char *)p)
+ : "r" (p), "r" (val)
+ : "cc", "memory");
+
+ return prev;
+}
+
+static __always_inline unsigned long
+__xchg_u8_relaxed(u8 *p, unsigned long val)
+{
+ unsigned long prev;
+
+ __asm__ __volatile__(
+"1: lbarx %0,0,%2\n"
+" stbcx. %3,0,%2\n"
+" bne- 1b"
+ : "=&r" (prev), "+m" (*p)
+ : "r" (p), "r" (val)
+ : "cc");
+
+ return prev;
+}
+
+static __always_inline unsigned long
+__xchg_u16_local(volatile void *p, unsigned long val)
+{
+ unsigned long prev;
+
+ __asm__ __volatile__(
+"1: lharx %0,0,%2 \n"
+" sthcx. %3,0,%2 \n\
+ bne- 1b"
+ : "=&r" (prev), "+m" (*(volatile unsigned short *)p)
+ : "r" (p), "r" (val)
+ : "cc", "memory");
+
+ return prev;
+}
+
+static __always_inline unsigned long
+__xchg_u16_relaxed(u16 *p, unsigned long val)
+{
+ unsigned long prev;
+
+ __asm__ __volatile__(
+"1: lharx %0,0,%2\n"
+" sthcx. %3,0,%2\n"
+" bne- 1b"
+ : "=&r" (prev), "+m" (*p)
+ : "r" (p), "r" (val)
+ : "cc");
+
+ return prev;
+}
+#endif
static __always_inline unsigned long
__xchg_u32_local(volatile void *p, unsigned long val)
@@ -198,11 +264,12 @@ __xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
(__typeof__(*(ptr))) __xchg_relaxed((ptr), \
(unsigned long)_x_, sizeof(*(ptr))); \
})
+
/*
* Compare and exchange - if *p == old, set it to new,
* and return the old value of *p.
*/
-
+#ifndef CONFIG_PPC_LBARX_LWARX
CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
CMPXCHG_GEN(u8, _local, , , "memory");
CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
@@ -211,6 +278,173 @@ CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
CMPXCHG_GEN(u16, _local, , , "memory");
CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
CMPXCHG_GEN(u16, _relaxed, , , "cc");
+#else
+static __always_inline unsigned long
+__cmpxchg_u8(volatile unsigned char *p, unsigned long old, unsigned long new)
+{
+ unsigned int prev;
+
+ __asm__ __volatile__ (
+ PPC_ATOMIC_ENTRY_BARRIER
+"1: lbarx %0,0,%2 # __cmpxchg_u8\n\
+ cmpw 0,%0,%3\n\
+ bne- 2f\n"
+" stbcx. %4,0,%2\n\
+ bne- 1b"
+ PPC_ATOMIC_EXIT_BARRIER
+ "\n\
+2:"
+ : "=&r" (prev), "+m" (*p)
+ : "r" (p), "r" (old), "r" (new)
+ : "cc", "memory");
+
+ return prev;
+}
+
+static __always_inline unsigned long
+__cmpxchg_u8_local(volatile unsigned char *p, unsigned long old,
+ unsigned long new)
+{
+ unsigned int prev;
+
+ __asm__ __volatile__ (
+"1: lbarx %0,0,%2 # __cmpxchg_u8\n\
+ cmpw 0,%0,%3\n\
+ bne- 2f\n"
+" stbcx. %4,0,%2\n\
+ bne- 1b"
+ "\n\
+2:"
+ : "=&r" (prev), "+m" (*p)
+ : "r" (p), "r" (old), "r" (new)
+ : "cc", "memory");
+
+ return prev;
+}
+
+static __always_inline unsigned long
+__cmpxchg_u8_relaxed(u8 *p, unsigned long old, unsigned long new)
+{
+ unsigned long prev;
+
+ __asm__ __volatile__ (
+"1: lbarx %0,0,%2 # __cmpxchg_u8_relaxed\n"
+" cmpw 0,%0,%3\n"
+" bne- 2f\n"
+" stbcx. %4,0,%2\n"
+" bne- 1b\n"
+"2:"
+ : "=&r" (prev), "+m" (*p)
+ : "r" (p), "r" (old), "r" (new)
+ : "cc");
+
+ return prev;
+}
+
+static __always_inline unsigned long
+__cmpxchg_u8_acquire(u8 *p, unsigned long old, unsigned long new)
+{
+ unsigned long prev;
+
+ __asm__ __volatile__ (
+"1: lbarx %0,0,%2 # __cmpxchg_u8_acquire\n"
+" cmpw 0,%0,%3\n"
+" bne- 2f\n"
+" stbcx. %4,0,%2\n"
+" bne- 1b\n"
+ PPC_ACQUIRE_BARRIER
+ "\n"
+"2:"
+ : "=&r" (prev), "+m" (*p)
+ : "r" (p), "r" (old), "r" (new)
+ : "cc", "memory");
+
+ return prev;
+}
+
+static __always_inline unsigned long
+__cmpxchg_u16(volatile unsigned short *p, unsigned long old, unsigned long new)
+{
+ unsigned int prev;
+
+ __asm__ __volatile__ (
+ PPC_ATOMIC_ENTRY_BARRIER
+"1: lharx %0,0,%2 # __cmpxchg_u16\n\
+ cmpw 0,%0,%3\n\
+ bne- 2f\n"
+" sthcx. %4,0,%2\n\
+ bne- 1b"
+ PPC_ATOMIC_EXIT_BARRIER
+ "\n\
+2:"
+ : "=&r" (prev), "+m" (*p)
+ : "r" (p), "r" (old), "r" (new)
+ : "cc", "memory");
+
+ return prev;
+}
+
+static __always_inline unsigned long
+__cmpxchg_u16_local(volatile unsigned short *p, unsigned long old,
+ unsigned long new)
+{
+ unsigned int prev;
+
+ __asm__ __volatile__ (
+"1: lharx %0,0,%2 # __cmpxchg_u16\n\
+ cmpw 0,%0,%3\n\
+ bne- 2f\n"
+" sthcx. %4,0,%2\n\
+ bne- 1b"
+ "\n\
+2:"
+ : "=&r" (prev), "+m" (*p)
+ : "r" (p), "r" (old), "r" (new)
+ : "cc", "memory");
+
+ return prev;
+}
+
+static __always_inline unsigned long
+__cmpxchg_u16_relaxed(u16 *p, unsigned long old, unsigned long new)
+{
+ unsigned long prev;
+
+ __asm__ __volatile__ (
+"1: lharx %0,0,%2 # __cmpxchg_u16_relaxed\n"
+" cmpw 0,%0,%3\n"
+" bne- 2f\n"
+" sthcx. %4,0,%2\n"
+" bne- 1b\n"
+"2:"
+ : "=&r" (prev), "+m" (*p)
+ : "r" (p), "r" (old), "r" (new)
+ : "cc");
+
+ return prev;
+}
+
+static __always_inline unsigned long
+__cmpxchg_u16_acquire(u16 *p, unsigned long old, unsigned long new)
+{
+ unsigned long prev;
+
+ __asm__ __volatile__ (
+"1: lharx %0,0,%2 # __cmpxchg_u16_acquire\n"
+" cmpw 0,%0,%3\n"
+" bne- 2f\n"
+" sthcx. %4,0,%2\n"
+" bne- 1b\n"
+ PPC_ACQUIRE_BARRIER
+ "\n"
+"2:"
+ : "=&r" (prev), "+m" (*p)
+ : "r" (p), "r" (old), "r" (new)
+ : "cc", "memory");
+
+ return prev;
+}
+#endif
static __always_inline unsigned long
__cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype
index c194c4ae8bc7..2f8c8d61dba4 100644
--- a/arch/powerpc/platforms/Kconfig.cputype
+++ b/arch/powerpc/platforms/Kconfig.cputype
@@ -118,6 +118,7 @@ config GENERIC_CPU
bool "Generic (POWER8 and above)"
depends on PPC64 && CPU_LITTLE_ENDIAN
select ARCH_HAS_FAST_MULTIPLIER
+ select PPC_LBARX_LWARX
config GENERIC_CPU
bool "Generic 32 bits powerpc"
@@ -139,16 +140,19 @@ config POWER7_CPU
bool "POWER7"
depends on PPC_BOOK3S_64
select ARCH_HAS_FAST_MULTIPLIER
+ select PPC_LBARX_LWARX
config POWER8_CPU
bool "POWER8"
depends on PPC_BOOK3S_64
select ARCH_HAS_FAST_MULTIPLIER
+ select PPC_LBARX_LWARX
config POWER9_CPU
bool "POWER9"
depends on PPC_BOOK3S_64
select ARCH_HAS_FAST_MULTIPLIER
+ select PPC_LBARX_LWARX
config E5500_CPU
bool "Freescale e5500"
@@ -157,6 +161,7 @@ config E5500_CPU
config E6500_CPU
bool "Freescale e6500"
depends on E500
+ select PPC_LBARX_LWARX
config 860_CPU
bool "8xx family"
--
2.23.0
^ permalink raw reply related
* Re: [PATCH] powerpc: add compile-time support for lbarx, lwarx
From: Gabriel Paubert @ 2020-11-07 7:12 UTC (permalink / raw)
To: Nicholas Piggin; +Cc: linuxppc-dev
In-Reply-To: <20201107032328.2454582-1-npiggin@gmail.com>
On Sat, Nov 07, 2020 at 01:23:28PM +1000, Nicholas Piggin wrote:
> ISA v2.06 (POWER7 and up) as well as e6500 support lbarx and lwarx.
Hmm, lwarx exists since original Power AFAIR, s/lwarx/lharx/ perhaps?
Same for the title of the patch and the CONFIG variable.
Gabriel
> Add a compile option that allows code to use it, and add support in
> cmpxchg and xchg 8 and 16 bit values.
>
> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
> ---
> arch/powerpc/Kconfig | 3 +
> arch/powerpc/include/asm/cmpxchg.h | 236 ++++++++++++++++++++++++-
> arch/powerpc/platforms/Kconfig.cputype | 5 +
> 3 files changed, 243 insertions(+), 1 deletion(-)
>
> diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
> index e9f13fe08492..d231af06f75a 100644
> --- a/arch/powerpc/Kconfig
> +++ b/arch/powerpc/Kconfig
> @@ -266,6 +266,9 @@ config PPC_BARRIER_NOSPEC
> default y
> depends on PPC_BOOK3S_64 || PPC_FSL_BOOK3E
>
> +config PPC_LBARX_LWARX
> + bool
> +
> config EARLY_PRINTK
> bool
> default y
> diff --git a/arch/powerpc/include/asm/cmpxchg.h b/arch/powerpc/include/asm/cmpxchg.h
> index cf091c4c22e5..17fd996dc0d4 100644
> --- a/arch/powerpc/include/asm/cmpxchg.h
> +++ b/arch/powerpc/include/asm/cmpxchg.h
> @@ -77,10 +77,76 @@ u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new) \
> * the previous value stored there.
> */
>
> +#ifndef CONFIG_PPC_LBARX_LWARX
> XCHG_GEN(u8, _local, "memory");
> XCHG_GEN(u8, _relaxed, "cc");
> XCHG_GEN(u16, _local, "memory");
> XCHG_GEN(u16, _relaxed, "cc");
> +#else
> +static __always_inline unsigned long
> +__xchg_u8_local(volatile void *p, unsigned long val)
> +{
> + unsigned long prev;
> +
> + __asm__ __volatile__(
> +"1: lbarx %0,0,%2 \n"
> +" stbcx. %3,0,%2 \n\
> + bne- 1b"
> + : "=&r" (prev), "+m" (*(volatile unsigned char *)p)
> + : "r" (p), "r" (val)
> + : "cc", "memory");
> +
> + return prev;
> +}
> +
> +static __always_inline unsigned long
> +__xchg_u8_relaxed(u8 *p, unsigned long val)
> +{
> + unsigned long prev;
> +
> + __asm__ __volatile__(
> +"1: lbarx %0,0,%2\n"
> +" stbcx. %3,0,%2\n"
> +" bne- 1b"
> + : "=&r" (prev), "+m" (*p)
> + : "r" (p), "r" (val)
> + : "cc");
> +
> + return prev;
> +}
> +
> +static __always_inline unsigned long
> +__xchg_u16_local(volatile void *p, unsigned long val)
> +{
> + unsigned long prev;
> +
> + __asm__ __volatile__(
> +"1: lharx %0,0,%2 \n"
> +" sthcx. %3,0,%2 \n\
> + bne- 1b"
> + : "=&r" (prev), "+m" (*(volatile unsigned short *)p)
> + : "r" (p), "r" (val)
> + : "cc", "memory");
> +
> + return prev;
> +}
> +
> +static __always_inline unsigned long
> +__xchg_u16_relaxed(u16 *p, unsigned long val)
> +{
> + unsigned long prev;
> +
> + __asm__ __volatile__(
> +"1: lharx %0,0,%2\n"
> +" sthcx. %3,0,%2\n"
> +" bne- 1b"
> + : "=&r" (prev), "+m" (*p)
> + : "r" (p), "r" (val)
> + : "cc");
> +
> + return prev;
> +}
> +#endif
>
> static __always_inline unsigned long
> __xchg_u32_local(volatile void *p, unsigned long val)
> @@ -198,11 +264,12 @@ __xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
> (__typeof__(*(ptr))) __xchg_relaxed((ptr), \
> (unsigned long)_x_, sizeof(*(ptr))); \
> })
> +
> /*
> * Compare and exchange - if *p == old, set it to new,
> * and return the old value of *p.
> */
> -
> +#ifndef CONFIG_PPC_LBARX_LWARX
> CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
> CMPXCHG_GEN(u8, _local, , , "memory");
> CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
> @@ -211,6 +278,173 @@ CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
> CMPXCHG_GEN(u16, _local, , , "memory");
> CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
> CMPXCHG_GEN(u16, _relaxed, , , "cc");
> +#else
> +static __always_inline unsigned long
> +__cmpxchg_u8(volatile unsigned char *p, unsigned long old, unsigned long new)
> +{
> + unsigned int prev;
> +
> + __asm__ __volatile__ (
> + PPC_ATOMIC_ENTRY_BARRIER
> +"1: lbarx %0,0,%2 # __cmpxchg_u8\n\
> + cmpw 0,%0,%3\n\
> + bne- 2f\n"
> +" stbcx. %4,0,%2\n\
> + bne- 1b"
> + PPC_ATOMIC_EXIT_BARRIER
> + "\n\
> +2:"
> + : "=&r" (prev), "+m" (*p)
> + : "r" (p), "r" (old), "r" (new)
> + : "cc", "memory");
> +
> + return prev;
> +}
> +
> +static __always_inline unsigned long
> +__cmpxchg_u8_local(volatile unsigned char *p, unsigned long old,
> + unsigned long new)
> +{
> + unsigned int prev;
> +
> + __asm__ __volatile__ (
> +"1: lbarx %0,0,%2 # __cmpxchg_u8\n\
> + cmpw 0,%0,%3\n\
> + bne- 2f\n"
> +" stbcx. %4,0,%2\n\
> + bne- 1b"
> + "\n\
> +2:"
> + : "=&r" (prev), "+m" (*p)
> + : "r" (p), "r" (old), "r" (new)
> + : "cc", "memory");
> +
> + return prev;
> +}
> +
> +static __always_inline unsigned long
> +__cmpxchg_u8_relaxed(u8 *p, unsigned long old, unsigned long new)
> +{
> + unsigned long prev;
> +
> + __asm__ __volatile__ (
> +"1: lbarx %0,0,%2 # __cmpxchg_u8_relaxed\n"
> +" cmpw 0,%0,%3\n"
> +" bne- 2f\n"
> +" stbcx. %4,0,%2\n"
> +" bne- 1b\n"
> +"2:"
> + : "=&r" (prev), "+m" (*p)
> + : "r" (p), "r" (old), "r" (new)
> + : "cc");
> +
> + return prev;
> +}
> +
> +static __always_inline unsigned long
> +__cmpxchg_u8_acquire(u8 *p, unsigned long old, unsigned long new)
> +{
> + unsigned long prev;
> +
> + __asm__ __volatile__ (
> +"1: lbarx %0,0,%2 # __cmpxchg_u8_acquire\n"
> +" cmpw 0,%0,%3\n"
> +" bne- 2f\n"
> +" stbcx. %4,0,%2\n"
> +" bne- 1b\n"
> + PPC_ACQUIRE_BARRIER
> + "\n"
> +"2:"
> + : "=&r" (prev), "+m" (*p)
> + : "r" (p), "r" (old), "r" (new)
> + : "cc", "memory");
> +
> + return prev;
> +}
> +
> +static __always_inline unsigned long
> +__cmpxchg_u16(volatile unsigned short *p, unsigned long old, unsigned long new)
> +{
> + unsigned int prev;
> +
> + __asm__ __volatile__ (
> + PPC_ATOMIC_ENTRY_BARRIER
> +"1: lharx %0,0,%2 # __cmpxchg_u16\n\
> + cmpw 0,%0,%3\n\
> + bne- 2f\n"
> +" sthcx. %4,0,%2\n\
> + bne- 1b"
> + PPC_ATOMIC_EXIT_BARRIER
> + "\n\
> +2:"
> + : "=&r" (prev), "+m" (*p)
> + : "r" (p), "r" (old), "r" (new)
> + : "cc", "memory");
> +
> + return prev;
> +}
> +
> +static __always_inline unsigned long
> +__cmpxchg_u16_local(volatile unsigned short *p, unsigned long old,
> + unsigned long new)
> +{
> + unsigned int prev;
> +
> + __asm__ __volatile__ (
> +"1: lharx %0,0,%2 # __cmpxchg_u16\n\
> + cmpw 0,%0,%3\n\
> + bne- 2f\n"
> +" sthcx. %4,0,%2\n\
> + bne- 1b"
> + "\n\
> +2:"
> + : "=&r" (prev), "+m" (*p)
> + : "r" (p), "r" (old), "r" (new)
> + : "cc", "memory");
> +
> + return prev;
> +}
> +
> +static __always_inline unsigned long
> +__cmpxchg_u16_relaxed(u16 *p, unsigned long old, unsigned long new)
> +{
> + unsigned long prev;
> +
> + __asm__ __volatile__ (
> +"1: lharx %0,0,%2 # __cmpxchg_u16_relaxed\n"
> +" cmpw 0,%0,%3\n"
> +" bne- 2f\n"
> +" sthcx. %4,0,%2\n"
> +" bne- 1b\n"
> +"2:"
> + : "=&r" (prev), "+m" (*p)
> + : "r" (p), "r" (old), "r" (new)
> + : "cc");
> +
> + return prev;
> +}
> +
> +static __always_inline unsigned long
> +__cmpxchg_u16_acquire(u16 *p, unsigned long old, unsigned long new)
> +{
> + unsigned long prev;
> +
> + __asm__ __volatile__ (
> +"1: lharx %0,0,%2 # __cmpxchg_u16_acquire\n"
> +" cmpw 0,%0,%3\n"
> +" bne- 2f\n"
> +" sthcx. %4,0,%2\n"
> +" bne- 1b\n"
> + PPC_ACQUIRE_BARRIER
> + "\n"
> +"2:"
> + : "=&r" (prev), "+m" (*p)
> + : "r" (p), "r" (old), "r" (new)
> + : "cc", "memory");
> +
> + return prev;
> +}
> +#endif
>
> static __always_inline unsigned long
> __cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
> diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype
> index c194c4ae8bc7..2f8c8d61dba4 100644
> --- a/arch/powerpc/platforms/Kconfig.cputype
> +++ b/arch/powerpc/platforms/Kconfig.cputype
> @@ -118,6 +118,7 @@ config GENERIC_CPU
> bool "Generic (POWER8 and above)"
> depends on PPC64 && CPU_LITTLE_ENDIAN
> select ARCH_HAS_FAST_MULTIPLIER
> + select PPC_LBARX_LWARX
>
> config GENERIC_CPU
> bool "Generic 32 bits powerpc"
> @@ -139,16 +140,19 @@ config POWER7_CPU
> bool "POWER7"
> depends on PPC_BOOK3S_64
> select ARCH_HAS_FAST_MULTIPLIER
> + select PPC_LBARX_LWARX
>
> config POWER8_CPU
> bool "POWER8"
> depends on PPC_BOOK3S_64
> select ARCH_HAS_FAST_MULTIPLIER
> + select PPC_LBARX_LWARX
>
> config POWER9_CPU
> bool "POWER9"
> depends on PPC_BOOK3S_64
> select ARCH_HAS_FAST_MULTIPLIER
> + select PPC_LBARX_LWARX
>
> config E5500_CPU
> bool "Freescale e5500"
> @@ -157,6 +161,7 @@ config E5500_CPU
> config E6500_CPU
> bool "Freescale e6500"
> depends on E500
> + select PPC_LBARX_LWARX
>
> config 860_CPU
> bool "8xx family"
> --
> 2.23.0
>
^ permalink raw reply
* Re: [PATCH] KVM: PPC: Book3S: Assign boolean values to a bool variable
From: Greg Kurz @ 2020-11-07 7:36 UTC (permalink / raw)
To: xiakaixu1987; +Cc: Kaixu Xia, linuxppc-dev, linux-kernel, kvm-ppc
In-Reply-To: <1604730382-5810-1-git-send-email-kaixuxia@tencent.com>
On Sat, 7 Nov 2020 14:26:22 +0800
xiakaixu1987@gmail.com wrote:
> From: Kaixu Xia <kaixuxia@tencent.com>
>
> Fix the following coccinelle warnings:
>
> ./arch/powerpc/kvm/book3s_xics.c:476:3-15: WARNING: Assignment of 0/1 to bool variable
> ./arch/powerpc/kvm/book3s_xics.c:504:3-15: WARNING: Assignment of 0/1 to bool variable
>
> Reported-by: Tosk Robot <tencent_os_robot@tencent.com>
> Signed-off-by: Kaixu Xia <kaixuxia@tencent.com>
> ---
Reviewed-by: Greg Kurz <groug@kaod.org>
> arch/powerpc/kvm/book3s_xics.c | 4 ++--
> 1 file changed, 2 insertions(+), 2 deletions(-)
>
> diff --git a/arch/powerpc/kvm/book3s_xics.c b/arch/powerpc/kvm/book3s_xics.c
> index 5fee5a11550d..303e3cb096db 100644
> --- a/arch/powerpc/kvm/book3s_xics.c
> +++ b/arch/powerpc/kvm/book3s_xics.c
> @@ -473,7 +473,7 @@ static void icp_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
> arch_spin_unlock(&ics->lock);
> local_irq_restore(flags);
> new_irq = reject;
> - check_resend = 0;
> + check_resend = false;
> goto again;
> }
> } else {
> @@ -501,7 +501,7 @@ static void icp_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp,
> state->resend = 0;
> arch_spin_unlock(&ics->lock);
> local_irq_restore(flags);
> - check_resend = 0;
> + check_resend = false;
> goto again;
> }
> }
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox