diff --git a/arch/riscv/include/asm/trap_trace.h b/arch/riscv/include/asm/trap_trace.h new file mode 100644 index 0000000000000..a7a3d5824ea34 --- /dev/null +++ b/arch/riscv/include/asm/trap_trace.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef _ASM_RISCV_TRAP_TRACE_H +#define _ASM_RISCV_TRAP_TRACE_H + +struct pt_regs; + +void riscv_trap_trace(struct pt_regs *regs); +void riscv_trap_trace_dump_cpu(unsigned int cpu); +void riscv_trap_trace_enable(void); +void riscv_trap_trace_disable(void); + +#endif diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index ebe1c3588177b..e00e452cd345c 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -72,6 +72,7 @@ obj-y += vendor_extensions.o obj-y += vendor_extensions/ obj-y += probes/ obj-y += tests/ +obj-y += trap_trace.o obj-$(CONFIG_MMU) += vdso.o vdso/ obj-$(CONFIG_RISCV_USER_CFI) += vdso_cfi/ diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 61b7a77166971..bb923f421da76 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -210,6 +210,9 @@ SYM_CODE_START(handle_exception) /* Load the kernel shadow call stack pointer if coming from userspace */ scs_load_current_if_from_user s1, s5 + move a0, sp + call riscv_trap_trace + #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE move a0, sp call riscv_v_context_nesting_start diff --git a/arch/riscv/kernel/trap_trace.c b/arch/riscv/kernel/trap_trace.c new file mode 100644 index 0000000000000..d634051483cb6 --- /dev/null +++ b/arch/riscv/kernel/trap_trace.c @@ -0,0 +1,142 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#include +#include +#include +#include +#include + +#include +#include + +#define RISCV_TRAP_TRACE_ORDER 4 +#define RISCV_TRAP_TRACE_SIZE (1U << RISCV_TRAP_TRACE_ORDER) +#define RISCV_TRAP_TRACE_MASK (RISCV_TRAP_TRACE_SIZE - 1) + +struct riscv_trap_trace_entry { + unsigned long seq; + unsigned long epc; + unsigned long cause; + unsigned long badaddr; + unsigned long a0; + unsigned long a1; + unsigned long a2; + unsigned long a3; + unsigned long a4; + unsigned long vl; + unsigned long vstart; + unsigned long vtype; + unsigned long v0[4] __attribute__((aligned(32))); +}; + +struct riscv_trap_trace_cpu { + unsigned long enabled; + unsigned long next; + struct riscv_trap_trace_entry entry[RISCV_TRAP_TRACE_SIZE]; +}; + +static DEFINE_PER_CPU(struct riscv_trap_trace_cpu, riscv_trap_trace_data); + +noinline void riscv_trap_trace_enable(void) +{ + struct riscv_trap_trace_cpu *trace; + + trace = this_cpu_ptr(&riscv_trap_trace_data); + WRITE_ONCE(trace->enabled, 1); +} + +noinline void riscv_trap_trace_disable(void) +{ + struct riscv_trap_trace_cpu *trace; + + trace = this_cpu_ptr(&riscv_trap_trace_data); + WRITE_ONCE(trace->enabled, 0); +} + +/* + * Called from entry.S after pt_regs has been populated. + * + * Keep this function small: no allocation, locks, or printk. + */ +noinline void riscv_trap_trace(struct pt_regs *regs) +{ + struct riscv_trap_trace_cpu *trace; + struct riscv_trap_trace_entry *entry; + unsigned long seq; + + trace = this_cpu_ptr(&riscv_trap_trace_data); + if (!READ_ONCE(trace->enabled)) + return; + + seq = trace->next++; + entry = &trace->entry[seq & RISCV_TRAP_TRACE_MASK]; + + entry->epc = regs->epc; + entry->cause = regs->cause; + entry->badaddr = regs->badaddr; + entry->a0 = regs->a0; + entry->a1 = regs->a1; + entry->a2 = regs->a2; + entry->a3 = regs->a3; + entry->a4 = regs->a4; + + /* At this stage SR_VS is disabled (at least in 7.3, some patchsets + * change that) */ + csr_set(CSR_SSTATUS, SR_VS); + entry->vl = csr_read(CSR_VL); + entry->vstart = csr_read(CSR_VSTART); + entry->vtype = csr_read(CSR_VTYPE); + + asm volatile( + " .option push\n\t" + " .option arch, +v\n\t" + " vs1r.v v0, (%0)\n\t" + " .option pop\n\t" + :: "r"(&entry->v0) : "memory"); + + csr_clear(CSR_SSTATUS, SR_VS); + + /* Publish the sequence number after writing the record. */ + barrier(); + WRITE_ONCE(entry->seq, seq); +} + +/* + * Call this from a suitable diagnostic path after detecting corruption. + * Pass a CPU number to dump that CPU's buffer. + */ +void riscv_trap_trace_dump_cpu(unsigned int cpu) +{ + struct riscv_trap_trace_cpu *trace; + unsigned long end, start, seq; + unsigned int i; + + if (cpu >= nr_cpu_ids || !cpu_online(cpu)) { + pr_err("riscv-trace: CPU %u is not online\n", cpu); + return; + } + + trace = per_cpu_ptr(&riscv_trap_trace_data, cpu); + end = READ_ONCE(trace->next); + start = end > RISCV_TRAP_TRACE_SIZE ? + end - RISCV_TRAP_TRACE_SIZE : 0; + + pr_err("riscv-trace: CPU %u, records %lu..%lu\n", + cpu, start, end); + + for (seq = start; seq < end; seq++) { + struct riscv_trap_trace_entry e; + + i = seq & RISCV_TRAP_TRACE_MASK; + e = trace->entry[i]; + + if (READ_ONCE(trace->entry[i].seq) != seq) + continue; + + pr_err("riscv-trace: cpu=%u seq=%lu epc=%#lx cause=%#lx badaddr=%#lx a0=%#lx a1=%#lx a2=%#lx a3=%#lx a4=%#lx vl=%#lx vstart=%#lx vtype=%#lx\n", + cpu, seq, e.epc, e.cause, e.badaddr, e.a0, e.a1, e.a2, e.a3, e.a4, e.vl, e.vstart, e.vtype); + pr_err("riscv-trace: cpu=%i seq=%lu v0=%lx %lx %lx %lx\n", + cpu, seq, e.v0[0], e.v0[1], e.v0[2], e.v0[3]); + } +} + diff --git a/arch/riscv/lib/riscv_v_helpers.c b/arch/riscv/lib/riscv_v_helpers.c index 7bbdfc6d4552d..585c6d59908ff 100644 --- a/arch/riscv/lib/riscv_v_helpers.c +++ b/arch/riscv/lib/riscv_v_helpers.c @@ -13,26 +13,48 @@ #include #endif +#include + #ifdef CONFIG_MMU size_t riscv_v_usercopy_threshold = CONFIG_RISCV_ISA_V_UCOPY_THRESHOLD; -int __asm_vector_usercopy(void *dst, void *src, size_t n); -int __asm_vector_usercopy_sum_enabled(void *dst, void *src, size_t n); +int __asm_vector_usercopy(void *dst, void *src, size_t n, uint64_t *dbg); +int __asm_vector_usercopy_sum_enabled(void *dst, void *src, size_t n, uint64_t *dbg); +int __asm_usercopy_check(void *dst, void *src, size_t n); +int __asm_usercopy_check_sum_enabled(void *dst, void *src, size_t n); int fallback_scalar_usercopy(void *dst, void *src, size_t n); int fallback_scalar_usercopy_sum_enabled(void *dst, void *src, size_t n); asmlinkage int enter_vector_usercopy(void *dst, void *src, size_t n, bool enable_sum) { - size_t remain, copied; + size_t remain, copied, checked; + + /* 0 => cpu@start */ + /* 1 => cpu@end */ + uint64_t dbg[2]; + /* skip has_vector() check because it has been done by the asm */ if (!may_use_simd()) goto fallback; kernel_vector_begin(); - remain = enable_sum ? __asm_vector_usercopy(dst, src, n) : - __asm_vector_usercopy_sum_enabled(dst, src, n); + riscv_trap_trace_enable(); + remain = enable_sum ? __asm_vector_usercopy(dst, src, n, dbg) : + __asm_vector_usercopy_sum_enabled(dst, src, n, dbg); + riscv_trap_trace_disable(); kernel_vector_end(); + copied = n - remain; + checked = enable_sum ? __asm_usercopy_check(dst, src, copied) : __asm_usercopy_check_sum_enabled(dst, src, copied); + + if (checked == -1) { + pr_warn("RVV usercopy check can't be performed for dst=%px src=%px n=%zu remain=%zu copied=%zu enable_sum=%i\n", dst, src, n, remain, copied, enable_sum); + } else if (checked != copied) { + pr_err("RVV usercopy copied wrong data at %zu for for dst=%px src=%px n=%zu remain=%zu copied=%zu enable_sum=%i\n", checked, dst, src, n, remain, copied, enable_sum); + pr_err("Starting on CPU %llu, ended on CPU %llu\n", dbg[0], dbg[1]); + riscv_trap_trace_dump_cpu(dbg[0]); + } + if (remain) { copied = n - remain; dst += copied; diff --git a/arch/riscv/lib/uaccess_vector.S b/arch/riscv/lib/uaccess_vector.S index 03b5560609a22..c13dc58e6172d 100644 --- a/arch/riscv/lib/uaccess_vector.S +++ b/arch/riscv/lib/uaccess_vector.S @@ -9,7 +9,7 @@ #define pSrc a1 #define iNum a2 -#define iVL a3 +#define iVL a4 #define ELEM_LMUL_SETTING m8 #define vData v0 @@ -36,6 +36,8 @@ SYM_FUNC_START(__asm_vector_usercopy) SYM_FUNC_END(__asm_vector_usercopy) SYM_FUNC_START(__asm_vector_usercopy_sum_enabled) + REG_L t0, TASK_TI_CPU(tp) + /* t0 now contains the Linux logical CPU ID */ loop: vsetvli iVL, iNum, e8, ELEM_LMUL_SETTING, ta, ma fixup vle8.v vData, (pSrc), 10f @@ -48,6 +50,9 @@ loop: /* Exception fixup for vector load is shared with normal exit */ 10: mv a0, iNum + sd t0, 0(a3) + REG_L t0, TASK_TI_CPU(tp) + sd t0, 8(a3) ret /* Exception fixup code for vector store. */ @@ -59,3 +64,41 @@ loop: sub iNum, iNum, t2 j 10b SYM_FUNC_END(__asm_vector_usercopy_sum_enabled) + +SYM_FUNC_START(__asm_usercopy_check) + /* Enable access to user memory */ + li t6, SR_SUM + csrs CSR_STATUS, t6 + mv t6, ra + + call __asm_usercopy_check_sum_enabled + + /* Disable access to user memory */ + mv ra, t6 + li t6, SR_SUM + csrc CSR_STATUS, t6 + ret +SYM_FUNC_END(__asm_usercopy_check) + +SYM_FUNC_START(__asm_usercopy_check_sum_enabled) + li t0, 0 # offset +1: + beq t0, a2, 2f # end -> equal + + fixup lbu t1, 0(a0), 3f + fixup lbu t2, 0(a1), 3f + + bne t1, t2, 2f # difference at t0 + + addi a0, a0, 1 + addi a1, a1, 1 + addi t0, t0, 1 + j 1b + +2: + mv a0, t0 + ret + +3: li a0, -1 + ret +SYM_FUNC_END(__asm_usercopy_check_sum_enabled)