* Re: [PATCH v4 03/16] powerpc: Use a datatype for instructions
From: Balamuruhan S @ 2020-03-23 6:07 UTC (permalink / raw)
To: Jordan Niethe, linuxppc-dev; +Cc: alistair, dja, npiggin
In-Reply-To: <20200320051809.24332-4-jniethe5@gmail.com>
On Fri, 2020-03-20 at 16:17 +1100, Jordan Niethe wrote:
> Currently unsigned ints are used to represent instructions on powerpc.
> This has worked well as instructions have always been 4 byte words.
> However, a future ISA version will introduce some changes to
> instructions that mean this scheme will no longer work as well. This
> change is Prefixed Instructions. A prefixed instruction is made up of a
> word prefix followed by a word suffix to make an 8 byte double word
> instruction. No matter the endianess of the system the prefix always
> comes first. Prefixed instructions are only planned for powerpc64.
>
> Introduce a ppc_inst type to represent both prefixed and word
> instructions on powerpc64 while keeping it possible to exclusively have
> word instructions on powerpc32, A latter patch will expand the type to
> include prefixed instructions but for now just typedef it to a u32.
>
> Later patches will introduce helper functions and macros for
> manipulating the instructions so that powerpc64 and powerpc32 might
> maintain separate type definitions.
>
> Signed-off-by: Jordan Niethe <jniethe5@gmail.com>
> ---
> arch/powerpc/include/asm/code-patching.h | 31 +++++------
> arch/powerpc/include/asm/inst.h | 53 +++++++++++++++++++
> arch/powerpc/include/asm/sstep.h | 5 +-
> arch/powerpc/kernel/align.c | 2 +-
> arch/powerpc/kernel/hw_breakpoint.c | 3 +-
> arch/powerpc/kernel/kprobes.c | 2 +-
> arch/powerpc/kernel/mce_power.c | 5 +-
> arch/powerpc/kernel/optprobes.c | 10 ++--
> arch/powerpc/kernel/trace/ftrace.c | 66 ++++++++++++------------
> arch/powerpc/kvm/emulate_loadstore.c | 1 +
> arch/powerpc/lib/code-patching.c | 54 +++++++++----------
> arch/powerpc/lib/sstep.c | 4 +-
> arch/powerpc/lib/test_emulate_step.c | 9 ++--
> arch/powerpc/xmon/xmon.c | 12 ++---
> 14 files changed, 160 insertions(+), 97 deletions(-)
> create mode 100644 arch/powerpc/include/asm/inst.h
>
> diff --git a/arch/powerpc/include/asm/code-patching.h
> b/arch/powerpc/include/asm/code-patching.h
> index 898b54262881..cb5106f92d67 100644
> --- a/arch/powerpc/include/asm/code-patching.h
> +++ b/arch/powerpc/include/asm/code-patching.h
> @@ -11,6 +11,7 @@
> #include <linux/string.h>
> #include <linux/kallsyms.h>
> #include <asm/asm-compat.h>
> +#include <asm/inst.h>
>
> /* Flags for create_branch:
> * "b" == create_branch(addr, target, 0);
> @@ -22,27 +23,27 @@
> #define BRANCH_ABSOLUTE 0x2
>
> bool is_offset_in_branch_range(long offset);
> -unsigned int create_branch(const unsigned int *addr,
> +ppc_inst create_branch(const ppc_inst *addr,
> unsigned long target, int flags);
> -unsigned int create_cond_branch(const unsigned int *addr,
> +unsigned int create_cond_branch(const ppc_inst *addr,
> unsigned long target, int flags);
> -int patch_branch(unsigned int *addr, unsigned long target, int flags);
> -int patch_instruction(unsigned int *addr, unsigned int instr);
> -int raw_patch_instruction(unsigned int *addr, unsigned int instr);
> +int patch_branch(ppc_inst *addr, unsigned long target, int flags);
> +int patch_instruction(ppc_inst *addr, ppc_inst instr);
> +int raw_patch_instruction(ppc_inst *addr, ppc_inst instr);
>
> static inline unsigned long patch_site_addr(s32 *site)
> {
> return (unsigned long)site + *site;
> }
>
> -static inline int patch_instruction_site(s32 *site, unsigned int instr)
> +static inline int patch_instruction_site(s32 *site, ppc_inst instr)
> {
> - return patch_instruction((unsigned int *)patch_site_addr(site), instr);
> + return patch_instruction((ppc_inst *)patch_site_addr(site), instr);
> }
>
> static inline int patch_branch_site(s32 *site, unsigned long target, int
> flags)
> {
> - return patch_branch((unsigned int *)patch_site_addr(site), target,
> flags);
> + return patch_branch((ppc_inst *)patch_site_addr(site), target, flags);
> }
>
> static inline int modify_instruction(unsigned int *addr, unsigned int clr,
> @@ -56,13 +57,13 @@ static inline int modify_instruction_site(s32 *site,
> unsigned int clr, unsigned
> return modify_instruction((unsigned int *)patch_site_addr(site), clr,
> set);
> }
>
> -int instr_is_relative_branch(unsigned int instr);
> -int instr_is_relative_link_branch(unsigned int instr);
> -int instr_is_branch_to_addr(const unsigned int *instr, unsigned long addr);
> -unsigned long branch_target(const unsigned int *instr);
> -unsigned int translate_branch(const unsigned int *dest,
> - const unsigned int *src);
> -extern bool is_conditional_branch(unsigned int instr);
> +int instr_is_relative_branch(ppc_inst instr);
> +int instr_is_relative_link_branch(ppc_inst instr);
> +int instr_is_branch_to_addr(const ppc_inst *instr, unsigned long addr);
> +unsigned long branch_target(const ppc_inst *instr);
> +ppc_inst translate_branch(const ppc_inst *dest,
> + const ppc_inst *src);
> +extern bool is_conditional_branch(ppc_inst instr);
> #ifdef CONFIG_PPC_BOOK3E_64
> void __patch_exception(int exc, unsigned long addr);
> #define patch_exception(exc, name) do { \
> diff --git a/arch/powerpc/include/asm/inst.h
> b/arch/powerpc/include/asm/inst.h
> new file mode 100644
> index 000000000000..7c8596ee411e
> --- /dev/null
> +++ b/arch/powerpc/include/asm/inst.h
> @@ -0,0 +1,53 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +#ifndef _ASM_INST_H
> +#define _ASM_INST_H
> +
> +/*
> + * Instruction data type for POWER
> + */
> +
> +typedef u32 ppc_inst;
> +
> +#define PPC_INST(x) (x)
> +
> +static inline int ppc_inst_len(ppc_inst x)
should we consider `size_t` which is of type unsigned int ?
-- Bala
> +{
> + return sizeof(ppc_inst);
> +}
> +
> +static inline int ppc_inst_opcode(ppc_inst x)
> +{
> + return x >> 26;
> +}
> +
> +static inline u32 ppc_inst_word(ppc_inst x)
> +{
> + return x;
> +}
> +
> +static inline ppc_inst ppc_inst_read(const ppc_inst *ptr)
> +{
> + return *(ppc_inst *)ptr;
> +}
> +
> +static inline void ppc_inst_write(void *ptr, ppc_inst x)
> +{
> + *(ppc_inst *)ptr = x;
> +}
> +
> +static inline bool ppc_inst_equal(ppc_inst x, ppc_inst y)
> +{
> + return x == y;
> +}
> +
> +static inline bool ppc_inst_null(ppc_inst x)
> +{
> + return x == 0;
> +}
> +
> +static inline u32 ppc_inst_mask(ppc_inst x, u32 mask)
> +{
> + return ppc_inst_word(x) & mask;
> +}
> +
> +#endif /* _ASM_INST_H */
> diff --git a/arch/powerpc/include/asm/sstep.h
> b/arch/powerpc/include/asm/sstep.h
> index 769f055509c9..9353916fcba7 100644
> --- a/arch/powerpc/include/asm/sstep.h
> +++ b/arch/powerpc/include/asm/sstep.h
> @@ -2,6 +2,7 @@
> /*
> * Copyright (C) 2004 Paul Mackerras <paulus@au.ibm.com>, IBM
> */
> +#include <asm/inst.h>
>
> struct pt_regs;
>
> @@ -132,7 +133,7 @@ union vsx_reg {
> * otherwise.
> */
> extern int analyse_instr(struct instruction_op *op, const struct pt_regs
> *regs,
> - unsigned int instr);
> + ppc_inst instr);
>
> /*
> * Emulate an instruction that can be executed just by updating
> @@ -149,7 +150,7 @@ void emulate_update_regs(struct pt_regs *reg, struct
> instruction_op *op);
> * 0 if it could not be emulated, or -1 for an instruction that
> * should not be emulated (rfid, mtmsrd clearing MSR_RI, etc.).
> */
> -extern int emulate_step(struct pt_regs *regs, unsigned int instr);
> +extern int emulate_step(struct pt_regs *regs, ppc_inst instr);
>
> /*
> * Emulate a load or store instruction by reading/writing the
> diff --git a/arch/powerpc/kernel/align.c b/arch/powerpc/kernel/align.c
> index 92045ed64976..34594aaa44de 100644
> --- a/arch/powerpc/kernel/align.c
> +++ b/arch/powerpc/kernel/align.c
> @@ -293,7 +293,7 @@ static int emulate_spe(struct pt_regs *regs, unsigned int
> reg,
>
> int fix_alignment(struct pt_regs *regs)
> {
> - unsigned int instr;
> + ppc_inst instr;
> struct instruction_op op;
> int r, type;
>
> diff --git a/arch/powerpc/kernel/hw_breakpoint.c
> b/arch/powerpc/kernel/hw_breakpoint.c
> index 2462cd7c565c..06b97353d231 100644
> --- a/arch/powerpc/kernel/hw_breakpoint.c
> +++ b/arch/powerpc/kernel/hw_breakpoint.c
> @@ -24,6 +24,7 @@
> #include <asm/debug.h>
> #include <asm/debugfs.h>
> #include <asm/hvcall.h>
> +#include <asm/inst.h>
> #include <linux/uaccess.h>
>
> /*
> @@ -243,7 +244,7 @@ dar_range_overlaps(unsigned long dar, int size, struct
> arch_hw_breakpoint *info)
> static bool stepping_handler(struct pt_regs *regs, struct perf_event *bp,
> struct arch_hw_breakpoint *info)
> {
> - unsigned int instr = 0;
> + ppc_inst instr = 0;
> int ret, type, size;
> struct instruction_op op;
> unsigned long addr = info->address;
> diff --git a/arch/powerpc/kernel/kprobes.c b/arch/powerpc/kernel/kprobes.c
> index 337516df17d4..e7205adc9820 100644
> --- a/arch/powerpc/kernel/kprobes.c
> +++ b/arch/powerpc/kernel/kprobes.c
> @@ -225,7 +225,7 @@ NOKPROBE_SYMBOL(arch_prepare_kretprobe);
> static int try_to_emulate(struct kprobe *p, struct pt_regs *regs)
> {
> int ret;
> - unsigned int insn = *p->ainsn.insn;
> + ppc_inst insn = *p->ainsn.insn;
>
> /* regs->nip is also adjusted if emulate_step returns 1 */
> ret = emulate_step(regs, insn);
> diff --git a/arch/powerpc/kernel/mce_power.c
> b/arch/powerpc/kernel/mce_power.c
> index 1cbf7f1a4e3d..e65616bb3a3e 100644
> --- a/arch/powerpc/kernel/mce_power.c
> +++ b/arch/powerpc/kernel/mce_power.c
> @@ -20,6 +20,7 @@
> #include <asm/sstep.h>
> #include <asm/exception-64s.h>
> #include <asm/extable.h>
> +#include <asm/inst.h>
>
> /*
> * Convert an address related to an mm to a PFN. NOTE: we are in real
> @@ -365,7 +366,7 @@ static int mce_find_instr_ea_and_phys(struct pt_regs
> *regs, uint64_t *addr,
> * in real-mode is tricky and can lead to recursive
> * faults
> */
> - int instr;
> + ppc_inst instr;
> unsigned long pfn, instr_addr;
> struct instruction_op op;
> struct pt_regs tmp = *regs;
> @@ -373,7 +374,7 @@ static int mce_find_instr_ea_and_phys(struct pt_regs
> *regs, uint64_t *addr,
> pfn = addr_to_pfn(regs, regs->nip);
> if (pfn != ULONG_MAX) {
> instr_addr = (pfn << PAGE_SHIFT) + (regs->nip & ~PAGE_MASK);
> - instr = *(unsigned int *)(instr_addr);
> + instr = *(ppc_inst *)(instr_addr);
> if (!analyse_instr(&op, &tmp, instr)) {
> pfn = addr_to_pfn(regs, op.ea);
> *addr = op.ea;
> diff --git a/arch/powerpc/kernel/optprobes.c
> b/arch/powerpc/kernel/optprobes.c
> index 024f7aad1952..f5e8cce438a3 100644
> --- a/arch/powerpc/kernel/optprobes.c
> +++ b/arch/powerpc/kernel/optprobes.c
> @@ -189,8 +189,8 @@ void patch_imm64_load_insns(unsigned long val,
> kprobe_opcode_t *addr)
>
> int arch_prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe
> *p)
> {
> - kprobe_opcode_t *buff, branch_op_callback, branch_emulate_step;
> - kprobe_opcode_t *op_callback_addr, *emulate_step_addr;
> + ppc_inst branch_op_callback, branch_emulate_step;
> + kprobe_opcode_t *op_callback_addr, *emulate_step_addr, *buff;
> long b_offset;
> unsigned long nip, size;
> int rc, i;
> @@ -251,11 +251,11 @@ int arch_prepare_optimized_kprobe(struct
> optimized_kprobe *op, struct kprobe *p)
> goto error;
> }
>
> - branch_op_callback = create_branch((unsigned int *)buff +
> TMPL_CALL_HDLR_IDX,
> + branch_op_callback = create_branch((ppc_inst *)buff +
> TMPL_CALL_HDLR_IDX,
> (unsigned long)op_callback_addr,
> BRANCH_SET_LINK);
>
> - branch_emulate_step = create_branch((unsigned int *)buff +
> TMPL_EMULATE_IDX,
> + branch_emulate_step = create_branch((ppc_inst *)buff +
> TMPL_EMULATE_IDX,
> (unsigned long)emulate_step_addr,
> BRANCH_SET_LINK);
>
> @@ -316,7 +316,7 @@ void arch_optimize_kprobes(struct list_head *oplist)
> memcpy(op->optinsn.copied_insn, op->kp.addr,
> RELATIVEJUMP_SIZE);
> patch_instruction(op->kp.addr,
> - create_branch((unsigned int *)op->kp.addr,
> + create_branch((ppc_inst *)op->kp.addr,
> (unsigned long)op->optinsn.insn, 0));
> list_del_init(&op->list);
> }
> diff --git a/arch/powerpc/kernel/trace/ftrace.c
> b/arch/powerpc/kernel/trace/ftrace.c
> index 7ea0ca044b65..5787ccffb4df 100644
> --- a/arch/powerpc/kernel/trace/ftrace.c
> +++ b/arch/powerpc/kernel/trace/ftrace.c
> @@ -27,6 +27,7 @@
> #include <asm/code-patching.h>
> #include <asm/ftrace.h>
> #include <asm/syscall.h>
> +#include <asm/inst.h>
>
>
> #ifdef CONFIG_DYNAMIC_FTRACE
> @@ -40,23 +41,23 @@
> #define NUM_FTRACE_TRAMPS 8
> static unsigned long ftrace_tramps[NUM_FTRACE_TRAMPS];
>
> -static unsigned int
> +static ppc_inst
> ftrace_call_replace(unsigned long ip, unsigned long addr, int link)
> {
> - unsigned int op;
> + ppc_inst op;
>
> addr = ppc_function_entry((void *)addr);
>
> /* if (link) set op to 'bl' else 'b' */
> - op = create_branch((unsigned int *)ip, addr, link ? 1 : 0);
> + op = create_branch((ppc_inst *)ip, addr, link ? 1 : 0);
>
> return op;
> }
>
> static int
> -ftrace_modify_code(unsigned long ip, unsigned int old, unsigned int new)
> +ftrace_modify_code(unsigned long ip, ppc_inst old, ppc_inst new)
> {
> - unsigned int replaced;
> + ppc_inst replaced;
>
> /*
> * Note:
> @@ -78,7 +79,7 @@ ftrace_modify_code(unsigned long ip, unsigned int old,
> unsigned int new)
> }
>
> /* replace the text with the new text */
> - if (patch_instruction((unsigned int *)ip, new))
> + if (patch_instruction((ppc_inst *)ip, new))
> return -EPERM;
>
> return 0;
> @@ -87,25 +88,25 @@ ftrace_modify_code(unsigned long ip, unsigned int old,
> unsigned int new)
> /*
> * Helper functions that are the same for both PPC64 and PPC32.
> */
> -static int test_24bit_addr(unsigned long ip, unsigned long addr)
> +static ppc_inst test_24bit_addr(unsigned long ip, unsigned long addr)
> {
> addr = ppc_function_entry((void *)addr);
>
> /* use the create_branch to verify that this offset can be branched */
> - return create_branch((unsigned int *)ip, addr, 0);
> + return create_branch((ppc_inst *)ip, addr, 0);
> }
>
> -static int is_bl_op(unsigned int op)
> +static int is_bl_op(ppc_inst op)
> {
> return (op & 0xfc000003) == 0x48000001;
> }
>
> -static int is_b_op(unsigned int op)
> +static int is_b_op(ppc_inst op)
> {
> return (op & 0xfc000003) == 0x48000000;
> }
>
> -static unsigned long find_bl_target(unsigned long ip, unsigned int op)
> +static unsigned long find_bl_target(unsigned long ip, ppc_inst op)
> {
> int offset;
>
> @@ -125,7 +126,7 @@ __ftrace_make_nop(struct module *mod,
> {
> unsigned long entry, ptr, tramp;
> unsigned long ip = rec->ip;
> - unsigned int op, pop;
> + ppc_inst op, pop;
>
> /* read where this goes */
> if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
> @@ -204,7 +205,7 @@ __ftrace_make_nop(struct module *mod,
> }
> #endif /* CONFIG_MPROFILE_KERNEL */
>
> - if (patch_instruction((unsigned int *)ip, pop)) {
> + if (patch_instruction((ppc_inst *)ip, pop)) {
> pr_err("Patching NOP failed.\n");
> return -EPERM;
> }
> @@ -217,7 +218,7 @@ static int
> __ftrace_make_nop(struct module *mod,
> struct dyn_ftrace *rec, unsigned long addr)
> {
> - unsigned int op;
> + ppc_inst op;
> unsigned int jmp[4];
> unsigned long ip = rec->ip;
> unsigned long tramp;
> @@ -276,7 +277,7 @@ __ftrace_make_nop(struct module *mod,
>
> op = PPC_INST_NOP;
>
> - if (patch_instruction((unsigned int *)ip, op))
> + if (patch_instruction((ppc_inst *)ip, op))
> return -EPERM;
>
> return 0;
> @@ -322,7 +323,8 @@ static int add_ftrace_tramp(unsigned long tramp)
> */
> static int setup_mcount_compiler_tramp(unsigned long tramp)
> {
> - int i, op;
> + int i;
> + ppc_inst op;
> unsigned long ptr;
> static unsigned long ftrace_plt_tramps[NUM_FTRACE_TRAMPS];
>
> @@ -388,7 +390,7 @@ static int setup_mcount_compiler_tramp(unsigned long
> tramp)
> static int __ftrace_make_nop_kernel(struct dyn_ftrace *rec, unsigned long
> addr)
> {
> unsigned long tramp, ip = rec->ip;
> - unsigned int op;
> + ppc_inst op;
>
> /* Read where this goes */
> if (probe_kernel_read(&op, (void *)ip, sizeof(int))) {
> @@ -416,7 +418,7 @@ static int __ftrace_make_nop_kernel(struct dyn_ftrace
> *rec, unsigned long addr)
> }
> }
>
> - if (patch_instruction((unsigned int *)ip, PPC_INST_NOP)) {
> + if (patch_instruction((ppc_inst *)ip, PPC_INST_NOP)) {
> pr_err("Patching NOP failed.\n");
> return -EPERM;
> }
> @@ -428,7 +430,7 @@ int ftrace_make_nop(struct module *mod,
> struct dyn_ftrace *rec, unsigned long addr)
> {
> unsigned long ip = rec->ip;
> - unsigned int old, new;
> + ppc_inst old, new;
>
> /*
> * If the calling address is more that 24 bits away,
> @@ -481,7 +483,7 @@ int ftrace_make_nop(struct module *mod,
> */
> #ifndef CONFIG_MPROFILE_KERNEL
> static int
> -expected_nop_sequence(void *ip, unsigned int op0, unsigned int op1)
> +expected_nop_sequence(void *ip, ppc_inst op0, ppc_inst op1)
> {
> /*
> * We expect to see:
> @@ -510,7 +512,7 @@ expected_nop_sequence(void *ip, unsigned int op0,
> unsigned int op1)
> static int
> __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
> {
> - unsigned int op[2];
> + ppc_inst op[2];
> void *ip = (void *)rec->ip;
> unsigned long entry, ptr, tramp;
> struct module *mod = rec->arch.mod;
> @@ -574,7 +576,7 @@ __ftrace_make_call(struct dyn_ftrace *rec, unsigned long
> addr)
> static int
> __ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
> {
> - unsigned int op;
> + ppc_inst op;
> unsigned long ip = rec->ip;
>
> /* read where this goes */
> @@ -594,7 +596,7 @@ __ftrace_make_call(struct dyn_ftrace *rec, unsigned long
> addr)
> }
>
> /* create the branch to the trampoline */
> - op = create_branch((unsigned int *)ip,
> + op = create_branch((ppc_inst *)ip,
> rec->arch.mod->arch.tramp, BRANCH_SET_LINK);
> if (!op) {
> pr_err("REL24 out of range!\n");
> @@ -613,7 +615,7 @@ __ftrace_make_call(struct dyn_ftrace *rec, unsigned long
> addr)
>
> static int __ftrace_make_call_kernel(struct dyn_ftrace *rec, unsigned long
> addr)
> {
> - unsigned int op;
> + ppc_inst op;
> void *ip = (void *)rec->ip;
> unsigned long tramp, entry, ptr;
>
> @@ -661,7 +663,7 @@ static int __ftrace_make_call_kernel(struct dyn_ftrace
> *rec, unsigned long addr)
> int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
> {
> unsigned long ip = rec->ip;
> - unsigned int old, new;
> + ppc_inst old, new;
>
> /*
> * If the calling address is more that 24 bits away,
> @@ -700,7 +702,7 @@ static int
> __ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
> unsigned long addr)
> {
> - unsigned int op;
> + ppc_inst op;
> unsigned long ip = rec->ip;
> unsigned long entry, ptr, tramp;
> struct module *mod = rec->arch.mod;
> @@ -748,7 +750,7 @@ __ftrace_modify_call(struct dyn_ftrace *rec, unsigned
> long old_addr,
> /* The new target may be within range */
> if (test_24bit_addr(ip, addr)) {
> /* within range */
> - if (patch_branch((unsigned int *)ip, addr, BRANCH_SET_LINK)) {
> + if (patch_branch((ppc_inst *)ip, addr, BRANCH_SET_LINK)) {
> pr_err("REL24 out of range!\n");
> return -EINVAL;
> }
> @@ -776,7 +778,7 @@ __ftrace_modify_call(struct dyn_ftrace *rec, unsigned
> long old_addr,
> }
>
> /* Ensure branch is within 24 bits */
> - if (!create_branch((unsigned int *)ip, tramp, BRANCH_SET_LINK)) {
> + if (!create_branch((ppc_inst *)ip, tramp, BRANCH_SET_LINK)) {
> pr_err("Branch out of range\n");
> return -EINVAL;
> }
> @@ -794,7 +796,7 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned
> long old_addr,
> unsigned long addr)
> {
> unsigned long ip = rec->ip;
> - unsigned int old, new;
> + ppc_inst old, new;
>
> /*
> * If the calling address is more that 24 bits away,
> @@ -834,7 +836,7 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned
> long old_addr,
> int ftrace_update_ftrace_func(ftrace_func_t func)
> {
> unsigned long ip = (unsigned long)(&ftrace_call);
> - unsigned int old, new;
> + ppc_inst old, new;
> int ret;
>
> old = *(unsigned int *)&ftrace_call;
> @@ -919,7 +921,7 @@ int ftrace_enable_ftrace_graph_caller(void)
> unsigned long ip = (unsigned long)(&ftrace_graph_call);
> unsigned long addr = (unsigned long)(&ftrace_graph_caller);
> unsigned long stub = (unsigned long)(&ftrace_graph_stub);
> - unsigned int old, new;
> + ppc_inst old, new;
>
> old = ftrace_call_replace(ip, stub, 0);
> new = ftrace_call_replace(ip, addr, 0);
> @@ -932,7 +934,7 @@ int ftrace_disable_ftrace_graph_caller(void)
> unsigned long ip = (unsigned long)(&ftrace_graph_call);
> unsigned long addr = (unsigned long)(&ftrace_graph_caller);
> unsigned long stub = (unsigned long)(&ftrace_graph_stub);
> - unsigned int old, new;
> + ppc_inst old, new;
>
> old = ftrace_call_replace(ip, addr, 0);
> new = ftrace_call_replace(ip, stub, 0);
> diff --git a/arch/powerpc/kvm/emulate_loadstore.c
> b/arch/powerpc/kvm/emulate_loadstore.c
> index 1139bc56e004..1c9bcbfeb924 100644
> --- a/arch/powerpc/kvm/emulate_loadstore.c
> +++ b/arch/powerpc/kvm/emulate_loadstore.c
> @@ -21,6 +21,7 @@
> #include <asm/disassemble.h>
> #include <asm/ppc-opcode.h>
> #include <asm/sstep.h>
> +#include <asm/inst.h>
> #include "timing.h"
> #include "trace.h"
>
> diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-
> patching.c
> index 3345f039a876..8492b9e2b8db 100644
> --- a/arch/powerpc/lib/code-patching.c
> +++ b/arch/powerpc/lib/code-patching.c
> @@ -17,9 +17,10 @@
> #include <asm/page.h>
> #include <asm/code-patching.h>
> #include <asm/setup.h>
> +#include <asm/inst.h>
>
> -static int __patch_instruction(unsigned int *exec_addr, unsigned int instr,
> - unsigned int *patch_addr)
> +static int __patch_instruction(ppc_inst *exec_addr, ppc_inst instr,
> + ppc_inst *patch_addr)
> {
> int err = 0;
>
> @@ -33,7 +34,7 @@ static int __patch_instruction(unsigned int *exec_addr,
> unsigned int instr,
> return 0;
> }
>
> -int raw_patch_instruction(unsigned int *addr, unsigned int instr)
> +int raw_patch_instruction(ppc_inst *addr, ppc_inst instr)
> {
> return __patch_instruction(addr, instr, addr);
> }
> @@ -136,10 +137,10 @@ static inline int unmap_patch_area(unsigned long addr)
> return 0;
> }
>
> -static int do_patch_instruction(unsigned int *addr, unsigned int instr)
> +static int do_patch_instruction(ppc_inst *addr, ppc_inst instr)
> {
> int err;
> - unsigned int *patch_addr = NULL;
> + ppc_inst *patch_addr = NULL;
> unsigned long flags;
> unsigned long text_poke_addr;
> unsigned long kaddr = (unsigned long)addr;
> @@ -176,7 +177,7 @@ static int do_patch_instruction(unsigned int *addr,
> unsigned int instr)
> }
> #else /* !CONFIG_STRICT_KERNEL_RWX */
>
> -static int do_patch_instruction(unsigned int *addr, unsigned int instr)
> +static int do_patch_instruction(ppc_inst *addr, ppc_inst instr)
> {
> return raw_patch_instruction(addr, instr);
> }
> @@ -194,7 +195,7 @@ int patch_instruction(unsigned int *addr, unsigned int
> instr)
> }
> NOKPROBE_SYMBOL(patch_instruction);
>
> -int patch_branch(unsigned int *addr, unsigned long target, int flags)
> +int patch_branch(ppc_inst *addr, unsigned long target, int flags)
> {
> return patch_instruction(addr, create_branch(addr, target, flags));
> }
> @@ -225,7 +226,7 @@ bool is_offset_in_branch_range(long offset)
> * Helper to check if a given instruction is a conditional branch
> * Derived from the conditional checks in analyse_instr()
> */
> -bool is_conditional_branch(unsigned int instr)
> +bool is_conditional_branch(ppc_inst instr)
> {
> unsigned int opcode = instr >> 26;
>
> @@ -243,10 +244,10 @@ bool is_conditional_branch(unsigned int instr)
> }
> NOKPROBE_SYMBOL(is_conditional_branch);
>
> -unsigned int create_branch(const unsigned int *addr,
> +ppc_inst create_branch(const ppc_inst *addr,
> unsigned long target, int flags)
> {
> - unsigned int instruction;
> + ppc_inst instruction;
> long offset;
>
> offset = target;
> @@ -266,7 +267,7 @@ unsigned int create_branch(const unsigned int *addr,
> unsigned int create_cond_branch(const unsigned int *addr,
> unsigned long target, int flags)
> {
> - unsigned int instruction;
> + ppc_inst instruction;
> long offset;
>
> offset = target;
> @@ -283,22 +284,22 @@ unsigned int create_cond_branch(const unsigned int
> *addr,
> return instruction;
> }
>
> -static unsigned int branch_opcode(unsigned int instr)
> +static unsigned int branch_opcode(ppc_inst instr)
> {
> return (instr >> 26) & 0x3F;
> }
>
> -static int instr_is_branch_iform(unsigned int instr)
> +static int instr_is_branch_iform(ppc_inst instr)
> {
> return branch_opcode(instr) == 18;
> }
>
> -static int instr_is_branch_bform(unsigned int instr)
> +static int instr_is_branch_bform(ppc_inst instr)
> {
> return branch_opcode(instr) == 16;
> }
>
> -int instr_is_relative_branch(unsigned int instr)
> +int instr_is_relative_branch(ppc_inst instr)
> {
> if (instr & BRANCH_ABSOLUTE)
> return 0;
> @@ -306,12 +307,12 @@ int instr_is_relative_branch(unsigned int instr)
> return instr_is_branch_iform(instr) || instr_is_branch_bform(instr);
> }
>
> -int instr_is_relative_link_branch(unsigned int instr)
> +int instr_is_relative_link_branch(ppc_inst instr)
> {
> return instr_is_relative_branch(instr) && (instr & BRANCH_SET_LINK);
> }
>
> -static unsigned long branch_iform_target(const unsigned int *instr)
> +static unsigned long branch_iform_target(const ppc_inst *instr)
> {
> signed long imm;
>
> @@ -327,7 +328,7 @@ static unsigned long branch_iform_target(const unsigned
> int *instr)
> return (unsigned long)imm;
> }
>
> -static unsigned long branch_bform_target(const unsigned int *instr)
> +static unsigned long branch_bform_target(const ppc_inst *instr)
> {
> signed long imm;
>
> @@ -343,7 +344,7 @@ static unsigned long branch_bform_target(const unsigned
> int *instr)
> return (unsigned long)imm;
> }
>
> -unsigned long branch_target(const unsigned int *instr)
> +unsigned long branch_target(const ppc_inst *instr)
> {
> if (instr_is_branch_iform(*instr))
> return branch_iform_target(instr);
> @@ -353,7 +354,7 @@ unsigned long branch_target(const unsigned int *instr)
> return 0;
> }
>
> -int instr_is_branch_to_addr(const unsigned int *instr, unsigned long addr)
> +int instr_is_branch_to_addr(const ppc_inst *instr, unsigned long addr)
> {
> if (instr_is_branch_iform(*instr) || instr_is_branch_bform(*instr))
> return branch_target(instr) == addr;
> @@ -361,7 +362,7 @@ int instr_is_branch_to_addr(const unsigned int *instr,
> unsigned long addr)
> return 0;
> }
>
> -unsigned int translate_branch(const unsigned int *dest, const unsigned int
> *src)
> +ppc_inst translate_branch(const ppc_inst *dest, const ppc_inst *src)
> {
> unsigned long target;
>
> @@ -403,7 +404,7 @@ static void __init test_trampoline(void)
>
> static void __init test_branch_iform(void)
> {
> - unsigned int instr;
> + ppc_inst instr;
> unsigned long addr;
>
> addr = (unsigned long)&instr;
> @@ -478,11 +479,11 @@ static void __init test_branch_iform(void)
>
> static void __init test_create_function_call(void)
> {
> - unsigned int *iptr;
> + ppc_inst *iptr;
> unsigned long dest;
>
> /* Check we can create a function call */
> - iptr = (unsigned int *)ppc_function_entry(test_trampoline);
> + iptr = (ppc_inst *)ppc_function_entry(test_trampoline);
> dest = ppc_function_entry(test_create_function_call);
> patch_instruction(iptr, create_branch(iptr, dest, BRANCH_SET_LINK));
> check(instr_is_branch_to_addr(iptr, dest));
> @@ -491,7 +492,8 @@ static void __init test_create_function_call(void)
> static void __init test_branch_bform(void)
> {
> unsigned long addr;
> - unsigned int *iptr, instr, flags;
> + ppc_inst *iptr, instr;
> + unsigned int flags;
>
> iptr = &instr;
> addr = (unsigned long)iptr;
> @@ -561,7 +563,7 @@ static void __init test_branch_bform(void)
> static void __init test_translate_branch(void)
> {
> unsigned long addr;
> - unsigned int *p, *q;
> + ppc_inst *p, *q;
> void *buf;
>
> buf = vmalloc(PAGE_ALIGN(0x2000000 + 1));
> diff --git a/arch/powerpc/lib/sstep.c b/arch/powerpc/lib/sstep.c
> index c077acb983a1..1d9c766a89fe 100644
> --- a/arch/powerpc/lib/sstep.c
> +++ b/arch/powerpc/lib/sstep.c
> @@ -1163,7 +1163,7 @@ static nokprobe_inline int trap_compare(long v1, long
> v2)
> * otherwise.
> */
> int analyse_instr(struct instruction_op *op, const struct pt_regs *regs,
> - unsigned int instr)
> + ppc_inst instr)
> {
> unsigned int opcode, ra, rb, rc, rd, spr, u;
> unsigned long int imm;
> @@ -3101,7 +3101,7 @@ NOKPROBE_SYMBOL(emulate_loadstore);
> * or -1 if the instruction is one that should not be stepped,
> * such as an rfid, or a mtmsrd that would clear MSR_RI.
> */
> -int emulate_step(struct pt_regs *regs, unsigned int instr)
> +int emulate_step(struct pt_regs *regs, ppc_inst instr)
> {
> struct instruction_op op;
> int r, err, type;
> diff --git a/arch/powerpc/lib/test_emulate_step.c
> b/arch/powerpc/lib/test_emulate_step.c
> index 42347067739c..158efc8a0f53 100644
> --- a/arch/powerpc/lib/test_emulate_step.c
> +++ b/arch/powerpc/lib/test_emulate_step.c
> @@ -460,7 +460,7 @@ struct compute_test {
> struct {
> char *descr;
> unsigned long flags;
> - unsigned int instr;
> + ppc_inst instr;
> struct pt_regs regs;
> } subtests[MAX_SUBTESTS + 1];
> };
> @@ -841,7 +841,7 @@ static struct compute_test compute_tests[] = {
> };
>
> static int __init emulate_compute_instr(struct pt_regs *regs,
> - unsigned int instr)
> + ppc_inst instr)
> {
> struct instruction_op op;
>
> @@ -859,7 +859,7 @@ static int __init emulate_compute_instr(struct pt_regs
> *regs,
> }
>
> static int __init execute_compute_instr(struct pt_regs *regs,
> - unsigned int instr)
> + ppc_inst instr)
> {
> extern int exec_instr(struct pt_regs *regs);
> extern s32 patch__exec_instr;
> @@ -890,7 +890,8 @@ static void __init run_tests_compute(void)
> unsigned long flags;
> struct compute_test *test;
> struct pt_regs *regs, exp, got;
> - unsigned int i, j, k, instr;
> + unsigned int i, j, k;
> + ppc_inst instr;
> bool ignore_gpr, ignore_xer, ignore_ccr, passed;
>
> for (i = 0; i < ARRAY_SIZE(compute_tests); i++) {
> diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
> index 7875d1a37770..a0bc442f9557 100644
> --- a/arch/powerpc/xmon/xmon.c
> +++ b/arch/powerpc/xmon/xmon.c
> @@ -892,7 +892,7 @@ static struct bpt *new_breakpoint(unsigned long a)
> static void insert_bpts(void)
> {
> int i;
> - unsigned int instr;
> + ppc_inst instr;
> struct bpt *bp;
>
> bp = bpts;
> @@ -914,7 +914,7 @@ static void insert_bpts(void)
> patch_instruction(bp->instr, instr);
> if (bp->enabled & BP_CIABR)
> continue;
> - if (patch_instruction((unsigned int *)bp->address,
> + if (patch_instruction((ppc_inst *)bp->address,
> bpinstr) != 0) {
> printf("Couldn't write instruction at %lx, "
> "disabling breakpoint there\n", bp->address);
> @@ -943,7 +943,7 @@ static void remove_bpts(void)
> {
> int i;
> struct bpt *bp;
> - unsigned instr;
> + ppc_inst instr;
>
> bp = bpts;
> for (i = 0; i < NBPTS; ++i, ++bp) {
> @@ -952,7 +952,7 @@ static void remove_bpts(void)
> if (mread(bp->address, &instr, 4) == 4
> && instr == bpinstr
> && patch_instruction(
> - (unsigned int *)bp->address, bp->instr[0]) != 0)
> + (ppc_inst *)bp->address, bp->instr[0]) != 0)
> printf("Couldn't remove breakpoint at %lx\n",
> bp->address);
> }
> @@ -1159,7 +1159,7 @@ static int do_step(struct pt_regs *regs)
> */
> static int do_step(struct pt_regs *regs)
> {
> - unsigned int instr;
> + ppc_inst instr;
> int stepped;
>
> force_enable_xmon();
> @@ -1325,7 +1325,7 @@ csum(void)
> */
> static long check_bp_loc(unsigned long addr)
> {
> - unsigned int instr;
> + ppc_inst instr;
>
> addr &= ~3;
> if (!is_kernel_addr(addr)) {
^ permalink raw reply
* Re: [PATCH v4 02/16] xmon: Move out-of-line instructions to text section
From: Balamuruhan S @ 2020-03-23 5:59 UTC (permalink / raw)
To: Jordan Niethe, linuxppc-dev; +Cc: alistair, dja, npiggin
In-Reply-To: <20200320051809.24332-3-jniethe5@gmail.com>
On Fri, 2020-03-20 at 16:17 +1100, Jordan Niethe wrote:
> To execute an instruction out of line after a breakpoint, the NIP is
> set
> to the address of struct bpt::instr. Here a copy of the instruction
> that
> was replaced with a breakpoint is kept, along with a trap so normal
> flow
> can be resumed after XOLing. The struct bpt's are located within the
> data section. This is problematic as the data section may be marked
> as
> no execute.
>
> Instead of each struct bpt holding the instructions to be XOL'd, make
> a
> new array, bpt_table[], with enough space to hold instructions for
> the
> number of supported breakpoints. Place this array in the text
> section.
> Make struct bpt::instr a pointer to the instructions in bpt_table[]
> associated with that breakpoint. This association is a simple
> mapping:
> bpts[n] -> bpt_table[n * words per breakpoint].
Can it separate commits ?
* introduce the array bpt_table[] and make struct bpt::instr a
pointer to the instructions in bpt_table[].
* place the array in text section.
> Currently we only need
> the copied instruction followed by a trap, so 2 words per breakpoint.
>
> Signed-off-by: Jordan Niethe <jniethe5@gmail.com>
> ---
> v4: New to series
> ---
> arch/powerpc/kernel/vmlinux.lds.S | 2 +-
> arch/powerpc/xmon/xmon.c | 22 +++++++++++++---------
> 2 files changed, 14 insertions(+), 10 deletions(-)
>
> diff --git a/arch/powerpc/kernel/vmlinux.lds.S
> b/arch/powerpc/kernel/vmlinux.lds.S
> index b4c89a1acebb..e90845b8c300 100644
> --- a/arch/powerpc/kernel/vmlinux.lds.S
> +++ b/arch/powerpc/kernel/vmlinux.lds.S
> @@ -86,7 +86,7 @@ SECTIONS
> ALIGN_FUNCTION();
> #endif
> /* careful! __ftr_alt_* sections need to be close to
> .text */
> - *(.text.hot TEXT_MAIN .text.fixup .text.unlikely .fixup
> __ftr_alt_* .ref.text);
> + *(.text.hot TEXT_MAIN .text.fixup .text.unlikely .fixup
> __ftr_alt_* .ref.text .text.xmon_bpts);
> #ifdef CONFIG_PPC64
> *(.tramp.ftrace.text);
> #endif
> diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
> index 02e3bd62cab4..7875d1a37770 100644
> --- a/arch/powerpc/xmon/xmon.c
> +++ b/arch/powerpc/xmon/xmon.c
> @@ -97,7 +97,7 @@ static long *xmon_fault_jmp[NR_CPUS];
> /* Breakpoint stuff */
> struct bpt {
> unsigned long address;
> - unsigned int instr[2];
> + unsigned int *instr;
> atomic_t ref_count;
> int enabled;
> unsigned long pad;
> @@ -109,6 +109,7 @@ struct bpt {
> #define BP_DABR 4
>
> #define NBPTS 256
> +#define BPT_WORDS 2
> static struct bpt bpts[NBPTS];
> static struct bpt dabr;
> static struct bpt *iabr;
> @@ -116,6 +117,8 @@ static unsigned bpinstr = 0x7fe00008; /* trap
> */
>
> #define BP_NUM(bp) ((bp) - bpts + 1)
>
> +static unsigned int __section(.text.xmon_bpts) bpt_table[NBPTS *
> BPT_WORDS];
> +
> /* Prototypes */
> static int cmds(struct pt_regs *);
> static int mread(unsigned long, void *, int);
> @@ -852,16 +855,16 @@ static struct bpt *at_breakpoint(unsigned long
> pc)
> static struct bpt *in_breakpoint_table(unsigned long nip, unsigned
> long *offp)
> {
> unsigned long off;
> + unsigned long bp_off;
>
> - off = nip - (unsigned long) bpts;
> - if (off >= sizeof(bpts))
> + off = nip - (unsigned long) bpt_table;
> + if (off >= sizeof(bpt_table))
> return NULL;
> - off %= sizeof(struct bpt);
> - if (off != offsetof(struct bpt, instr[0])
> - && off != offsetof(struct bpt, instr[1]))
> + bp_off = off % (sizeof(unsigned int) * BPT_WORDS);
> + if (bp_off != 0 && bp_off != 4)
> return NULL;
> - *offp = off - offsetof(struct bpt, instr[0]);
> - return (struct bpt *) (nip - off);
> + *offp = bp_off;
> + return bpts + ((off - bp_off) / (sizeof(unsigned int) *
> BPT_WORDS));
`(off - bp_off) / (sizeof(unsigned int) * BPT_WORDS)` seems to be the
actual breakpoint offset. Can we have something like,
#define NBPTS 256
#define BPT_WORDS 2
#define BPT_WORDS_SIZE (sizeof(unsigned int) * BPT_WORDS)
#define BPT_OFFSET(off, bp_word_off) ((off - bp_word_off) / BPT_WOR
DS_SIZE);
:::
:::
:::
bp_word_off = off % BPT_WORDS_SIZE;
if (bp_word_off != 0 && bp_word_off != 4)
return NULL;
*offp = bp_word_off;
return bpts + BPT_OFFSET(off, bp_word_off);
-- Bala
> }
static struct bpt *new_breakpoint(unsigned long a)
@@ -876,7
> +879,8 @@ static struct bpt *new_breakpoint(unsigned long a)
for (bp
> = bpts; bp < &bpts[NBPTS]; ++bp) {
if (!bp->enabled &&
> atomic_read(&bp->ref_count) == 0) {
bp->address =
> a;
- patch_instruction(&bp->instr[1], bpinstr);
+
> bp->instr = bpt_table + ((bp - bpts) * BPT_WORDS);
+
> patch_instruction(bp->instr + 1, bpinstr);
> return bp;
}
}
^ permalink raw reply
* Re: [PATCH v4 02/16] xmon: Move out-of-line instructions to text section
From: Balamuruhan S @ 2020-03-23 6:05 UTC (permalink / raw)
To: Jordan Niethe, linuxppc-dev; +Cc: alistair, dja, npiggin
In-Reply-To: <20200320051809.24332-3-jniethe5@gmail.com>
On Fri, 2020-03-20 at 16:17 +1100, Jordan Niethe wrote:
> To execute an instruction out of line after a breakpoint, the NIP is
> set
> to the address of struct bpt::instr. Here a copy of the instruction
> that
> was replaced with a breakpoint is kept, along with a trap so normal
> flow
> can be resumed after XOLing. The struct bpt's are located within the
> data section. This is problematic as the data section may be marked
> as
> no execute.
>
> Instead of each struct bpt holding the instructions to be XOL'd, make
> a
> new array, bpt_table[], with enough space to hold instructions for
> the
> number of supported breakpoints. Place this array in the text
> section.
> Make struct bpt::instr a pointer to the instructions in bpt_table[]
> associated with that breakpoint. This association is a simple
> mapping:
> bpts[n] -> bpt_table[n * words per breakpoint].
Can we have it in separate commits ?
* introduce the array bpt_table[] and make struct bpt::instr a
pointer to the instructions in bpt_table[].
* place the array in text section.
> Currently we only need
> the copied instruction followed by a trap, so 2 words per breakpoint.
>
> Signed-off-by: Jordan Niethe <jniethe5@gmail.com>
> ---
> v4: New to series
> ---
> arch/powerpc/kernel/vmlinux.lds.S | 2 +-
> arch/powerpc/xmon/xmon.c | 22 +++++++++++++---------
> 2 files changed, 14 insertions(+), 10 deletions(-)
>
> diff --git a/arch/powerpc/kernel/vmlinux.lds.S
> b/arch/powerpc/kernel/vmlinux.lds.S
> index b4c89a1acebb..e90845b8c300 100644
> --- a/arch/powerpc/kernel/vmlinux.lds.S
> +++ b/arch/powerpc/kernel/vmlinux.lds.S
> @@ -86,7 +86,7 @@ SECTIONS
> ALIGN_FUNCTION();
> #endif
> /* careful! __ftr_alt_* sections need to be close to
> .text */
> - *(.text.hot TEXT_MAIN .text.fixup .text.unlikely .fixup
> __ftr_alt_* .ref.text);
> + *(.text.hot TEXT_MAIN .text.fixup .text.unlikely .fixup
> __ftr_alt_* .ref.text .text.xmon_bpts);
> #ifdef CONFIG_PPC64
> *(.tramp.ftrace.text);
> #endif
> diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
> index 02e3bd62cab4..7875d1a37770 100644
> --- a/arch/powerpc/xmon/xmon.c
> +++ b/arch/powerpc/xmon/xmon.c
> @@ -97,7 +97,7 @@ static long *xmon_fault_jmp[NR_CPUS];
> /* Breakpoint stuff */
> struct bpt {
> unsigned long address;
> - unsigned int instr[2];
> + unsigned int *instr;
> atomic_t ref_count;
> int enabled;
> unsigned long pad;
> @@ -109,6 +109,7 @@ struct bpt {
> #define BP_DABR 4
>
> #define NBPTS 256
> +#define BPT_WORDS 2
> static struct bpt bpts[NBPTS];
> static struct bpt dabr;
> static struct bpt *iabr;
> @@ -116,6 +117,8 @@ static unsigned bpinstr = 0x7fe00008; /* trap
> */
>
> #define BP_NUM(bp) ((bp) - bpts + 1)
>
> +static unsigned int __section(.text.xmon_bpts) bpt_table[NBPTS *
> BPT_WORDS];
> +
> /* Prototypes */
> static int cmds(struct pt_regs *);
> static int mread(unsigned long, void *, int);
> @@ -852,16 +855,16 @@ static struct bpt *at_breakpoint(unsigned long
> pc)
> static struct bpt *in_breakpoint_table(unsigned long nip, unsigned
> long *offp)
> {
> unsigned long off;
> + unsigned long bp_off;
>
> - off = nip - (unsigned long) bpts;
> - if (off >= sizeof(bpts))
> + off = nip - (unsigned long) bpt_table;
> + if (off >= sizeof(bpt_table))
> return NULL;
> - off %= sizeof(struct bpt);
> - if (off != offsetof(struct bpt, instr[0])
> - && off != offsetof(struct bpt, instr[1]))
> + bp_off = off % (sizeof(unsigned int) * BPT_WORDS);
> + if (bp_off != 0 && bp_off != 4)
> return NULL;
> - *offp = off - offsetof(struct bpt, instr[0]);
> - return (struct bpt *) (nip - off);
> + *offp = bp_off;
> + return bpts + ((off - bp_off) / (sizeof(unsigned int) *
> BPT_WORDS));
`(off - bp_off) / (sizeof(unsigned int) * BPT_WORDS)` seems to be the
actual breakpoint offset. Can we have something like,
#define NBPTS 256
#define BPT_WORDS 2
#define BPT_WORDS_SIZE (sizeof(unsigned int) * BPT_WORDS)
#define BPT_OFFSET(off, bp_word_off) ((off - bp_word_off) / \ BPT_WORDS_SIZE)
;
:::
:::
:::
bp_word_off = off % BPT_WORDS_SIZE;
if (bp_word_off != 0 && bp_word_off != 4)
return NULL;
*offp = bp_word_off;
return bpts + BPT_OFFSET(off, bp_word_off);
-- Bala
^ permalink raw reply
* [PATCH 1/2] Fix "[v3, 25/32] powerpc/64: system call implement entry/exit logic in C"
From: Nicholas Piggin @ 2020-03-23 5:44 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
Changes Christophe asked for to help ppc32 implementation, and move a
stray hunk into a later patch.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/asm-prototypes.h | 4 ---
arch/powerpc/include/asm/cputime.h | 38 +++++++++++++----------
2 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h
index 4b3609554e76..ab59a4904254 100644
--- a/arch/powerpc/include/asm/asm-prototypes.h
+++ b/arch/powerpc/include/asm/asm-prototypes.h
@@ -97,12 +97,8 @@ ppc_select(int n, fd_set __user *inp, fd_set __user *outp, fd_set __user *exp,
unsigned long __init early_init(unsigned long dt_ptr);
void __init machine_init(u64 dt_ptr);
#endif
-#ifdef CONFIG_PPC64
long system_call_exception(long r3, long r4, long r5, long r6, long r7, long r8, unsigned long r0, struct pt_regs *regs);
notrace unsigned long syscall_exit_prepare(unsigned long r3, struct pt_regs *regs);
-notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr);
-notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsigned long msr);
-#endif
long ppc_fadvise64_64(int fd, int advice, u32 offset_high, u32 offset_low,
u32 len_high, u32 len_low);
diff --git a/arch/powerpc/include/asm/cputime.h b/arch/powerpc/include/asm/cputime.h
index 6639a6847cc0..0fccd5ea1e9a 100644
--- a/arch/powerpc/include/asm/cputime.h
+++ b/arch/powerpc/include/asm/cputime.h
@@ -43,8 +43,26 @@ static inline unsigned long cputime_to_usecs(const cputime_t ct)
*/
#ifdef CONFIG_PPC64
#define get_accounting(tsk) (&get_paca()->accounting)
+#define raw_get_accounting(tsk) (&local_paca->accounting)
static inline void arch_vtime_task_switch(struct task_struct *tsk) { }
+#else
+#define get_accounting(tsk) (&task_thread_info(tsk)->accounting)
+#define raw_get_accounting(tsk) get_accounting(tsk)
+/*
+ * Called from the context switch with interrupts disabled, to charge all
+ * accumulated times to the current process, and to prepare accounting on
+ * the next process.
+ */
+static inline void arch_vtime_task_switch(struct task_struct *prev)
+{
+ struct cpu_accounting_data *acct = get_accounting(current);
+ struct cpu_accounting_data *acct0 = get_accounting(prev);
+
+ acct->starttime = acct0->starttime;
+}
+#endif
+
/*
* account_cpu_user_entry/exit runs "unreconciled", so can't trace,
* can't use use get_paca()
@@ -52,35 +70,21 @@ static inline void arch_vtime_task_switch(struct task_struct *tsk) { }
static notrace inline void account_cpu_user_entry(void)
{
unsigned long tb = mftb();
- struct cpu_accounting_data *acct = &local_paca->accounting;
+ struct cpu_accounting_data *acct = raw_get_accounting(current);
acct->utime += (tb - acct->starttime_user);
acct->starttime = tb;
}
+
static notrace inline void account_cpu_user_exit(void)
{
unsigned long tb = mftb();
- struct cpu_accounting_data *acct = &local_paca->accounting;
+ struct cpu_accounting_data *acct = raw_get_accounting(current);
acct->stime += (tb - acct->starttime);
acct->starttime_user = tb;
}
-#else
-#define get_accounting(tsk) (&task_thread_info(tsk)->accounting)
-/*
- * Called from the context switch with interrupts disabled, to charge all
- * accumulated times to the current process, and to prepare accounting on
- * the next process.
- */
-static inline void arch_vtime_task_switch(struct task_struct *prev)
-{
- struct cpu_accounting_data *acct = get_accounting(current);
- struct cpu_accounting_data *acct0 = get_accounting(prev);
-
- acct->starttime = acct0->starttime;
-}
-#endif
#endif /* __KERNEL__ */
#else /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
--
2.23.0
^ permalink raw reply related
* [RFC PATCH 1/3] powerpc/mm: Introduce temporary mm
From: Christopher M. Riedl @ 2020-03-23 4:52 UTC (permalink / raw)
To: linuxppc-dev
In-Reply-To: <20200323045205.20314-1-cmr@informatik.wtf>
x86 supports the notion of a temporary mm which restricts access to
temporary PTEs to a single CPU. A temporary mm is useful for situations
where a CPU needs to perform sensitive operations (such as patching a
STRICT_KERNEL_RWX kernel) requiring temporary mappings without exposing
said mappings to other CPUs. A side benefit is that other CPU TLBs do
not need to be flushed when the temporary mm is torn down.
Mappings in the temporary mm can be set in the userspace portion of the
address-space.
Interrupts must be disabled while the temporary mm is in use. HW
breakpoints, which may have been set by userspace as watchpoints on
addresses now within the temporary mm, are saved and disabled when
loading the temporary mm. The HW breakpoints are restored when unloading
the temporary mm. All HW breakpoints are indiscriminately disabled while
the temporary mm is in use.
Based on x86 implementation:
commit cefa929c034e
("x86/mm: Introduce temporary mm structs")
Signed-off-by: Christopher M. Riedl <cmr@informatik.wtf>
---
arch/powerpc/include/asm/debug.h | 1 +
arch/powerpc/include/asm/mmu_context.h | 56 +++++++++++++++++++++++++-
arch/powerpc/kernel/process.c | 5 +++
3 files changed, 61 insertions(+), 1 deletion(-)
diff --git a/arch/powerpc/include/asm/debug.h b/arch/powerpc/include/asm/debug.h
index 7756026b95ca..b945bc16c932 100644
--- a/arch/powerpc/include/asm/debug.h
+++ b/arch/powerpc/include/asm/debug.h
@@ -45,6 +45,7 @@ static inline int debugger_break_match(struct pt_regs *regs) { return 0; }
static inline int debugger_fault_handler(struct pt_regs *regs) { return 0; }
#endif
+void __get_breakpoint(struct arch_hw_breakpoint *brk);
void __set_breakpoint(struct arch_hw_breakpoint *brk);
bool ppc_breakpoint_available(void);
#ifdef CONFIG_PPC_ADV_DEBUG_REGS
diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h
index 360367c579de..3e6381d04c28 100644
--- a/arch/powerpc/include/asm/mmu_context.h
+++ b/arch/powerpc/include/asm/mmu_context.h
@@ -7,9 +7,10 @@
#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/spinlock.h>
-#include <asm/mmu.h>
+#include <asm/mmu.h>
#include <asm/cputable.h>
#include <asm/cputhreads.h>
+#include <asm/hw_breakpoint.h>
/*
* Most if the context management is out of line
@@ -270,5 +271,58 @@ static inline int arch_dup_mmap(struct mm_struct *oldmm,
return 0;
}
+struct temp_mm {
+ struct mm_struct *temp;
+ struct mm_struct *prev;
+ bool is_kernel_thread;
+ struct arch_hw_breakpoint brk;
+};
+
+static inline void init_temp_mm(struct temp_mm *temp_mm, struct mm_struct *mm)
+{
+ temp_mm->temp = mm;
+ temp_mm->prev = NULL;
+ temp_mm->is_kernel_thread = false;
+ memset(&temp_mm->brk, 0, sizeof(temp_mm->brk));
+}
+
+static inline void use_temporary_mm(struct temp_mm *temp_mm)
+{
+ lockdep_assert_irqs_disabled();
+
+ temp_mm->is_kernel_thread = current->mm == NULL;
+ if (temp_mm->is_kernel_thread)
+ temp_mm->prev = current->active_mm;
+ else
+ temp_mm->prev = current->mm;
+
+ /*
+ * Hash requires a non-NULL current->mm to allocate a userspace address
+ * when handling a page fault. Does not appear to hurt in Radix either.
+ */
+ current->mm = temp_mm->temp;
+ switch_mm_irqs_off(NULL, temp_mm->temp, current);
+
+ if (ppc_breakpoint_available()) {
+ __get_breakpoint(&temp_mm->brk);
+ if (temp_mm->brk.type != 0)
+ hw_breakpoint_disable();
+ }
+}
+
+static inline void unuse_temporary_mm(struct temp_mm *temp_mm)
+{
+ lockdep_assert_irqs_disabled();
+
+ if (temp_mm->is_kernel_thread)
+ current->mm = NULL;
+ else
+ current->mm = temp_mm->prev;
+ switch_mm_irqs_off(NULL, temp_mm->prev, current);
+
+ if (ppc_breakpoint_available() && temp_mm->brk.type != 0)
+ __set_breakpoint(&temp_mm->brk);
+}
+
#endif /* __KERNEL__ */
#endif /* __ASM_POWERPC_MMU_CONTEXT_H */
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index fad50db9dcf2..5e5cf33fc358 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -793,6 +793,11 @@ static inline int set_breakpoint_8xx(struct arch_hw_breakpoint *brk)
return 0;
}
+void __get_breakpoint(struct arch_hw_breakpoint *brk)
+{
+ memcpy(brk, this_cpu_ptr(¤t_brk), sizeof(*brk));
+}
+
void __set_breakpoint(struct arch_hw_breakpoint *brk)
{
memcpy(this_cpu_ptr(¤t_brk), brk, sizeof(*brk));
--
2.25.1
^ permalink raw reply related
* [RFC PATCH 3/3] powerpc/lib: Use a temporary mm for code patching
From: Christopher M. Riedl @ 2020-03-23 4:52 UTC (permalink / raw)
To: linuxppc-dev
In-Reply-To: <20200323045205.20314-1-cmr@informatik.wtf>
Currently, code patching a STRICT_KERNEL_RWX exposes the temporary
mappings to other CPUs. These mappings should be kept local to the CPU
doing the patching. Use the pre-initialized temporary mm and patching
address for this purpose. Also add a check after patching to ensure the
patch succeeded.
Based on x86 implementation:
commit b3fd8e83ada0
("x86/alternatives: Use temporary mm for text poking")
Signed-off-by: Christopher M. Riedl <cmr@informatik.wtf>
---
arch/powerpc/lib/code-patching.c | 128 ++++++++++++++-----------------
1 file changed, 57 insertions(+), 71 deletions(-)
diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c
index 18b88ecfc5a8..f156132e8975 100644
--- a/arch/powerpc/lib/code-patching.c
+++ b/arch/powerpc/lib/code-patching.c
@@ -19,6 +19,7 @@
#include <asm/page.h>
#include <asm/code-patching.h>
#include <asm/setup.h>
+#include <asm/mmu_context.h>
static int __patch_instruction(unsigned int *exec_addr, unsigned int instr,
unsigned int *patch_addr)
@@ -65,99 +66,79 @@ void __init poking_init(void)
pte_unmap_unlock(ptep, ptl);
}
-static DEFINE_PER_CPU(struct vm_struct *, text_poke_area);
-
-static int text_area_cpu_up(unsigned int cpu)
-{
- struct vm_struct *area;
-
- area = get_vm_area(PAGE_SIZE, VM_ALLOC);
- if (!area) {
- WARN_ONCE(1, "Failed to create text area for cpu %d\n",
- cpu);
- return -1;
- }
- this_cpu_write(text_poke_area, area);
-
- return 0;
-}
-
-static int text_area_cpu_down(unsigned int cpu)
-{
- free_vm_area(this_cpu_read(text_poke_area));
- return 0;
-}
-
-/*
- * Run as a late init call. This allows all the boot time patching to be done
- * simply by patching the code, and then we're called here prior to
- * mark_rodata_ro(), which happens after all init calls are run. Although
- * BUG_ON() is rude, in this case it should only happen if ENOMEM, and we judge
- * it as being preferable to a kernel that will crash later when someone tries
- * to use patch_instruction().
- */
-static int __init setup_text_poke_area(void)
-{
- BUG_ON(!cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
- "powerpc/text_poke:online", text_area_cpu_up,
- text_area_cpu_down));
-
- return 0;
-}
-late_initcall(setup_text_poke_area);
+struct patch_mapping {
+ spinlock_t *ptl; /* for protecting pte table */
+ struct temp_mm temp_mm;
+};
/*
* This can be called for kernel text or a module.
*/
-static int map_patch_area(void *addr, unsigned long text_poke_addr)
+static int map_patch(const void *addr, struct patch_mapping *patch_mapping)
{
- unsigned long pfn;
- int err;
+ struct page *page;
+ pte_t pte, *ptep;
+ pgprot_t pgprot;
if (is_vmalloc_addr(addr))
- pfn = vmalloc_to_pfn(addr);
+ page = vmalloc_to_page(addr);
else
- pfn = __pa_symbol(addr) >> PAGE_SHIFT;
+ page = virt_to_page(addr);
- err = map_kernel_page(text_poke_addr, (pfn << PAGE_SHIFT), PAGE_KERNEL);
+ if (radix_enabled())
+ pgprot = __pgprot(pgprot_val(PAGE_KERNEL));
+ else
+ pgprot = PAGE_SHARED;
- pr_devel("Mapped addr %lx with pfn %lx:%d\n", text_poke_addr, pfn, err);
- if (err)
+ ptep = get_locked_pte(patching_mm, patching_addr, &patch_mapping->ptl);
+ if (unlikely(!ptep)) {
+ pr_warn("map patch: failed to allocate pte for patching\n");
return -1;
+ }
+
+ pte = mk_pte(page, pgprot);
+ set_pte_at(patching_mm, patching_addr, ptep, pte);
+
+ init_temp_mm(&patch_mapping->temp_mm, patching_mm);
+ use_temporary_mm(&patch_mapping->temp_mm);
return 0;
}
-static inline int unmap_patch_area(unsigned long addr)
+static int unmap_patch(struct patch_mapping *patch_mapping)
{
pte_t *ptep;
pmd_t *pmdp;
pud_t *pudp;
pgd_t *pgdp;
- pgdp = pgd_offset_k(addr);
+ pgdp = pgd_offset(patching_mm, patching_addr);
if (unlikely(!pgdp))
return -EINVAL;
- pudp = pud_offset(pgdp, addr);
+ pudp = pud_offset(pgdp, patching_addr);
if (unlikely(!pudp))
return -EINVAL;
- pmdp = pmd_offset(pudp, addr);
+ pmdp = pmd_offset(pudp, patching_addr);
if (unlikely(!pmdp))
return -EINVAL;
- ptep = pte_offset_kernel(pmdp, addr);
+ ptep = pte_offset_kernel(pmdp, patching_addr);
if (unlikely(!ptep))
return -EINVAL;
- pr_devel("clearing mm %p, pte %p, addr %lx\n", &init_mm, ptep, addr);
+ /*
+ * In hash, pte_clear flushes the tlb
+ */
+ pte_clear(patching_mm, patching_addr, ptep);
+ unuse_temporary_mm(&patch_mapping->temp_mm);
/*
- * In hash, pte_clear flushes the tlb, in radix, we have to
+ * In radix, we have to explicitly flush the tlb (no-op in hash)
*/
- pte_clear(&init_mm, addr, ptep);
- flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
+ local_flush_tlb_mm(patching_mm);
+ pte_unmap_unlock(ptep, patch_mapping->ptl);
return 0;
}
@@ -167,33 +148,38 @@ static int do_patch_instruction(unsigned int *addr, unsigned int instr)
int err;
unsigned int *patch_addr = NULL;
unsigned long flags;
- unsigned long text_poke_addr;
- unsigned long kaddr = (unsigned long)addr;
+ struct patch_mapping patch_mapping;
/*
- * During early early boot patch_instruction is called
- * when text_poke_area is not ready, but we still need
- * to allow patching. We just do the plain old patching
+ * The patching_mm is initialized before calling mark_rodata_ro. Prior
+ * to this, patch_instruction is called when we don't have (and don't
+ * need) the patching_mm so just do plain old patching.
*/
- if (!this_cpu_read(text_poke_area))
+ if (!patching_mm)
return raw_patch_instruction(addr, instr);
local_irq_save(flags);
- text_poke_addr = (unsigned long)__this_cpu_read(text_poke_area)->addr;
- if (map_patch_area(addr, text_poke_addr)) {
- err = -1;
+ err = map_patch(addr, &patch_mapping);
+ if (err)
goto out;
- }
- patch_addr = (unsigned int *)(text_poke_addr) +
- ((kaddr & ~PAGE_MASK) / sizeof(unsigned int));
+ patch_addr = (unsigned int *)(patching_addr) +
+ (offset_in_page((unsigned long)addr) /
+ sizeof(unsigned int));
__patch_instruction(addr, instr, patch_addr);
- err = unmap_patch_area(text_poke_addr);
+ err = unmap_patch(&patch_mapping);
if (err)
- pr_warn("failed to unmap %lx\n", text_poke_addr);
+ pr_warn("unmap patch: failed to unmap patch\n");
+
+ /*
+ * Something is wrong if what we just wrote doesn't match what we
+ * think we just wrote.
+ * XXX: BUG_ON() instead?
+ */
+ WARN_ON(memcmp(addr, &instr, sizeof(instr)));
out:
local_irq_restore(flags);
--
2.25.1
^ permalink raw reply related
* [RFC PATCH 0/3] Use per-CPU temporary mappings for patching
From: Christopher M. Riedl @ 2020-03-23 4:52 UTC (permalink / raw)
To: linuxppc-dev
When compiled with CONFIG_STRICT_KERNEL_RWX, the kernel must create
temporary mappings when patching itself. These mappings temporarily
override the strict RWX text protections to permit a write. Currently,
powerpc allocates a per-CPU VM area for patching. Patching occurs as
follows:
1. Map page of text to be patched to per-CPU VM area w/
PAGE_KERNEL protection
2. Patch text
3. Remove the temporary mapping
While the VM area is per-CPU, the mapping is actually inserted into the
kernel page tables. Presumably, this could allow another CPU to access
the normally write-protected text - either malicously or accidentally -
via this same mapping if the address of the VM area is known. Ideally,
the mapping should be kept local to the CPU doing the patching (or any
other sensitive operations requiring temporarily overriding memory
protections) [0].
x86 introduced "temporary mm" structs which allow the creation of
mappings local to a particular CPU [1]. This series intends to bring the
notion of a temporary mm to powerpc and harden powerpc by using such a
mapping for patching a kernel with strict RWX permissions.
The first patch introduces the temporary mm struct and API for powerpc
along with a new function to retrieve a current hw breakpoint.
The second patch uses the `poking_init` init hook added by the x86
patches to initialize a temporary mm and patching address. The patching
address is randomized between 0 and DEFAULT_MAP_WINDOW-PAGE_SIZE. The
upper limit is necessary due to how the hash MMU operates - by default
the space above DEFAULT_MAP_WINDOW is not available. For now, both hash
and radix randomize inside this range. The number of possible random
addresses is dependent on PAGE_SIZE and limited by DEFAULT_MAP_WINDOW.
Bits of entropy with 64K page size on BOOK3S_64:
bits-o-entropy = log2(DEFAULT_MAP_WINDOW_USER64 / PAGE_SIZE)
PAGE_SIZE=64K, DEFAULT_MAP_WINDOW_USER64=128TB
bits-o-entropy = log2(128TB / 64K)
bits-o-entropy = 31
Currently, randomization occurs only once during initialization at boot.
The third patch replaces the VM area with the temporary mm in the
patching code. The page for patching has to be mapped PAGE_SHARED with
the hash MMU since hash prevents the kernel from accessing userspace
pages with PAGE_PRIVILEGED bit set. There is on-going work on my side to
explore if this is actually necessary in the hash codepath.
Testing so far is limited to booting on QEMU (power8 and power9 targets)
and a POWER8 VM along with setting some simple xmon breakpoints (which
makes use of code-patching). A POC lkdtm test is in-progress to actually
exploit the existing vulnerability (ie. the mapping during patching is
exposed in kernel page tables and accessible by other CPUS) - this will
accompany a future v1 of this series.
[0]: https://github.com/linuxppc/issues/issues/224
[1]: https://lore.kernel.org/kernel-hardening/20190426232303.28381-1-nadav.amit@gmail.com/
Christopher M. Riedl (3):
powerpc/mm: Introduce temporary mm
powerpc/lib: Initialize a temporary mm for code patching
powerpc/lib: Use a temporary mm for code patching
arch/powerpc/include/asm/debug.h | 1 +
arch/powerpc/include/asm/mmu_context.h | 56 +++++++++-
arch/powerpc/kernel/process.c | 5 +
arch/powerpc/lib/code-patching.c | 140 ++++++++++++++-----------
4 files changed, 137 insertions(+), 65 deletions(-)
--
2.25.1
^ permalink raw reply
* [RFC PATCH 2/3] powerpc/lib: Initialize a temporary mm for code patching
From: Christopher M. Riedl @ 2020-03-23 4:52 UTC (permalink / raw)
To: linuxppc-dev
In-Reply-To: <20200323045205.20314-1-cmr@informatik.wtf>
When code patching a STRICT_KERNEL_RWX kernel the page containing the
address to be patched is temporarily mapped with permissive memory
protections. Currently, a per-cpu vmalloc patch area is used for this
purpose. While the patch area is per-cpu, the temporary page mapping is
inserted into the kernel page tables for the duration of the patching.
The mapping is exposed to CPUs other than the patching CPU - this is
undesirable from a hardening perspective.
Use the `poking_init` init hook to prepare a temporary mm and patching
address. Initialize the temporary mm by copying the init mm. Choose a
randomized patching address inside the temporary mm userspace address
portion. The next patch uses the temporary mm and patching address for
code patching.
Based on x86 implementation:
commit 4fc19708b165
("x86/alternatives: Initialize temporary mm for patching")
Signed-off-by: Christopher M. Riedl <cmr@informatik.wtf>
---
arch/powerpc/lib/code-patching.c | 26 ++++++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c
index 3345f039a876..18b88ecfc5a8 100644
--- a/arch/powerpc/lib/code-patching.c
+++ b/arch/powerpc/lib/code-patching.c
@@ -11,6 +11,8 @@
#include <linux/cpuhotplug.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
+#include <linux/sched/task.h>
+#include <linux/random.h>
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
@@ -39,6 +41,30 @@ int raw_patch_instruction(unsigned int *addr, unsigned int instr)
}
#ifdef CONFIG_STRICT_KERNEL_RWX
+
+__ro_after_init struct mm_struct *patching_mm;
+__ro_after_init unsigned long patching_addr;
+
+void __init poking_init(void)
+{
+ spinlock_t *ptl; /* for protecting pte table */
+ pte_t *ptep;
+
+ patching_mm = copy_init_mm();
+ BUG_ON(!patching_mm);
+
+ /*
+ * In hash we cannot go above DEFAULT_MAP_WINDOW easily.
+ * XXX: Do we want additional bits of entropy for radix?
+ */
+ patching_addr = (get_random_long() & PAGE_MASK) %
+ (DEFAULT_MAP_WINDOW - PAGE_SIZE);
+
+ ptep = get_locked_pte(patching_mm, patching_addr, &ptl);
+ BUG_ON(!ptep);
+ pte_unmap_unlock(ptep, ptl);
+}
+
static DEFINE_PER_CPU(struct vm_struct *, text_poke_area);
static int text_area_cpu_up(unsigned int cpu)
--
2.25.1
^ permalink raw reply related
* Re: [PATCH 2/2] KVM: PPC: Book3S HV: H_SVM_INIT_START must call UV_RETURN
From: Bharata B Rao @ 2020-03-23 4:21 UTC (permalink / raw)
To: Laurent Dufour; +Cc: linux-kernel, kvm-ppc, linuxppc-dev
In-Reply-To: <f6a71da6-6363-8cba-8215-78b23a046443@linux.ibm.com>
On Fri, Mar 20, 2020 at 03:36:05PM +0100, Laurent Dufour wrote:
> Le 20/03/2020 à 12:24, Bharata B Rao a écrit :
> > On Fri, Mar 20, 2020 at 11:26:43AM +0100, Laurent Dufour wrote:
> > > When the call to UV_REGISTER_MEM_SLOT is failing, for instance because
> > > there is not enough free secured memory, the Hypervisor (HV) has to call
> > > UV_RETURN to report the error to the Ultravisor (UV). Then the UV will call
> > > H_SVM_INIT_ABORT to abort the securing phase and go back to the calling VM.
> > >
> > > If the kvm->arch.secure_guest is not set, in the return path rfid is called
> > > but there is no valid context to get back to the SVM since the Hcall has
> > > been routed by the Ultravisor.
> > >
> > > Move the setting of kvm->arch.secure_guest earlier in
> > > kvmppc_h_svm_init_start() so in the return path, UV_RETURN will be called
> > > instead of rfid.
> > >
> > > Cc: Bharata B Rao <bharata@linux.ibm.com>
> > > Cc: Paul Mackerras <paulus@ozlabs.org>
> > > Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
> > > Cc: Michael Ellerman <mpe@ellerman.id.au>
> > > Signed-off-by: Laurent Dufour <ldufour@linux.ibm.com>
> > > ---
> > > arch/powerpc/kvm/book3s_hv_uvmem.c | 3 ++-
> > > 1 file changed, 2 insertions(+), 1 deletion(-)
> > >
> > > diff --git a/arch/powerpc/kvm/book3s_hv_uvmem.c b/arch/powerpc/kvm/book3s_hv_uvmem.c
> > > index 79b1202b1c62..68dff151315c 100644
> > > --- a/arch/powerpc/kvm/book3s_hv_uvmem.c
> > > +++ b/arch/powerpc/kvm/book3s_hv_uvmem.c
> > > @@ -209,6 +209,8 @@ unsigned long kvmppc_h_svm_init_start(struct kvm *kvm)
> > > int ret = H_SUCCESS;
> > > int srcu_idx;
> > > + kvm->arch.secure_guest = KVMPPC_SECURE_INIT_START;
> > > +
> > > if (!kvmppc_uvmem_bitmap)
> > > return H_UNSUPPORTED;
> > > @@ -233,7 +235,6 @@ unsigned long kvmppc_h_svm_init_start(struct kvm *kvm)
> > > goto out;
> > > }
> > > }
> > > - kvm->arch.secure_guest |= KVMPPC_SECURE_INIT_START;
> >
> > There is an assumption that memory slots would have been registered with UV
> > if KVMPPC_SECURE_INIT_START has been done. KVM_PPC_SVM_OFF ioctl will skip
> > unregistration and other steps during reboot if KVMPPC_SECURE_INIT_START
> > hasn't been done.
> >
> > Have you checked if that path isn't affected by this change?
>
> I checked that and didn't find any issue there.
>
> My only concern was that block:
> kvm_for_each_vcpu(i, vcpu, kvm) {
> spin_lock(&vcpu->arch.vpa_update_lock);
> unpin_vpa_reset(kvm, &vcpu->arch.dtl);
> unpin_vpa_reset(kvm, &vcpu->arch.slb_shadow);
> unpin_vpa_reset(kvm, &vcpu->arch.vpa);
> spin_unlock(&vcpu->arch.vpa_update_lock);
> }
>
> But that seems to be safe.
Yes, looks like.
>
> However I'm not a familiar with the KVM's code, do you think an additional
> KVMPPC_SECURE_INIT_* value needed here?
May be not as long as UV can handle the unexpected uv_unregister_mem_slot()
calls, we are good.
Regards,
Bharata.
^ permalink raw reply
* Re: [PATCH] powerpc/64: ftrace don't trace real mode
From: Nicholas Piggin @ 2020-03-23 4:10 UTC (permalink / raw)
To: linuxppc-dev, Naveen N. Rao
In-Reply-To: <1584893587.qg26sf1ty0.naveen@linux.ibm.com>
Naveen N. Rao's on March 23, 2020 2:17 am:
> Nicholas Piggin wrote:
>> Naveen N. Rao's on March 21, 2020 4:39 am:
>>> Hi Nick,
>>>
>>> Nicholas Piggin wrote:
>>>> This warns and prevents tracing attempted in a real-mode context.
>>>
>>> Is this something you're seeing often? Last time we looked at this, KVM
>>> was the biggest offender and we introduced paca->ftrace_enabled as a way
>>> to disable ftrace while in KVM code.
>>
>> Not often but it has a tendancy to blow up the least tested code at the
>> worst times :)
>>
>> Machine check is bad, I'm sure HMI too but I haven't tested that yet.
>>
>> I've fixed up most of it with annotations, this is obviously an extra
>> safety not something to rely on like ftrace_enabled. Probably even the
>> WARN_ON here is dangerous here, but I don't want to leave these bugs
>> in there.
>
> Ok, makes sense.
>
>>
>> Although the machine check and hmi code touch a fair bit of stuff and
>> annotating is a bit fragile. It might actually be better if the
>> paca->ftrace_enabled could be a nesting counter, then we could use it
>> in machine checks too and avoid a lot of annotations.
>
> I'm not too familiar with MC/HMI, but I suppose those aren't re-entrant?
> If those have access to an emergency stack, can we save/restore
> ftrace_enabled state across the handlers?
Yeah that's true. They're not highly reentrant though, we could just
make it an 8 bit counter, it's nicer than saving / restoring from stack
(but I guess I could do that too).
>
> We're primarily disabling ftrace across idle/offline/KVM right now. I'm
> not sure if nesting is useful there.
>
>>
>>> While this is cheap when handling ftrace_regs_caller() as done in this
>>> patch, for simple function tracing (see below), we will have to grab the
>>> MSR which will slow things down slightly.
>>
>> mfmsr is not too bad these days.
>
> That's great!
Thanks,
Nick
^ permalink raw reply
* [PATCH v4 9/9] Documentation/powerpc: VAS API
From: Haren Myneni @ 2020-03-23 4:06 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
Power9 introduced Virtual Accelerator Switchboard (VAS) which allows
userspace to communicate with Nest Accelerator (NX) directly. But
kernel has to establish channel to NX for userspace. This document
describes user space API that application can use to establish
communication channel.
Signed-off-by: Sukadev Bhattiprolu <sukadev@linux.ibm.com>
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
Documentation/powerpc/index.rst | 1 +
Documentation/powerpc/vas-api.rst | 249 ++++++++++++++++++++++++++++++++++++++
2 files changed, 250 insertions(+)
create mode 100644 Documentation/powerpc/vas-api.rst
diff --git a/Documentation/powerpc/index.rst b/Documentation/powerpc/index.rst
index 0d45f0f..afe2d5e 100644
--- a/Documentation/powerpc/index.rst
+++ b/Documentation/powerpc/index.rst
@@ -30,6 +30,7 @@ powerpc
syscall64-abi
transactional_memory
ultravisor
+ vas-api
.. only:: subproject and html
diff --git a/Documentation/powerpc/vas-api.rst b/Documentation/powerpc/vas-api.rst
new file mode 100644
index 0000000..bd7042d
--- /dev/null
+++ b/Documentation/powerpc/vas-api.rst
@@ -0,0 +1,249 @@
+.. SPDX-License-Identifier: GPL-2.0
+.. _VAS-API:
+
+===================================================
+Virtual Accelerator Switchboard (VAS) userspace API
+===================================================
+
+Introduction
+============
+
+Power9 processor introduced Virtual Accelerator Switchboard (VAS) which
+allows both userspace and kernel communicate to co-processor
+(hardware accelerator) referred to as the Nest Accelerator (NX). The NX
+unit comprises of one or more hardware engines or co-processor types
+such as 842 compression, GZIP compression and encryption. On power9,
+userspace applications will have access to only GZIP Compression engine
+which supports ZLIB and GZIP compression algorithms in the hardware.
+
+To communicate with NX, kernel has to establish a channel or window and
+then requests can be submitted directly without kernel involvement.
+Requests to the GZIP engine must be formatted as a co-processor Request
+Block (CRB) and these CRBs must be submitted to the NX using COPY/PASTE
+instructions to paste the CRB to hardware address that is associated with
+the engine's request queue.
+
+The GZIP engine provides two priority levels of requests: Normal and
+High. Only Normal requests are supported from userspace right now.
+
+This document explains userspace API that is used to interact with
+kernel to setup channel / window which can be used to send compression
+requests directly to NX accelerator.
+
+
+Overview
+========
+
+Application access to the GZIP engine is provided through
+/dev/crypto/nx-gzip device node implemented by the VAS/NX device driver.
+An application must open the /dev/crypto/nx-gzip device to obtain a file
+descriptor (fd). Then should issue VAS_TX_WIN_OPEN ioctl with this fd to
+establish connection to the engine. It means send window is opened on GZIP
+engine for this process. Once a connection is established, the application
+should use the mmap() system call to map the hardware address of engine's
+request queue into the application's virtual address space.
+
+The application can then submit one or more requests to the the engine by
+using copy/paste instructions and pasting the CRBs to the virtual address
+(aka paste_address) returned by mmap(). User space can close the
+established connection or send window by closing the file descriptior
+(close(fd)) or upon the process exit.
+
+Note that applications can send several requests with the same window or
+can establish multiple windows, but one window for each file descriptor.
+
+Following sections provide additional details and references about the
+individual steps.
+
+NX-GZIP Device Node
+===================
+
+There is one /dev/crypto/nx-gzip node in the system and it provides
+access to all GZIP engines in the system. The only valid operations on
+/dev/crypto/nx-gzip are:
+
+ * open() the device for read and write.
+ * issue VAS_TX_WIN_OPEN ioctl
+ * mmap() the engine's request queue into application's virtual
+ address space (i.e. get a paste_address for the co-processor
+ engine).
+ * close the device node.
+
+Other file operations on this device node are undefined.
+
+Note that the copy and paste operations go directly to the hardware and
+do not go through this device. Refer COPY/PASTE document for more
+details.
+
+Although a system may have several instances of the NX co-processor
+engines (typically, one per P9 chip) there is just one
+/dev/crypto/nx-gzip device node in the system. When the nx-gzip device
+node is opened, Kernel opens send window on a suitable instance of NX
+accelerator. It finds CPU on which the user process is executing and
+determine the NX instance for the corresponding chip on which this CPU
+belongs.
+
+Applications may chose a specific instance of the NX co-processor using
+the vas_id field in the VAS_TX_WIN_OPEN ioctl as detailed below.
+
+A userspace library libnxz is available here but still in development:
+ https://github.com/abalib/power-gzip
+
+Applications that use inflate / deflate calls can link with libnxz
+instead of libz and use NX GZIP compression without any modification.
+
+Open /dev/crypto/nx-gzip
+========================
+
+The nx-gzip device should be opened for read and write. No special
+privileges are needed to open the device. Each window corresponds to one
+file descriptor. So if the userspace process needs multiple windows,
+several open calls have to be issued.
+
+See open(2) system call man pages for other details such as return values,
+error codes and restrictions.
+
+VAS_TX_WIN_OPEN ioctl
+=====================
+
+Applications should use the VAS_TX_WIN_OPEN ioctl as follows to establish
+a connection with NX co-processor engine:
+
+ ::
+ struct vas_tx_win_open_attr {
+ __u32 version;
+ __s16 vas_id; /* specific instance of vas or -1
+ for default */
+ __u16 reserved1;
+ __u64 flags; /* For future use */
+ __u64 reserved2[6];
+ };
+
+ version: The version field must be currently set to 1.
+ vas_id: If '-1' is passed, kernel will make a best-effort attempt
+ to assign an optimal instance of NX for the process. To
+ select the specific VAS instance, refer
+ "Discovery of available VAS engines" section below.
+
+ flags, reserved1 and reserved2[6] fields are for future extension
+ and must be set to 0.
+
+ The attributes attr for the VAS_TX_WIN_OPEN ioctl are defined as
+ follows:
+ #define VAS_MAGIC 'v'
+ #define VAS_TX_WIN_OPEN _IOW(VAS_MAGIC, 1,
+ struct vas_tx_win_open_attr)
+
+ struct vas_tx_win_open_attr attr;
+ rc = ioctl(fd, VAS_TX_WIN_OPEN, &attr);
+
+ The VAS_TX_WIN_OPEN ioctl returns 0 on success. On errors, it
+ returns -1 and sets the errno variable to indicate the error.
+
+ Error conditions:
+ EINVAL fd does not refer to a valid VAS device.
+ EINVAL Invalid vas ID
+ EINVAL version is not set with proper value
+ EEXIST Window is already opened for the given fd
+ ENOMEM Memory is not available to allocate window
+ ENOSPC System has too many active windows (connections)
+ opened
+ EINVAL reserved fields are not set to 0.
+
+ See the ioctl(2) man page for more details, error codes and
+ restrictions.
+
+mmap() NX-GZIP device
+=====================
+
+The mmap() system call for a NX-GZIP device fd returns a paste_address
+that the application can use to copy/paste its CRB to the hardware engines.
+ ::
+
+ paste_addr = mmap(addr, size, prot, flags, fd, offset);
+
+ Only restrictions on mmap for a NX-GZIP device fd are:
+ * size should be 4K page size
+ * offset parameter should be 0ULL
+
+ Refer to mmap(2) man page for additional details/restrictions.
+ In addition to the error conditions listed on the mmap(2) man
+ page, can also fail with one of the following error codes:
+
+ EINVAL fd is not associated with an open window
+ (i.e mmap() does not follow a successful call
+ to the VAS_TX_WIN_OPEN ioctl).
+ EINVAL offset field is not 0ULL.
+
+Discovery of available VAS engines
+==================================
+
+Each available VAS instance in the system will have a device tree node
+like /proc/device-tree/vas@* or /proc/device-tree/xscom@*/vas@*.
+Determine the chip or VAS instance and use the corresponding ibm,vas-id
+property value in this node to select specific VAS instance.
+
+Copy/Paste operations
+=====================
+
+Applications should use the copy and paste instructions to send CRB to NX.
+Refer section 4.4 in PowerISA for Copy/Paste instructions:
+https://openpowerfoundation.org/?resource_lib=power-isa-version-3-0
+
+CRB Specification and use NX
+============================
+
+Applications should format requests to the co-processor using the
+co-processor Request Block (CRBs). Refer NX-GZIP user's manual for the format
+of CRB and use NX from userspace such as sending requests and checking
+request status.
+
+NX-GZIP User's Manual:
+https://github.com/libnxz/power-gzip/blob/master/power_nx_gzip_um.pdf
+
+Simple example
+==============
+
+ ::
+ int use_nx_gzip()
+ {
+ int rc, fd;
+ void *addr;
+ struct vas_setup_attr txattr;
+
+ fd = open("/dev/crypto/nx-gzip", O_RDWR);
+ if (fd < 0) {
+ fprintf(stderr, "open nx-gzip failed\n");
+ return -1;
+ }
+ memset(&txattr, 0, sizeof(txattr));
+ txattr.version = 1;
+ txattr.vas_id = -1
+ rc = ioctl(fd, VAS_TX_WIN_OPEN,
+ (unsigned long)&txattr);
+ if (rc < 0) {
+ fprintf(stderr, "ioctl() n %d, error %d\n",
+ rc, errno);
+ return rc;
+ }
+ addr = mmap(NULL, 4096, PROT_READ|PROT_WRITE,
+ MAP_SHARED, fd, 0ULL);
+ if (addr == MAP_FAILED) {
+ fprintf(stderr, "mmap() failed, errno %d\n",
+ errno);
+ return -errno;
+ }
+ do {
+ //Format CRB request with compression or
+ //uncompression
+ // Refer tests for vas_copy/vas_paste
+ vas_copy((&crb, 0, 1);
+ vas_paste(addr, 0, 1);
+ // Poll on csb.flags with timeout
+ // csb address is listed in CRB
+ } while (true)
+ close(fd) or window can be closed upon process exit
+ }
+
+ Refer https://github.com/abalib/power-gzip for tests or more
+ use cases.
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 8/9] crypto/nx: Remove 'pid' in vas_tx_win_attr struct
From: Haren Myneni @ 2020-03-23 4:06 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
When window is opened, pid reference is taken for user space
windows. Not needed for kernel windows. So remove 'pid' in
vas_tx_win_attr struct.
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
arch/powerpc/include/asm/vas.h | 1 -
drivers/crypto/nx/nx-common-powernv.c | 1 -
2 files changed, 2 deletions(-)
diff --git a/arch/powerpc/include/asm/vas.h b/arch/powerpc/include/asm/vas.h
index e064953..994db6f 100644
--- a/arch/powerpc/include/asm/vas.h
+++ b/arch/powerpc/include/asm/vas.h
@@ -86,7 +86,6 @@ struct vas_tx_win_attr {
int wcreds_max;
int lpid;
int pidr; /* hardware PID (from SPRN_PID) */
- int pid; /* linux process id */
int pswid;
int rsvd_txbuf_count;
int tc_mode;
diff --git a/drivers/crypto/nx/nx-common-powernv.c b/drivers/crypto/nx/nx-common-powernv.c
index f570691..38333e4 100644
--- a/drivers/crypto/nx/nx-common-powernv.c
+++ b/drivers/crypto/nx/nx-common-powernv.c
@@ -692,7 +692,6 @@ static struct vas_window *nx_alloc_txwin(struct nx_coproc *coproc)
*/
vas_init_tx_win_attr(&txattr, coproc->ct);
txattr.lpid = 0; /* lpid is 0 for kernel requests */
- txattr.pid = 0; /* pid is 0 for kernel requests */
/*
* Open a VAS send window which is used to send request to NX.
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 7/9] crypto/nx: Enable and setup GZIP compression type
From: Haren Myneni @ 2020-03-23 4:05 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
Changes to probe GZIP device-tree nodes, open RX windows and setup
GZIP compression type. No plans to provide GZIP usage in kernel right
now, but this patch enables GZIP for user space usage.
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
drivers/crypto/nx/nx-common-powernv.c | 43 ++++++++++++++++++++++++++++++-----
1 file changed, 37 insertions(+), 6 deletions(-)
diff --git a/drivers/crypto/nx/nx-common-powernv.c b/drivers/crypto/nx/nx-common-powernv.c
index 82dfa60..f570691 100644
--- a/drivers/crypto/nx/nx-common-powernv.c
+++ b/drivers/crypto/nx/nx-common-powernv.c
@@ -65,6 +65,7 @@ struct nx_coproc {
* Using same values as in skiboot or coprocessor type representing
* in NX workbook.
*/
+#define NX_CT_GZIP (2) /* on P9 and later */
#define NX_CT_842 (3)
static int (*nx842_powernv_exec)(const unsigned char *in,
@@ -819,6 +820,9 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
if (type == NX_CT_842)
ret = nx_set_ct(coproc, priority, VAS_COP_TYPE_842_HIPRI,
VAS_COP_TYPE_842);
+ else if (type == NX_CT_GZIP)
+ ret = nx_set_ct(coproc, priority, VAS_COP_TYPE_GZIP_HIPRI,
+ VAS_COP_TYPE_GZIP);
if (ret)
goto err_out;
@@ -867,12 +871,16 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
return ret;
}
-static int __init nx_coproc_init(int chip_id, int ct_842)
+static int __init nx_coproc_init(int chip_id, int ct_842, int ct_gzip)
{
int ret = 0;
if (opal_check_token(OPAL_NX_COPROC_INIT)) {
ret = opal_nx_coproc_init(chip_id, ct_842);
+
+ if (!ret)
+ ret = opal_nx_coproc_init(chip_id, ct_gzip);
+
if (ret) {
ret = opal_error_code(ret);
pr_err("Failed to initialize NX for chip(%d): %d\n",
@@ -902,8 +910,8 @@ static int __init find_nx_device_tree(struct device_node *dn, int chip_id,
static int __init nx_powernv_probe_vas(struct device_node *pn)
{
int chip_id, vasid, ret = 0;
+ int ct_842 = 0, ct_gzip = 0;
struct device_node *dn;
- int ct_842 = 0;
chip_id = of_get_ibm_chip_id(pn);
if (chip_id < 0) {
@@ -920,19 +928,24 @@ static int __init nx_powernv_probe_vas(struct device_node *pn)
for_each_child_of_node(pn, dn) {
ret = find_nx_device_tree(dn, chip_id, vasid, NX_CT_842,
"ibm,p9-nx-842", &ct_842);
+
+ if (!ret)
+ ret = find_nx_device_tree(dn, chip_id, vasid,
+ NX_CT_GZIP, "ibm,p9-nx-gzip", &ct_gzip);
+
if (ret)
return ret;
}
- if (!ct_842) {
- pr_err("NX842 FIFO nodes are missing\n");
+ if (!ct_842 || !ct_gzip) {
+ pr_err("NX FIFO nodes are missing\n");
return -EINVAL;
}
/*
* Initialize NX instance for both high and normal priority FIFOs.
*/
- ret = nx_coproc_init(chip_id, ct_842);
+ ret = nx_coproc_init(chip_id, ct_842, ct_gzip);
return ret;
}
@@ -1072,10 +1085,19 @@ static __init int nx_compress_powernv_init(void)
nx842_powernv_exec = nx842_exec_icswx;
} else {
/*
+ * Register VAS user space API for NX GZIP so
+ * that user space can use GZIP engine.
+ * 842 compression is supported only in kernel.
+ */
+ ret = vas_register_coproc_api(THIS_MODULE);
+
+ /*
* GZIP is not supported in kernel right now.
* So open tx windows only for 842.
*/
- ret = nx_open_percpu_txwins();
+ if (!ret)
+ ret = nx_open_percpu_txwins();
+
if (ret) {
nx_delete_coprocs();
return ret;
@@ -1096,6 +1118,15 @@ static __init int nx_compress_powernv_init(void)
static void __exit nx_compress_powernv_exit(void)
{
+ /*
+ * GZIP engine is supported only in power9 or later and nx842_ct
+ * is used on power8 (icswx).
+ * VAS API for NX GZIP is registered during init for user space
+ * use. So delete this API use for GZIP engine.
+ */
+ if (!nx842_ct)
+ vas_unregister_coproc_api();
+
crypto_unregister_alg(&nx842_powernv_alg);
nx_delete_coprocs();
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 6/9] crypto/NX: Make enable code generic to add new GZIP compression type
From: Haren Myneni @ 2020-03-23 4:04 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
Make setup and enable code generic to support new GZIP compression type.
Changed nx842 reference to nx and moved some code to new functions.
Functionality is not changed except sparse warning fix - setting NULL
instead of 0 for per_cpu send window in nx_delete_coprocs().
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
drivers/crypto/nx/nx-common-powernv.c | 161 +++++++++++++++++++++-------------
1 file changed, 101 insertions(+), 60 deletions(-)
diff --git a/drivers/crypto/nx/nx-common-powernv.c b/drivers/crypto/nx/nx-common-powernv.c
index f42881f..82dfa60 100644
--- a/drivers/crypto/nx/nx-common-powernv.c
+++ b/drivers/crypto/nx/nx-common-powernv.c
@@ -40,9 +40,9 @@ struct nx842_workmem {
char padding[WORKMEM_ALIGN]; /* unused, to allow alignment */
} __packed __aligned(WORKMEM_ALIGN);
-struct nx842_coproc {
+struct nx_coproc {
unsigned int chip_id;
- unsigned int ct;
+ unsigned int ct; /* Can be 842 or GZIP high/normal*/
unsigned int ci; /* Coprocessor instance, used with icswx */
struct {
struct vas_window *rxwin;
@@ -58,9 +58,15 @@ struct nx842_coproc {
static DEFINE_PER_CPU(struct vas_window *, cpu_txwin);
/* no cpu hotplug on powernv, so this list never changes after init */
-static LIST_HEAD(nx842_coprocs);
+static LIST_HEAD(nx_coprocs);
static unsigned int nx842_ct; /* used in icswx function */
+/*
+ * Using same values as in skiboot or coprocessor type representing
+ * in NX workbook.
+ */
+#define NX_CT_842 (3)
+
static int (*nx842_powernv_exec)(const unsigned char *in,
unsigned int inlen, unsigned char *out,
unsigned int *outlenp, void *workmem, int fc);
@@ -666,15 +672,15 @@ static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen,
wmem, CCW_FC_842_DECOMP_CRC);
}
-static inline void nx842_add_coprocs_list(struct nx842_coproc *coproc,
+static inline void nx_add_coprocs_list(struct nx_coproc *coproc,
int chipid)
{
coproc->chip_id = chipid;
INIT_LIST_HEAD(&coproc->list);
- list_add(&coproc->list, &nx842_coprocs);
+ list_add(&coproc->list, &nx_coprocs);
}
-static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc)
+static struct vas_window *nx_alloc_txwin(struct nx_coproc *coproc)
{
struct vas_window *txwin = NULL;
struct vas_tx_win_attr txattr;
@@ -704,9 +710,9 @@ static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc)
* cpu_txwin is used in copy/paste operation for each compression /
* decompression request.
*/
-static int nx842_open_percpu_txwins(void)
+static int nx_open_percpu_txwins(void)
{
- struct nx842_coproc *coproc, *n;
+ struct nx_coproc *coproc, *n;
unsigned int i, chip_id;
for_each_possible_cpu(i) {
@@ -714,17 +720,18 @@ static int nx842_open_percpu_txwins(void)
chip_id = cpu_to_chip_id(i);
- list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
+ list_for_each_entry_safe(coproc, n, &nx_coprocs, list) {
/*
* Kernel requests use only high priority FIFOs. So
* open send windows for these FIFOs.
+ * GZIP is not supported in kernel right now.
*/
if (coproc->ct != VAS_COP_TYPE_842_HIPRI)
continue;
if (coproc->chip_id == chip_id) {
- txwin = nx842_alloc_txwin(coproc);
+ txwin = nx_alloc_txwin(coproc);
if (IS_ERR(txwin))
return PTR_ERR(txwin);
@@ -743,13 +750,28 @@ static int nx842_open_percpu_txwins(void)
return 0;
}
+static int __init nx_set_ct(struct nx_coproc *coproc, const char *priority,
+ int high, int normal)
+{
+ if (!strcmp(priority, "High"))
+ coproc->ct = high;
+ else if (!strcmp(priority, "Normal"))
+ coproc->ct = normal;
+ else {
+ pr_err("Invalid RxFIFO priority value\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
- int vasid, int *ct)
+ int vasid, int type, int *ct)
{
struct vas_window *rxwin = NULL;
struct vas_rx_win_attr rxattr;
- struct nx842_coproc *coproc;
u32 lpid, pid, tid, fifo_size;
+ struct nx_coproc *coproc;
u64 rx_fifo;
const char *priority;
int ret;
@@ -794,15 +816,12 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
if (!coproc)
return -ENOMEM;
- if (!strcmp(priority, "High"))
- coproc->ct = VAS_COP_TYPE_842_HIPRI;
- else if (!strcmp(priority, "Normal"))
- coproc->ct = VAS_COP_TYPE_842;
- else {
- pr_err("Invalid RxFIFO priority value\n");
- ret = -EINVAL;
+ if (type == NX_CT_842)
+ ret = nx_set_ct(coproc, priority, VAS_COP_TYPE_842_HIPRI,
+ VAS_COP_TYPE_842);
+
+ if (ret)
goto err_out;
- }
vas_init_rx_win_attr(&rxattr, coproc->ct);
rxattr.rx_fifo = (void *)rx_fifo;
@@ -830,7 +849,7 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
coproc->vas.rxwin = rxwin;
coproc->vas.id = vasid;
- nx842_add_coprocs_list(coproc, chip_id);
+ nx_add_coprocs_list(coproc, chip_id);
/*
* (lpid, pid, tid) combination has to be unique for each
@@ -848,13 +867,43 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
return ret;
}
+static int __init nx_coproc_init(int chip_id, int ct_842)
+{
+ int ret = 0;
-static int __init nx842_powernv_probe_vas(struct device_node *pn)
+ if (opal_check_token(OPAL_NX_COPROC_INIT)) {
+ ret = opal_nx_coproc_init(chip_id, ct_842);
+ if (ret) {
+ ret = opal_error_code(ret);
+ pr_err("Failed to initialize NX for chip(%d): %d\n",
+ chip_id, ret);
+ }
+ } else
+ pr_warn("Firmware doesn't support NX initialization\n");
+
+ return ret;
+}
+
+static int __init find_nx_device_tree(struct device_node *dn, int chip_id,
+ int vasid, int type, char *devname,
+ int *ct)
+{
+ int ret = 0;
+
+ if (of_device_is_compatible(dn, devname)) {
+ ret = vas_cfg_coproc_info(dn, chip_id, vasid, type, ct);
+ if (ret)
+ of_node_put(dn);
+ }
+
+ return ret;
+}
+
+static int __init nx_powernv_probe_vas(struct device_node *pn)
{
- struct device_node *dn;
int chip_id, vasid, ret = 0;
- int nx_fifo_found = 0;
- int uninitialized_var(ct);
+ struct device_node *dn;
+ int ct_842 = 0;
chip_id = of_get_ibm_chip_id(pn);
if (chip_id < 0) {
@@ -869,17 +918,13 @@ static int __init nx842_powernv_probe_vas(struct device_node *pn)
}
for_each_child_of_node(pn, dn) {
- if (of_device_is_compatible(dn, "ibm,p9-nx-842")) {
- ret = vas_cfg_coproc_info(dn, chip_id, vasid, &ct);
- if (ret) {
- of_node_put(dn);
- return ret;
- }
- nx_fifo_found++;
- }
+ ret = find_nx_device_tree(dn, chip_id, vasid, NX_CT_842,
+ "ibm,p9-nx-842", &ct_842);
+ if (ret)
+ return ret;
}
- if (!nx_fifo_found) {
+ if (!ct_842) {
pr_err("NX842 FIFO nodes are missing\n");
return -EINVAL;
}
@@ -887,22 +932,14 @@ static int __init nx842_powernv_probe_vas(struct device_node *pn)
/*
* Initialize NX instance for both high and normal priority FIFOs.
*/
- if (opal_check_token(OPAL_NX_COPROC_INIT)) {
- ret = opal_nx_coproc_init(chip_id, ct);
- if (ret) {
- pr_err("Failed to initialize NX for chip(%d): %d\n",
- chip_id, ret);
- ret = opal_error_code(ret);
- }
- } else
- pr_warn("Firmware doesn't support NX initialization\n");
+ ret = nx_coproc_init(chip_id, ct_842);
return ret;
}
static int __init nx842_powernv_probe(struct device_node *dn)
{
- struct nx842_coproc *coproc;
+ struct nx_coproc *coproc;
unsigned int ct, ci;
int chip_id;
@@ -928,7 +965,7 @@ static int __init nx842_powernv_probe(struct device_node *dn)
coproc->ct = ct;
coproc->ci = ci;
- nx842_add_coprocs_list(coproc, chip_id);
+ nx_add_coprocs_list(coproc, chip_id);
pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci);
@@ -941,9 +978,9 @@ static int __init nx842_powernv_probe(struct device_node *dn)
return 0;
}
-static void nx842_delete_coprocs(void)
+static void nx_delete_coprocs(void)
{
- struct nx842_coproc *coproc, *n;
+ struct nx_coproc *coproc, *n;
struct vas_window *txwin;
int i;
@@ -955,10 +992,10 @@ static void nx842_delete_coprocs(void)
if (txwin)
vas_win_close(txwin);
- per_cpu(cpu_txwin, i) = 0;
+ per_cpu(cpu_txwin, i) = NULL;
}
- list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
+ list_for_each_entry_safe(coproc, n, &nx_coprocs, list) {
if (coproc->vas.rxwin)
vas_win_close(coproc->vas.rxwin);
@@ -1002,7 +1039,7 @@ static int nx842_powernv_crypto_init(struct crypto_tfm *tfm)
.coa_decompress = nx842_crypto_decompress } }
};
-static __init int nx842_powernv_init(void)
+static __init int nx_compress_powernv_init(void)
{
struct device_node *dn;
int ret;
@@ -1017,15 +1054,15 @@ static __init int nx842_powernv_init(void)
BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT);
for_each_compatible_node(dn, NULL, "ibm,power9-nx") {
- ret = nx842_powernv_probe_vas(dn);
+ ret = nx_powernv_probe_vas(dn);
if (ret) {
- nx842_delete_coprocs();
+ nx_delete_coprocs();
of_node_put(dn);
return ret;
}
}
- if (list_empty(&nx842_coprocs)) {
+ if (list_empty(&nx_coprocs)) {
for_each_compatible_node(dn, NULL, "ibm,power-nx")
nx842_powernv_probe(dn);
@@ -1034,9 +1071,13 @@ static __init int nx842_powernv_init(void)
nx842_powernv_exec = nx842_exec_icswx;
} else {
- ret = nx842_open_percpu_txwins();
+ /*
+ * GZIP is not supported in kernel right now.
+ * So open tx windows only for 842.
+ */
+ ret = nx_open_percpu_txwins();
if (ret) {
- nx842_delete_coprocs();
+ nx_delete_coprocs();
return ret;
}
@@ -1045,18 +1086,18 @@ static __init int nx842_powernv_init(void)
ret = crypto_register_alg(&nx842_powernv_alg);
if (ret) {
- nx842_delete_coprocs();
+ nx_delete_coprocs();
return ret;
}
return 0;
}
-module_init(nx842_powernv_init);
+module_init(nx_compress_powernv_init);
-static void __exit nx842_powernv_exit(void)
+static void __exit nx_compress_powernv_exit(void)
{
crypto_unregister_alg(&nx842_powernv_alg);
- nx842_delete_coprocs();
+ nx_delete_coprocs();
}
-module_exit(nx842_powernv_exit);
+module_exit(nx_compress_powernv_exit);
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 5/9] crypto/nx: Rename nx-842-powernv file name to nx-common-powernv
From: Haren Myneni @ 2020-03-23 4:03 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
Rename nx-842-powernv.c to nx-common-powernv.c to add code for setup
and enable new GZIP compression type. The actual functionality is not
changed in this patch.
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
drivers/crypto/nx/Makefile | 2 +-
drivers/crypto/nx/nx-842-powernv.c | 1062 ---------------------------------
drivers/crypto/nx/nx-common-powernv.c | 1062 +++++++++++++++++++++++++++++++++
3 files changed, 1063 insertions(+), 1063 deletions(-)
delete mode 100644 drivers/crypto/nx/nx-842-powernv.c
create mode 100644 drivers/crypto/nx/nx-common-powernv.c
diff --git a/drivers/crypto/nx/Makefile b/drivers/crypto/nx/Makefile
index 015155d..bc89a20 100644
--- a/drivers/crypto/nx/Makefile
+++ b/drivers/crypto/nx/Makefile
@@ -15,4 +15,4 @@ obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_PSERIES) += nx-compress-pseries.o nx-compres
obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_POWERNV) += nx-compress-powernv.o nx-compress.o
nx-compress-objs := nx-842.o
nx-compress-pseries-objs := nx-842-pseries.o
-nx-compress-powernv-objs := nx-842-powernv.o
+nx-compress-powernv-objs := nx-common-powernv.o
diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c
deleted file mode 100644
index 8e63326..0000000
--- a/drivers/crypto/nx/nx-842-powernv.c
+++ /dev/null
@@ -1,1062 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Driver for IBM PowerNV 842 compression accelerator
- *
- * Copyright (C) 2015 Dan Streetman, IBM Corp
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include "nx-842.h"
-
-#include <linux/timer.h>
-
-#include <asm/prom.h>
-#include <asm/icswx.h>
-#include <asm/vas.h>
-#include <asm/reg.h>
-#include <asm/opal-api.h>
-#include <asm/opal.h>
-
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
-MODULE_DESCRIPTION("842 H/W Compression driver for IBM PowerNV processors");
-MODULE_ALIAS_CRYPTO("842");
-MODULE_ALIAS_CRYPTO("842-nx");
-
-#define WORKMEM_ALIGN (CRB_ALIGN)
-#define CSB_WAIT_MAX (5000) /* ms */
-#define VAS_RETRIES (10)
-
-struct nx842_workmem {
- /* Below fields must be properly aligned */
- struct coprocessor_request_block crb; /* CRB_ALIGN align */
- struct data_descriptor_entry ddl_in[DDL_LEN_MAX]; /* DDE_ALIGN align */
- struct data_descriptor_entry ddl_out[DDL_LEN_MAX]; /* DDE_ALIGN align */
- /* Above fields must be properly aligned */
-
- ktime_t start;
-
- char padding[WORKMEM_ALIGN]; /* unused, to allow alignment */
-} __packed __aligned(WORKMEM_ALIGN);
-
-struct nx842_coproc {
- unsigned int chip_id;
- unsigned int ct;
- unsigned int ci; /* Coprocessor instance, used with icswx */
- struct {
- struct vas_window *rxwin;
- int id;
- } vas;
- struct list_head list;
-};
-
-/*
- * Send the request to NX engine on the chip for the corresponding CPU
- * where the process is executing. Use with VAS function.
- */
-static DEFINE_PER_CPU(struct vas_window *, cpu_txwin);
-
-/* no cpu hotplug on powernv, so this list never changes after init */
-static LIST_HEAD(nx842_coprocs);
-static unsigned int nx842_ct; /* used in icswx function */
-
-static int (*nx842_powernv_exec)(const unsigned char *in,
- unsigned int inlen, unsigned char *out,
- unsigned int *outlenp, void *workmem, int fc);
-
-/**
- * setup_indirect_dde - Setup an indirect DDE
- *
- * The DDE is setup with the the DDE count, byte count, and address of
- * first direct DDE in the list.
- */
-static void setup_indirect_dde(struct data_descriptor_entry *dde,
- struct data_descriptor_entry *ddl,
- unsigned int dde_count, unsigned int byte_count)
-{
- dde->flags = 0;
- dde->count = dde_count;
- dde->index = 0;
- dde->length = cpu_to_be32(byte_count);
- dde->address = cpu_to_be64(nx842_get_pa(ddl));
-}
-
-/**
- * setup_direct_dde - Setup single DDE from buffer
- *
- * The DDE is setup with the buffer and length. The buffer must be properly
- * aligned. The used length is returned.
- * Returns:
- * N Successfully set up DDE with N bytes
- */
-static unsigned int setup_direct_dde(struct data_descriptor_entry *dde,
- unsigned long pa, unsigned int len)
-{
- unsigned int l = min_t(unsigned int, len, LEN_ON_PAGE(pa));
-
- dde->flags = 0;
- dde->count = 0;
- dde->index = 0;
- dde->length = cpu_to_be32(l);
- dde->address = cpu_to_be64(pa);
-
- return l;
-}
-
-/**
- * setup_ddl - Setup DDL from buffer
- *
- * Returns:
- * 0 Successfully set up DDL
- */
-static int setup_ddl(struct data_descriptor_entry *dde,
- struct data_descriptor_entry *ddl,
- unsigned char *buf, unsigned int len,
- bool in)
-{
- unsigned long pa = nx842_get_pa(buf);
- int i, ret, total_len = len;
-
- if (!IS_ALIGNED(pa, DDE_BUFFER_ALIGN)) {
- pr_debug("%s buffer pa 0x%lx not 0x%x-byte aligned\n",
- in ? "input" : "output", pa, DDE_BUFFER_ALIGN);
- return -EINVAL;
- }
-
- /* only need to check last mult; since buffer must be
- * DDE_BUFFER_ALIGN aligned, and that is a multiple of
- * DDE_BUFFER_SIZE_MULT, and pre-last page DDE buffers
- * are guaranteed a multiple of DDE_BUFFER_SIZE_MULT.
- */
- if (len % DDE_BUFFER_LAST_MULT) {
- pr_debug("%s buffer len 0x%x not a multiple of 0x%x\n",
- in ? "input" : "output", len, DDE_BUFFER_LAST_MULT);
- if (in)
- return -EINVAL;
- len = round_down(len, DDE_BUFFER_LAST_MULT);
- }
-
- /* use a single direct DDE */
- if (len <= LEN_ON_PAGE(pa)) {
- ret = setup_direct_dde(dde, pa, len);
- WARN_ON(ret < len);
- return 0;
- }
-
- /* use the DDL */
- for (i = 0; i < DDL_LEN_MAX && len > 0; i++) {
- ret = setup_direct_dde(&ddl[i], pa, len);
- buf += ret;
- len -= ret;
- pa = nx842_get_pa(buf);
- }
-
- if (len > 0) {
- pr_debug("0x%x total %s bytes 0x%x too many for DDL.\n",
- total_len, in ? "input" : "output", len);
- if (in)
- return -EMSGSIZE;
- total_len -= len;
- }
- setup_indirect_dde(dde, ddl, i, total_len);
-
- return 0;
-}
-
-#define CSB_ERR(csb, msg, ...) \
- pr_err("ERROR: " msg " : %02x %02x %02x %02x %08x\n", \
- ##__VA_ARGS__, (csb)->flags, \
- (csb)->cs, (csb)->cc, (csb)->ce, \
- be32_to_cpu((csb)->count))
-
-#define CSB_ERR_ADDR(csb, msg, ...) \
- CSB_ERR(csb, msg " at %lx", ##__VA_ARGS__, \
- (unsigned long)be64_to_cpu((csb)->address))
-
-/**
- * wait_for_csb
- */
-static int wait_for_csb(struct nx842_workmem *wmem,
- struct coprocessor_status_block *csb)
-{
- ktime_t start = wmem->start, now = ktime_get();
- ktime_t timeout = ktime_add_ms(start, CSB_WAIT_MAX);
-
- while (!(READ_ONCE(csb->flags) & CSB_V)) {
- cpu_relax();
- now = ktime_get();
- if (ktime_after(now, timeout))
- break;
- }
-
- /* hw has updated csb and output buffer */
- barrier();
-
- /* check CSB flags */
- if (!(csb->flags & CSB_V)) {
- CSB_ERR(csb, "CSB still not valid after %ld us, giving up",
- (long)ktime_us_delta(now, start));
- return -ETIMEDOUT;
- }
- if (csb->flags & CSB_F) {
- CSB_ERR(csb, "Invalid CSB format");
- return -EPROTO;
- }
- if (csb->flags & CSB_CH) {
- CSB_ERR(csb, "Invalid CSB chaining state");
- return -EPROTO;
- }
-
- /* verify CSB completion sequence is 0 */
- if (csb->cs) {
- CSB_ERR(csb, "Invalid CSB completion sequence");
- return -EPROTO;
- }
-
- /* check CSB Completion Code */
- switch (csb->cc) {
- /* no error */
- case CSB_CC_SUCCESS:
- break;
- case CSB_CC_TPBC_GT_SPBC:
- /* not an error, but the compressed data is
- * larger than the uncompressed data :(
- */
- break;
-
- /* input data errors */
- case CSB_CC_OPERAND_OVERLAP:
- /* input and output buffers overlap */
- CSB_ERR(csb, "Operand Overlap error");
- return -EINVAL;
- case CSB_CC_INVALID_OPERAND:
- CSB_ERR(csb, "Invalid operand");
- return -EINVAL;
- case CSB_CC_NOSPC:
- /* output buffer too small */
- return -ENOSPC;
- case CSB_CC_ABORT:
- CSB_ERR(csb, "Function aborted");
- return -EINTR;
- case CSB_CC_CRC_MISMATCH:
- CSB_ERR(csb, "CRC mismatch");
- return -EINVAL;
- case CSB_CC_TEMPL_INVALID:
- CSB_ERR(csb, "Compressed data template invalid");
- return -EINVAL;
- case CSB_CC_TEMPL_OVERFLOW:
- CSB_ERR(csb, "Compressed data template shows data past end");
- return -EINVAL;
- case CSB_CC_EXCEED_BYTE_COUNT: /* P9 or later */
- /*
- * DDE byte count exceeds the limit specified in Maximum
- * byte count register.
- */
- CSB_ERR(csb, "DDE byte count exceeds the limit");
- return -EINVAL;
-
- /* these should not happen */
- case CSB_CC_INVALID_ALIGN:
- /* setup_ddl should have detected this */
- CSB_ERR_ADDR(csb, "Invalid alignment");
- return -EINVAL;
- case CSB_CC_DATA_LENGTH:
- /* setup_ddl should have detected this */
- CSB_ERR(csb, "Invalid data length");
- return -EINVAL;
- case CSB_CC_WR_TRANSLATION:
- case CSB_CC_TRANSLATION:
- case CSB_CC_TRANSLATION_DUP1:
- case CSB_CC_TRANSLATION_DUP2:
- case CSB_CC_TRANSLATION_DUP3:
- case CSB_CC_TRANSLATION_DUP4:
- case CSB_CC_TRANSLATION_DUP5:
- case CSB_CC_TRANSLATION_DUP6:
- /* should not happen, we use physical addrs */
- CSB_ERR_ADDR(csb, "Translation error");
- return -EPROTO;
- case CSB_CC_WR_PROTECTION:
- case CSB_CC_PROTECTION:
- case CSB_CC_PROTECTION_DUP1:
- case CSB_CC_PROTECTION_DUP2:
- case CSB_CC_PROTECTION_DUP3:
- case CSB_CC_PROTECTION_DUP4:
- case CSB_CC_PROTECTION_DUP5:
- case CSB_CC_PROTECTION_DUP6:
- /* should not happen, we use physical addrs */
- CSB_ERR_ADDR(csb, "Protection error");
- return -EPROTO;
- case CSB_CC_PRIVILEGE:
- /* shouldn't happen, we're in HYP mode */
- CSB_ERR(csb, "Insufficient Privilege error");
- return -EPROTO;
- case CSB_CC_EXCESSIVE_DDE:
- /* shouldn't happen, setup_ddl doesn't use many dde's */
- CSB_ERR(csb, "Too many DDEs in DDL");
- return -EINVAL;
- case CSB_CC_TRANSPORT:
- case CSB_CC_INVALID_CRB: /* P9 or later */
- /* shouldn't happen, we setup CRB correctly */
- CSB_ERR(csb, "Invalid CRB");
- return -EINVAL;
- case CSB_CC_INVALID_DDE: /* P9 or later */
- /*
- * shouldn't happen, setup_direct/indirect_dde creates
- * DDE right
- */
- CSB_ERR(csb, "Invalid DDE");
- return -EINVAL;
- case CSB_CC_SEGMENTED_DDL:
- /* shouldn't happen, setup_ddl creates DDL right */
- CSB_ERR(csb, "Segmented DDL error");
- return -EINVAL;
- case CSB_CC_DDE_OVERFLOW:
- /* shouldn't happen, setup_ddl creates DDL right */
- CSB_ERR(csb, "DDE overflow error");
- return -EINVAL;
- case CSB_CC_SESSION:
- /* should not happen with ICSWX */
- CSB_ERR(csb, "Session violation error");
- return -EPROTO;
- case CSB_CC_CHAIN:
- /* should not happen, we don't use chained CRBs */
- CSB_ERR(csb, "Chained CRB error");
- return -EPROTO;
- case CSB_CC_SEQUENCE:
- /* should not happen, we don't use chained CRBs */
- CSB_ERR(csb, "CRB sequence number error");
- return -EPROTO;
- case CSB_CC_UNKNOWN_CODE:
- CSB_ERR(csb, "Unknown subfunction code");
- return -EPROTO;
-
- /* hardware errors */
- case CSB_CC_RD_EXTERNAL:
- case CSB_CC_RD_EXTERNAL_DUP1:
- case CSB_CC_RD_EXTERNAL_DUP2:
- case CSB_CC_RD_EXTERNAL_DUP3:
- CSB_ERR_ADDR(csb, "Read error outside coprocessor");
- return -EPROTO;
- case CSB_CC_WR_EXTERNAL:
- CSB_ERR_ADDR(csb, "Write error outside coprocessor");
- return -EPROTO;
- case CSB_CC_INTERNAL:
- CSB_ERR(csb, "Internal error in coprocessor");
- return -EPROTO;
- case CSB_CC_PROVISION:
- CSB_ERR(csb, "Storage provision error");
- return -EPROTO;
- case CSB_CC_HW:
- CSB_ERR(csb, "Correctable hardware error");
- return -EPROTO;
- case CSB_CC_HW_EXPIRED_TIMER: /* P9 or later */
- CSB_ERR(csb, "Job did not finish within allowed time");
- return -EPROTO;
-
- default:
- CSB_ERR(csb, "Invalid CC %d", csb->cc);
- return -EPROTO;
- }
-
- /* check Completion Extension state */
- if (csb->ce & CSB_CE_TERMINATION) {
- CSB_ERR(csb, "CSB request was terminated");
- return -EPROTO;
- }
- if (csb->ce & CSB_CE_INCOMPLETE) {
- CSB_ERR(csb, "CSB request not complete");
- return -EPROTO;
- }
- if (!(csb->ce & CSB_CE_TPBC)) {
- CSB_ERR(csb, "TPBC not provided, unknown target length");
- return -EPROTO;
- }
-
- /* successful completion */
- pr_debug_ratelimited("Processed %u bytes in %lu us\n",
- be32_to_cpu(csb->count),
- (unsigned long)ktime_us_delta(now, start));
-
- return 0;
-}
-
-static int nx842_config_crb(const unsigned char *in, unsigned int inlen,
- unsigned char *out, unsigned int outlen,
- struct nx842_workmem *wmem)
-{
- struct coprocessor_request_block *crb;
- struct coprocessor_status_block *csb;
- u64 csb_addr;
- int ret;
-
- crb = &wmem->crb;
- csb = &crb->csb;
-
- /* Clear any previous values */
- memset(crb, 0, sizeof(*crb));
-
- /* set up DDLs */
- ret = setup_ddl(&crb->source, wmem->ddl_in,
- (unsigned char *)in, inlen, true);
- if (ret)
- return ret;
-
- ret = setup_ddl(&crb->target, wmem->ddl_out,
- out, outlen, false);
- if (ret)
- return ret;
-
- /* set up CRB's CSB addr */
- csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS;
- csb_addr |= CRB_CSB_AT; /* Addrs are phys */
- crb->csb_addr = cpu_to_be64(csb_addr);
-
- return 0;
-}
-
-/**
- * nx842_exec_icswx - compress/decompress data using the 842 algorithm
- *
- * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
- * This compresses or decompresses the provided input buffer into the provided
- * output buffer.
- *
- * Upon return from this function @outlen contains the length of the
- * output data. If there is an error then @outlen will be 0 and an
- * error will be specified by the return code from this function.
- *
- * The @workmem buffer should only be used by one function call at a time.
- *
- * @in: input buffer pointer
- * @inlen: input buffer size
- * @out: output buffer pointer
- * @outlenp: output buffer size pointer
- * @workmem: working memory buffer pointer, size determined by
- * nx842_powernv_driver.workmem_size
- * @fc: function code, see CCW Function Codes in nx-842.h
- *
- * Returns:
- * 0 Success, output of length @outlenp stored in the buffer at @out
- * -ENODEV Hardware unavailable
- * -ENOSPC Output buffer is to small
- * -EMSGSIZE Input buffer too large
- * -EINVAL buffer constraints do not fix nx842_constraints
- * -EPROTO hardware error during operation
- * -ETIMEDOUT hardware did not complete operation in reasonable time
- * -EINTR operation was aborted
- */
-static int nx842_exec_icswx(const unsigned char *in, unsigned int inlen,
- unsigned char *out, unsigned int *outlenp,
- void *workmem, int fc)
-{
- struct coprocessor_request_block *crb;
- struct coprocessor_status_block *csb;
- struct nx842_workmem *wmem;
- int ret;
- u32 ccw;
- unsigned int outlen = *outlenp;
-
- wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);
-
- *outlenp = 0;
-
- /* shoudn't happen, we don't load without a coproc */
- if (!nx842_ct) {
- pr_err_ratelimited("coprocessor CT is 0");
- return -ENODEV;
- }
-
- ret = nx842_config_crb(in, inlen, out, outlen, wmem);
- if (ret)
- return ret;
-
- crb = &wmem->crb;
- csb = &crb->csb;
-
- /* set up CCW */
- ccw = 0;
- ccw = SET_FIELD(CCW_CT, ccw, nx842_ct);
- ccw = SET_FIELD(CCW_CI_842, ccw, 0); /* use 0 for hw auto-selection */
- ccw = SET_FIELD(CCW_FC_842, ccw, fc);
-
- wmem->start = ktime_get();
-
- /* do ICSWX */
- ret = icswx(cpu_to_be32(ccw), crb);
-
- pr_debug_ratelimited("icswx CR %x ccw %x crb->ccw %x\n", ret,
- (unsigned int)ccw,
- (unsigned int)be32_to_cpu(crb->ccw));
-
- /*
- * NX842 coprocessor sets 3rd bit in CR register with XER[S0].
- * XER[S0] is the integer summary overflow bit which is nothing
- * to do NX. Since this bit can be set with other return values,
- * mask this bit.
- */
- ret &= ~ICSWX_XERS0;
-
- switch (ret) {
- case ICSWX_INITIATED:
- ret = wait_for_csb(wmem, csb);
- break;
- case ICSWX_BUSY:
- pr_debug_ratelimited("842 Coprocessor busy\n");
- ret = -EBUSY;
- break;
- case ICSWX_REJECTED:
- pr_err_ratelimited("ICSWX rejected\n");
- ret = -EPROTO;
- break;
- }
-
- if (!ret)
- *outlenp = be32_to_cpu(csb->count);
-
- return ret;
-}
-
-/**
- * nx842_exec_vas - compress/decompress data using the 842 algorithm
- *
- * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
- * This compresses or decompresses the provided input buffer into the provided
- * output buffer.
- *
- * Upon return from this function @outlen contains the length of the
- * output data. If there is an error then @outlen will be 0 and an
- * error will be specified by the return code from this function.
- *
- * The @workmem buffer should only be used by one function call at a time.
- *
- * @in: input buffer pointer
- * @inlen: input buffer size
- * @out: output buffer pointer
- * @outlenp: output buffer size pointer
- * @workmem: working memory buffer pointer, size determined by
- * nx842_powernv_driver.workmem_size
- * @fc: function code, see CCW Function Codes in nx-842.h
- *
- * Returns:
- * 0 Success, output of length @outlenp stored in the buffer
- * at @out
- * -ENODEV Hardware unavailable
- * -ENOSPC Output buffer is to small
- * -EMSGSIZE Input buffer too large
- * -EINVAL buffer constraints do not fix nx842_constraints
- * -EPROTO hardware error during operation
- * -ETIMEDOUT hardware did not complete operation in reasonable time
- * -EINTR operation was aborted
- */
-static int nx842_exec_vas(const unsigned char *in, unsigned int inlen,
- unsigned char *out, unsigned int *outlenp,
- void *workmem, int fc)
-{
- struct coprocessor_request_block *crb;
- struct coprocessor_status_block *csb;
- struct nx842_workmem *wmem;
- struct vas_window *txwin;
- int ret, i = 0;
- u32 ccw;
- unsigned int outlen = *outlenp;
-
- wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);
-
- *outlenp = 0;
-
- crb = &wmem->crb;
- csb = &crb->csb;
-
- ret = nx842_config_crb(in, inlen, out, outlen, wmem);
- if (ret)
- return ret;
-
- ccw = 0;
- ccw = SET_FIELD(CCW_FC_842, ccw, fc);
- crb->ccw = cpu_to_be32(ccw);
-
- do {
- wmem->start = ktime_get();
- preempt_disable();
- txwin = this_cpu_read(cpu_txwin);
-
- /*
- * VAS copy CRB into L2 cache. Refer <asm/vas.h>.
- * @crb and @offset.
- */
- vas_copy_crb(crb, 0);
-
- /*
- * VAS paste previously copied CRB to NX.
- * @txwin, @offset and @last (must be true).
- */
- ret = vas_paste_crb(txwin, 0, 1);
- preempt_enable();
- /*
- * Retry copy/paste function for VAS failures.
- */
- } while (ret && (i++ < VAS_RETRIES));
-
- if (ret) {
- pr_err_ratelimited("VAS copy/paste failed\n");
- return ret;
- }
-
- ret = wait_for_csb(wmem, csb);
- if (!ret)
- *outlenp = be32_to_cpu(csb->count);
-
- return ret;
-}
-
-/**
- * nx842_powernv_compress - Compress data using the 842 algorithm
- *
- * Compression provided by the NX842 coprocessor on IBM PowerNV systems.
- * The input buffer is compressed and the result is stored in the
- * provided output buffer.
- *
- * Upon return from this function @outlen contains the length of the
- * compressed data. If there is an error then @outlen will be 0 and an
- * error will be specified by the return code from this function.
- *
- * @in: input buffer pointer
- * @inlen: input buffer size
- * @out: output buffer pointer
- * @outlenp: output buffer size pointer
- * @workmem: working memory buffer pointer, size determined by
- * nx842_powernv_driver.workmem_size
- *
- * Returns: see @nx842_powernv_exec()
- */
-static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen,
- unsigned char *out, unsigned int *outlenp,
- void *wmem)
-{
- return nx842_powernv_exec(in, inlen, out, outlenp,
- wmem, CCW_FC_842_COMP_CRC);
-}
-
-/**
- * nx842_powernv_decompress - Decompress data using the 842 algorithm
- *
- * Decompression provided by the NX842 coprocessor on IBM PowerNV systems.
- * The input buffer is decompressed and the result is stored in the
- * provided output buffer.
- *
- * Upon return from this function @outlen contains the length of the
- * decompressed data. If there is an error then @outlen will be 0 and an
- * error will be specified by the return code from this function.
- *
- * @in: input buffer pointer
- * @inlen: input buffer size
- * @out: output buffer pointer
- * @outlenp: output buffer size pointer
- * @workmem: working memory buffer pointer, size determined by
- * nx842_powernv_driver.workmem_size
- *
- * Returns: see @nx842_powernv_exec()
- */
-static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen,
- unsigned char *out, unsigned int *outlenp,
- void *wmem)
-{
- return nx842_powernv_exec(in, inlen, out, outlenp,
- wmem, CCW_FC_842_DECOMP_CRC);
-}
-
-static inline void nx842_add_coprocs_list(struct nx842_coproc *coproc,
- int chipid)
-{
- coproc->chip_id = chipid;
- INIT_LIST_HEAD(&coproc->list);
- list_add(&coproc->list, &nx842_coprocs);
-}
-
-static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc)
-{
- struct vas_window *txwin = NULL;
- struct vas_tx_win_attr txattr;
-
- /*
- * Kernel requests will be high priority. So open send
- * windows only for high priority RxFIFO entries.
- */
- vas_init_tx_win_attr(&txattr, coproc->ct);
- txattr.lpid = 0; /* lpid is 0 for kernel requests */
- txattr.pid = 0; /* pid is 0 for kernel requests */
-
- /*
- * Open a VAS send window which is used to send request to NX.
- */
- txwin = vas_tx_win_open(coproc->vas.id, coproc->ct, &txattr);
- if (IS_ERR(txwin))
- pr_err("ibm,nx-842: Can not open TX window: %ld\n",
- PTR_ERR(txwin));
-
- return txwin;
-}
-
-/*
- * Identify chip ID for each CPU, open send wndow for the corresponding NX
- * engine and save txwin in percpu cpu_txwin.
- * cpu_txwin is used in copy/paste operation for each compression /
- * decompression request.
- */
-static int nx842_open_percpu_txwins(void)
-{
- struct nx842_coproc *coproc, *n;
- unsigned int i, chip_id;
-
- for_each_possible_cpu(i) {
- struct vas_window *txwin = NULL;
-
- chip_id = cpu_to_chip_id(i);
-
- list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
- /*
- * Kernel requests use only high priority FIFOs. So
- * open send windows for these FIFOs.
- */
-
- if (coproc->ct != VAS_COP_TYPE_842_HIPRI)
- continue;
-
- if (coproc->chip_id == chip_id) {
- txwin = nx842_alloc_txwin(coproc);
- if (IS_ERR(txwin))
- return PTR_ERR(txwin);
-
- per_cpu(cpu_txwin, i) = txwin;
- break;
- }
- }
-
- if (!per_cpu(cpu_txwin, i)) {
- /* shouldn't happen, Each chip will have NX engine */
- pr_err("NX engine is not available for CPU %d\n", i);
- return -EINVAL;
- }
- }
-
- return 0;
-}
-
-static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
- int vasid, int *ct)
-{
- struct vas_window *rxwin = NULL;
- struct vas_rx_win_attr rxattr;
- struct nx842_coproc *coproc;
- u32 lpid, pid, tid, fifo_size;
- u64 rx_fifo;
- const char *priority;
- int ret;
-
- ret = of_property_read_u64(dn, "rx-fifo-address", &rx_fifo);
- if (ret) {
- pr_err("Missing rx-fifo-address property\n");
- return ret;
- }
-
- ret = of_property_read_u32(dn, "rx-fifo-size", &fifo_size);
- if (ret) {
- pr_err("Missing rx-fifo-size property\n");
- return ret;
- }
-
- ret = of_property_read_u32(dn, "lpid", &lpid);
- if (ret) {
- pr_err("Missing lpid property\n");
- return ret;
- }
-
- ret = of_property_read_u32(dn, "pid", &pid);
- if (ret) {
- pr_err("Missing pid property\n");
- return ret;
- }
-
- ret = of_property_read_u32(dn, "tid", &tid);
- if (ret) {
- pr_err("Missing tid property\n");
- return ret;
- }
-
- ret = of_property_read_string(dn, "priority", &priority);
- if (ret) {
- pr_err("Missing priority property\n");
- return ret;
- }
-
- coproc = kzalloc(sizeof(*coproc), GFP_KERNEL);
- if (!coproc)
- return -ENOMEM;
-
- if (!strcmp(priority, "High"))
- coproc->ct = VAS_COP_TYPE_842_HIPRI;
- else if (!strcmp(priority, "Normal"))
- coproc->ct = VAS_COP_TYPE_842;
- else {
- pr_err("Invalid RxFIFO priority value\n");
- ret = -EINVAL;
- goto err_out;
- }
-
- vas_init_rx_win_attr(&rxattr, coproc->ct);
- rxattr.rx_fifo = (void *)rx_fifo;
- rxattr.rx_fifo_size = fifo_size;
- rxattr.lnotify_lpid = lpid;
- rxattr.lnotify_pid = pid;
- rxattr.lnotify_tid = tid;
- /*
- * Maximum RX window credits can not be more than #CRBs in
- * RxFIFO. Otherwise, can get checkstop if RxFIFO overruns.
- */
- rxattr.wcreds_max = fifo_size / CRB_SIZE;
-
- /*
- * Open a VAS receice window which is used to configure RxFIFO
- * for NX.
- */
- rxwin = vas_rx_win_open(vasid, coproc->ct, &rxattr);
- if (IS_ERR(rxwin)) {
- ret = PTR_ERR(rxwin);
- pr_err("setting RxFIFO with VAS failed: %d\n",
- ret);
- goto err_out;
- }
-
- coproc->vas.rxwin = rxwin;
- coproc->vas.id = vasid;
- nx842_add_coprocs_list(coproc, chip_id);
-
- /*
- * (lpid, pid, tid) combination has to be unique for each
- * coprocessor instance in the system. So to make it
- * unique, skiboot uses coprocessor type such as 842 or
- * GZIP for pid and provides this value to kernel in pid
- * device-tree property.
- */
- *ct = pid;
-
- return 0;
-
-err_out:
- kfree(coproc);
- return ret;
-}
-
-
-static int __init nx842_powernv_probe_vas(struct device_node *pn)
-{
- struct device_node *dn;
- int chip_id, vasid, ret = 0;
- int nx_fifo_found = 0;
- int uninitialized_var(ct);
-
- chip_id = of_get_ibm_chip_id(pn);
- if (chip_id < 0) {
- pr_err("ibm,chip-id missing\n");
- return -EINVAL;
- }
-
- vasid = chip_to_vas_id(chip_id);
- if (vasid < 0) {
- pr_err("Unable to map chip_id %d to vasid\n", chip_id);
- return -EINVAL;
- }
-
- for_each_child_of_node(pn, dn) {
- if (of_device_is_compatible(dn, "ibm,p9-nx-842")) {
- ret = vas_cfg_coproc_info(dn, chip_id, vasid, &ct);
- if (ret) {
- of_node_put(dn);
- return ret;
- }
- nx_fifo_found++;
- }
- }
-
- if (!nx_fifo_found) {
- pr_err("NX842 FIFO nodes are missing\n");
- return -EINVAL;
- }
-
- /*
- * Initialize NX instance for both high and normal priority FIFOs.
- */
- if (opal_check_token(OPAL_NX_COPROC_INIT)) {
- ret = opal_nx_coproc_init(chip_id, ct);
- if (ret) {
- pr_err("Failed to initialize NX for chip(%d): %d\n",
- chip_id, ret);
- ret = opal_error_code(ret);
- }
- } else
- pr_warn("Firmware doesn't support NX initialization\n");
-
- return ret;
-}
-
-static int __init nx842_powernv_probe(struct device_node *dn)
-{
- struct nx842_coproc *coproc;
- unsigned int ct, ci;
- int chip_id;
-
- chip_id = of_get_ibm_chip_id(dn);
- if (chip_id < 0) {
- pr_err("ibm,chip-id missing\n");
- return -EINVAL;
- }
-
- if (of_property_read_u32(dn, "ibm,842-coprocessor-type", &ct)) {
- pr_err("ibm,842-coprocessor-type missing\n");
- return -EINVAL;
- }
-
- if (of_property_read_u32(dn, "ibm,842-coprocessor-instance", &ci)) {
- pr_err("ibm,842-coprocessor-instance missing\n");
- return -EINVAL;
- }
-
- coproc = kzalloc(sizeof(*coproc), GFP_KERNEL);
- if (!coproc)
- return -ENOMEM;
-
- coproc->ct = ct;
- coproc->ci = ci;
- nx842_add_coprocs_list(coproc, chip_id);
-
- pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci);
-
- if (!nx842_ct)
- nx842_ct = ct;
- else if (nx842_ct != ct)
- pr_err("NX842 chip %d, CT %d != first found CT %d\n",
- chip_id, ct, nx842_ct);
-
- return 0;
-}
-
-static void nx842_delete_coprocs(void)
-{
- struct nx842_coproc *coproc, *n;
- struct vas_window *txwin;
- int i;
-
- /*
- * close percpu txwins that are opened for the corresponding coproc.
- */
- for_each_possible_cpu(i) {
- txwin = per_cpu(cpu_txwin, i);
- if (txwin)
- vas_win_close(txwin);
-
- per_cpu(cpu_txwin, i) = 0;
- }
-
- list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
- if (coproc->vas.rxwin)
- vas_win_close(coproc->vas.rxwin);
-
- list_del(&coproc->list);
- kfree(coproc);
- }
-}
-
-static struct nx842_constraints nx842_powernv_constraints = {
- .alignment = DDE_BUFFER_ALIGN,
- .multiple = DDE_BUFFER_LAST_MULT,
- .minimum = DDE_BUFFER_LAST_MULT,
- .maximum = (DDL_LEN_MAX - 1) * PAGE_SIZE,
-};
-
-static struct nx842_driver nx842_powernv_driver = {
- .name = KBUILD_MODNAME,
- .owner = THIS_MODULE,
- .workmem_size = sizeof(struct nx842_workmem),
- .constraints = &nx842_powernv_constraints,
- .compress = nx842_powernv_compress,
- .decompress = nx842_powernv_decompress,
-};
-
-static int nx842_powernv_crypto_init(struct crypto_tfm *tfm)
-{
- return nx842_crypto_init(tfm, &nx842_powernv_driver);
-}
-
-static struct crypto_alg nx842_powernv_alg = {
- .cra_name = "842",
- .cra_driver_name = "842-nx",
- .cra_priority = 300,
- .cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
- .cra_ctxsize = sizeof(struct nx842_crypto_ctx),
- .cra_module = THIS_MODULE,
- .cra_init = nx842_powernv_crypto_init,
- .cra_exit = nx842_crypto_exit,
- .cra_u = { .compress = {
- .coa_compress = nx842_crypto_compress,
- .coa_decompress = nx842_crypto_decompress } }
-};
-
-static __init int nx842_powernv_init(void)
-{
- struct device_node *dn;
- int ret;
-
- /* verify workmem size/align restrictions */
- BUILD_BUG_ON(WORKMEM_ALIGN % CRB_ALIGN);
- BUILD_BUG_ON(CRB_ALIGN % DDE_ALIGN);
- BUILD_BUG_ON(CRB_SIZE % DDE_ALIGN);
- /* verify buffer size/align restrictions */
- BUILD_BUG_ON(PAGE_SIZE % DDE_BUFFER_ALIGN);
- BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT);
- BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT);
-
- for_each_compatible_node(dn, NULL, "ibm,power9-nx") {
- ret = nx842_powernv_probe_vas(dn);
- if (ret) {
- nx842_delete_coprocs();
- of_node_put(dn);
- return ret;
- }
- }
-
- if (list_empty(&nx842_coprocs)) {
- for_each_compatible_node(dn, NULL, "ibm,power-nx")
- nx842_powernv_probe(dn);
-
- if (!nx842_ct)
- return -ENODEV;
-
- nx842_powernv_exec = nx842_exec_icswx;
- } else {
- ret = nx842_open_percpu_txwins();
- if (ret) {
- nx842_delete_coprocs();
- return ret;
- }
-
- nx842_powernv_exec = nx842_exec_vas;
- }
-
- ret = crypto_register_alg(&nx842_powernv_alg);
- if (ret) {
- nx842_delete_coprocs();
- return ret;
- }
-
- return 0;
-}
-module_init(nx842_powernv_init);
-
-static void __exit nx842_powernv_exit(void)
-{
- crypto_unregister_alg(&nx842_powernv_alg);
-
- nx842_delete_coprocs();
-}
-module_exit(nx842_powernv_exit);
diff --git a/drivers/crypto/nx/nx-common-powernv.c b/drivers/crypto/nx/nx-common-powernv.c
new file mode 100644
index 0000000..f42881f
--- /dev/null
+++ b/drivers/crypto/nx/nx-common-powernv.c
@@ -0,0 +1,1062 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for IBM PowerNV compression accelerator
+ *
+ * Copyright (C) 2015 Dan Streetman, IBM Corp
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include "nx-842.h"
+
+#include <linux/timer.h>
+
+#include <asm/prom.h>
+#include <asm/icswx.h>
+#include <asm/vas.h>
+#include <asm/reg.h>
+#include <asm/opal-api.h>
+#include <asm/opal.h>
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
+MODULE_DESCRIPTION("H/W Compression driver for IBM PowerNV processors");
+MODULE_ALIAS_CRYPTO("842");
+MODULE_ALIAS_CRYPTO("842-nx");
+
+#define WORKMEM_ALIGN (CRB_ALIGN)
+#define CSB_WAIT_MAX (5000) /* ms */
+#define VAS_RETRIES (10)
+
+struct nx842_workmem {
+ /* Below fields must be properly aligned */
+ struct coprocessor_request_block crb; /* CRB_ALIGN align */
+ struct data_descriptor_entry ddl_in[DDL_LEN_MAX]; /* DDE_ALIGN align */
+ struct data_descriptor_entry ddl_out[DDL_LEN_MAX]; /* DDE_ALIGN align */
+ /* Above fields must be properly aligned */
+
+ ktime_t start;
+
+ char padding[WORKMEM_ALIGN]; /* unused, to allow alignment */
+} __packed __aligned(WORKMEM_ALIGN);
+
+struct nx842_coproc {
+ unsigned int chip_id;
+ unsigned int ct;
+ unsigned int ci; /* Coprocessor instance, used with icswx */
+ struct {
+ struct vas_window *rxwin;
+ int id;
+ } vas;
+ struct list_head list;
+};
+
+/*
+ * Send the request to NX engine on the chip for the corresponding CPU
+ * where the process is executing. Use with VAS function.
+ */
+static DEFINE_PER_CPU(struct vas_window *, cpu_txwin);
+
+/* no cpu hotplug on powernv, so this list never changes after init */
+static LIST_HEAD(nx842_coprocs);
+static unsigned int nx842_ct; /* used in icswx function */
+
+static int (*nx842_powernv_exec)(const unsigned char *in,
+ unsigned int inlen, unsigned char *out,
+ unsigned int *outlenp, void *workmem, int fc);
+
+/**
+ * setup_indirect_dde - Setup an indirect DDE
+ *
+ * The DDE is setup with the the DDE count, byte count, and address of
+ * first direct DDE in the list.
+ */
+static void setup_indirect_dde(struct data_descriptor_entry *dde,
+ struct data_descriptor_entry *ddl,
+ unsigned int dde_count, unsigned int byte_count)
+{
+ dde->flags = 0;
+ dde->count = dde_count;
+ dde->index = 0;
+ dde->length = cpu_to_be32(byte_count);
+ dde->address = cpu_to_be64(nx842_get_pa(ddl));
+}
+
+/**
+ * setup_direct_dde - Setup single DDE from buffer
+ *
+ * The DDE is setup with the buffer and length. The buffer must be properly
+ * aligned. The used length is returned.
+ * Returns:
+ * N Successfully set up DDE with N bytes
+ */
+static unsigned int setup_direct_dde(struct data_descriptor_entry *dde,
+ unsigned long pa, unsigned int len)
+{
+ unsigned int l = min_t(unsigned int, len, LEN_ON_PAGE(pa));
+
+ dde->flags = 0;
+ dde->count = 0;
+ dde->index = 0;
+ dde->length = cpu_to_be32(l);
+ dde->address = cpu_to_be64(pa);
+
+ return l;
+}
+
+/**
+ * setup_ddl - Setup DDL from buffer
+ *
+ * Returns:
+ * 0 Successfully set up DDL
+ */
+static int setup_ddl(struct data_descriptor_entry *dde,
+ struct data_descriptor_entry *ddl,
+ unsigned char *buf, unsigned int len,
+ bool in)
+{
+ unsigned long pa = nx842_get_pa(buf);
+ int i, ret, total_len = len;
+
+ if (!IS_ALIGNED(pa, DDE_BUFFER_ALIGN)) {
+ pr_debug("%s buffer pa 0x%lx not 0x%x-byte aligned\n",
+ in ? "input" : "output", pa, DDE_BUFFER_ALIGN);
+ return -EINVAL;
+ }
+
+ /* only need to check last mult; since buffer must be
+ * DDE_BUFFER_ALIGN aligned, and that is a multiple of
+ * DDE_BUFFER_SIZE_MULT, and pre-last page DDE buffers
+ * are guaranteed a multiple of DDE_BUFFER_SIZE_MULT.
+ */
+ if (len % DDE_BUFFER_LAST_MULT) {
+ pr_debug("%s buffer len 0x%x not a multiple of 0x%x\n",
+ in ? "input" : "output", len, DDE_BUFFER_LAST_MULT);
+ if (in)
+ return -EINVAL;
+ len = round_down(len, DDE_BUFFER_LAST_MULT);
+ }
+
+ /* use a single direct DDE */
+ if (len <= LEN_ON_PAGE(pa)) {
+ ret = setup_direct_dde(dde, pa, len);
+ WARN_ON(ret < len);
+ return 0;
+ }
+
+ /* use the DDL */
+ for (i = 0; i < DDL_LEN_MAX && len > 0; i++) {
+ ret = setup_direct_dde(&ddl[i], pa, len);
+ buf += ret;
+ len -= ret;
+ pa = nx842_get_pa(buf);
+ }
+
+ if (len > 0) {
+ pr_debug("0x%x total %s bytes 0x%x too many for DDL.\n",
+ total_len, in ? "input" : "output", len);
+ if (in)
+ return -EMSGSIZE;
+ total_len -= len;
+ }
+ setup_indirect_dde(dde, ddl, i, total_len);
+
+ return 0;
+}
+
+#define CSB_ERR(csb, msg, ...) \
+ pr_err("ERROR: " msg " : %02x %02x %02x %02x %08x\n", \
+ ##__VA_ARGS__, (csb)->flags, \
+ (csb)->cs, (csb)->cc, (csb)->ce, \
+ be32_to_cpu((csb)->count))
+
+#define CSB_ERR_ADDR(csb, msg, ...) \
+ CSB_ERR(csb, msg " at %lx", ##__VA_ARGS__, \
+ (unsigned long)be64_to_cpu((csb)->address))
+
+/**
+ * wait_for_csb
+ */
+static int wait_for_csb(struct nx842_workmem *wmem,
+ struct coprocessor_status_block *csb)
+{
+ ktime_t start = wmem->start, now = ktime_get();
+ ktime_t timeout = ktime_add_ms(start, CSB_WAIT_MAX);
+
+ while (!(READ_ONCE(csb->flags) & CSB_V)) {
+ cpu_relax();
+ now = ktime_get();
+ if (ktime_after(now, timeout))
+ break;
+ }
+
+ /* hw has updated csb and output buffer */
+ barrier();
+
+ /* check CSB flags */
+ if (!(csb->flags & CSB_V)) {
+ CSB_ERR(csb, "CSB still not valid after %ld us, giving up",
+ (long)ktime_us_delta(now, start));
+ return -ETIMEDOUT;
+ }
+ if (csb->flags & CSB_F) {
+ CSB_ERR(csb, "Invalid CSB format");
+ return -EPROTO;
+ }
+ if (csb->flags & CSB_CH) {
+ CSB_ERR(csb, "Invalid CSB chaining state");
+ return -EPROTO;
+ }
+
+ /* verify CSB completion sequence is 0 */
+ if (csb->cs) {
+ CSB_ERR(csb, "Invalid CSB completion sequence");
+ return -EPROTO;
+ }
+
+ /* check CSB Completion Code */
+ switch (csb->cc) {
+ /* no error */
+ case CSB_CC_SUCCESS:
+ break;
+ case CSB_CC_TPBC_GT_SPBC:
+ /* not an error, but the compressed data is
+ * larger than the uncompressed data :(
+ */
+ break;
+
+ /* input data errors */
+ case CSB_CC_OPERAND_OVERLAP:
+ /* input and output buffers overlap */
+ CSB_ERR(csb, "Operand Overlap error");
+ return -EINVAL;
+ case CSB_CC_INVALID_OPERAND:
+ CSB_ERR(csb, "Invalid operand");
+ return -EINVAL;
+ case CSB_CC_NOSPC:
+ /* output buffer too small */
+ return -ENOSPC;
+ case CSB_CC_ABORT:
+ CSB_ERR(csb, "Function aborted");
+ return -EINTR;
+ case CSB_CC_CRC_MISMATCH:
+ CSB_ERR(csb, "CRC mismatch");
+ return -EINVAL;
+ case CSB_CC_TEMPL_INVALID:
+ CSB_ERR(csb, "Compressed data template invalid");
+ return -EINVAL;
+ case CSB_CC_TEMPL_OVERFLOW:
+ CSB_ERR(csb, "Compressed data template shows data past end");
+ return -EINVAL;
+ case CSB_CC_EXCEED_BYTE_COUNT: /* P9 or later */
+ /*
+ * DDE byte count exceeds the limit specified in Maximum
+ * byte count register.
+ */
+ CSB_ERR(csb, "DDE byte count exceeds the limit");
+ return -EINVAL;
+
+ /* these should not happen */
+ case CSB_CC_INVALID_ALIGN:
+ /* setup_ddl should have detected this */
+ CSB_ERR_ADDR(csb, "Invalid alignment");
+ return -EINVAL;
+ case CSB_CC_DATA_LENGTH:
+ /* setup_ddl should have detected this */
+ CSB_ERR(csb, "Invalid data length");
+ return -EINVAL;
+ case CSB_CC_WR_TRANSLATION:
+ case CSB_CC_TRANSLATION:
+ case CSB_CC_TRANSLATION_DUP1:
+ case CSB_CC_TRANSLATION_DUP2:
+ case CSB_CC_TRANSLATION_DUP3:
+ case CSB_CC_TRANSLATION_DUP4:
+ case CSB_CC_TRANSLATION_DUP5:
+ case CSB_CC_TRANSLATION_DUP6:
+ /* should not happen, we use physical addrs */
+ CSB_ERR_ADDR(csb, "Translation error");
+ return -EPROTO;
+ case CSB_CC_WR_PROTECTION:
+ case CSB_CC_PROTECTION:
+ case CSB_CC_PROTECTION_DUP1:
+ case CSB_CC_PROTECTION_DUP2:
+ case CSB_CC_PROTECTION_DUP3:
+ case CSB_CC_PROTECTION_DUP4:
+ case CSB_CC_PROTECTION_DUP5:
+ case CSB_CC_PROTECTION_DUP6:
+ /* should not happen, we use physical addrs */
+ CSB_ERR_ADDR(csb, "Protection error");
+ return -EPROTO;
+ case CSB_CC_PRIVILEGE:
+ /* shouldn't happen, we're in HYP mode */
+ CSB_ERR(csb, "Insufficient Privilege error");
+ return -EPROTO;
+ case CSB_CC_EXCESSIVE_DDE:
+ /* shouldn't happen, setup_ddl doesn't use many dde's */
+ CSB_ERR(csb, "Too many DDEs in DDL");
+ return -EINVAL;
+ case CSB_CC_TRANSPORT:
+ case CSB_CC_INVALID_CRB: /* P9 or later */
+ /* shouldn't happen, we setup CRB correctly */
+ CSB_ERR(csb, "Invalid CRB");
+ return -EINVAL;
+ case CSB_CC_INVALID_DDE: /* P9 or later */
+ /*
+ * shouldn't happen, setup_direct/indirect_dde creates
+ * DDE right
+ */
+ CSB_ERR(csb, "Invalid DDE");
+ return -EINVAL;
+ case CSB_CC_SEGMENTED_DDL:
+ /* shouldn't happen, setup_ddl creates DDL right */
+ CSB_ERR(csb, "Segmented DDL error");
+ return -EINVAL;
+ case CSB_CC_DDE_OVERFLOW:
+ /* shouldn't happen, setup_ddl creates DDL right */
+ CSB_ERR(csb, "DDE overflow error");
+ return -EINVAL;
+ case CSB_CC_SESSION:
+ /* should not happen with ICSWX */
+ CSB_ERR(csb, "Session violation error");
+ return -EPROTO;
+ case CSB_CC_CHAIN:
+ /* should not happen, we don't use chained CRBs */
+ CSB_ERR(csb, "Chained CRB error");
+ return -EPROTO;
+ case CSB_CC_SEQUENCE:
+ /* should not happen, we don't use chained CRBs */
+ CSB_ERR(csb, "CRB sequence number error");
+ return -EPROTO;
+ case CSB_CC_UNKNOWN_CODE:
+ CSB_ERR(csb, "Unknown subfunction code");
+ return -EPROTO;
+
+ /* hardware errors */
+ case CSB_CC_RD_EXTERNAL:
+ case CSB_CC_RD_EXTERNAL_DUP1:
+ case CSB_CC_RD_EXTERNAL_DUP2:
+ case CSB_CC_RD_EXTERNAL_DUP3:
+ CSB_ERR_ADDR(csb, "Read error outside coprocessor");
+ return -EPROTO;
+ case CSB_CC_WR_EXTERNAL:
+ CSB_ERR_ADDR(csb, "Write error outside coprocessor");
+ return -EPROTO;
+ case CSB_CC_INTERNAL:
+ CSB_ERR(csb, "Internal error in coprocessor");
+ return -EPROTO;
+ case CSB_CC_PROVISION:
+ CSB_ERR(csb, "Storage provision error");
+ return -EPROTO;
+ case CSB_CC_HW:
+ CSB_ERR(csb, "Correctable hardware error");
+ return -EPROTO;
+ case CSB_CC_HW_EXPIRED_TIMER: /* P9 or later */
+ CSB_ERR(csb, "Job did not finish within allowed time");
+ return -EPROTO;
+
+ default:
+ CSB_ERR(csb, "Invalid CC %d", csb->cc);
+ return -EPROTO;
+ }
+
+ /* check Completion Extension state */
+ if (csb->ce & CSB_CE_TERMINATION) {
+ CSB_ERR(csb, "CSB request was terminated");
+ return -EPROTO;
+ }
+ if (csb->ce & CSB_CE_INCOMPLETE) {
+ CSB_ERR(csb, "CSB request not complete");
+ return -EPROTO;
+ }
+ if (!(csb->ce & CSB_CE_TPBC)) {
+ CSB_ERR(csb, "TPBC not provided, unknown target length");
+ return -EPROTO;
+ }
+
+ /* successful completion */
+ pr_debug_ratelimited("Processed %u bytes in %lu us\n",
+ be32_to_cpu(csb->count),
+ (unsigned long)ktime_us_delta(now, start));
+
+ return 0;
+}
+
+static int nx842_config_crb(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int outlen,
+ struct nx842_workmem *wmem)
+{
+ struct coprocessor_request_block *crb;
+ struct coprocessor_status_block *csb;
+ u64 csb_addr;
+ int ret;
+
+ crb = &wmem->crb;
+ csb = &crb->csb;
+
+ /* Clear any previous values */
+ memset(crb, 0, sizeof(*crb));
+
+ /* set up DDLs */
+ ret = setup_ddl(&crb->source, wmem->ddl_in,
+ (unsigned char *)in, inlen, true);
+ if (ret)
+ return ret;
+
+ ret = setup_ddl(&crb->target, wmem->ddl_out,
+ out, outlen, false);
+ if (ret)
+ return ret;
+
+ /* set up CRB's CSB addr */
+ csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS;
+ csb_addr |= CRB_CSB_AT; /* Addrs are phys */
+ crb->csb_addr = cpu_to_be64(csb_addr);
+
+ return 0;
+}
+
+/**
+ * nx842_exec_icswx - compress/decompress data using the 842 algorithm
+ *
+ * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
+ * This compresses or decompresses the provided input buffer into the provided
+ * output buffer.
+ *
+ * Upon return from this function @outlen contains the length of the
+ * output data. If there is an error then @outlen will be 0 and an
+ * error will be specified by the return code from this function.
+ *
+ * The @workmem buffer should only be used by one function call at a time.
+ *
+ * @in: input buffer pointer
+ * @inlen: input buffer size
+ * @out: output buffer pointer
+ * @outlenp: output buffer size pointer
+ * @workmem: working memory buffer pointer, size determined by
+ * nx842_powernv_driver.workmem_size
+ * @fc: function code, see CCW Function Codes in nx-842.h
+ *
+ * Returns:
+ * 0 Success, output of length @outlenp stored in the buffer at @out
+ * -ENODEV Hardware unavailable
+ * -ENOSPC Output buffer is to small
+ * -EMSGSIZE Input buffer too large
+ * -EINVAL buffer constraints do not fix nx842_constraints
+ * -EPROTO hardware error during operation
+ * -ETIMEDOUT hardware did not complete operation in reasonable time
+ * -EINTR operation was aborted
+ */
+static int nx842_exec_icswx(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int *outlenp,
+ void *workmem, int fc)
+{
+ struct coprocessor_request_block *crb;
+ struct coprocessor_status_block *csb;
+ struct nx842_workmem *wmem;
+ int ret;
+ u32 ccw;
+ unsigned int outlen = *outlenp;
+
+ wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);
+
+ *outlenp = 0;
+
+ /* shoudn't happen, we don't load without a coproc */
+ if (!nx842_ct) {
+ pr_err_ratelimited("coprocessor CT is 0");
+ return -ENODEV;
+ }
+
+ ret = nx842_config_crb(in, inlen, out, outlen, wmem);
+ if (ret)
+ return ret;
+
+ crb = &wmem->crb;
+ csb = &crb->csb;
+
+ /* set up CCW */
+ ccw = 0;
+ ccw = SET_FIELD(CCW_CT, ccw, nx842_ct);
+ ccw = SET_FIELD(CCW_CI_842, ccw, 0); /* use 0 for hw auto-selection */
+ ccw = SET_FIELD(CCW_FC_842, ccw, fc);
+
+ wmem->start = ktime_get();
+
+ /* do ICSWX */
+ ret = icswx(cpu_to_be32(ccw), crb);
+
+ pr_debug_ratelimited("icswx CR %x ccw %x crb->ccw %x\n", ret,
+ (unsigned int)ccw,
+ (unsigned int)be32_to_cpu(crb->ccw));
+
+ /*
+ * NX842 coprocessor sets 3rd bit in CR register with XER[S0].
+ * XER[S0] is the integer summary overflow bit which is nothing
+ * to do NX. Since this bit can be set with other return values,
+ * mask this bit.
+ */
+ ret &= ~ICSWX_XERS0;
+
+ switch (ret) {
+ case ICSWX_INITIATED:
+ ret = wait_for_csb(wmem, csb);
+ break;
+ case ICSWX_BUSY:
+ pr_debug_ratelimited("842 Coprocessor busy\n");
+ ret = -EBUSY;
+ break;
+ case ICSWX_REJECTED:
+ pr_err_ratelimited("ICSWX rejected\n");
+ ret = -EPROTO;
+ break;
+ }
+
+ if (!ret)
+ *outlenp = be32_to_cpu(csb->count);
+
+ return ret;
+}
+
+/**
+ * nx842_exec_vas - compress/decompress data using the 842 algorithm
+ *
+ * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
+ * This compresses or decompresses the provided input buffer into the provided
+ * output buffer.
+ *
+ * Upon return from this function @outlen contains the length of the
+ * output data. If there is an error then @outlen will be 0 and an
+ * error will be specified by the return code from this function.
+ *
+ * The @workmem buffer should only be used by one function call at a time.
+ *
+ * @in: input buffer pointer
+ * @inlen: input buffer size
+ * @out: output buffer pointer
+ * @outlenp: output buffer size pointer
+ * @workmem: working memory buffer pointer, size determined by
+ * nx842_powernv_driver.workmem_size
+ * @fc: function code, see CCW Function Codes in nx-842.h
+ *
+ * Returns:
+ * 0 Success, output of length @outlenp stored in the buffer
+ * at @out
+ * -ENODEV Hardware unavailable
+ * -ENOSPC Output buffer is to small
+ * -EMSGSIZE Input buffer too large
+ * -EINVAL buffer constraints do not fix nx842_constraints
+ * -EPROTO hardware error during operation
+ * -ETIMEDOUT hardware did not complete operation in reasonable time
+ * -EINTR operation was aborted
+ */
+static int nx842_exec_vas(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int *outlenp,
+ void *workmem, int fc)
+{
+ struct coprocessor_request_block *crb;
+ struct coprocessor_status_block *csb;
+ struct nx842_workmem *wmem;
+ struct vas_window *txwin;
+ int ret, i = 0;
+ u32 ccw;
+ unsigned int outlen = *outlenp;
+
+ wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);
+
+ *outlenp = 0;
+
+ crb = &wmem->crb;
+ csb = &crb->csb;
+
+ ret = nx842_config_crb(in, inlen, out, outlen, wmem);
+ if (ret)
+ return ret;
+
+ ccw = 0;
+ ccw = SET_FIELD(CCW_FC_842, ccw, fc);
+ crb->ccw = cpu_to_be32(ccw);
+
+ do {
+ wmem->start = ktime_get();
+ preempt_disable();
+ txwin = this_cpu_read(cpu_txwin);
+
+ /*
+ * VAS copy CRB into L2 cache. Refer <asm/vas.h>.
+ * @crb and @offset.
+ */
+ vas_copy_crb(crb, 0);
+
+ /*
+ * VAS paste previously copied CRB to NX.
+ * @txwin, @offset and @last (must be true).
+ */
+ ret = vas_paste_crb(txwin, 0, 1);
+ preempt_enable();
+ /*
+ * Retry copy/paste function for VAS failures.
+ */
+ } while (ret && (i++ < VAS_RETRIES));
+
+ if (ret) {
+ pr_err_ratelimited("VAS copy/paste failed\n");
+ return ret;
+ }
+
+ ret = wait_for_csb(wmem, csb);
+ if (!ret)
+ *outlenp = be32_to_cpu(csb->count);
+
+ return ret;
+}
+
+/**
+ * nx842_powernv_compress - Compress data using the 842 algorithm
+ *
+ * Compression provided by the NX842 coprocessor on IBM PowerNV systems.
+ * The input buffer is compressed and the result is stored in the
+ * provided output buffer.
+ *
+ * Upon return from this function @outlen contains the length of the
+ * compressed data. If there is an error then @outlen will be 0 and an
+ * error will be specified by the return code from this function.
+ *
+ * @in: input buffer pointer
+ * @inlen: input buffer size
+ * @out: output buffer pointer
+ * @outlenp: output buffer size pointer
+ * @workmem: working memory buffer pointer, size determined by
+ * nx842_powernv_driver.workmem_size
+ *
+ * Returns: see @nx842_powernv_exec()
+ */
+static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int *outlenp,
+ void *wmem)
+{
+ return nx842_powernv_exec(in, inlen, out, outlenp,
+ wmem, CCW_FC_842_COMP_CRC);
+}
+
+/**
+ * nx842_powernv_decompress - Decompress data using the 842 algorithm
+ *
+ * Decompression provided by the NX842 coprocessor on IBM PowerNV systems.
+ * The input buffer is decompressed and the result is stored in the
+ * provided output buffer.
+ *
+ * Upon return from this function @outlen contains the length of the
+ * decompressed data. If there is an error then @outlen will be 0 and an
+ * error will be specified by the return code from this function.
+ *
+ * @in: input buffer pointer
+ * @inlen: input buffer size
+ * @out: output buffer pointer
+ * @outlenp: output buffer size pointer
+ * @workmem: working memory buffer pointer, size determined by
+ * nx842_powernv_driver.workmem_size
+ *
+ * Returns: see @nx842_powernv_exec()
+ */
+static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int *outlenp,
+ void *wmem)
+{
+ return nx842_powernv_exec(in, inlen, out, outlenp,
+ wmem, CCW_FC_842_DECOMP_CRC);
+}
+
+static inline void nx842_add_coprocs_list(struct nx842_coproc *coproc,
+ int chipid)
+{
+ coproc->chip_id = chipid;
+ INIT_LIST_HEAD(&coproc->list);
+ list_add(&coproc->list, &nx842_coprocs);
+}
+
+static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc)
+{
+ struct vas_window *txwin = NULL;
+ struct vas_tx_win_attr txattr;
+
+ /*
+ * Kernel requests will be high priority. So open send
+ * windows only for high priority RxFIFO entries.
+ */
+ vas_init_tx_win_attr(&txattr, coproc->ct);
+ txattr.lpid = 0; /* lpid is 0 for kernel requests */
+ txattr.pid = 0; /* pid is 0 for kernel requests */
+
+ /*
+ * Open a VAS send window which is used to send request to NX.
+ */
+ txwin = vas_tx_win_open(coproc->vas.id, coproc->ct, &txattr);
+ if (IS_ERR(txwin))
+ pr_err("ibm,nx-842: Can not open TX window: %ld\n",
+ PTR_ERR(txwin));
+
+ return txwin;
+}
+
+/*
+ * Identify chip ID for each CPU, open send wndow for the corresponding NX
+ * engine and save txwin in percpu cpu_txwin.
+ * cpu_txwin is used in copy/paste operation for each compression /
+ * decompression request.
+ */
+static int nx842_open_percpu_txwins(void)
+{
+ struct nx842_coproc *coproc, *n;
+ unsigned int i, chip_id;
+
+ for_each_possible_cpu(i) {
+ struct vas_window *txwin = NULL;
+
+ chip_id = cpu_to_chip_id(i);
+
+ list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
+ /*
+ * Kernel requests use only high priority FIFOs. So
+ * open send windows for these FIFOs.
+ */
+
+ if (coproc->ct != VAS_COP_TYPE_842_HIPRI)
+ continue;
+
+ if (coproc->chip_id == chip_id) {
+ txwin = nx842_alloc_txwin(coproc);
+ if (IS_ERR(txwin))
+ return PTR_ERR(txwin);
+
+ per_cpu(cpu_txwin, i) = txwin;
+ break;
+ }
+ }
+
+ if (!per_cpu(cpu_txwin, i)) {
+ /* shouldn't happen, Each chip will have NX engine */
+ pr_err("NX engine is not available for CPU %d\n", i);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
+ int vasid, int *ct)
+{
+ struct vas_window *rxwin = NULL;
+ struct vas_rx_win_attr rxattr;
+ struct nx842_coproc *coproc;
+ u32 lpid, pid, tid, fifo_size;
+ u64 rx_fifo;
+ const char *priority;
+ int ret;
+
+ ret = of_property_read_u64(dn, "rx-fifo-address", &rx_fifo);
+ if (ret) {
+ pr_err("Missing rx-fifo-address property\n");
+ return ret;
+ }
+
+ ret = of_property_read_u32(dn, "rx-fifo-size", &fifo_size);
+ if (ret) {
+ pr_err("Missing rx-fifo-size property\n");
+ return ret;
+ }
+
+ ret = of_property_read_u32(dn, "lpid", &lpid);
+ if (ret) {
+ pr_err("Missing lpid property\n");
+ return ret;
+ }
+
+ ret = of_property_read_u32(dn, "pid", &pid);
+ if (ret) {
+ pr_err("Missing pid property\n");
+ return ret;
+ }
+
+ ret = of_property_read_u32(dn, "tid", &tid);
+ if (ret) {
+ pr_err("Missing tid property\n");
+ return ret;
+ }
+
+ ret = of_property_read_string(dn, "priority", &priority);
+ if (ret) {
+ pr_err("Missing priority property\n");
+ return ret;
+ }
+
+ coproc = kzalloc(sizeof(*coproc), GFP_KERNEL);
+ if (!coproc)
+ return -ENOMEM;
+
+ if (!strcmp(priority, "High"))
+ coproc->ct = VAS_COP_TYPE_842_HIPRI;
+ else if (!strcmp(priority, "Normal"))
+ coproc->ct = VAS_COP_TYPE_842;
+ else {
+ pr_err("Invalid RxFIFO priority value\n");
+ ret = -EINVAL;
+ goto err_out;
+ }
+
+ vas_init_rx_win_attr(&rxattr, coproc->ct);
+ rxattr.rx_fifo = (void *)rx_fifo;
+ rxattr.rx_fifo_size = fifo_size;
+ rxattr.lnotify_lpid = lpid;
+ rxattr.lnotify_pid = pid;
+ rxattr.lnotify_tid = tid;
+ /*
+ * Maximum RX window credits can not be more than #CRBs in
+ * RxFIFO. Otherwise, can get checkstop if RxFIFO overruns.
+ */
+ rxattr.wcreds_max = fifo_size / CRB_SIZE;
+
+ /*
+ * Open a VAS receice window which is used to configure RxFIFO
+ * for NX.
+ */
+ rxwin = vas_rx_win_open(vasid, coproc->ct, &rxattr);
+ if (IS_ERR(rxwin)) {
+ ret = PTR_ERR(rxwin);
+ pr_err("setting RxFIFO with VAS failed: %d\n",
+ ret);
+ goto err_out;
+ }
+
+ coproc->vas.rxwin = rxwin;
+ coproc->vas.id = vasid;
+ nx842_add_coprocs_list(coproc, chip_id);
+
+ /*
+ * (lpid, pid, tid) combination has to be unique for each
+ * coprocessor instance in the system. So to make it
+ * unique, skiboot uses coprocessor type such as 842 or
+ * GZIP for pid and provides this value to kernel in pid
+ * device-tree property.
+ */
+ *ct = pid;
+
+ return 0;
+
+err_out:
+ kfree(coproc);
+ return ret;
+}
+
+
+static int __init nx842_powernv_probe_vas(struct device_node *pn)
+{
+ struct device_node *dn;
+ int chip_id, vasid, ret = 0;
+ int nx_fifo_found = 0;
+ int uninitialized_var(ct);
+
+ chip_id = of_get_ibm_chip_id(pn);
+ if (chip_id < 0) {
+ pr_err("ibm,chip-id missing\n");
+ return -EINVAL;
+ }
+
+ vasid = chip_to_vas_id(chip_id);
+ if (vasid < 0) {
+ pr_err("Unable to map chip_id %d to vasid\n", chip_id);
+ return -EINVAL;
+ }
+
+ for_each_child_of_node(pn, dn) {
+ if (of_device_is_compatible(dn, "ibm,p9-nx-842")) {
+ ret = vas_cfg_coproc_info(dn, chip_id, vasid, &ct);
+ if (ret) {
+ of_node_put(dn);
+ return ret;
+ }
+ nx_fifo_found++;
+ }
+ }
+
+ if (!nx_fifo_found) {
+ pr_err("NX842 FIFO nodes are missing\n");
+ return -EINVAL;
+ }
+
+ /*
+ * Initialize NX instance for both high and normal priority FIFOs.
+ */
+ if (opal_check_token(OPAL_NX_COPROC_INIT)) {
+ ret = opal_nx_coproc_init(chip_id, ct);
+ if (ret) {
+ pr_err("Failed to initialize NX for chip(%d): %d\n",
+ chip_id, ret);
+ ret = opal_error_code(ret);
+ }
+ } else
+ pr_warn("Firmware doesn't support NX initialization\n");
+
+ return ret;
+}
+
+static int __init nx842_powernv_probe(struct device_node *dn)
+{
+ struct nx842_coproc *coproc;
+ unsigned int ct, ci;
+ int chip_id;
+
+ chip_id = of_get_ibm_chip_id(dn);
+ if (chip_id < 0) {
+ pr_err("ibm,chip-id missing\n");
+ return -EINVAL;
+ }
+
+ if (of_property_read_u32(dn, "ibm,842-coprocessor-type", &ct)) {
+ pr_err("ibm,842-coprocessor-type missing\n");
+ return -EINVAL;
+ }
+
+ if (of_property_read_u32(dn, "ibm,842-coprocessor-instance", &ci)) {
+ pr_err("ibm,842-coprocessor-instance missing\n");
+ return -EINVAL;
+ }
+
+ coproc = kzalloc(sizeof(*coproc), GFP_KERNEL);
+ if (!coproc)
+ return -ENOMEM;
+
+ coproc->ct = ct;
+ coproc->ci = ci;
+ nx842_add_coprocs_list(coproc, chip_id);
+
+ pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci);
+
+ if (!nx842_ct)
+ nx842_ct = ct;
+ else if (nx842_ct != ct)
+ pr_err("NX842 chip %d, CT %d != first found CT %d\n",
+ chip_id, ct, nx842_ct);
+
+ return 0;
+}
+
+static void nx842_delete_coprocs(void)
+{
+ struct nx842_coproc *coproc, *n;
+ struct vas_window *txwin;
+ int i;
+
+ /*
+ * close percpu txwins that are opened for the corresponding coproc.
+ */
+ for_each_possible_cpu(i) {
+ txwin = per_cpu(cpu_txwin, i);
+ if (txwin)
+ vas_win_close(txwin);
+
+ per_cpu(cpu_txwin, i) = 0;
+ }
+
+ list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
+ if (coproc->vas.rxwin)
+ vas_win_close(coproc->vas.rxwin);
+
+ list_del(&coproc->list);
+ kfree(coproc);
+ }
+}
+
+static struct nx842_constraints nx842_powernv_constraints = {
+ .alignment = DDE_BUFFER_ALIGN,
+ .multiple = DDE_BUFFER_LAST_MULT,
+ .minimum = DDE_BUFFER_LAST_MULT,
+ .maximum = (DDL_LEN_MAX - 1) * PAGE_SIZE,
+};
+
+static struct nx842_driver nx842_powernv_driver = {
+ .name = KBUILD_MODNAME,
+ .owner = THIS_MODULE,
+ .workmem_size = sizeof(struct nx842_workmem),
+ .constraints = &nx842_powernv_constraints,
+ .compress = nx842_powernv_compress,
+ .decompress = nx842_powernv_decompress,
+};
+
+static int nx842_powernv_crypto_init(struct crypto_tfm *tfm)
+{
+ return nx842_crypto_init(tfm, &nx842_powernv_driver);
+}
+
+static struct crypto_alg nx842_powernv_alg = {
+ .cra_name = "842",
+ .cra_driver_name = "842-nx",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
+ .cra_ctxsize = sizeof(struct nx842_crypto_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_init = nx842_powernv_crypto_init,
+ .cra_exit = nx842_crypto_exit,
+ .cra_u = { .compress = {
+ .coa_compress = nx842_crypto_compress,
+ .coa_decompress = nx842_crypto_decompress } }
+};
+
+static __init int nx842_powernv_init(void)
+{
+ struct device_node *dn;
+ int ret;
+
+ /* verify workmem size/align restrictions */
+ BUILD_BUG_ON(WORKMEM_ALIGN % CRB_ALIGN);
+ BUILD_BUG_ON(CRB_ALIGN % DDE_ALIGN);
+ BUILD_BUG_ON(CRB_SIZE % DDE_ALIGN);
+ /* verify buffer size/align restrictions */
+ BUILD_BUG_ON(PAGE_SIZE % DDE_BUFFER_ALIGN);
+ BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT);
+ BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT);
+
+ for_each_compatible_node(dn, NULL, "ibm,power9-nx") {
+ ret = nx842_powernv_probe_vas(dn);
+ if (ret) {
+ nx842_delete_coprocs();
+ of_node_put(dn);
+ return ret;
+ }
+ }
+
+ if (list_empty(&nx842_coprocs)) {
+ for_each_compatible_node(dn, NULL, "ibm,power-nx")
+ nx842_powernv_probe(dn);
+
+ if (!nx842_ct)
+ return -ENODEV;
+
+ nx842_powernv_exec = nx842_exec_icswx;
+ } else {
+ ret = nx842_open_percpu_txwins();
+ if (ret) {
+ nx842_delete_coprocs();
+ return ret;
+ }
+
+ nx842_powernv_exec = nx842_exec_vas;
+ }
+
+ ret = crypto_register_alg(&nx842_powernv_alg);
+ if (ret) {
+ nx842_delete_coprocs();
+ return ret;
+ }
+
+ return 0;
+}
+module_init(nx842_powernv_init);
+
+static void __exit nx842_powernv_exit(void)
+{
+ crypto_unregister_alg(&nx842_powernv_alg);
+
+ nx842_delete_coprocs();
+}
+module_exit(nx842_powernv_exit);
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 4/9] crypto/nx: Initialize coproc entry with kzalloc
From: Haren Myneni @ 2020-03-23 4:03 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
coproc entry is initialized during NX probe on power9, but not on P8.
nx842_delete_coprocs() is used for both and frees receive window if it
is allocated. Getting crash for rmmod on P8 since coproc->vas.rxwin
is not initialized.
This patch replaces kmalloc with kzalloc in nx842_powernv_probe()
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
drivers/crypto/nx/nx-842-powernv.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c
index c037a24..8e63326 100644
--- a/drivers/crypto/nx/nx-842-powernv.c
+++ b/drivers/crypto/nx/nx-842-powernv.c
@@ -922,7 +922,7 @@ static int __init nx842_powernv_probe(struct device_node *dn)
return -EINVAL;
}
- coproc = kmalloc(sizeof(*coproc), GFP_KERNEL);
+ coproc = kzalloc(sizeof(*coproc), GFP_KERNEL);
if (!coproc)
return -ENOMEM;
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 3/9] powerpc/vas: Add VAS user space API
From: Haren Myneni @ 2020-03-23 4:02 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
On power9, userspace can send GZIP compression requests directly to NX
once kernel establishes NX channel / window with VAS. This patch provides
user space API which allows user space to establish channel using open
VAS_TX_WIN_OPEN ioctl, mmap and close operations.
Each window corresponds to file descriptor and application can open
multiple windows. After the window is opened, VAS_TX_WIN_OPEN icoctl to
open a window on specific VAS instance, mmap() system call to map
the hardware address of engine's request queue into the application's
virtual address space.
Then the application can then submit one or more requests to the the
engine by using the copy/paste instructions and pasting the CRBs to
the virtual address (aka paste_address) returned by mmap().
Only NX GZIP coprocessor type is supported right now and allow GZIP
engine access via /dev/crypto/nx-gzip device node.
Signed-off-by: Sukadev Bhattiprolu <sukadev@linux.vnet.ibm.com>
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
arch/powerpc/include/asm/vas.h | 11 ++
arch/powerpc/platforms/powernv/Makefile | 2 +-
arch/powerpc/platforms/powernv/vas-api.c | 257 ++++++++++++++++++++++++++++
arch/powerpc/platforms/powernv/vas-window.c | 6 +-
arch/powerpc/platforms/powernv/vas.h | 2 +
5 files changed, 274 insertions(+), 4 deletions(-)
create mode 100644 arch/powerpc/platforms/powernv/vas-api.c
diff --git a/arch/powerpc/include/asm/vas.h b/arch/powerpc/include/asm/vas.h
index f93e6b0..e064953 100644
--- a/arch/powerpc/include/asm/vas.h
+++ b/arch/powerpc/include/asm/vas.h
@@ -163,4 +163,15 @@ struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
*/
int vas_paste_crb(struct vas_window *win, int offset, bool re);
+/*
+ * Register / unregister coprocessor type to VAS API which will be exported
+ * to user space. Applications can use this API to open / close window
+ * which can be used to send / receive requests directly to cooprcessor.
+ *
+ * Only NX GZIP coprocessor type is supported now, but this API can be
+ * used for others in future.
+ */
+int vas_register_coproc_api(struct module *mod);
+void vas_unregister_coproc_api(void);
+
#endif /* __ASM_POWERPC_VAS_H */
diff --git a/arch/powerpc/platforms/powernv/Makefile b/arch/powerpc/platforms/powernv/Makefile
index 395789f..fe3f0fb 100644
--- a/arch/powerpc/platforms/powernv/Makefile
+++ b/arch/powerpc/platforms/powernv/Makefile
@@ -17,7 +17,7 @@ obj-$(CONFIG_MEMORY_FAILURE) += opal-memory-errors.o
obj-$(CONFIG_OPAL_PRD) += opal-prd.o
obj-$(CONFIG_PERF_EVENTS) += opal-imc.o
obj-$(CONFIG_PPC_MEMTRACE) += memtrace.o
-obj-$(CONFIG_PPC_VAS) += vas.o vas-window.o vas-debug.o vas-fault.o
+obj-$(CONFIG_PPC_VAS) += vas.o vas-window.o vas-debug.o vas-fault.o vas-api.o
obj-$(CONFIG_OCXL_BASE) += ocxl.o
obj-$(CONFIG_SCOM_DEBUGFS) += opal-xscom.o
obj-$(CONFIG_PPC_SECURE_BOOT) += opal-secvar.o
diff --git a/arch/powerpc/platforms/powernv/vas-api.c b/arch/powerpc/platforms/powernv/vas-api.c
new file mode 100644
index 0000000..7d049af
--- /dev/null
+++ b/arch/powerpc/platforms/powernv/vas-api.c
@@ -0,0 +1,257 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * VAS user space API for its accelerators (Only NX-GZIP is supported now)
+ * Copyright (C) 2019 Haren Myneni, IBM Corp
+ */
+
+#include <linux/kernel.h>
+#include <linux/device.h>
+#include <linux/cdev.h>
+#include <linux/fs.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <asm/vas.h>
+#include <uapi/asm/vas-api.h>
+#include "vas.h"
+
+/*
+ * The driver creates the device node that can be used as follows:
+ * For NX-GZIP
+ *
+ * fd = open("/dev/crypto/nx-gzip", O_RDWR);
+ * rc = ioctl(fd, VAS_TX_WIN_OPEN, &attr);
+ * paste_addr = mmap(NULL, PAGE_SIZE, prot, MAP_SHARED, fd, 0ULL).
+ * vas_copy(&crb, 0, 1);
+ * vas_paste(paste_addr, 0, 1);
+ * close(fd) or exit process to close window.
+ *
+ * where "vas_copy" and "vas_paste" are defined in copy-paste.h.
+ * copy/paste returns to the user space directly. So refer NX hardware
+ * documententation for exact copy/paste usage and completion / error
+ * conditions.
+ */
+
+static char *coproc_dev_name = "nx-gzip";
+
+/*
+ * Wrapper object for the nx-gzip device - there is just one instance of
+ * this node for the whole system.
+ */
+static struct coproc_dev {
+ struct cdev cdev;
+ struct device *device;
+ char *name;
+ dev_t devt;
+ struct class *class;
+} coproc_device;
+
+static char *coproc_devnode(struct device *dev, umode_t *mode)
+{
+ return kasprintf(GFP_KERNEL, "crypto/%s", dev_name(dev));
+}
+
+static int coproc_open(struct inode *inode, struct file *fp)
+{
+ /*
+ * vas_window is allocated and assigned to fp->private_data
+ * in ioctl. Nothing to do here for NX GZIP.
+ */
+ return 0;
+}
+
+static int coproc_ioc_tx_win_open(struct file *fp, unsigned long arg)
+{
+ void __user *uptr = (void __user *)arg;
+ struct vas_tx_win_attr txattr = {};
+ struct vas_tx_win_open_attr uattr;
+ struct vas_window *txwin;
+ int rc, vasid;
+
+ /*
+ * One window for file descriptor
+ */
+ if (fp->private_data)
+ return -EEXIST;
+
+ rc = copy_from_user(&uattr, uptr, sizeof(uattr));
+ if (rc) {
+ pr_err("%s(): copy_from_user() returns %d\n", __func__, rc);
+ return -EFAULT;
+ }
+
+ if (uattr.version != 1) {
+ pr_err("Invalid version\n");
+ return -EINVAL;
+ }
+
+ vasid = uattr.vas_id;
+
+ vas_init_tx_win_attr(&txattr, VAS_COP_TYPE_GZIP);
+
+ txattr.lpid = mfspr(SPRN_LPID);
+ txattr.pidr = mfspr(SPRN_PID);
+ txattr.user_win = true;
+ txattr.rsvd_txbuf_count = false;
+ txattr.pswid = false;
+
+ pr_devel("Pid %d: Opening txwin, PIDR %ld\n", txattr.pidr,
+ mfspr(SPRN_PID));
+
+ txwin = vas_tx_win_open(vasid, VAS_COP_TYPE_GZIP, &txattr);
+ if (IS_ERR(txwin)) {
+ pr_err("%s() vas_tx_win_open() failed, %ld\n", __func__,
+ PTR_ERR(txwin));
+ return PTR_ERR(txwin);
+ }
+
+ fp->private_data = txwin;
+
+ return 0;
+}
+
+static int coproc_release(struct inode *inode, struct file *fp)
+{
+ struct vas_window *txwin = fp->private_data;
+
+ if (txwin) {
+ vas_win_close(txwin);
+ fp->private_data = NULL;
+ }
+
+ /*
+ * We don't know here if user has other receive windows
+ * open, so we can't really call clear_thread_tidr().
+ * So, once the process calls set_thread_tidr(), the
+ * TIDR value sticks around until process exits, resulting
+ * in an extra copy in restore_sprs().
+ */
+
+ return 0;
+}
+
+static int coproc_mmap(struct file *fp, struct vm_area_struct *vma)
+{
+ struct vas_window *txwin = fp->private_data;
+ unsigned long pfn;
+ u64 paste_addr;
+ pgprot_t prot;
+ int rc;
+
+ if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
+ pr_debug("%s(): size 0x%zx, PAGE_SIZE 0x%zx\n", __func__,
+ (vma->vm_end - vma->vm_start), PAGE_SIZE);
+ return -EINVAL;
+ }
+
+ /* Ensure instance has an open send window */
+ if (!txwin) {
+ pr_err("%s(): No send window open?\n", __func__);
+ return -EINVAL;
+ }
+
+ vas_win_paste_addr(txwin, &paste_addr, NULL);
+ pfn = paste_addr >> PAGE_SHIFT;
+
+ /* flags, page_prot from cxl_mmap(), except we want cachable */
+ vma->vm_flags |= VM_IO | VM_PFNMAP;
+ vma->vm_page_prot = pgprot_cached(vma->vm_page_prot);
+
+ prot = __pgprot(pgprot_val(vma->vm_page_prot) | _PAGE_DIRTY);
+
+ rc = remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,
+ vma->vm_end - vma->vm_start, prot);
+
+ pr_devel("%s(): paste addr %llx at %lx, rc %d\n", __func__,
+ paste_addr, vma->vm_start, rc);
+
+ return rc;
+}
+
+static long coproc_ioctl(struct file *fp, unsigned int cmd, unsigned long arg)
+{
+ switch (cmd) {
+ case VAS_TX_WIN_OPEN:
+ return coproc_ioc_tx_win_open(fp, arg);
+ default:
+ return -EINVAL;
+ }
+}
+
+static struct file_operations coproc_fops = {
+ .open = coproc_open,
+ .release = coproc_release,
+ .mmap = coproc_mmap,
+ .unlocked_ioctl = coproc_ioctl,
+};
+
+/*
+ * Supporting only nx-gzip coprocessor type now, but this API code
+ * extended to other coprocessor types later.
+ */
+int vas_register_coproc_api(struct module *mod)
+{
+ int rc = -EINVAL;
+ dev_t devno;
+
+ rc = alloc_chrdev_region(&coproc_device.devt, 1, 1, coproc_dev_name);
+ if (rc) {
+ pr_err("Unable to allocate coproc major number: %i\n", rc);
+ return rc;
+ }
+
+ pr_devel("%s device allocated, dev [%i,%i]\n", coproc_dev_name,
+ MAJOR(coproc_device.devt), MINOR(coproc_device.devt));
+
+ coproc_device.class = class_create(mod, coproc_dev_name);
+ if (IS_ERR(coproc_device.class)) {
+ rc = PTR_ERR(coproc_device.class);
+ pr_err("Unable to create %s class %d\n", coproc_dev_name, rc);
+ goto err_class;
+ }
+ coproc_device.class->devnode = coproc_devnode;
+
+ coproc_fops.owner = mod;
+ cdev_init(&coproc_device.cdev, &coproc_fops);
+
+ devno = MKDEV(MAJOR(coproc_device.devt), 0);
+ rc = cdev_add(&coproc_device.cdev, devno, 1);
+ if (rc) {
+ pr_err("cdev_add() failed %d\n", rc);
+ goto err_cdev;
+ }
+
+ coproc_device.device = device_create(coproc_device.class, NULL,
+ devno, NULL, coproc_dev_name, MINOR(devno));
+ if (IS_ERR(coproc_device.device)) {
+ rc = PTR_ERR(coproc_device.device);
+ pr_err("Unable to create coproc-%d %d\n", MINOR(devno), rc);
+ goto err;
+ }
+
+ pr_devel("%s: Added dev [%d,%d]\n", __func__, MAJOR(devno),
+ MINOR(devno));
+
+ return 0;
+
+err:
+ cdev_del(&coproc_device.cdev);
+err_cdev:
+ class_destroy(coproc_device.class);
+err_class:
+ unregister_chrdev_region(coproc_device.devt, 1);
+ return rc;
+}
+EXPORT_SYMBOL_GPL(vas_register_coproc_api);
+
+void vas_unregister_coproc_api(void)
+{
+ dev_t devno;
+
+ cdev_del(&coproc_device.cdev);
+ devno = MKDEV(MAJOR(coproc_device.devt), 0);
+ device_destroy(coproc_device.class, devno);
+
+ class_destroy(coproc_device.class);
+ unregister_chrdev_region(coproc_device.devt, 1);
+}
+EXPORT_SYMBOL_GPL(vas_unregister_coproc_api);
diff --git a/arch/powerpc/platforms/powernv/vas-window.c b/arch/powerpc/platforms/powernv/vas-window.c
index 4ce421f..7b5187b 100644
--- a/arch/powerpc/platforms/powernv/vas-window.c
+++ b/arch/powerpc/platforms/powernv/vas-window.c
@@ -26,7 +26,7 @@
* Compute the paste address region for the window @window using the
* ->paste_base_addr and ->paste_win_id_shift we got from device tree.
*/
-static void compute_paste_address(struct vas_window *window, u64 *addr, int *len)
+void vas_win_paste_addr(struct vas_window *window, u64 *addr, int *len)
{
int winid;
u64 base, shift;
@@ -80,7 +80,7 @@ static void *map_paste_region(struct vas_window *txwin)
goto free_name;
txwin->paste_addr_name = name;
- compute_paste_address(txwin, &start, &len);
+ vas_win_paste_addr(txwin, &start, &len);
if (!request_mem_region(start, len, name)) {
pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
@@ -138,7 +138,7 @@ static void unmap_paste_region(struct vas_window *window)
u64 busaddr_start;
if (window->paste_kaddr) {
- compute_paste_address(window, &busaddr_start, &len);
+ vas_win_paste_addr(window, &busaddr_start, &len);
unmap_region(window->paste_kaddr, busaddr_start, len);
window->paste_kaddr = NULL;
kfree(window->paste_addr_name);
diff --git a/arch/powerpc/platforms/powernv/vas.h b/arch/powerpc/platforms/powernv/vas.h
index 8c39a7d..a10abed 100644
--- a/arch/powerpc/platforms/powernv/vas.h
+++ b/arch/powerpc/platforms/powernv/vas.h
@@ -431,6 +431,8 @@ struct vas_winctx {
extern void vas_return_credit(struct vas_window *window, bool tx);
extern struct vas_window *vas_pswid_to_window(struct vas_instance *vinst,
uint32_t pswid);
+extern void vas_win_paste_addr(struct vas_window *window, u64 *addr,
+ int *len);
static inline int vas_window_pid(struct vas_window *window)
{
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 2/9] powerpc/vas: Define VAS_TX_WIN_OPEN ioctl API
From: Haren Myneni @ 2020-03-23 4:01 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
Define the VAS_TX_WIN_OPEN ioctl interface for NX GZIP access
from user space. This interface is used to open GZIP send window and
mmap region which can be used by userspace to send requests to NX
directly with copy/paste instructions.
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
Documentation/userspace-api/ioctl/ioctl-number.rst | 1 +
arch/powerpc/include/uapi/asm/vas-api.h | 22 ++++++++++++++++++++++
2 files changed, 23 insertions(+)
create mode 100644 arch/powerpc/include/uapi/asm/vas-api.h
diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst
index 2e91370..deabc73 100644
--- a/Documentation/userspace-api/ioctl/ioctl-number.rst
+++ b/Documentation/userspace-api/ioctl/ioctl-number.rst
@@ -287,6 +287,7 @@ Code Seq# Include File Comments
'v' 00-1F linux/fs.h conflict!
'v' 00-0F linux/sonypi.h conflict!
'v' 00-0F media/v4l2-subdev.h conflict!
+'v' 20-27 arch/powerpc/include/uapi/asm/vas-api.h VAS API
'v' C0-FF linux/meye.h conflict!
'w' all CERN SCI driver
'y' 00-1F packet based user level communications
diff --git a/arch/powerpc/include/uapi/asm/vas-api.h b/arch/powerpc/include/uapi/asm/vas-api.h
new file mode 100644
index 0000000..fe95d67
--- /dev/null
+++ b/arch/powerpc/include/uapi/asm/vas-api.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */
+/*
+ * Copyright 2019 IBM Corp.
+ */
+
+#ifndef _UAPI_MISC_VAS_H
+#define _UAPI_MISC_VAS_H
+
+#include <asm/ioctl.h>
+
+#define VAS_MAGIC 'v'
+#define VAS_TX_WIN_OPEN _IOW(VAS_MAGIC, 0x20, struct vas_tx_win_open_attr)
+
+struct vas_tx_win_open_attr {
+ __u32 version;
+ __s16 vas_id; /* specific instance of vas or -1 for default */
+ __u16 reserved1;
+ __u64 flags; /* Future use */
+ __u64 reserved2[6];
+};
+
+#endif /* _UAPI_MISC_VAS_H */
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 1/9] powerpc/vas: Initialize window attributes for GZIP coprocessor type
From: Haren Myneni @ 2020-03-23 4:00 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
Initialize send and receive window attributes for GZIP high and
normal priority types.
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
arch/powerpc/platforms/powernv/vas-window.c | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/arch/powerpc/platforms/powernv/vas-window.c b/arch/powerpc/platforms/powernv/vas-window.c
index a1d3fb1..4ce421f 100644
--- a/arch/powerpc/platforms/powernv/vas-window.c
+++ b/arch/powerpc/platforms/powernv/vas-window.c
@@ -817,7 +817,8 @@ void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop)
{
memset(rxattr, 0, sizeof(*rxattr));
- if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) {
+ if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
+ cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
rxattr->pin_win = true;
rxattr->nx_win = true;
rxattr->fault_win = false;
@@ -892,7 +893,8 @@ void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, enum vas_cop_type cop)
{
memset(txattr, 0, sizeof(*txattr));
- if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) {
+ if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
+ cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
txattr->rej_no_credit = false;
txattr->rx_wcred_mode = true;
txattr->tx_wcred_mode = true;
@@ -976,9 +978,14 @@ static bool tx_win_args_valid(enum vas_cop_type cop,
if (attr->wcreds_max > VAS_TX_WCREDS_MAX)
return false;
- if (attr->user_win &&
- (cop != VAS_COP_TYPE_FTW || attr->rsvd_txbuf_count))
- return false;
+ if (attr->user_win) {
+ if (attr->rsvd_txbuf_count)
+ return false;
+
+ if (cop != VAS_COP_TYPE_FTW && cop != VAS_COP_TYPE_GZIP &&
+ cop != VAS_COP_TYPE_GZIP_HIPRI)
+ return false;
+ }
return true;
}
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 0/9] crypto/nx: Enable GZIP engine and provide userpace API
From: Haren Myneni @ 2020-03-23 3:41 UTC (permalink / raw)
To: herbert, mpe, dja; +Cc: mikey, sukadev, linuxppc-dev, linux-crypto, npiggin
Power9 processor supports Virtual Accelerator Switchboard (VAS) which
allows kernel and userspace to send compression requests to Nest
Accelerator (NX) directly. The NX unit comprises of 2 842 compression
engines and 1 GZIP engine. Linux kernel already has 842 compression
support on kernel. This patch series adds GZIP compression support
from user space. The GZIP Compression engine implements the ZLIB and
GZIP compression algorithms. No plans of adding NX-GZIP compression
support in kernel right now.
Applications can send requests to NX directly with COPY/PASTE
instructions. But kernel has to establish channel / window on NX-GZIP
device for the userspace. So userspace access to the GZIP engine is
provided through /dev/crypto/nx-gzip device with several operations.
An application must open the this device to obtain a file descriptor (fd).
Using the fd, application should issue the VAS_TX_WIN_OPEN ioctl to
establish a connection to the engine. Once window is opened, should use
mmap() system call to map the hardware address of engine's request queue
into the application's virtual address space. Then user space forms the
request as co-processor Request Block (CRB) and paste this CRB on the
mapped HW address using COPY/PASTE instructions. Application can poll
on status flags (part of CRB) with timeout for request completion.
For VAS_TX_WIN_OPEN ioctl, if user space passes vas_id = -1 (struct
vas_tx_win_open_attr), kernel determines the VAS instance on the
corresponding chip based on the CPU on which the process is executing.
Otherwise, the specified VAS instance is used if application passes the
proper VAS instance (vas_id listed in /proc/device-tree/vas@*/ibm,vas_id).
Process can open multiple windows with different FDs or can send several
requests to NX on the same window at the same time.
A userspace library libnxz is available:
https://github.com/abalib/power-gzip
Applications that use inflate/deflate calls can link with libNXz and use
NX GZIP compression without any modification.
Tested the available 842 compression on power8 and power9 system to make
sure no regression and tested GZIP compression on power9 with tests
available in the above link.
Thanks to Bulent Abali for nxz library and tests development.
Changelog:
V2:
- Move user space API code to powerpc as suggested. Also this API
can be extended to any other coprocessor type that VAS can support
in future. Example: Fast thread wakeup feature from VAS
- Rebased to 5.6-rc3
V3:
- Fix sparse warnings (patches 3&6)
V4:
- Remove unused coproc_instid and add only window address in
fp->private_data.
- Add NX User's manual and Copy/paste links in VAS API documentation
in patch and other changes as Daniel Axtens suggested
Haren Myneni (9):
powerpc/vas: Initialize window attributes for GZIP coprocessor type
powerpc/vas: Define VAS_TX_WIN_OPEN ioctl API
powerpc/vas: Add VAS user space API
crypto/nx: Initialize coproc entry with kzalloc
crypto/nx: Rename nx-842-powernv file name to nx-common-powernv
crypto/NX: Make enable code generic to add new GZIP compression type
crypto/nx: Enable and setup GZIP compresstion type
crypto/nx: Remove 'pid' in vas_tx_win_attr struct
Documentation/powerpc: VAS API
Documentation/powerpc/index.rst | 1 +
Documentation/powerpc/vas-api.rst | 249 +++++
Documentation/userspace-api/ioctl/ioctl-number.rst | 1 +
arch/powerpc/include/asm/vas.h | 12 +-
arch/powerpc/include/uapi/asm/vas-api.h | 22 +
arch/powerpc/platforms/powernv/Makefile | 2 +-
arch/powerpc/platforms/powernv/vas-api.c | 257 +++++
arch/powerpc/platforms/powernv/vas-window.c | 23 +-
arch/powerpc/platforms/powernv/vas.h | 2 +
drivers/crypto/nx/Makefile | 2 +-
drivers/crypto/nx/nx-842-powernv.c | 1062 ------------------
drivers/crypto/nx/nx-common-powernv.c | 1133 ++++++++++++++++++++
12 files changed, 1693 insertions(+), 1073 deletions(-)
create mode 100644 Documentation/powerpc/vas-api.rst
create mode 100644 arch/powerpc/include/uapi/asm/vas-api.h
create mode 100644 arch/powerpc/platforms/powernv/vas-api.c
delete mode 100644 drivers/crypto/nx/nx-842-powernv.c
create mode 100644 drivers/crypto/nx/nx-common-powernv.c
--
1.8.3.1
^ permalink raw reply
* Re: [patch V3 13/20] Documentation: Add lock ordering and nesting documentation
From: Paul E. McKenney @ 2020-03-23 2:55 UTC (permalink / raw)
To: Thomas Gleixner
Cc: linux-usb, linux-ia64, Peter Zijlstra, linux-pci,
Sebastian Siewior, Oleg Nesterov, Guo Ren, Joel Fernandes,
Vincent Chen, Ingo Molnar, Davidlohr Bueso, linux-acpi,
Brian Cain, Jonathan Corbet, linux-hexagon, Rafael J. Wysocki,
linux-csky, Linus Torvalds, Darren Hart, Zhang Rui, Len Brown,
Fenghua Yu, Arnd Bergmann, linux-pm, linuxppc-dev, Greentime Hu,
Bjorn Helgaas, Kurt Schwemmer, platform-driver-x86, Kalle Valo,
kbuild test robot, Felipe Balbi, Michal Simek, Tony Luck, Nick Hu,
Geoff Levand, netdev, Randy Dunlap, linux-wireless, LKML,
Davidlohr Bueso, Greg Kroah-Hartman, Logan Gunthorpe,
David S. Miller, Andy Shevchenko
In-Reply-To: <20200321113242.026561244@linutronix.de>
On Sat, Mar 21, 2020 at 12:25:57PM +0100, Thomas Gleixner wrote:
> From: Thomas Gleixner <tglx@linutronix.de>
>
> The kernel provides a variety of locking primitives. The nesting of these
> lock types and the implications of them on RT enabled kernels is nowhere
> documented.
>
> Add initial documentation.
>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> Cc: "Paul E . McKenney" <paulmck@kernel.org>
> Cc: Jonathan Corbet <corbet@lwn.net>
> Cc: Davidlohr Bueso <dave@stgolabs.net>
> Cc: Randy Dunlap <rdunlap@infradead.org>
> ---
> V3: Addressed review comments from Paul, Jonathan, Davidlohr
> V2: Addressed review comments from Randy
> ---
> Documentation/locking/index.rst | 1
> Documentation/locking/locktypes.rst | 299 ++++++++++++++++++++++++++++++++++++
> 2 files changed, 300 insertions(+)
> create mode 100644 Documentation/locking/locktypes.rst
>
> --- a/Documentation/locking/index.rst
> +++ b/Documentation/locking/index.rst
> @@ -7,6 +7,7 @@ locking
> .. toctree::
> :maxdepth: 1
>
> + locktypes
> lockdep-design
> lockstat
> locktorture
> --- /dev/null
> +++ b/Documentation/locking/locktypes.rst
> @@ -0,0 +1,299 @@
[ . . . Adding your example execution sequences . . . ]
> +PREEMPT_RT kernels preserve all other spinlock_t semantics:
> +
> + - Tasks holding a spinlock_t do not migrate. Non-PREEMPT_RT kernels
> + avoid migration by disabling preemption. PREEMPT_RT kernels instead
> + disable migration, which ensures that pointers to per-CPU variables
> + remain valid even if the task is preempted.
> +
> + - Task state is preserved across spinlock acquisition, ensuring that the
> + task-state rules apply to all kernel configurations. Non-PREEMPT_RT
> + kernels leave task state untouched. However, PREEMPT_RT must change
> + task state if the task blocks during acquisition. Therefore, it saves
> + the current task state before blocking and the corresponding lock wakeup
> + restores it.
> +
> + Other types of wakeups would normally unconditionally set the task state
> + to RUNNING, but that does not work here because the task must remain
> + blocked until the lock becomes available. Therefore, when a non-lock
> + wakeup attempts to awaken a task blocked waiting for a spinlock, it
> + instead sets the saved state to RUNNING. Then, when the lock
> + acquisition completes, the lock wakeup sets the task state to the saved
> + state, in this case setting it to RUNNING.
In the normal case where the task sleeps through the entire lock
acquisition, the sequence of events is as follows:
state = UNINTERRUPTIBLE
lock()
block()
real_state = state
state = SLEEPONLOCK
lock wakeup
state = real_state == UNINTERRUPTIBLE
This sequence of events can occur when the task acquires spinlocks
on its way to sleeping, for example, in a call to wait_event().
The non-lock wakeup can occur when a wakeup races with this wait_event(),
which can result in the following sequence of events:
state = UNINTERRUPTIBLE
lock()
block()
real_state = state
state = SLEEPONLOCK
non lock wakeup
real_state = RUNNING
lock wakeup
state = real_state == RUNNING
Without this real_state subterfuge, the wakeup might be lost.
[ . . . and continuing where I left off earlier . . . ]
> +bit spinlocks
> +-------------
> +
> +Bit spinlocks are problematic for PREEMPT_RT as they cannot be easily
> +substituted by an RT-mutex based implementation for obvious reasons.
> +
> +The semantics of bit spinlocks are preserved on PREEMPT_RT kernels and the
> +caveats vs. raw_spinlock_t apply.
> +
> +Some bit spinlocks are substituted by regular spinlock_t for PREEMPT_RT but
> +this requires conditional (#ifdef'ed) code changes at the usage site while
> +the spinlock_t substitution is simply done by the compiler and the
> +conditionals are restricted to header files and core implementation of the
> +locking primitives and the usage sites do not require any changes.
PREEMPT_RT cannot substitute bit spinlocks because a single bit is
too small to accommodate an RT-mutex. Therefore, the semantics of bit
spinlocks are preserved on PREEMPT_RT kernels, so that the raw_spinlock_t
caveats also apply to bit spinlocks.
Some bit spinlocks are replaced with regular spinlock_t for PREEMPT_RT
using conditional (#ifdef'ed) code changes at the usage site.
In contrast, usage-site changes are not needed for the spinlock_t
substitution. Instead, conditionals in header files and the core locking
implemementation enable the compiler to do the substitution transparently.
> +Lock type nesting rules
> +=======================
> +
> +The most basic rules are:
> +
> + - Lock types of the same lock category (sleeping, spinning) can nest
> + arbitrarily as long as they respect the general lock ordering rules to
> + prevent deadlocks.
- Lock types in the same category (sleeping, spinning) can nest
arbitrarily as long as they respect the general deadlock-avoidance
ordering rules.
[ Give or take lockdep eventually complaining about too-deep nesting,
but that is probably not worth mentioning here. Leave that caveat
to the lockdep documentation. ]
> + - Sleeping lock types cannot nest inside spinning lock types.
> +
> + - Spinning lock types can nest inside sleeping lock types.
> +
> +These rules apply in general independent of CONFIG_PREEMPT_RT.
These constraints apply both in CONFIG_PREEMPT_RT and otherwise.
> +As PREEMPT_RT changes the lock category of spinlock_t and rwlock_t from
> +spinning to sleeping this has obviously restrictions how they can nest with
> +raw_spinlock_t.
> +
> +This results in the following nest ordering:
The fact that PREEMPT_RT changes the lock category of spinlock_t and
rwlock_t from spinning to sleeping means that they cannot be acquired
while holding a raw spinlock. This results in the following nesting
ordering:
> + 1) Sleeping locks
> + 2) spinlock_t and rwlock_t
> + 3) raw_spinlock_t and bit spinlocks
> +
> +Lockdep is aware of these constraints to ensure that they are respected.
Lockdep will complain if these constraints are violated, both in
CONFIG_PREEMPT_RT and otherwise.
> +Owner semantics
> +===============
> +
> +Most lock types in the Linux kernel have strict owner semantics, i.e. the
> +context (task) which acquires a lock has to release it.
The aforementioned lock types have strict owner semantics: The context
(task) that acquired the lock must release it.
> +There are two exceptions:
> +
> + - semaphores
> + - rwsems
> +
> +semaphores have no owner semantics for historical reason, and as such
> +trylock and release operations can be called from any context. They are
> +often used for both serialization and waiting purposes. That's generally
> +discouraged and should be replaced by separate serialization and wait
> +mechanisms, such as mutexes and completions.
semaphores lack owner semantics for historical reasons, so their trylock
and release operations may be called from any context. They are often
used for both serialization and waiting, but new use cases should
instead use separate serialization and wait mechanisms, such as mutexes
and completions.
> +rwsems have grown interfaces which allow non owner release for special
> +purposes. This usage is problematic on PREEMPT_RT because PREEMPT_RT
> +substitutes all locking primitives except semaphores with RT-mutex based
> +implementations to provide priority inheritance for all lock types except
> +the truly spinning ones. Priority inheritance on ownerless locks is
> +obviously impossible.
> +
> +For now the rwsem non-owner release excludes code which utilizes it from
> +being used on PREEMPT_RT enabled kernels. In same cases this can be
> +mitigated by disabling portions of the code, in other cases the complete
> +functionality has to be disabled until a workable solution has been found.
rwsems have grown special-purpose interfaces that allow non-owner release.
This non-owner release prevents PREEMPT_RT from substituting RT-mutex
implementations, for example, by defeating priority inheritance.
After all, if the lock has no owner, whose priority should be boosted?
As a result, PREEMPT_RT does not currently support rwsem, which in turn
means that code using it must therefore be disabled until a workable
solution presents itself.
[ Note: Not as confident as I would like to be in the above. ]
Thanx, Paul
^ permalink raw reply
* Re: [PATCH v8 12/14] powerpc/vas: Return credits after handling fault
From: Nicholas Piggin @ 2020-03-23 2:44 UTC (permalink / raw)
To: Haren Myneni, mpe; +Cc: mikey, ajd, hch, oohall, sukadev, linuxppc-dev, herbert
In-Reply-To: <1584598755.9256.15264.camel@hbabu-laptop>
Haren Myneni's on March 19, 2020 4:19 pm:
>
> NX expects OS to return credit for send window after processing each
> fault. Also credit has to be returned even for fault window.
And this should be merged in the fault handler function.
>
> Signed-off-by: Sukadev Bhattiprolu <sukadev@linux.vnet.ibm.com>
> Signed-off-by: Haren Myneni <haren@linux.ibm.com>
> ---
> arch/powerpc/platforms/powernv/vas-fault.c | 9 +++++++++
> arch/powerpc/platforms/powernv/vas-window.c | 17 +++++++++++++++++
> arch/powerpc/platforms/powernv/vas.h | 1 +
> 3 files changed, 27 insertions(+)
>
> diff --git a/arch/powerpc/platforms/powernv/vas-fault.c b/arch/powerpc/platforms/powernv/vas-fault.c
> index 40e1de4..292f7ba 100644
> --- a/arch/powerpc/platforms/powernv/vas-fault.c
> +++ b/arch/powerpc/platforms/powernv/vas-fault.c
> @@ -238,6 +238,10 @@ irqreturn_t vas_fault_thread_fn(int irq, void *data)
> memcpy(crb, fifo, CRB_SIZE);
> entry->stamp.nx.pswid = cpu_to_be32(FIFO_INVALID_ENTRY);
> entry->ccw |= cpu_to_be32(CCW0_INVALID);
> + /*
> + * Return credit for the fault window.
> + */
None of the comments in this patch are useful.
> + vas_return_credit(vinst->fault_win, 0);
Can you use true/false for bools?
> mutex_unlock(&vinst->mutex);
>
> pr_devel("VAS[%d] fault_fifo %p, fifo %p, fault_crbs %d\n",
> @@ -267,6 +271,11 @@ irqreturn_t vas_fault_thread_fn(int irq, void *data)
> }
>
> update_csb(window, crb);
> + /*
> + * Return credit for send window after processing
> + * fault CRB.
> + */
Any chance of a little bit of explanation how the credit system works?
Or is it in the code somewhere already?
I don't suppose there is a chance to batch credit updates with multiple
faults? (maybe the MMIO is insignificant)
> + vas_return_credit(window, 1);
> } while (true);
> }
>
Thanks,
Nick
^ permalink raw reply
* Re: [PATCH v8 11/14] powerpc/vas: Do not use default credits for receive window
From: Nicholas Piggin @ 2020-03-23 2:40 UTC (permalink / raw)
To: Haren Myneni, mpe; +Cc: mikey, ajd, hch, oohall, sukadev, linuxppc-dev, herbert
In-Reply-To: <1584598722.9256.15262.camel@hbabu-laptop>
Haren Myneni's on March 19, 2020 4:18 pm:
>
> System checkstops if RxFIFO overruns with more requests than the
> maximum possible number of CRBs allowed in FIFO at any time. So
> max credits value (rxattr.wcreds_max) is set and is passed to
> vas_rx_win_open() by the the driver.
This seems like it should be a bug fix or merged in the NX fault
window register patch or something.
Thanks,
Nick
^ permalink raw reply
* Re: [PATCH v8 09/14] powerpc/vas: Update CSB and notify process for fault CRBs
From: Nicholas Piggin @ 2020-03-23 2:37 UTC (permalink / raw)
To: Haren Myneni, mpe; +Cc: mikey, ajd, hch, oohall, sukadev, linuxppc-dev, herbert
In-Reply-To: <1584598648.9256.15258.camel@hbabu-laptop>
Haren Myneni's on March 19, 2020 4:17 pm:
>
> For each fault CRB, update fault address in CRB (fault_storage_addr)
> and translation error status in CSB so that user space can touch the
> fault address and resend the request. If the user space passed invalid
> CSB address send signal to process with SIGSEGV.
This is where the actual fault handling is done? Does this need to be
split from the other patch? Why not merge them and put it after the
reference counting one?
I'll wait until comments and questions on the first fault handling patch
are resolved before I look at this one.
Thanks,
Nick
>
> Signed-off-by: Sukadev Bhattiprolu <sukadev@linux.vnet.ibm.com>
> Signed-off-by: Haren Myneni <haren@linux.ibm.com>
> ---
> arch/powerpc/platforms/powernv/vas-fault.c | 115 +++++++++++++++++++++++++++++
> 1 file changed, 115 insertions(+)
>
> diff --git a/arch/powerpc/platforms/powernv/vas-fault.c b/arch/powerpc/platforms/powernv/vas-fault.c
> index 1c6d5cc..6eceac5d 100644
> --- a/arch/powerpc/platforms/powernv/vas-fault.c
> +++ b/arch/powerpc/platforms/powernv/vas-fault.c
> @@ -11,6 +11,7 @@
> #include <linux/slab.h>
> #include <linux/uaccess.h>
> #include <linux/kthread.h>
> +#include <linux/sched/signal.h>
> #include <linux/mmu_context.h>
> #include <asm/icswx.h>
>
> @@ -26,6 +27,119 @@
> #define VAS_FAULT_WIN_FIFO_SIZE (4 << 20)
>
> /*
> + * Update the CSB to indicate a translation error.
> + *
> + * If we are unable to update the CSB means copy_to_user failed due to
> + * invalid csb_addr, send a signal to the process.
> + *
> + * Remaining settings in the CSB are based on wait_for_csb() of
> + * NX-GZIP.
> + */
> +static void update_csb(struct vas_window *window,
> + struct coprocessor_request_block *crb)
> +{
> + struct coprocessor_status_block csb;
> + struct kernel_siginfo info;
> + struct task_struct *tsk;
> + void __user *csb_addr;
> + struct pid *pid;
> + int rc;
> +
> + /*
> + * NX user space windows can not be opened for task->mm=NULL
> + * and faults will not be generated for kernel requests.
> + */
> + if (WARN_ON_ONCE(!window->mm || !window->user_win))
> + return;
> +
> + csb_addr = (void __user *)be64_to_cpu(crb->csb_addr);
> +
> + memset(&csb, 0, sizeof(csb));
> + csb.cc = CSB_CC_TRANSLATION;
> + csb.ce = CSB_CE_TERMINATION;
> + csb.cs = 0;
> + csb.count = 0;
> +
> + /*
> + * NX operates and returns in BE format as defined CRB struct.
> + * So return fault_storage_addr in BE as NX pastes in FIFO and
> + * expects user space to convert to CPU format.
> + */
> + csb.address = crb->stamp.nx.fault_storage_addr;
> + csb.flags = 0;
> +
> + pid = window->pid;
> + tsk = get_pid_task(pid, PIDTYPE_PID);
> + /*
> + * Send window will be closed after processing all NX requests
> + * and process exits after closing all windows. In multi-thread
> + * applications, thread may not exists, but does not close FD
> + * (means send window) upon exit. Parent thread (tgid) can use
> + * and close the window later.
> + * pid and mm references are taken when window is opened by
> + * process (pid). So tgid is used only when child thread opens
> + * a window and exits without closing it in multithread tasks.
> + */
> + if (!tsk) {
> + pid = window->tgid;
> + tsk = get_pid_task(pid, PIDTYPE_PID);
> + /*
> + * Parent thread will be closing window during its exit.
> + * So should not get here.
> + */
> + if (WARN_ON_ONCE(!tsk))
> + return;
> + }
> +
> + /* Return if the task is exiting. */
> + if (tsk->flags & PF_EXITING) {
> + put_task_struct(tsk);
> + return;
> + }
> +
> + use_mm(window->mm);
> + rc = copy_to_user(csb_addr, &csb, sizeof(csb));
> + /*
> + * User space polls on csb.flags (first byte). So add barrier
> + * then copy first byte with csb flags update.
> + */
> + if (!rc) {
> + csb.flags = CSB_V;
> + smp_mb();
> + rc = copy_to_user(csb_addr, &csb, sizeof(u8));
> + }
> + unuse_mm(window->mm);
> + put_task_struct(tsk);
> +
> + /* Success */
> + if (!rc)
> + return;
> +
> + pr_debug("Invalid CSB address 0x%p signalling pid(%d)\n",
> + csb_addr, pid_vnr(pid));
> +
> + clear_siginfo(&info);
> + info.si_signo = SIGSEGV;
> + info.si_errno = EFAULT;
> + info.si_code = SEGV_MAPERR;
> + info.si_addr = csb_addr;
> +
> + /*
> + * process will be polling on csb.flags after request is sent to
> + * NX. So generally CSB update should not fail except when an
> + * application does not follow the process properly. So an error
> + * message will be displayed and leave it to user space whether
> + * to ignore or handle this signal.
> + */
> + rcu_read_lock();
> + rc = kill_pid_info(SIGSEGV, &info, pid);
> + rcu_read_unlock();
> +
> + pr_devel("%s(): pid %d kill_proc_info() rc %d\n", __func__,
> + pid_vnr(pid), rc);
> +}
> +
> +/*
> * Process valid CRBs in fault FIFO.
> */
> irqreturn_t vas_fault_thread_fn(int irq, void *data)
> @@ -111,6 +225,7 @@ irqreturn_t vas_fault_thread_fn(int irq, void *data)
> return IRQ_HANDLED;
> }
>
> + update_csb(window, crb);
> } while (true);
> }
>
> --
> 1.8.3.1
>
>
>
>
^ permalink raw reply
* Re: [PATCH v8 08/14] powerpc/vas: Take reference to PID and mm for user space windows
From: Nicholas Piggin @ 2020-03-23 2:34 UTC (permalink / raw)
To: Haren Myneni, mpe; +Cc: mikey, ajd, hch, oohall, sukadev, linuxppc-dev, herbert
In-Reply-To: <1584598613.9256.15257.camel@hbabu-laptop>
Haren Myneni's on March 19, 2020 4:16 pm:
>
> When process opens a window, its pid and tgid will be saved in vas_window
> struct. This window will be closed when the process exits. Kernel handles
> NX faults by updating CSB or send SEGV signal to pid if user space csb_addr
> is invalid.
Bit of a nitpick, but can you use articles consistently ("the", "a")? I
won't keep nitpicking changelogs but I think they could be made easier
to read. I'm happy to help proof read and suggest things offline when
you're happy with the technical content of them, let me know.
>
> In multi-thread applications, a window can be opened by child thread, but
> it will not be closed when this thread exits. Expects parent to clean up
> all resources including NX windows. Child thread can send requests using
> this window and can be killed before they are completed. But the pid
> assigned to this thread can be reused for other task while requests are
> pending. If the csb_addr passed in these requests is invalid, kernel will
> end up sending signal to the wrong task.
>
> To prevent reusing the pid, take references to pid and mm when the window
> is opened and release them during window close.
We went over this together a while back, but task management isn't
something I look at every day and it's complicated and easy to introduce
bugs. I suggest if we can get the changelog and comments written well
and understandable for someone who does not know or care about vas,
then cc linux-kernel and the maintainers, and hopefully someone will
take a look. It's not a large patch so if assumptions and concurrency
etc is documented, then it shouldn't be too much work.
Thanks,
Nick
>
> Signed-off-by: Haren Myneni <haren@linux.ibm.com>
> ---
> arch/powerpc/platforms/powernv/vas-debug.c | 2 +-
> arch/powerpc/platforms/powernv/vas-window.c | 53 ++++++++++++++++++++++++++---
> arch/powerpc/platforms/powernv/vas.h | 9 ++++-
> 3 files changed, 57 insertions(+), 7 deletions(-)
>
> diff --git a/arch/powerpc/platforms/powernv/vas-debug.c b/arch/powerpc/platforms/powernv/vas-debug.c
> index 09e63df..ef9a717 100644
> --- a/arch/powerpc/platforms/powernv/vas-debug.c
> +++ b/arch/powerpc/platforms/powernv/vas-debug.c
> @@ -38,7 +38,7 @@ static int info_show(struct seq_file *s, void *private)
>
> seq_printf(s, "Type: %s, %s\n", cop_to_str(window->cop),
> window->tx_win ? "Send" : "Receive");
> - seq_printf(s, "Pid : %d\n", window->pid);
> + seq_printf(s, "Pid : %d\n", vas_window_pid(window));
>
> unlock:
> mutex_unlock(&vas_mutex);
> diff --git a/arch/powerpc/platforms/powernv/vas-window.c b/arch/powerpc/platforms/powernv/vas-window.c
> index acb6a22..e7641a5 100644
> --- a/arch/powerpc/platforms/powernv/vas-window.c
> +++ b/arch/powerpc/platforms/powernv/vas-window.c
> @@ -12,6 +12,8 @@
> #include <linux/log2.h>
> #include <linux/rcupdate.h>
> #include <linux/cred.h>
> +#include <linux/sched/mm.h>
> +#include <linux/mmu_context.h>
> #include <asm/switch_to.h>
> #include <asm/ppc-opcode.h>
> #include "vas.h"
> @@ -876,8 +878,6 @@ struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop,
> rxwin->user_win = rxattr->user_win;
> rxwin->cop = cop;
> rxwin->wcreds_max = rxattr->wcreds_max ?: VAS_WCREDS_DEFAULT;
> - if (rxattr->user_win)
> - rxwin->pid = task_pid_vnr(current);
>
> init_winctx_for_rxwin(rxwin, rxattr, &winctx);
> init_winctx_regs(rxwin, &winctx);
> @@ -1027,7 +1027,6 @@ struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
> txwin->tx_win = 1;
> txwin->rxwin = rxwin;
> txwin->nx_win = txwin->rxwin->nx_win;
> - txwin->pid = attr->pid;
> txwin->user_win = attr->user_win;
> txwin->wcreds_max = attr->wcreds_max ?: VAS_WCREDS_DEFAULT;
>
> @@ -1068,8 +1067,43 @@ struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
> goto free_window;
> }
>
> - set_vinst_win(vinst, txwin);
> + if (txwin->user_win) {
> + /*
> + * Window opened by child thread may not be closed when
> + * it exits. So take reference to its pid and release it
> + * when the window is free by parent thread.
> + * Acquire a reference to the task's pid to make sure
> + * pid will not be re-used - needed only for multithread
> + * applications.
> + */
> + txwin->pid = get_task_pid(current, PIDTYPE_PID);
> + /*
> + * Acquire a reference to the task's mm.
> + */
> + txwin->mm = get_task_mm(current);
>
> + if (!txwin->mm) {
> + put_pid(txwin->pid);
> + pr_err("VAS: pid(%d): mm_struct is not found\n",
> + current->pid);
> + rc = -EPERM;
> + goto free_window;
> + }
> +
> + mmgrab(txwin->mm);
> + mmput(txwin->mm);
> + mm_context_add_copro(txwin->mm);
> + /*
> + * Process closes window during exit. In the case of
> + * multithread application, child can open window and
> + * can exit without closing it. Expects parent thread
> + * to use and close the window. So do not need to take
> + * pid reference for parent thread.
> + */
> + txwin->tgid = find_get_pid(task_tgid_vnr(current));
> + }
> +
> + set_vinst_win(vinst, txwin);
> return txwin;
>
> free_window:
> @@ -1266,8 +1300,17 @@ int vas_win_close(struct vas_window *window)
> poll_window_castout(window);
>
> /* if send window, drop reference to matching receive window */
> - if (window->tx_win)
> + if (window->tx_win) {
> + if (window->user_win) {
> + /* Drop references to pid and mm */
> + put_pid(window->pid);
> + if (window->mm) {
> + mm_context_remove_copro(window->mm);
> + mmdrop(window->mm);
> + }
> + }
> put_rx_win(window->rxwin);
> + }
>
> vas_window_free(window);
>
> diff --git a/arch/powerpc/platforms/powernv/vas.h b/arch/powerpc/platforms/powernv/vas.h
> index 310b8a0..16aa8ec 100644
> --- a/arch/powerpc/platforms/powernv/vas.h
> +++ b/arch/powerpc/platforms/powernv/vas.h
> @@ -353,7 +353,9 @@ struct vas_window {
> bool user_win; /* True if user space window */
> void *hvwc_map; /* HV window context */
> void *uwc_map; /* OS/User window context */
> - pid_t pid; /* Linux process id of owner */
> + struct pid *pid; /* Linux process id of owner */
> + struct pid *tgid; /* Thread group ID of owner */
> + struct mm_struct *mm; /* Linux process mm_struct */
> int wcreds_max; /* Window credits */
>
> char *dbgname;
> @@ -431,6 +433,11 @@ struct vas_winctx {
> extern struct vas_window *vas_pswid_to_window(struct vas_instance *vinst,
> uint32_t pswid);
>
> +static inline int vas_window_pid(struct vas_window *window)
> +{
> + return pid_vnr(window->pid);
> +}
> +
> static inline void vas_log_write(struct vas_window *win, char *name,
> void *regptr, u64 val)
> {
> --
> 1.8.3.1
>
>
>
>
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox