* Re: [PATCH v4 02/16] xmon: Move out-of-line instructions to text section
From: Balamuruhan S @ 2020-03-23 6:05 UTC (permalink / raw)
To: Jordan Niethe, linuxppc-dev; +Cc: alistair, dja, npiggin
In-Reply-To: <20200320051809.24332-3-jniethe5@gmail.com>
On Fri, 2020-03-20 at 16:17 +1100, Jordan Niethe wrote:
> To execute an instruction out of line after a breakpoint, the NIP is
> set
> to the address of struct bpt::instr. Here a copy of the instruction
> that
> was replaced with a breakpoint is kept, along with a trap so normal
> flow
> can be resumed after XOLing. The struct bpt's are located within the
> data section. This is problematic as the data section may be marked
> as
> no execute.
>
> Instead of each struct bpt holding the instructions to be XOL'd, make
> a
> new array, bpt_table[], with enough space to hold instructions for
> the
> number of supported breakpoints. Place this array in the text
> section.
> Make struct bpt::instr a pointer to the instructions in bpt_table[]
> associated with that breakpoint. This association is a simple
> mapping:
> bpts[n] -> bpt_table[n * words per breakpoint].
Can we have it in separate commits ?
* introduce the array bpt_table[] and make struct bpt::instr a
pointer to the instructions in bpt_table[].
* place the array in text section.
> Currently we only need
> the copied instruction followed by a trap, so 2 words per breakpoint.
>
> Signed-off-by: Jordan Niethe <jniethe5@gmail.com>
> ---
> v4: New to series
> ---
> arch/powerpc/kernel/vmlinux.lds.S | 2 +-
> arch/powerpc/xmon/xmon.c | 22 +++++++++++++---------
> 2 files changed, 14 insertions(+), 10 deletions(-)
>
> diff --git a/arch/powerpc/kernel/vmlinux.lds.S
> b/arch/powerpc/kernel/vmlinux.lds.S
> index b4c89a1acebb..e90845b8c300 100644
> --- a/arch/powerpc/kernel/vmlinux.lds.S
> +++ b/arch/powerpc/kernel/vmlinux.lds.S
> @@ -86,7 +86,7 @@ SECTIONS
> ALIGN_FUNCTION();
> #endif
> /* careful! __ftr_alt_* sections need to be close to
> .text */
> - *(.text.hot TEXT_MAIN .text.fixup .text.unlikely .fixup
> __ftr_alt_* .ref.text);
> + *(.text.hot TEXT_MAIN .text.fixup .text.unlikely .fixup
> __ftr_alt_* .ref.text .text.xmon_bpts);
> #ifdef CONFIG_PPC64
> *(.tramp.ftrace.text);
> #endif
> diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
> index 02e3bd62cab4..7875d1a37770 100644
> --- a/arch/powerpc/xmon/xmon.c
> +++ b/arch/powerpc/xmon/xmon.c
> @@ -97,7 +97,7 @@ static long *xmon_fault_jmp[NR_CPUS];
> /* Breakpoint stuff */
> struct bpt {
> unsigned long address;
> - unsigned int instr[2];
> + unsigned int *instr;
> atomic_t ref_count;
> int enabled;
> unsigned long pad;
> @@ -109,6 +109,7 @@ struct bpt {
> #define BP_DABR 4
>
> #define NBPTS 256
> +#define BPT_WORDS 2
> static struct bpt bpts[NBPTS];
> static struct bpt dabr;
> static struct bpt *iabr;
> @@ -116,6 +117,8 @@ static unsigned bpinstr = 0x7fe00008; /* trap
> */
>
> #define BP_NUM(bp) ((bp) - bpts + 1)
>
> +static unsigned int __section(.text.xmon_bpts) bpt_table[NBPTS *
> BPT_WORDS];
> +
> /* Prototypes */
> static int cmds(struct pt_regs *);
> static int mread(unsigned long, void *, int);
> @@ -852,16 +855,16 @@ static struct bpt *at_breakpoint(unsigned long
> pc)
> static struct bpt *in_breakpoint_table(unsigned long nip, unsigned
> long *offp)
> {
> unsigned long off;
> + unsigned long bp_off;
>
> - off = nip - (unsigned long) bpts;
> - if (off >= sizeof(bpts))
> + off = nip - (unsigned long) bpt_table;
> + if (off >= sizeof(bpt_table))
> return NULL;
> - off %= sizeof(struct bpt);
> - if (off != offsetof(struct bpt, instr[0])
> - && off != offsetof(struct bpt, instr[1]))
> + bp_off = off % (sizeof(unsigned int) * BPT_WORDS);
> + if (bp_off != 0 && bp_off != 4)
> return NULL;
> - *offp = off - offsetof(struct bpt, instr[0]);
> - return (struct bpt *) (nip - off);
> + *offp = bp_off;
> + return bpts + ((off - bp_off) / (sizeof(unsigned int) *
> BPT_WORDS));
`(off - bp_off) / (sizeof(unsigned int) * BPT_WORDS)` seems to be the
actual breakpoint offset. Can we have something like,
#define NBPTS 256
#define BPT_WORDS 2
#define BPT_WORDS_SIZE (sizeof(unsigned int) * BPT_WORDS)
#define BPT_OFFSET(off, bp_word_off) ((off - bp_word_off) / \ BPT_WORDS_SIZE)
;
:::
:::
:::
bp_word_off = off % BPT_WORDS_SIZE;
if (bp_word_off != 0 && bp_word_off != 4)
return NULL;
*offp = bp_word_off;
return bpts + BPT_OFFSET(off, bp_word_off);
-- Bala
^ permalink raw reply
* [PATCH 1/2] Fix "[v3, 25/32] powerpc/64: system call implement entry/exit logic in C"
From: Nicholas Piggin @ 2020-03-23 5:44 UTC (permalink / raw)
To: linuxppc-dev; +Cc: Nicholas Piggin
Changes Christophe asked for to help ppc32 implementation, and move a
stray hunk into a later patch.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/asm-prototypes.h | 4 ---
arch/powerpc/include/asm/cputime.h | 38 +++++++++++++----------
2 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h
index 4b3609554e76..ab59a4904254 100644
--- a/arch/powerpc/include/asm/asm-prototypes.h
+++ b/arch/powerpc/include/asm/asm-prototypes.h
@@ -97,12 +97,8 @@ ppc_select(int n, fd_set __user *inp, fd_set __user *outp, fd_set __user *exp,
unsigned long __init early_init(unsigned long dt_ptr);
void __init machine_init(u64 dt_ptr);
#endif
-#ifdef CONFIG_PPC64
long system_call_exception(long r3, long r4, long r5, long r6, long r7, long r8, unsigned long r0, struct pt_regs *regs);
notrace unsigned long syscall_exit_prepare(unsigned long r3, struct pt_regs *regs);
-notrace unsigned long interrupt_exit_user_prepare(struct pt_regs *regs, unsigned long msr);
-notrace unsigned long interrupt_exit_kernel_prepare(struct pt_regs *regs, unsigned long msr);
-#endif
long ppc_fadvise64_64(int fd, int advice, u32 offset_high, u32 offset_low,
u32 len_high, u32 len_low);
diff --git a/arch/powerpc/include/asm/cputime.h b/arch/powerpc/include/asm/cputime.h
index 6639a6847cc0..0fccd5ea1e9a 100644
--- a/arch/powerpc/include/asm/cputime.h
+++ b/arch/powerpc/include/asm/cputime.h
@@ -43,8 +43,26 @@ static inline unsigned long cputime_to_usecs(const cputime_t ct)
*/
#ifdef CONFIG_PPC64
#define get_accounting(tsk) (&get_paca()->accounting)
+#define raw_get_accounting(tsk) (&local_paca->accounting)
static inline void arch_vtime_task_switch(struct task_struct *tsk) { }
+#else
+#define get_accounting(tsk) (&task_thread_info(tsk)->accounting)
+#define raw_get_accounting(tsk) get_accounting(tsk)
+/*
+ * Called from the context switch with interrupts disabled, to charge all
+ * accumulated times to the current process, and to prepare accounting on
+ * the next process.
+ */
+static inline void arch_vtime_task_switch(struct task_struct *prev)
+{
+ struct cpu_accounting_data *acct = get_accounting(current);
+ struct cpu_accounting_data *acct0 = get_accounting(prev);
+
+ acct->starttime = acct0->starttime;
+}
+#endif
+
/*
* account_cpu_user_entry/exit runs "unreconciled", so can't trace,
* can't use use get_paca()
@@ -52,35 +70,21 @@ static inline void arch_vtime_task_switch(struct task_struct *tsk) { }
static notrace inline void account_cpu_user_entry(void)
{
unsigned long tb = mftb();
- struct cpu_accounting_data *acct = &local_paca->accounting;
+ struct cpu_accounting_data *acct = raw_get_accounting(current);
acct->utime += (tb - acct->starttime_user);
acct->starttime = tb;
}
+
static notrace inline void account_cpu_user_exit(void)
{
unsigned long tb = mftb();
- struct cpu_accounting_data *acct = &local_paca->accounting;
+ struct cpu_accounting_data *acct = raw_get_accounting(current);
acct->stime += (tb - acct->starttime);
acct->starttime_user = tb;
}
-#else
-#define get_accounting(tsk) (&task_thread_info(tsk)->accounting)
-/*
- * Called from the context switch with interrupts disabled, to charge all
- * accumulated times to the current process, and to prepare accounting on
- * the next process.
- */
-static inline void arch_vtime_task_switch(struct task_struct *prev)
-{
- struct cpu_accounting_data *acct = get_accounting(current);
- struct cpu_accounting_data *acct0 = get_accounting(prev);
-
- acct->starttime = acct0->starttime;
-}
-#endif
#endif /* __KERNEL__ */
#else /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
--
2.23.0
^ permalink raw reply related
* [RFC PATCH 1/3] powerpc/mm: Introduce temporary mm
From: Christopher M. Riedl @ 2020-03-23 4:52 UTC (permalink / raw)
To: linuxppc-dev
In-Reply-To: <20200323045205.20314-1-cmr@informatik.wtf>
x86 supports the notion of a temporary mm which restricts access to
temporary PTEs to a single CPU. A temporary mm is useful for situations
where a CPU needs to perform sensitive operations (such as patching a
STRICT_KERNEL_RWX kernel) requiring temporary mappings without exposing
said mappings to other CPUs. A side benefit is that other CPU TLBs do
not need to be flushed when the temporary mm is torn down.
Mappings in the temporary mm can be set in the userspace portion of the
address-space.
Interrupts must be disabled while the temporary mm is in use. HW
breakpoints, which may have been set by userspace as watchpoints on
addresses now within the temporary mm, are saved and disabled when
loading the temporary mm. The HW breakpoints are restored when unloading
the temporary mm. All HW breakpoints are indiscriminately disabled while
the temporary mm is in use.
Based on x86 implementation:
commit cefa929c034e
("x86/mm: Introduce temporary mm structs")
Signed-off-by: Christopher M. Riedl <cmr@informatik.wtf>
---
arch/powerpc/include/asm/debug.h | 1 +
arch/powerpc/include/asm/mmu_context.h | 56 +++++++++++++++++++++++++-
arch/powerpc/kernel/process.c | 5 +++
3 files changed, 61 insertions(+), 1 deletion(-)
diff --git a/arch/powerpc/include/asm/debug.h b/arch/powerpc/include/asm/debug.h
index 7756026b95ca..b945bc16c932 100644
--- a/arch/powerpc/include/asm/debug.h
+++ b/arch/powerpc/include/asm/debug.h
@@ -45,6 +45,7 @@ static inline int debugger_break_match(struct pt_regs *regs) { return 0; }
static inline int debugger_fault_handler(struct pt_regs *regs) { return 0; }
#endif
+void __get_breakpoint(struct arch_hw_breakpoint *brk);
void __set_breakpoint(struct arch_hw_breakpoint *brk);
bool ppc_breakpoint_available(void);
#ifdef CONFIG_PPC_ADV_DEBUG_REGS
diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h
index 360367c579de..3e6381d04c28 100644
--- a/arch/powerpc/include/asm/mmu_context.h
+++ b/arch/powerpc/include/asm/mmu_context.h
@@ -7,9 +7,10 @@
#include <linux/mm.h>
#include <linux/sched.h>
#include <linux/spinlock.h>
-#include <asm/mmu.h>
+#include <asm/mmu.h>
#include <asm/cputable.h>
#include <asm/cputhreads.h>
+#include <asm/hw_breakpoint.h>
/*
* Most if the context management is out of line
@@ -270,5 +271,58 @@ static inline int arch_dup_mmap(struct mm_struct *oldmm,
return 0;
}
+struct temp_mm {
+ struct mm_struct *temp;
+ struct mm_struct *prev;
+ bool is_kernel_thread;
+ struct arch_hw_breakpoint brk;
+};
+
+static inline void init_temp_mm(struct temp_mm *temp_mm, struct mm_struct *mm)
+{
+ temp_mm->temp = mm;
+ temp_mm->prev = NULL;
+ temp_mm->is_kernel_thread = false;
+ memset(&temp_mm->brk, 0, sizeof(temp_mm->brk));
+}
+
+static inline void use_temporary_mm(struct temp_mm *temp_mm)
+{
+ lockdep_assert_irqs_disabled();
+
+ temp_mm->is_kernel_thread = current->mm == NULL;
+ if (temp_mm->is_kernel_thread)
+ temp_mm->prev = current->active_mm;
+ else
+ temp_mm->prev = current->mm;
+
+ /*
+ * Hash requires a non-NULL current->mm to allocate a userspace address
+ * when handling a page fault. Does not appear to hurt in Radix either.
+ */
+ current->mm = temp_mm->temp;
+ switch_mm_irqs_off(NULL, temp_mm->temp, current);
+
+ if (ppc_breakpoint_available()) {
+ __get_breakpoint(&temp_mm->brk);
+ if (temp_mm->brk.type != 0)
+ hw_breakpoint_disable();
+ }
+}
+
+static inline void unuse_temporary_mm(struct temp_mm *temp_mm)
+{
+ lockdep_assert_irqs_disabled();
+
+ if (temp_mm->is_kernel_thread)
+ current->mm = NULL;
+ else
+ current->mm = temp_mm->prev;
+ switch_mm_irqs_off(NULL, temp_mm->prev, current);
+
+ if (ppc_breakpoint_available() && temp_mm->brk.type != 0)
+ __set_breakpoint(&temp_mm->brk);
+}
+
#endif /* __KERNEL__ */
#endif /* __ASM_POWERPC_MMU_CONTEXT_H */
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index fad50db9dcf2..5e5cf33fc358 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -793,6 +793,11 @@ static inline int set_breakpoint_8xx(struct arch_hw_breakpoint *brk)
return 0;
}
+void __get_breakpoint(struct arch_hw_breakpoint *brk)
+{
+ memcpy(brk, this_cpu_ptr(¤t_brk), sizeof(*brk));
+}
+
void __set_breakpoint(struct arch_hw_breakpoint *brk)
{
memcpy(this_cpu_ptr(¤t_brk), brk, sizeof(*brk));
--
2.25.1
^ permalink raw reply related
* [RFC PATCH 3/3] powerpc/lib: Use a temporary mm for code patching
From: Christopher M. Riedl @ 2020-03-23 4:52 UTC (permalink / raw)
To: linuxppc-dev
In-Reply-To: <20200323045205.20314-1-cmr@informatik.wtf>
Currently, code patching a STRICT_KERNEL_RWX exposes the temporary
mappings to other CPUs. These mappings should be kept local to the CPU
doing the patching. Use the pre-initialized temporary mm and patching
address for this purpose. Also add a check after patching to ensure the
patch succeeded.
Based on x86 implementation:
commit b3fd8e83ada0
("x86/alternatives: Use temporary mm for text poking")
Signed-off-by: Christopher M. Riedl <cmr@informatik.wtf>
---
arch/powerpc/lib/code-patching.c | 128 ++++++++++++++-----------------
1 file changed, 57 insertions(+), 71 deletions(-)
diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c
index 18b88ecfc5a8..f156132e8975 100644
--- a/arch/powerpc/lib/code-patching.c
+++ b/arch/powerpc/lib/code-patching.c
@@ -19,6 +19,7 @@
#include <asm/page.h>
#include <asm/code-patching.h>
#include <asm/setup.h>
+#include <asm/mmu_context.h>
static int __patch_instruction(unsigned int *exec_addr, unsigned int instr,
unsigned int *patch_addr)
@@ -65,99 +66,79 @@ void __init poking_init(void)
pte_unmap_unlock(ptep, ptl);
}
-static DEFINE_PER_CPU(struct vm_struct *, text_poke_area);
-
-static int text_area_cpu_up(unsigned int cpu)
-{
- struct vm_struct *area;
-
- area = get_vm_area(PAGE_SIZE, VM_ALLOC);
- if (!area) {
- WARN_ONCE(1, "Failed to create text area for cpu %d\n",
- cpu);
- return -1;
- }
- this_cpu_write(text_poke_area, area);
-
- return 0;
-}
-
-static int text_area_cpu_down(unsigned int cpu)
-{
- free_vm_area(this_cpu_read(text_poke_area));
- return 0;
-}
-
-/*
- * Run as a late init call. This allows all the boot time patching to be done
- * simply by patching the code, and then we're called here prior to
- * mark_rodata_ro(), which happens after all init calls are run. Although
- * BUG_ON() is rude, in this case it should only happen if ENOMEM, and we judge
- * it as being preferable to a kernel that will crash later when someone tries
- * to use patch_instruction().
- */
-static int __init setup_text_poke_area(void)
-{
- BUG_ON(!cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
- "powerpc/text_poke:online", text_area_cpu_up,
- text_area_cpu_down));
-
- return 0;
-}
-late_initcall(setup_text_poke_area);
+struct patch_mapping {
+ spinlock_t *ptl; /* for protecting pte table */
+ struct temp_mm temp_mm;
+};
/*
* This can be called for kernel text or a module.
*/
-static int map_patch_area(void *addr, unsigned long text_poke_addr)
+static int map_patch(const void *addr, struct patch_mapping *patch_mapping)
{
- unsigned long pfn;
- int err;
+ struct page *page;
+ pte_t pte, *ptep;
+ pgprot_t pgprot;
if (is_vmalloc_addr(addr))
- pfn = vmalloc_to_pfn(addr);
+ page = vmalloc_to_page(addr);
else
- pfn = __pa_symbol(addr) >> PAGE_SHIFT;
+ page = virt_to_page(addr);
- err = map_kernel_page(text_poke_addr, (pfn << PAGE_SHIFT), PAGE_KERNEL);
+ if (radix_enabled())
+ pgprot = __pgprot(pgprot_val(PAGE_KERNEL));
+ else
+ pgprot = PAGE_SHARED;
- pr_devel("Mapped addr %lx with pfn %lx:%d\n", text_poke_addr, pfn, err);
- if (err)
+ ptep = get_locked_pte(patching_mm, patching_addr, &patch_mapping->ptl);
+ if (unlikely(!ptep)) {
+ pr_warn("map patch: failed to allocate pte for patching\n");
return -1;
+ }
+
+ pte = mk_pte(page, pgprot);
+ set_pte_at(patching_mm, patching_addr, ptep, pte);
+
+ init_temp_mm(&patch_mapping->temp_mm, patching_mm);
+ use_temporary_mm(&patch_mapping->temp_mm);
return 0;
}
-static inline int unmap_patch_area(unsigned long addr)
+static int unmap_patch(struct patch_mapping *patch_mapping)
{
pte_t *ptep;
pmd_t *pmdp;
pud_t *pudp;
pgd_t *pgdp;
- pgdp = pgd_offset_k(addr);
+ pgdp = pgd_offset(patching_mm, patching_addr);
if (unlikely(!pgdp))
return -EINVAL;
- pudp = pud_offset(pgdp, addr);
+ pudp = pud_offset(pgdp, patching_addr);
if (unlikely(!pudp))
return -EINVAL;
- pmdp = pmd_offset(pudp, addr);
+ pmdp = pmd_offset(pudp, patching_addr);
if (unlikely(!pmdp))
return -EINVAL;
- ptep = pte_offset_kernel(pmdp, addr);
+ ptep = pte_offset_kernel(pmdp, patching_addr);
if (unlikely(!ptep))
return -EINVAL;
- pr_devel("clearing mm %p, pte %p, addr %lx\n", &init_mm, ptep, addr);
+ /*
+ * In hash, pte_clear flushes the tlb
+ */
+ pte_clear(patching_mm, patching_addr, ptep);
+ unuse_temporary_mm(&patch_mapping->temp_mm);
/*
- * In hash, pte_clear flushes the tlb, in radix, we have to
+ * In radix, we have to explicitly flush the tlb (no-op in hash)
*/
- pte_clear(&init_mm, addr, ptep);
- flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
+ local_flush_tlb_mm(patching_mm);
+ pte_unmap_unlock(ptep, patch_mapping->ptl);
return 0;
}
@@ -167,33 +148,38 @@ static int do_patch_instruction(unsigned int *addr, unsigned int instr)
int err;
unsigned int *patch_addr = NULL;
unsigned long flags;
- unsigned long text_poke_addr;
- unsigned long kaddr = (unsigned long)addr;
+ struct patch_mapping patch_mapping;
/*
- * During early early boot patch_instruction is called
- * when text_poke_area is not ready, but we still need
- * to allow patching. We just do the plain old patching
+ * The patching_mm is initialized before calling mark_rodata_ro. Prior
+ * to this, patch_instruction is called when we don't have (and don't
+ * need) the patching_mm so just do plain old patching.
*/
- if (!this_cpu_read(text_poke_area))
+ if (!patching_mm)
return raw_patch_instruction(addr, instr);
local_irq_save(flags);
- text_poke_addr = (unsigned long)__this_cpu_read(text_poke_area)->addr;
- if (map_patch_area(addr, text_poke_addr)) {
- err = -1;
+ err = map_patch(addr, &patch_mapping);
+ if (err)
goto out;
- }
- patch_addr = (unsigned int *)(text_poke_addr) +
- ((kaddr & ~PAGE_MASK) / sizeof(unsigned int));
+ patch_addr = (unsigned int *)(patching_addr) +
+ (offset_in_page((unsigned long)addr) /
+ sizeof(unsigned int));
__patch_instruction(addr, instr, patch_addr);
- err = unmap_patch_area(text_poke_addr);
+ err = unmap_patch(&patch_mapping);
if (err)
- pr_warn("failed to unmap %lx\n", text_poke_addr);
+ pr_warn("unmap patch: failed to unmap patch\n");
+
+ /*
+ * Something is wrong if what we just wrote doesn't match what we
+ * think we just wrote.
+ * XXX: BUG_ON() instead?
+ */
+ WARN_ON(memcmp(addr, &instr, sizeof(instr)));
out:
local_irq_restore(flags);
--
2.25.1
^ permalink raw reply related
* [RFC PATCH 0/3] Use per-CPU temporary mappings for patching
From: Christopher M. Riedl @ 2020-03-23 4:52 UTC (permalink / raw)
To: linuxppc-dev
When compiled with CONFIG_STRICT_KERNEL_RWX, the kernel must create
temporary mappings when patching itself. These mappings temporarily
override the strict RWX text protections to permit a write. Currently,
powerpc allocates a per-CPU VM area for patching. Patching occurs as
follows:
1. Map page of text to be patched to per-CPU VM area w/
PAGE_KERNEL protection
2. Patch text
3. Remove the temporary mapping
While the VM area is per-CPU, the mapping is actually inserted into the
kernel page tables. Presumably, this could allow another CPU to access
the normally write-protected text - either malicously or accidentally -
via this same mapping if the address of the VM area is known. Ideally,
the mapping should be kept local to the CPU doing the patching (or any
other sensitive operations requiring temporarily overriding memory
protections) [0].
x86 introduced "temporary mm" structs which allow the creation of
mappings local to a particular CPU [1]. This series intends to bring the
notion of a temporary mm to powerpc and harden powerpc by using such a
mapping for patching a kernel with strict RWX permissions.
The first patch introduces the temporary mm struct and API for powerpc
along with a new function to retrieve a current hw breakpoint.
The second patch uses the `poking_init` init hook added by the x86
patches to initialize a temporary mm and patching address. The patching
address is randomized between 0 and DEFAULT_MAP_WINDOW-PAGE_SIZE. The
upper limit is necessary due to how the hash MMU operates - by default
the space above DEFAULT_MAP_WINDOW is not available. For now, both hash
and radix randomize inside this range. The number of possible random
addresses is dependent on PAGE_SIZE and limited by DEFAULT_MAP_WINDOW.
Bits of entropy with 64K page size on BOOK3S_64:
bits-o-entropy = log2(DEFAULT_MAP_WINDOW_USER64 / PAGE_SIZE)
PAGE_SIZE=64K, DEFAULT_MAP_WINDOW_USER64=128TB
bits-o-entropy = log2(128TB / 64K)
bits-o-entropy = 31
Currently, randomization occurs only once during initialization at boot.
The third patch replaces the VM area with the temporary mm in the
patching code. The page for patching has to be mapped PAGE_SHARED with
the hash MMU since hash prevents the kernel from accessing userspace
pages with PAGE_PRIVILEGED bit set. There is on-going work on my side to
explore if this is actually necessary in the hash codepath.
Testing so far is limited to booting on QEMU (power8 and power9 targets)
and a POWER8 VM along with setting some simple xmon breakpoints (which
makes use of code-patching). A POC lkdtm test is in-progress to actually
exploit the existing vulnerability (ie. the mapping during patching is
exposed in kernel page tables and accessible by other CPUS) - this will
accompany a future v1 of this series.
[0]: https://github.com/linuxppc/issues/issues/224
[1]: https://lore.kernel.org/kernel-hardening/20190426232303.28381-1-nadav.amit@gmail.com/
Christopher M. Riedl (3):
powerpc/mm: Introduce temporary mm
powerpc/lib: Initialize a temporary mm for code patching
powerpc/lib: Use a temporary mm for code patching
arch/powerpc/include/asm/debug.h | 1 +
arch/powerpc/include/asm/mmu_context.h | 56 +++++++++-
arch/powerpc/kernel/process.c | 5 +
arch/powerpc/lib/code-patching.c | 140 ++++++++++++++-----------
4 files changed, 137 insertions(+), 65 deletions(-)
--
2.25.1
^ permalink raw reply
* [RFC PATCH 2/3] powerpc/lib: Initialize a temporary mm for code patching
From: Christopher M. Riedl @ 2020-03-23 4:52 UTC (permalink / raw)
To: linuxppc-dev
In-Reply-To: <20200323045205.20314-1-cmr@informatik.wtf>
When code patching a STRICT_KERNEL_RWX kernel the page containing the
address to be patched is temporarily mapped with permissive memory
protections. Currently, a per-cpu vmalloc patch area is used for this
purpose. While the patch area is per-cpu, the temporary page mapping is
inserted into the kernel page tables for the duration of the patching.
The mapping is exposed to CPUs other than the patching CPU - this is
undesirable from a hardening perspective.
Use the `poking_init` init hook to prepare a temporary mm and patching
address. Initialize the temporary mm by copying the init mm. Choose a
randomized patching address inside the temporary mm userspace address
portion. The next patch uses the temporary mm and patching address for
code patching.
Based on x86 implementation:
commit 4fc19708b165
("x86/alternatives: Initialize temporary mm for patching")
Signed-off-by: Christopher M. Riedl <cmr@informatik.wtf>
---
arch/powerpc/lib/code-patching.c | 26 ++++++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c
index 3345f039a876..18b88ecfc5a8 100644
--- a/arch/powerpc/lib/code-patching.c
+++ b/arch/powerpc/lib/code-patching.c
@@ -11,6 +11,8 @@
#include <linux/cpuhotplug.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
+#include <linux/sched/task.h>
+#include <linux/random.h>
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
@@ -39,6 +41,30 @@ int raw_patch_instruction(unsigned int *addr, unsigned int instr)
}
#ifdef CONFIG_STRICT_KERNEL_RWX
+
+__ro_after_init struct mm_struct *patching_mm;
+__ro_after_init unsigned long patching_addr;
+
+void __init poking_init(void)
+{
+ spinlock_t *ptl; /* for protecting pte table */
+ pte_t *ptep;
+
+ patching_mm = copy_init_mm();
+ BUG_ON(!patching_mm);
+
+ /*
+ * In hash we cannot go above DEFAULT_MAP_WINDOW easily.
+ * XXX: Do we want additional bits of entropy for radix?
+ */
+ patching_addr = (get_random_long() & PAGE_MASK) %
+ (DEFAULT_MAP_WINDOW - PAGE_SIZE);
+
+ ptep = get_locked_pte(patching_mm, patching_addr, &ptl);
+ BUG_ON(!ptep);
+ pte_unmap_unlock(ptep, ptl);
+}
+
static DEFINE_PER_CPU(struct vm_struct *, text_poke_area);
static int text_area_cpu_up(unsigned int cpu)
--
2.25.1
^ permalink raw reply related
* Re: [PATCH 2/2] KVM: PPC: Book3S HV: H_SVM_INIT_START must call UV_RETURN
From: Bharata B Rao @ 2020-03-23 4:21 UTC (permalink / raw)
To: Laurent Dufour; +Cc: linux-kernel, kvm-ppc, linuxppc-dev
In-Reply-To: <f6a71da6-6363-8cba-8215-78b23a046443@linux.ibm.com>
On Fri, Mar 20, 2020 at 03:36:05PM +0100, Laurent Dufour wrote:
> Le 20/03/2020 à 12:24, Bharata B Rao a écrit :
> > On Fri, Mar 20, 2020 at 11:26:43AM +0100, Laurent Dufour wrote:
> > > When the call to UV_REGISTER_MEM_SLOT is failing, for instance because
> > > there is not enough free secured memory, the Hypervisor (HV) has to call
> > > UV_RETURN to report the error to the Ultravisor (UV). Then the UV will call
> > > H_SVM_INIT_ABORT to abort the securing phase and go back to the calling VM.
> > >
> > > If the kvm->arch.secure_guest is not set, in the return path rfid is called
> > > but there is no valid context to get back to the SVM since the Hcall has
> > > been routed by the Ultravisor.
> > >
> > > Move the setting of kvm->arch.secure_guest earlier in
> > > kvmppc_h_svm_init_start() so in the return path, UV_RETURN will be called
> > > instead of rfid.
> > >
> > > Cc: Bharata B Rao <bharata@linux.ibm.com>
> > > Cc: Paul Mackerras <paulus@ozlabs.org>
> > > Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
> > > Cc: Michael Ellerman <mpe@ellerman.id.au>
> > > Signed-off-by: Laurent Dufour <ldufour@linux.ibm.com>
> > > ---
> > > arch/powerpc/kvm/book3s_hv_uvmem.c | 3 ++-
> > > 1 file changed, 2 insertions(+), 1 deletion(-)
> > >
> > > diff --git a/arch/powerpc/kvm/book3s_hv_uvmem.c b/arch/powerpc/kvm/book3s_hv_uvmem.c
> > > index 79b1202b1c62..68dff151315c 100644
> > > --- a/arch/powerpc/kvm/book3s_hv_uvmem.c
> > > +++ b/arch/powerpc/kvm/book3s_hv_uvmem.c
> > > @@ -209,6 +209,8 @@ unsigned long kvmppc_h_svm_init_start(struct kvm *kvm)
> > > int ret = H_SUCCESS;
> > > int srcu_idx;
> > > + kvm->arch.secure_guest = KVMPPC_SECURE_INIT_START;
> > > +
> > > if (!kvmppc_uvmem_bitmap)
> > > return H_UNSUPPORTED;
> > > @@ -233,7 +235,6 @@ unsigned long kvmppc_h_svm_init_start(struct kvm *kvm)
> > > goto out;
> > > }
> > > }
> > > - kvm->arch.secure_guest |= KVMPPC_SECURE_INIT_START;
> >
> > There is an assumption that memory slots would have been registered with UV
> > if KVMPPC_SECURE_INIT_START has been done. KVM_PPC_SVM_OFF ioctl will skip
> > unregistration and other steps during reboot if KVMPPC_SECURE_INIT_START
> > hasn't been done.
> >
> > Have you checked if that path isn't affected by this change?
>
> I checked that and didn't find any issue there.
>
> My only concern was that block:
> kvm_for_each_vcpu(i, vcpu, kvm) {
> spin_lock(&vcpu->arch.vpa_update_lock);
> unpin_vpa_reset(kvm, &vcpu->arch.dtl);
> unpin_vpa_reset(kvm, &vcpu->arch.slb_shadow);
> unpin_vpa_reset(kvm, &vcpu->arch.vpa);
> spin_unlock(&vcpu->arch.vpa_update_lock);
> }
>
> But that seems to be safe.
Yes, looks like.
>
> However I'm not a familiar with the KVM's code, do you think an additional
> KVMPPC_SECURE_INIT_* value needed here?
May be not as long as UV can handle the unexpected uv_unregister_mem_slot()
calls, we are good.
Regards,
Bharata.
^ permalink raw reply
* Re: [PATCH] powerpc/64: ftrace don't trace real mode
From: Nicholas Piggin @ 2020-03-23 4:10 UTC (permalink / raw)
To: linuxppc-dev, Naveen N. Rao
In-Reply-To: <1584893587.qg26sf1ty0.naveen@linux.ibm.com>
Naveen N. Rao's on March 23, 2020 2:17 am:
> Nicholas Piggin wrote:
>> Naveen N. Rao's on March 21, 2020 4:39 am:
>>> Hi Nick,
>>>
>>> Nicholas Piggin wrote:
>>>> This warns and prevents tracing attempted in a real-mode context.
>>>
>>> Is this something you're seeing often? Last time we looked at this, KVM
>>> was the biggest offender and we introduced paca->ftrace_enabled as a way
>>> to disable ftrace while in KVM code.
>>
>> Not often but it has a tendancy to blow up the least tested code at the
>> worst times :)
>>
>> Machine check is bad, I'm sure HMI too but I haven't tested that yet.
>>
>> I've fixed up most of it with annotations, this is obviously an extra
>> safety not something to rely on like ftrace_enabled. Probably even the
>> WARN_ON here is dangerous here, but I don't want to leave these bugs
>> in there.
>
> Ok, makes sense.
>
>>
>> Although the machine check and hmi code touch a fair bit of stuff and
>> annotating is a bit fragile. It might actually be better if the
>> paca->ftrace_enabled could be a nesting counter, then we could use it
>> in machine checks too and avoid a lot of annotations.
>
> I'm not too familiar with MC/HMI, but I suppose those aren't re-entrant?
> If those have access to an emergency stack, can we save/restore
> ftrace_enabled state across the handlers?
Yeah that's true. They're not highly reentrant though, we could just
make it an 8 bit counter, it's nicer than saving / restoring from stack
(but I guess I could do that too).
>
> We're primarily disabling ftrace across idle/offline/KVM right now. I'm
> not sure if nesting is useful there.
>
>>
>>> While this is cheap when handling ftrace_regs_caller() as done in this
>>> patch, for simple function tracing (see below), we will have to grab the
>>> MSR which will slow things down slightly.
>>
>> mfmsr is not too bad these days.
>
> That's great!
Thanks,
Nick
^ permalink raw reply
* [PATCH v4 9/9] Documentation/powerpc: VAS API
From: Haren Myneni @ 2020-03-23 4:06 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
Power9 introduced Virtual Accelerator Switchboard (VAS) which allows
userspace to communicate with Nest Accelerator (NX) directly. But
kernel has to establish channel to NX for userspace. This document
describes user space API that application can use to establish
communication channel.
Signed-off-by: Sukadev Bhattiprolu <sukadev@linux.ibm.com>
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
Documentation/powerpc/index.rst | 1 +
Documentation/powerpc/vas-api.rst | 249 ++++++++++++++++++++++++++++++++++++++
2 files changed, 250 insertions(+)
create mode 100644 Documentation/powerpc/vas-api.rst
diff --git a/Documentation/powerpc/index.rst b/Documentation/powerpc/index.rst
index 0d45f0f..afe2d5e 100644
--- a/Documentation/powerpc/index.rst
+++ b/Documentation/powerpc/index.rst
@@ -30,6 +30,7 @@ powerpc
syscall64-abi
transactional_memory
ultravisor
+ vas-api
.. only:: subproject and html
diff --git a/Documentation/powerpc/vas-api.rst b/Documentation/powerpc/vas-api.rst
new file mode 100644
index 0000000..bd7042d
--- /dev/null
+++ b/Documentation/powerpc/vas-api.rst
@@ -0,0 +1,249 @@
+.. SPDX-License-Identifier: GPL-2.0
+.. _VAS-API:
+
+===================================================
+Virtual Accelerator Switchboard (VAS) userspace API
+===================================================
+
+Introduction
+============
+
+Power9 processor introduced Virtual Accelerator Switchboard (VAS) which
+allows both userspace and kernel communicate to co-processor
+(hardware accelerator) referred to as the Nest Accelerator (NX). The NX
+unit comprises of one or more hardware engines or co-processor types
+such as 842 compression, GZIP compression and encryption. On power9,
+userspace applications will have access to only GZIP Compression engine
+which supports ZLIB and GZIP compression algorithms in the hardware.
+
+To communicate with NX, kernel has to establish a channel or window and
+then requests can be submitted directly without kernel involvement.
+Requests to the GZIP engine must be formatted as a co-processor Request
+Block (CRB) and these CRBs must be submitted to the NX using COPY/PASTE
+instructions to paste the CRB to hardware address that is associated with
+the engine's request queue.
+
+The GZIP engine provides two priority levels of requests: Normal and
+High. Only Normal requests are supported from userspace right now.
+
+This document explains userspace API that is used to interact with
+kernel to setup channel / window which can be used to send compression
+requests directly to NX accelerator.
+
+
+Overview
+========
+
+Application access to the GZIP engine is provided through
+/dev/crypto/nx-gzip device node implemented by the VAS/NX device driver.
+An application must open the /dev/crypto/nx-gzip device to obtain a file
+descriptor (fd). Then should issue VAS_TX_WIN_OPEN ioctl with this fd to
+establish connection to the engine. It means send window is opened on GZIP
+engine for this process. Once a connection is established, the application
+should use the mmap() system call to map the hardware address of engine's
+request queue into the application's virtual address space.
+
+The application can then submit one or more requests to the the engine by
+using copy/paste instructions and pasting the CRBs to the virtual address
+(aka paste_address) returned by mmap(). User space can close the
+established connection or send window by closing the file descriptior
+(close(fd)) or upon the process exit.
+
+Note that applications can send several requests with the same window or
+can establish multiple windows, but one window for each file descriptor.
+
+Following sections provide additional details and references about the
+individual steps.
+
+NX-GZIP Device Node
+===================
+
+There is one /dev/crypto/nx-gzip node in the system and it provides
+access to all GZIP engines in the system. The only valid operations on
+/dev/crypto/nx-gzip are:
+
+ * open() the device for read and write.
+ * issue VAS_TX_WIN_OPEN ioctl
+ * mmap() the engine's request queue into application's virtual
+ address space (i.e. get a paste_address for the co-processor
+ engine).
+ * close the device node.
+
+Other file operations on this device node are undefined.
+
+Note that the copy and paste operations go directly to the hardware and
+do not go through this device. Refer COPY/PASTE document for more
+details.
+
+Although a system may have several instances of the NX co-processor
+engines (typically, one per P9 chip) there is just one
+/dev/crypto/nx-gzip device node in the system. When the nx-gzip device
+node is opened, Kernel opens send window on a suitable instance of NX
+accelerator. It finds CPU on which the user process is executing and
+determine the NX instance for the corresponding chip on which this CPU
+belongs.
+
+Applications may chose a specific instance of the NX co-processor using
+the vas_id field in the VAS_TX_WIN_OPEN ioctl as detailed below.
+
+A userspace library libnxz is available here but still in development:
+ https://github.com/abalib/power-gzip
+
+Applications that use inflate / deflate calls can link with libnxz
+instead of libz and use NX GZIP compression without any modification.
+
+Open /dev/crypto/nx-gzip
+========================
+
+The nx-gzip device should be opened for read and write. No special
+privileges are needed to open the device. Each window corresponds to one
+file descriptor. So if the userspace process needs multiple windows,
+several open calls have to be issued.
+
+See open(2) system call man pages for other details such as return values,
+error codes and restrictions.
+
+VAS_TX_WIN_OPEN ioctl
+=====================
+
+Applications should use the VAS_TX_WIN_OPEN ioctl as follows to establish
+a connection with NX co-processor engine:
+
+ ::
+ struct vas_tx_win_open_attr {
+ __u32 version;
+ __s16 vas_id; /* specific instance of vas or -1
+ for default */
+ __u16 reserved1;
+ __u64 flags; /* For future use */
+ __u64 reserved2[6];
+ };
+
+ version: The version field must be currently set to 1.
+ vas_id: If '-1' is passed, kernel will make a best-effort attempt
+ to assign an optimal instance of NX for the process. To
+ select the specific VAS instance, refer
+ "Discovery of available VAS engines" section below.
+
+ flags, reserved1 and reserved2[6] fields are for future extension
+ and must be set to 0.
+
+ The attributes attr for the VAS_TX_WIN_OPEN ioctl are defined as
+ follows:
+ #define VAS_MAGIC 'v'
+ #define VAS_TX_WIN_OPEN _IOW(VAS_MAGIC, 1,
+ struct vas_tx_win_open_attr)
+
+ struct vas_tx_win_open_attr attr;
+ rc = ioctl(fd, VAS_TX_WIN_OPEN, &attr);
+
+ The VAS_TX_WIN_OPEN ioctl returns 0 on success. On errors, it
+ returns -1 and sets the errno variable to indicate the error.
+
+ Error conditions:
+ EINVAL fd does not refer to a valid VAS device.
+ EINVAL Invalid vas ID
+ EINVAL version is not set with proper value
+ EEXIST Window is already opened for the given fd
+ ENOMEM Memory is not available to allocate window
+ ENOSPC System has too many active windows (connections)
+ opened
+ EINVAL reserved fields are not set to 0.
+
+ See the ioctl(2) man page for more details, error codes and
+ restrictions.
+
+mmap() NX-GZIP device
+=====================
+
+The mmap() system call for a NX-GZIP device fd returns a paste_address
+that the application can use to copy/paste its CRB to the hardware engines.
+ ::
+
+ paste_addr = mmap(addr, size, prot, flags, fd, offset);
+
+ Only restrictions on mmap for a NX-GZIP device fd are:
+ * size should be 4K page size
+ * offset parameter should be 0ULL
+
+ Refer to mmap(2) man page for additional details/restrictions.
+ In addition to the error conditions listed on the mmap(2) man
+ page, can also fail with one of the following error codes:
+
+ EINVAL fd is not associated with an open window
+ (i.e mmap() does not follow a successful call
+ to the VAS_TX_WIN_OPEN ioctl).
+ EINVAL offset field is not 0ULL.
+
+Discovery of available VAS engines
+==================================
+
+Each available VAS instance in the system will have a device tree node
+like /proc/device-tree/vas@* or /proc/device-tree/xscom@*/vas@*.
+Determine the chip or VAS instance and use the corresponding ibm,vas-id
+property value in this node to select specific VAS instance.
+
+Copy/Paste operations
+=====================
+
+Applications should use the copy and paste instructions to send CRB to NX.
+Refer section 4.4 in PowerISA for Copy/Paste instructions:
+https://openpowerfoundation.org/?resource_lib=power-isa-version-3-0
+
+CRB Specification and use NX
+============================
+
+Applications should format requests to the co-processor using the
+co-processor Request Block (CRBs). Refer NX-GZIP user's manual for the format
+of CRB and use NX from userspace such as sending requests and checking
+request status.
+
+NX-GZIP User's Manual:
+https://github.com/libnxz/power-gzip/blob/master/power_nx_gzip_um.pdf
+
+Simple example
+==============
+
+ ::
+ int use_nx_gzip()
+ {
+ int rc, fd;
+ void *addr;
+ struct vas_setup_attr txattr;
+
+ fd = open("/dev/crypto/nx-gzip", O_RDWR);
+ if (fd < 0) {
+ fprintf(stderr, "open nx-gzip failed\n");
+ return -1;
+ }
+ memset(&txattr, 0, sizeof(txattr));
+ txattr.version = 1;
+ txattr.vas_id = -1
+ rc = ioctl(fd, VAS_TX_WIN_OPEN,
+ (unsigned long)&txattr);
+ if (rc < 0) {
+ fprintf(stderr, "ioctl() n %d, error %d\n",
+ rc, errno);
+ return rc;
+ }
+ addr = mmap(NULL, 4096, PROT_READ|PROT_WRITE,
+ MAP_SHARED, fd, 0ULL);
+ if (addr == MAP_FAILED) {
+ fprintf(stderr, "mmap() failed, errno %d\n",
+ errno);
+ return -errno;
+ }
+ do {
+ //Format CRB request with compression or
+ //uncompression
+ // Refer tests for vas_copy/vas_paste
+ vas_copy((&crb, 0, 1);
+ vas_paste(addr, 0, 1);
+ // Poll on csb.flags with timeout
+ // csb address is listed in CRB
+ } while (true)
+ close(fd) or window can be closed upon process exit
+ }
+
+ Refer https://github.com/abalib/power-gzip for tests or more
+ use cases.
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 8/9] crypto/nx: Remove 'pid' in vas_tx_win_attr struct
From: Haren Myneni @ 2020-03-23 4:06 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
When window is opened, pid reference is taken for user space
windows. Not needed for kernel windows. So remove 'pid' in
vas_tx_win_attr struct.
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
arch/powerpc/include/asm/vas.h | 1 -
drivers/crypto/nx/nx-common-powernv.c | 1 -
2 files changed, 2 deletions(-)
diff --git a/arch/powerpc/include/asm/vas.h b/arch/powerpc/include/asm/vas.h
index e064953..994db6f 100644
--- a/arch/powerpc/include/asm/vas.h
+++ b/arch/powerpc/include/asm/vas.h
@@ -86,7 +86,6 @@ struct vas_tx_win_attr {
int wcreds_max;
int lpid;
int pidr; /* hardware PID (from SPRN_PID) */
- int pid; /* linux process id */
int pswid;
int rsvd_txbuf_count;
int tc_mode;
diff --git a/drivers/crypto/nx/nx-common-powernv.c b/drivers/crypto/nx/nx-common-powernv.c
index f570691..38333e4 100644
--- a/drivers/crypto/nx/nx-common-powernv.c
+++ b/drivers/crypto/nx/nx-common-powernv.c
@@ -692,7 +692,6 @@ static struct vas_window *nx_alloc_txwin(struct nx_coproc *coproc)
*/
vas_init_tx_win_attr(&txattr, coproc->ct);
txattr.lpid = 0; /* lpid is 0 for kernel requests */
- txattr.pid = 0; /* pid is 0 for kernel requests */
/*
* Open a VAS send window which is used to send request to NX.
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 7/9] crypto/nx: Enable and setup GZIP compression type
From: Haren Myneni @ 2020-03-23 4:05 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
Changes to probe GZIP device-tree nodes, open RX windows and setup
GZIP compression type. No plans to provide GZIP usage in kernel right
now, but this patch enables GZIP for user space usage.
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
drivers/crypto/nx/nx-common-powernv.c | 43 ++++++++++++++++++++++++++++++-----
1 file changed, 37 insertions(+), 6 deletions(-)
diff --git a/drivers/crypto/nx/nx-common-powernv.c b/drivers/crypto/nx/nx-common-powernv.c
index 82dfa60..f570691 100644
--- a/drivers/crypto/nx/nx-common-powernv.c
+++ b/drivers/crypto/nx/nx-common-powernv.c
@@ -65,6 +65,7 @@ struct nx_coproc {
* Using same values as in skiboot or coprocessor type representing
* in NX workbook.
*/
+#define NX_CT_GZIP (2) /* on P9 and later */
#define NX_CT_842 (3)
static int (*nx842_powernv_exec)(const unsigned char *in,
@@ -819,6 +820,9 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
if (type == NX_CT_842)
ret = nx_set_ct(coproc, priority, VAS_COP_TYPE_842_HIPRI,
VAS_COP_TYPE_842);
+ else if (type == NX_CT_GZIP)
+ ret = nx_set_ct(coproc, priority, VAS_COP_TYPE_GZIP_HIPRI,
+ VAS_COP_TYPE_GZIP);
if (ret)
goto err_out;
@@ -867,12 +871,16 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
return ret;
}
-static int __init nx_coproc_init(int chip_id, int ct_842)
+static int __init nx_coproc_init(int chip_id, int ct_842, int ct_gzip)
{
int ret = 0;
if (opal_check_token(OPAL_NX_COPROC_INIT)) {
ret = opal_nx_coproc_init(chip_id, ct_842);
+
+ if (!ret)
+ ret = opal_nx_coproc_init(chip_id, ct_gzip);
+
if (ret) {
ret = opal_error_code(ret);
pr_err("Failed to initialize NX for chip(%d): %d\n",
@@ -902,8 +910,8 @@ static int __init find_nx_device_tree(struct device_node *dn, int chip_id,
static int __init nx_powernv_probe_vas(struct device_node *pn)
{
int chip_id, vasid, ret = 0;
+ int ct_842 = 0, ct_gzip = 0;
struct device_node *dn;
- int ct_842 = 0;
chip_id = of_get_ibm_chip_id(pn);
if (chip_id < 0) {
@@ -920,19 +928,24 @@ static int __init nx_powernv_probe_vas(struct device_node *pn)
for_each_child_of_node(pn, dn) {
ret = find_nx_device_tree(dn, chip_id, vasid, NX_CT_842,
"ibm,p9-nx-842", &ct_842);
+
+ if (!ret)
+ ret = find_nx_device_tree(dn, chip_id, vasid,
+ NX_CT_GZIP, "ibm,p9-nx-gzip", &ct_gzip);
+
if (ret)
return ret;
}
- if (!ct_842) {
- pr_err("NX842 FIFO nodes are missing\n");
+ if (!ct_842 || !ct_gzip) {
+ pr_err("NX FIFO nodes are missing\n");
return -EINVAL;
}
/*
* Initialize NX instance for both high and normal priority FIFOs.
*/
- ret = nx_coproc_init(chip_id, ct_842);
+ ret = nx_coproc_init(chip_id, ct_842, ct_gzip);
return ret;
}
@@ -1072,10 +1085,19 @@ static __init int nx_compress_powernv_init(void)
nx842_powernv_exec = nx842_exec_icswx;
} else {
/*
+ * Register VAS user space API for NX GZIP so
+ * that user space can use GZIP engine.
+ * 842 compression is supported only in kernel.
+ */
+ ret = vas_register_coproc_api(THIS_MODULE);
+
+ /*
* GZIP is not supported in kernel right now.
* So open tx windows only for 842.
*/
- ret = nx_open_percpu_txwins();
+ if (!ret)
+ ret = nx_open_percpu_txwins();
+
if (ret) {
nx_delete_coprocs();
return ret;
@@ -1096,6 +1118,15 @@ static __init int nx_compress_powernv_init(void)
static void __exit nx_compress_powernv_exit(void)
{
+ /*
+ * GZIP engine is supported only in power9 or later and nx842_ct
+ * is used on power8 (icswx).
+ * VAS API for NX GZIP is registered during init for user space
+ * use. So delete this API use for GZIP engine.
+ */
+ if (!nx842_ct)
+ vas_unregister_coproc_api();
+
crypto_unregister_alg(&nx842_powernv_alg);
nx_delete_coprocs();
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 6/9] crypto/NX: Make enable code generic to add new GZIP compression type
From: Haren Myneni @ 2020-03-23 4:04 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
Make setup and enable code generic to support new GZIP compression type.
Changed nx842 reference to nx and moved some code to new functions.
Functionality is not changed except sparse warning fix - setting NULL
instead of 0 for per_cpu send window in nx_delete_coprocs().
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
drivers/crypto/nx/nx-common-powernv.c | 161 +++++++++++++++++++++-------------
1 file changed, 101 insertions(+), 60 deletions(-)
diff --git a/drivers/crypto/nx/nx-common-powernv.c b/drivers/crypto/nx/nx-common-powernv.c
index f42881f..82dfa60 100644
--- a/drivers/crypto/nx/nx-common-powernv.c
+++ b/drivers/crypto/nx/nx-common-powernv.c
@@ -40,9 +40,9 @@ struct nx842_workmem {
char padding[WORKMEM_ALIGN]; /* unused, to allow alignment */
} __packed __aligned(WORKMEM_ALIGN);
-struct nx842_coproc {
+struct nx_coproc {
unsigned int chip_id;
- unsigned int ct;
+ unsigned int ct; /* Can be 842 or GZIP high/normal*/
unsigned int ci; /* Coprocessor instance, used with icswx */
struct {
struct vas_window *rxwin;
@@ -58,9 +58,15 @@ struct nx842_coproc {
static DEFINE_PER_CPU(struct vas_window *, cpu_txwin);
/* no cpu hotplug on powernv, so this list never changes after init */
-static LIST_HEAD(nx842_coprocs);
+static LIST_HEAD(nx_coprocs);
static unsigned int nx842_ct; /* used in icswx function */
+/*
+ * Using same values as in skiboot or coprocessor type representing
+ * in NX workbook.
+ */
+#define NX_CT_842 (3)
+
static int (*nx842_powernv_exec)(const unsigned char *in,
unsigned int inlen, unsigned char *out,
unsigned int *outlenp, void *workmem, int fc);
@@ -666,15 +672,15 @@ static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen,
wmem, CCW_FC_842_DECOMP_CRC);
}
-static inline void nx842_add_coprocs_list(struct nx842_coproc *coproc,
+static inline void nx_add_coprocs_list(struct nx_coproc *coproc,
int chipid)
{
coproc->chip_id = chipid;
INIT_LIST_HEAD(&coproc->list);
- list_add(&coproc->list, &nx842_coprocs);
+ list_add(&coproc->list, &nx_coprocs);
}
-static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc)
+static struct vas_window *nx_alloc_txwin(struct nx_coproc *coproc)
{
struct vas_window *txwin = NULL;
struct vas_tx_win_attr txattr;
@@ -704,9 +710,9 @@ static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc)
* cpu_txwin is used in copy/paste operation for each compression /
* decompression request.
*/
-static int nx842_open_percpu_txwins(void)
+static int nx_open_percpu_txwins(void)
{
- struct nx842_coproc *coproc, *n;
+ struct nx_coproc *coproc, *n;
unsigned int i, chip_id;
for_each_possible_cpu(i) {
@@ -714,17 +720,18 @@ static int nx842_open_percpu_txwins(void)
chip_id = cpu_to_chip_id(i);
- list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
+ list_for_each_entry_safe(coproc, n, &nx_coprocs, list) {
/*
* Kernel requests use only high priority FIFOs. So
* open send windows for these FIFOs.
+ * GZIP is not supported in kernel right now.
*/
if (coproc->ct != VAS_COP_TYPE_842_HIPRI)
continue;
if (coproc->chip_id == chip_id) {
- txwin = nx842_alloc_txwin(coproc);
+ txwin = nx_alloc_txwin(coproc);
if (IS_ERR(txwin))
return PTR_ERR(txwin);
@@ -743,13 +750,28 @@ static int nx842_open_percpu_txwins(void)
return 0;
}
+static int __init nx_set_ct(struct nx_coproc *coproc, const char *priority,
+ int high, int normal)
+{
+ if (!strcmp(priority, "High"))
+ coproc->ct = high;
+ else if (!strcmp(priority, "Normal"))
+ coproc->ct = normal;
+ else {
+ pr_err("Invalid RxFIFO priority value\n");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
- int vasid, int *ct)
+ int vasid, int type, int *ct)
{
struct vas_window *rxwin = NULL;
struct vas_rx_win_attr rxattr;
- struct nx842_coproc *coproc;
u32 lpid, pid, tid, fifo_size;
+ struct nx_coproc *coproc;
u64 rx_fifo;
const char *priority;
int ret;
@@ -794,15 +816,12 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
if (!coproc)
return -ENOMEM;
- if (!strcmp(priority, "High"))
- coproc->ct = VAS_COP_TYPE_842_HIPRI;
- else if (!strcmp(priority, "Normal"))
- coproc->ct = VAS_COP_TYPE_842;
- else {
- pr_err("Invalid RxFIFO priority value\n");
- ret = -EINVAL;
+ if (type == NX_CT_842)
+ ret = nx_set_ct(coproc, priority, VAS_COP_TYPE_842_HIPRI,
+ VAS_COP_TYPE_842);
+
+ if (ret)
goto err_out;
- }
vas_init_rx_win_attr(&rxattr, coproc->ct);
rxattr.rx_fifo = (void *)rx_fifo;
@@ -830,7 +849,7 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
coproc->vas.rxwin = rxwin;
coproc->vas.id = vasid;
- nx842_add_coprocs_list(coproc, chip_id);
+ nx_add_coprocs_list(coproc, chip_id);
/*
* (lpid, pid, tid) combination has to be unique for each
@@ -848,13 +867,43 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
return ret;
}
+static int __init nx_coproc_init(int chip_id, int ct_842)
+{
+ int ret = 0;
-static int __init nx842_powernv_probe_vas(struct device_node *pn)
+ if (opal_check_token(OPAL_NX_COPROC_INIT)) {
+ ret = opal_nx_coproc_init(chip_id, ct_842);
+ if (ret) {
+ ret = opal_error_code(ret);
+ pr_err("Failed to initialize NX for chip(%d): %d\n",
+ chip_id, ret);
+ }
+ } else
+ pr_warn("Firmware doesn't support NX initialization\n");
+
+ return ret;
+}
+
+static int __init find_nx_device_tree(struct device_node *dn, int chip_id,
+ int vasid, int type, char *devname,
+ int *ct)
+{
+ int ret = 0;
+
+ if (of_device_is_compatible(dn, devname)) {
+ ret = vas_cfg_coproc_info(dn, chip_id, vasid, type, ct);
+ if (ret)
+ of_node_put(dn);
+ }
+
+ return ret;
+}
+
+static int __init nx_powernv_probe_vas(struct device_node *pn)
{
- struct device_node *dn;
int chip_id, vasid, ret = 0;
- int nx_fifo_found = 0;
- int uninitialized_var(ct);
+ struct device_node *dn;
+ int ct_842 = 0;
chip_id = of_get_ibm_chip_id(pn);
if (chip_id < 0) {
@@ -869,17 +918,13 @@ static int __init nx842_powernv_probe_vas(struct device_node *pn)
}
for_each_child_of_node(pn, dn) {
- if (of_device_is_compatible(dn, "ibm,p9-nx-842")) {
- ret = vas_cfg_coproc_info(dn, chip_id, vasid, &ct);
- if (ret) {
- of_node_put(dn);
- return ret;
- }
- nx_fifo_found++;
- }
+ ret = find_nx_device_tree(dn, chip_id, vasid, NX_CT_842,
+ "ibm,p9-nx-842", &ct_842);
+ if (ret)
+ return ret;
}
- if (!nx_fifo_found) {
+ if (!ct_842) {
pr_err("NX842 FIFO nodes are missing\n");
return -EINVAL;
}
@@ -887,22 +932,14 @@ static int __init nx842_powernv_probe_vas(struct device_node *pn)
/*
* Initialize NX instance for both high and normal priority FIFOs.
*/
- if (opal_check_token(OPAL_NX_COPROC_INIT)) {
- ret = opal_nx_coproc_init(chip_id, ct);
- if (ret) {
- pr_err("Failed to initialize NX for chip(%d): %d\n",
- chip_id, ret);
- ret = opal_error_code(ret);
- }
- } else
- pr_warn("Firmware doesn't support NX initialization\n");
+ ret = nx_coproc_init(chip_id, ct_842);
return ret;
}
static int __init nx842_powernv_probe(struct device_node *dn)
{
- struct nx842_coproc *coproc;
+ struct nx_coproc *coproc;
unsigned int ct, ci;
int chip_id;
@@ -928,7 +965,7 @@ static int __init nx842_powernv_probe(struct device_node *dn)
coproc->ct = ct;
coproc->ci = ci;
- nx842_add_coprocs_list(coproc, chip_id);
+ nx_add_coprocs_list(coproc, chip_id);
pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci);
@@ -941,9 +978,9 @@ static int __init nx842_powernv_probe(struct device_node *dn)
return 0;
}
-static void nx842_delete_coprocs(void)
+static void nx_delete_coprocs(void)
{
- struct nx842_coproc *coproc, *n;
+ struct nx_coproc *coproc, *n;
struct vas_window *txwin;
int i;
@@ -955,10 +992,10 @@ static void nx842_delete_coprocs(void)
if (txwin)
vas_win_close(txwin);
- per_cpu(cpu_txwin, i) = 0;
+ per_cpu(cpu_txwin, i) = NULL;
}
- list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
+ list_for_each_entry_safe(coproc, n, &nx_coprocs, list) {
if (coproc->vas.rxwin)
vas_win_close(coproc->vas.rxwin);
@@ -1002,7 +1039,7 @@ static int nx842_powernv_crypto_init(struct crypto_tfm *tfm)
.coa_decompress = nx842_crypto_decompress } }
};
-static __init int nx842_powernv_init(void)
+static __init int nx_compress_powernv_init(void)
{
struct device_node *dn;
int ret;
@@ -1017,15 +1054,15 @@ static __init int nx842_powernv_init(void)
BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT);
for_each_compatible_node(dn, NULL, "ibm,power9-nx") {
- ret = nx842_powernv_probe_vas(dn);
+ ret = nx_powernv_probe_vas(dn);
if (ret) {
- nx842_delete_coprocs();
+ nx_delete_coprocs();
of_node_put(dn);
return ret;
}
}
- if (list_empty(&nx842_coprocs)) {
+ if (list_empty(&nx_coprocs)) {
for_each_compatible_node(dn, NULL, "ibm,power-nx")
nx842_powernv_probe(dn);
@@ -1034,9 +1071,13 @@ static __init int nx842_powernv_init(void)
nx842_powernv_exec = nx842_exec_icswx;
} else {
- ret = nx842_open_percpu_txwins();
+ /*
+ * GZIP is not supported in kernel right now.
+ * So open tx windows only for 842.
+ */
+ ret = nx_open_percpu_txwins();
if (ret) {
- nx842_delete_coprocs();
+ nx_delete_coprocs();
return ret;
}
@@ -1045,18 +1086,18 @@ static __init int nx842_powernv_init(void)
ret = crypto_register_alg(&nx842_powernv_alg);
if (ret) {
- nx842_delete_coprocs();
+ nx_delete_coprocs();
return ret;
}
return 0;
}
-module_init(nx842_powernv_init);
+module_init(nx_compress_powernv_init);
-static void __exit nx842_powernv_exit(void)
+static void __exit nx_compress_powernv_exit(void)
{
crypto_unregister_alg(&nx842_powernv_alg);
- nx842_delete_coprocs();
+ nx_delete_coprocs();
}
-module_exit(nx842_powernv_exit);
+module_exit(nx_compress_powernv_exit);
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 5/9] crypto/nx: Rename nx-842-powernv file name to nx-common-powernv
From: Haren Myneni @ 2020-03-23 4:03 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
Rename nx-842-powernv.c to nx-common-powernv.c to add code for setup
and enable new GZIP compression type. The actual functionality is not
changed in this patch.
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
drivers/crypto/nx/Makefile | 2 +-
drivers/crypto/nx/nx-842-powernv.c | 1062 ---------------------------------
drivers/crypto/nx/nx-common-powernv.c | 1062 +++++++++++++++++++++++++++++++++
3 files changed, 1063 insertions(+), 1063 deletions(-)
delete mode 100644 drivers/crypto/nx/nx-842-powernv.c
create mode 100644 drivers/crypto/nx/nx-common-powernv.c
diff --git a/drivers/crypto/nx/Makefile b/drivers/crypto/nx/Makefile
index 015155d..bc89a20 100644
--- a/drivers/crypto/nx/Makefile
+++ b/drivers/crypto/nx/Makefile
@@ -15,4 +15,4 @@ obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_PSERIES) += nx-compress-pseries.o nx-compres
obj-$(CONFIG_CRYPTO_DEV_NX_COMPRESS_POWERNV) += nx-compress-powernv.o nx-compress.o
nx-compress-objs := nx-842.o
nx-compress-pseries-objs := nx-842-pseries.o
-nx-compress-powernv-objs := nx-842-powernv.o
+nx-compress-powernv-objs := nx-common-powernv.o
diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c
deleted file mode 100644
index 8e63326..0000000
--- a/drivers/crypto/nx/nx-842-powernv.c
+++ /dev/null
@@ -1,1062 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Driver for IBM PowerNV 842 compression accelerator
- *
- * Copyright (C) 2015 Dan Streetman, IBM Corp
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include "nx-842.h"
-
-#include <linux/timer.h>
-
-#include <asm/prom.h>
-#include <asm/icswx.h>
-#include <asm/vas.h>
-#include <asm/reg.h>
-#include <asm/opal-api.h>
-#include <asm/opal.h>
-
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
-MODULE_DESCRIPTION("842 H/W Compression driver for IBM PowerNV processors");
-MODULE_ALIAS_CRYPTO("842");
-MODULE_ALIAS_CRYPTO("842-nx");
-
-#define WORKMEM_ALIGN (CRB_ALIGN)
-#define CSB_WAIT_MAX (5000) /* ms */
-#define VAS_RETRIES (10)
-
-struct nx842_workmem {
- /* Below fields must be properly aligned */
- struct coprocessor_request_block crb; /* CRB_ALIGN align */
- struct data_descriptor_entry ddl_in[DDL_LEN_MAX]; /* DDE_ALIGN align */
- struct data_descriptor_entry ddl_out[DDL_LEN_MAX]; /* DDE_ALIGN align */
- /* Above fields must be properly aligned */
-
- ktime_t start;
-
- char padding[WORKMEM_ALIGN]; /* unused, to allow alignment */
-} __packed __aligned(WORKMEM_ALIGN);
-
-struct nx842_coproc {
- unsigned int chip_id;
- unsigned int ct;
- unsigned int ci; /* Coprocessor instance, used with icswx */
- struct {
- struct vas_window *rxwin;
- int id;
- } vas;
- struct list_head list;
-};
-
-/*
- * Send the request to NX engine on the chip for the corresponding CPU
- * where the process is executing. Use with VAS function.
- */
-static DEFINE_PER_CPU(struct vas_window *, cpu_txwin);
-
-/* no cpu hotplug on powernv, so this list never changes after init */
-static LIST_HEAD(nx842_coprocs);
-static unsigned int nx842_ct; /* used in icswx function */
-
-static int (*nx842_powernv_exec)(const unsigned char *in,
- unsigned int inlen, unsigned char *out,
- unsigned int *outlenp, void *workmem, int fc);
-
-/**
- * setup_indirect_dde - Setup an indirect DDE
- *
- * The DDE is setup with the the DDE count, byte count, and address of
- * first direct DDE in the list.
- */
-static void setup_indirect_dde(struct data_descriptor_entry *dde,
- struct data_descriptor_entry *ddl,
- unsigned int dde_count, unsigned int byte_count)
-{
- dde->flags = 0;
- dde->count = dde_count;
- dde->index = 0;
- dde->length = cpu_to_be32(byte_count);
- dde->address = cpu_to_be64(nx842_get_pa(ddl));
-}
-
-/**
- * setup_direct_dde - Setup single DDE from buffer
- *
- * The DDE is setup with the buffer and length. The buffer must be properly
- * aligned. The used length is returned.
- * Returns:
- * N Successfully set up DDE with N bytes
- */
-static unsigned int setup_direct_dde(struct data_descriptor_entry *dde,
- unsigned long pa, unsigned int len)
-{
- unsigned int l = min_t(unsigned int, len, LEN_ON_PAGE(pa));
-
- dde->flags = 0;
- dde->count = 0;
- dde->index = 0;
- dde->length = cpu_to_be32(l);
- dde->address = cpu_to_be64(pa);
-
- return l;
-}
-
-/**
- * setup_ddl - Setup DDL from buffer
- *
- * Returns:
- * 0 Successfully set up DDL
- */
-static int setup_ddl(struct data_descriptor_entry *dde,
- struct data_descriptor_entry *ddl,
- unsigned char *buf, unsigned int len,
- bool in)
-{
- unsigned long pa = nx842_get_pa(buf);
- int i, ret, total_len = len;
-
- if (!IS_ALIGNED(pa, DDE_BUFFER_ALIGN)) {
- pr_debug("%s buffer pa 0x%lx not 0x%x-byte aligned\n",
- in ? "input" : "output", pa, DDE_BUFFER_ALIGN);
- return -EINVAL;
- }
-
- /* only need to check last mult; since buffer must be
- * DDE_BUFFER_ALIGN aligned, and that is a multiple of
- * DDE_BUFFER_SIZE_MULT, and pre-last page DDE buffers
- * are guaranteed a multiple of DDE_BUFFER_SIZE_MULT.
- */
- if (len % DDE_BUFFER_LAST_MULT) {
- pr_debug("%s buffer len 0x%x not a multiple of 0x%x\n",
- in ? "input" : "output", len, DDE_BUFFER_LAST_MULT);
- if (in)
- return -EINVAL;
- len = round_down(len, DDE_BUFFER_LAST_MULT);
- }
-
- /* use a single direct DDE */
- if (len <= LEN_ON_PAGE(pa)) {
- ret = setup_direct_dde(dde, pa, len);
- WARN_ON(ret < len);
- return 0;
- }
-
- /* use the DDL */
- for (i = 0; i < DDL_LEN_MAX && len > 0; i++) {
- ret = setup_direct_dde(&ddl[i], pa, len);
- buf += ret;
- len -= ret;
- pa = nx842_get_pa(buf);
- }
-
- if (len > 0) {
- pr_debug("0x%x total %s bytes 0x%x too many for DDL.\n",
- total_len, in ? "input" : "output", len);
- if (in)
- return -EMSGSIZE;
- total_len -= len;
- }
- setup_indirect_dde(dde, ddl, i, total_len);
-
- return 0;
-}
-
-#define CSB_ERR(csb, msg, ...) \
- pr_err("ERROR: " msg " : %02x %02x %02x %02x %08x\n", \
- ##__VA_ARGS__, (csb)->flags, \
- (csb)->cs, (csb)->cc, (csb)->ce, \
- be32_to_cpu((csb)->count))
-
-#define CSB_ERR_ADDR(csb, msg, ...) \
- CSB_ERR(csb, msg " at %lx", ##__VA_ARGS__, \
- (unsigned long)be64_to_cpu((csb)->address))
-
-/**
- * wait_for_csb
- */
-static int wait_for_csb(struct nx842_workmem *wmem,
- struct coprocessor_status_block *csb)
-{
- ktime_t start = wmem->start, now = ktime_get();
- ktime_t timeout = ktime_add_ms(start, CSB_WAIT_MAX);
-
- while (!(READ_ONCE(csb->flags) & CSB_V)) {
- cpu_relax();
- now = ktime_get();
- if (ktime_after(now, timeout))
- break;
- }
-
- /* hw has updated csb and output buffer */
- barrier();
-
- /* check CSB flags */
- if (!(csb->flags & CSB_V)) {
- CSB_ERR(csb, "CSB still not valid after %ld us, giving up",
- (long)ktime_us_delta(now, start));
- return -ETIMEDOUT;
- }
- if (csb->flags & CSB_F) {
- CSB_ERR(csb, "Invalid CSB format");
- return -EPROTO;
- }
- if (csb->flags & CSB_CH) {
- CSB_ERR(csb, "Invalid CSB chaining state");
- return -EPROTO;
- }
-
- /* verify CSB completion sequence is 0 */
- if (csb->cs) {
- CSB_ERR(csb, "Invalid CSB completion sequence");
- return -EPROTO;
- }
-
- /* check CSB Completion Code */
- switch (csb->cc) {
- /* no error */
- case CSB_CC_SUCCESS:
- break;
- case CSB_CC_TPBC_GT_SPBC:
- /* not an error, but the compressed data is
- * larger than the uncompressed data :(
- */
- break;
-
- /* input data errors */
- case CSB_CC_OPERAND_OVERLAP:
- /* input and output buffers overlap */
- CSB_ERR(csb, "Operand Overlap error");
- return -EINVAL;
- case CSB_CC_INVALID_OPERAND:
- CSB_ERR(csb, "Invalid operand");
- return -EINVAL;
- case CSB_CC_NOSPC:
- /* output buffer too small */
- return -ENOSPC;
- case CSB_CC_ABORT:
- CSB_ERR(csb, "Function aborted");
- return -EINTR;
- case CSB_CC_CRC_MISMATCH:
- CSB_ERR(csb, "CRC mismatch");
- return -EINVAL;
- case CSB_CC_TEMPL_INVALID:
- CSB_ERR(csb, "Compressed data template invalid");
- return -EINVAL;
- case CSB_CC_TEMPL_OVERFLOW:
- CSB_ERR(csb, "Compressed data template shows data past end");
- return -EINVAL;
- case CSB_CC_EXCEED_BYTE_COUNT: /* P9 or later */
- /*
- * DDE byte count exceeds the limit specified in Maximum
- * byte count register.
- */
- CSB_ERR(csb, "DDE byte count exceeds the limit");
- return -EINVAL;
-
- /* these should not happen */
- case CSB_CC_INVALID_ALIGN:
- /* setup_ddl should have detected this */
- CSB_ERR_ADDR(csb, "Invalid alignment");
- return -EINVAL;
- case CSB_CC_DATA_LENGTH:
- /* setup_ddl should have detected this */
- CSB_ERR(csb, "Invalid data length");
- return -EINVAL;
- case CSB_CC_WR_TRANSLATION:
- case CSB_CC_TRANSLATION:
- case CSB_CC_TRANSLATION_DUP1:
- case CSB_CC_TRANSLATION_DUP2:
- case CSB_CC_TRANSLATION_DUP3:
- case CSB_CC_TRANSLATION_DUP4:
- case CSB_CC_TRANSLATION_DUP5:
- case CSB_CC_TRANSLATION_DUP6:
- /* should not happen, we use physical addrs */
- CSB_ERR_ADDR(csb, "Translation error");
- return -EPROTO;
- case CSB_CC_WR_PROTECTION:
- case CSB_CC_PROTECTION:
- case CSB_CC_PROTECTION_DUP1:
- case CSB_CC_PROTECTION_DUP2:
- case CSB_CC_PROTECTION_DUP3:
- case CSB_CC_PROTECTION_DUP4:
- case CSB_CC_PROTECTION_DUP5:
- case CSB_CC_PROTECTION_DUP6:
- /* should not happen, we use physical addrs */
- CSB_ERR_ADDR(csb, "Protection error");
- return -EPROTO;
- case CSB_CC_PRIVILEGE:
- /* shouldn't happen, we're in HYP mode */
- CSB_ERR(csb, "Insufficient Privilege error");
- return -EPROTO;
- case CSB_CC_EXCESSIVE_DDE:
- /* shouldn't happen, setup_ddl doesn't use many dde's */
- CSB_ERR(csb, "Too many DDEs in DDL");
- return -EINVAL;
- case CSB_CC_TRANSPORT:
- case CSB_CC_INVALID_CRB: /* P9 or later */
- /* shouldn't happen, we setup CRB correctly */
- CSB_ERR(csb, "Invalid CRB");
- return -EINVAL;
- case CSB_CC_INVALID_DDE: /* P9 or later */
- /*
- * shouldn't happen, setup_direct/indirect_dde creates
- * DDE right
- */
- CSB_ERR(csb, "Invalid DDE");
- return -EINVAL;
- case CSB_CC_SEGMENTED_DDL:
- /* shouldn't happen, setup_ddl creates DDL right */
- CSB_ERR(csb, "Segmented DDL error");
- return -EINVAL;
- case CSB_CC_DDE_OVERFLOW:
- /* shouldn't happen, setup_ddl creates DDL right */
- CSB_ERR(csb, "DDE overflow error");
- return -EINVAL;
- case CSB_CC_SESSION:
- /* should not happen with ICSWX */
- CSB_ERR(csb, "Session violation error");
- return -EPROTO;
- case CSB_CC_CHAIN:
- /* should not happen, we don't use chained CRBs */
- CSB_ERR(csb, "Chained CRB error");
- return -EPROTO;
- case CSB_CC_SEQUENCE:
- /* should not happen, we don't use chained CRBs */
- CSB_ERR(csb, "CRB sequence number error");
- return -EPROTO;
- case CSB_CC_UNKNOWN_CODE:
- CSB_ERR(csb, "Unknown subfunction code");
- return -EPROTO;
-
- /* hardware errors */
- case CSB_CC_RD_EXTERNAL:
- case CSB_CC_RD_EXTERNAL_DUP1:
- case CSB_CC_RD_EXTERNAL_DUP2:
- case CSB_CC_RD_EXTERNAL_DUP3:
- CSB_ERR_ADDR(csb, "Read error outside coprocessor");
- return -EPROTO;
- case CSB_CC_WR_EXTERNAL:
- CSB_ERR_ADDR(csb, "Write error outside coprocessor");
- return -EPROTO;
- case CSB_CC_INTERNAL:
- CSB_ERR(csb, "Internal error in coprocessor");
- return -EPROTO;
- case CSB_CC_PROVISION:
- CSB_ERR(csb, "Storage provision error");
- return -EPROTO;
- case CSB_CC_HW:
- CSB_ERR(csb, "Correctable hardware error");
- return -EPROTO;
- case CSB_CC_HW_EXPIRED_TIMER: /* P9 or later */
- CSB_ERR(csb, "Job did not finish within allowed time");
- return -EPROTO;
-
- default:
- CSB_ERR(csb, "Invalid CC %d", csb->cc);
- return -EPROTO;
- }
-
- /* check Completion Extension state */
- if (csb->ce & CSB_CE_TERMINATION) {
- CSB_ERR(csb, "CSB request was terminated");
- return -EPROTO;
- }
- if (csb->ce & CSB_CE_INCOMPLETE) {
- CSB_ERR(csb, "CSB request not complete");
- return -EPROTO;
- }
- if (!(csb->ce & CSB_CE_TPBC)) {
- CSB_ERR(csb, "TPBC not provided, unknown target length");
- return -EPROTO;
- }
-
- /* successful completion */
- pr_debug_ratelimited("Processed %u bytes in %lu us\n",
- be32_to_cpu(csb->count),
- (unsigned long)ktime_us_delta(now, start));
-
- return 0;
-}
-
-static int nx842_config_crb(const unsigned char *in, unsigned int inlen,
- unsigned char *out, unsigned int outlen,
- struct nx842_workmem *wmem)
-{
- struct coprocessor_request_block *crb;
- struct coprocessor_status_block *csb;
- u64 csb_addr;
- int ret;
-
- crb = &wmem->crb;
- csb = &crb->csb;
-
- /* Clear any previous values */
- memset(crb, 0, sizeof(*crb));
-
- /* set up DDLs */
- ret = setup_ddl(&crb->source, wmem->ddl_in,
- (unsigned char *)in, inlen, true);
- if (ret)
- return ret;
-
- ret = setup_ddl(&crb->target, wmem->ddl_out,
- out, outlen, false);
- if (ret)
- return ret;
-
- /* set up CRB's CSB addr */
- csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS;
- csb_addr |= CRB_CSB_AT; /* Addrs are phys */
- crb->csb_addr = cpu_to_be64(csb_addr);
-
- return 0;
-}
-
-/**
- * nx842_exec_icswx - compress/decompress data using the 842 algorithm
- *
- * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
- * This compresses or decompresses the provided input buffer into the provided
- * output buffer.
- *
- * Upon return from this function @outlen contains the length of the
- * output data. If there is an error then @outlen will be 0 and an
- * error will be specified by the return code from this function.
- *
- * The @workmem buffer should only be used by one function call at a time.
- *
- * @in: input buffer pointer
- * @inlen: input buffer size
- * @out: output buffer pointer
- * @outlenp: output buffer size pointer
- * @workmem: working memory buffer pointer, size determined by
- * nx842_powernv_driver.workmem_size
- * @fc: function code, see CCW Function Codes in nx-842.h
- *
- * Returns:
- * 0 Success, output of length @outlenp stored in the buffer at @out
- * -ENODEV Hardware unavailable
- * -ENOSPC Output buffer is to small
- * -EMSGSIZE Input buffer too large
- * -EINVAL buffer constraints do not fix nx842_constraints
- * -EPROTO hardware error during operation
- * -ETIMEDOUT hardware did not complete operation in reasonable time
- * -EINTR operation was aborted
- */
-static int nx842_exec_icswx(const unsigned char *in, unsigned int inlen,
- unsigned char *out, unsigned int *outlenp,
- void *workmem, int fc)
-{
- struct coprocessor_request_block *crb;
- struct coprocessor_status_block *csb;
- struct nx842_workmem *wmem;
- int ret;
- u32 ccw;
- unsigned int outlen = *outlenp;
-
- wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);
-
- *outlenp = 0;
-
- /* shoudn't happen, we don't load without a coproc */
- if (!nx842_ct) {
- pr_err_ratelimited("coprocessor CT is 0");
- return -ENODEV;
- }
-
- ret = nx842_config_crb(in, inlen, out, outlen, wmem);
- if (ret)
- return ret;
-
- crb = &wmem->crb;
- csb = &crb->csb;
-
- /* set up CCW */
- ccw = 0;
- ccw = SET_FIELD(CCW_CT, ccw, nx842_ct);
- ccw = SET_FIELD(CCW_CI_842, ccw, 0); /* use 0 for hw auto-selection */
- ccw = SET_FIELD(CCW_FC_842, ccw, fc);
-
- wmem->start = ktime_get();
-
- /* do ICSWX */
- ret = icswx(cpu_to_be32(ccw), crb);
-
- pr_debug_ratelimited("icswx CR %x ccw %x crb->ccw %x\n", ret,
- (unsigned int)ccw,
- (unsigned int)be32_to_cpu(crb->ccw));
-
- /*
- * NX842 coprocessor sets 3rd bit in CR register with XER[S0].
- * XER[S0] is the integer summary overflow bit which is nothing
- * to do NX. Since this bit can be set with other return values,
- * mask this bit.
- */
- ret &= ~ICSWX_XERS0;
-
- switch (ret) {
- case ICSWX_INITIATED:
- ret = wait_for_csb(wmem, csb);
- break;
- case ICSWX_BUSY:
- pr_debug_ratelimited("842 Coprocessor busy\n");
- ret = -EBUSY;
- break;
- case ICSWX_REJECTED:
- pr_err_ratelimited("ICSWX rejected\n");
- ret = -EPROTO;
- break;
- }
-
- if (!ret)
- *outlenp = be32_to_cpu(csb->count);
-
- return ret;
-}
-
-/**
- * nx842_exec_vas - compress/decompress data using the 842 algorithm
- *
- * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
- * This compresses or decompresses the provided input buffer into the provided
- * output buffer.
- *
- * Upon return from this function @outlen contains the length of the
- * output data. If there is an error then @outlen will be 0 and an
- * error will be specified by the return code from this function.
- *
- * The @workmem buffer should only be used by one function call at a time.
- *
- * @in: input buffer pointer
- * @inlen: input buffer size
- * @out: output buffer pointer
- * @outlenp: output buffer size pointer
- * @workmem: working memory buffer pointer, size determined by
- * nx842_powernv_driver.workmem_size
- * @fc: function code, see CCW Function Codes in nx-842.h
- *
- * Returns:
- * 0 Success, output of length @outlenp stored in the buffer
- * at @out
- * -ENODEV Hardware unavailable
- * -ENOSPC Output buffer is to small
- * -EMSGSIZE Input buffer too large
- * -EINVAL buffer constraints do not fix nx842_constraints
- * -EPROTO hardware error during operation
- * -ETIMEDOUT hardware did not complete operation in reasonable time
- * -EINTR operation was aborted
- */
-static int nx842_exec_vas(const unsigned char *in, unsigned int inlen,
- unsigned char *out, unsigned int *outlenp,
- void *workmem, int fc)
-{
- struct coprocessor_request_block *crb;
- struct coprocessor_status_block *csb;
- struct nx842_workmem *wmem;
- struct vas_window *txwin;
- int ret, i = 0;
- u32 ccw;
- unsigned int outlen = *outlenp;
-
- wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);
-
- *outlenp = 0;
-
- crb = &wmem->crb;
- csb = &crb->csb;
-
- ret = nx842_config_crb(in, inlen, out, outlen, wmem);
- if (ret)
- return ret;
-
- ccw = 0;
- ccw = SET_FIELD(CCW_FC_842, ccw, fc);
- crb->ccw = cpu_to_be32(ccw);
-
- do {
- wmem->start = ktime_get();
- preempt_disable();
- txwin = this_cpu_read(cpu_txwin);
-
- /*
- * VAS copy CRB into L2 cache. Refer <asm/vas.h>.
- * @crb and @offset.
- */
- vas_copy_crb(crb, 0);
-
- /*
- * VAS paste previously copied CRB to NX.
- * @txwin, @offset and @last (must be true).
- */
- ret = vas_paste_crb(txwin, 0, 1);
- preempt_enable();
- /*
- * Retry copy/paste function for VAS failures.
- */
- } while (ret && (i++ < VAS_RETRIES));
-
- if (ret) {
- pr_err_ratelimited("VAS copy/paste failed\n");
- return ret;
- }
-
- ret = wait_for_csb(wmem, csb);
- if (!ret)
- *outlenp = be32_to_cpu(csb->count);
-
- return ret;
-}
-
-/**
- * nx842_powernv_compress - Compress data using the 842 algorithm
- *
- * Compression provided by the NX842 coprocessor on IBM PowerNV systems.
- * The input buffer is compressed and the result is stored in the
- * provided output buffer.
- *
- * Upon return from this function @outlen contains the length of the
- * compressed data. If there is an error then @outlen will be 0 and an
- * error will be specified by the return code from this function.
- *
- * @in: input buffer pointer
- * @inlen: input buffer size
- * @out: output buffer pointer
- * @outlenp: output buffer size pointer
- * @workmem: working memory buffer pointer, size determined by
- * nx842_powernv_driver.workmem_size
- *
- * Returns: see @nx842_powernv_exec()
- */
-static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen,
- unsigned char *out, unsigned int *outlenp,
- void *wmem)
-{
- return nx842_powernv_exec(in, inlen, out, outlenp,
- wmem, CCW_FC_842_COMP_CRC);
-}
-
-/**
- * nx842_powernv_decompress - Decompress data using the 842 algorithm
- *
- * Decompression provided by the NX842 coprocessor on IBM PowerNV systems.
- * The input buffer is decompressed and the result is stored in the
- * provided output buffer.
- *
- * Upon return from this function @outlen contains the length of the
- * decompressed data. If there is an error then @outlen will be 0 and an
- * error will be specified by the return code from this function.
- *
- * @in: input buffer pointer
- * @inlen: input buffer size
- * @out: output buffer pointer
- * @outlenp: output buffer size pointer
- * @workmem: working memory buffer pointer, size determined by
- * nx842_powernv_driver.workmem_size
- *
- * Returns: see @nx842_powernv_exec()
- */
-static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen,
- unsigned char *out, unsigned int *outlenp,
- void *wmem)
-{
- return nx842_powernv_exec(in, inlen, out, outlenp,
- wmem, CCW_FC_842_DECOMP_CRC);
-}
-
-static inline void nx842_add_coprocs_list(struct nx842_coproc *coproc,
- int chipid)
-{
- coproc->chip_id = chipid;
- INIT_LIST_HEAD(&coproc->list);
- list_add(&coproc->list, &nx842_coprocs);
-}
-
-static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc)
-{
- struct vas_window *txwin = NULL;
- struct vas_tx_win_attr txattr;
-
- /*
- * Kernel requests will be high priority. So open send
- * windows only for high priority RxFIFO entries.
- */
- vas_init_tx_win_attr(&txattr, coproc->ct);
- txattr.lpid = 0; /* lpid is 0 for kernel requests */
- txattr.pid = 0; /* pid is 0 for kernel requests */
-
- /*
- * Open a VAS send window which is used to send request to NX.
- */
- txwin = vas_tx_win_open(coproc->vas.id, coproc->ct, &txattr);
- if (IS_ERR(txwin))
- pr_err("ibm,nx-842: Can not open TX window: %ld\n",
- PTR_ERR(txwin));
-
- return txwin;
-}
-
-/*
- * Identify chip ID for each CPU, open send wndow for the corresponding NX
- * engine and save txwin in percpu cpu_txwin.
- * cpu_txwin is used in copy/paste operation for each compression /
- * decompression request.
- */
-static int nx842_open_percpu_txwins(void)
-{
- struct nx842_coproc *coproc, *n;
- unsigned int i, chip_id;
-
- for_each_possible_cpu(i) {
- struct vas_window *txwin = NULL;
-
- chip_id = cpu_to_chip_id(i);
-
- list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
- /*
- * Kernel requests use only high priority FIFOs. So
- * open send windows for these FIFOs.
- */
-
- if (coproc->ct != VAS_COP_TYPE_842_HIPRI)
- continue;
-
- if (coproc->chip_id == chip_id) {
- txwin = nx842_alloc_txwin(coproc);
- if (IS_ERR(txwin))
- return PTR_ERR(txwin);
-
- per_cpu(cpu_txwin, i) = txwin;
- break;
- }
- }
-
- if (!per_cpu(cpu_txwin, i)) {
- /* shouldn't happen, Each chip will have NX engine */
- pr_err("NX engine is not available for CPU %d\n", i);
- return -EINVAL;
- }
- }
-
- return 0;
-}
-
-static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
- int vasid, int *ct)
-{
- struct vas_window *rxwin = NULL;
- struct vas_rx_win_attr rxattr;
- struct nx842_coproc *coproc;
- u32 lpid, pid, tid, fifo_size;
- u64 rx_fifo;
- const char *priority;
- int ret;
-
- ret = of_property_read_u64(dn, "rx-fifo-address", &rx_fifo);
- if (ret) {
- pr_err("Missing rx-fifo-address property\n");
- return ret;
- }
-
- ret = of_property_read_u32(dn, "rx-fifo-size", &fifo_size);
- if (ret) {
- pr_err("Missing rx-fifo-size property\n");
- return ret;
- }
-
- ret = of_property_read_u32(dn, "lpid", &lpid);
- if (ret) {
- pr_err("Missing lpid property\n");
- return ret;
- }
-
- ret = of_property_read_u32(dn, "pid", &pid);
- if (ret) {
- pr_err("Missing pid property\n");
- return ret;
- }
-
- ret = of_property_read_u32(dn, "tid", &tid);
- if (ret) {
- pr_err("Missing tid property\n");
- return ret;
- }
-
- ret = of_property_read_string(dn, "priority", &priority);
- if (ret) {
- pr_err("Missing priority property\n");
- return ret;
- }
-
- coproc = kzalloc(sizeof(*coproc), GFP_KERNEL);
- if (!coproc)
- return -ENOMEM;
-
- if (!strcmp(priority, "High"))
- coproc->ct = VAS_COP_TYPE_842_HIPRI;
- else if (!strcmp(priority, "Normal"))
- coproc->ct = VAS_COP_TYPE_842;
- else {
- pr_err("Invalid RxFIFO priority value\n");
- ret = -EINVAL;
- goto err_out;
- }
-
- vas_init_rx_win_attr(&rxattr, coproc->ct);
- rxattr.rx_fifo = (void *)rx_fifo;
- rxattr.rx_fifo_size = fifo_size;
- rxattr.lnotify_lpid = lpid;
- rxattr.lnotify_pid = pid;
- rxattr.lnotify_tid = tid;
- /*
- * Maximum RX window credits can not be more than #CRBs in
- * RxFIFO. Otherwise, can get checkstop if RxFIFO overruns.
- */
- rxattr.wcreds_max = fifo_size / CRB_SIZE;
-
- /*
- * Open a VAS receice window which is used to configure RxFIFO
- * for NX.
- */
- rxwin = vas_rx_win_open(vasid, coproc->ct, &rxattr);
- if (IS_ERR(rxwin)) {
- ret = PTR_ERR(rxwin);
- pr_err("setting RxFIFO with VAS failed: %d\n",
- ret);
- goto err_out;
- }
-
- coproc->vas.rxwin = rxwin;
- coproc->vas.id = vasid;
- nx842_add_coprocs_list(coproc, chip_id);
-
- /*
- * (lpid, pid, tid) combination has to be unique for each
- * coprocessor instance in the system. So to make it
- * unique, skiboot uses coprocessor type such as 842 or
- * GZIP for pid and provides this value to kernel in pid
- * device-tree property.
- */
- *ct = pid;
-
- return 0;
-
-err_out:
- kfree(coproc);
- return ret;
-}
-
-
-static int __init nx842_powernv_probe_vas(struct device_node *pn)
-{
- struct device_node *dn;
- int chip_id, vasid, ret = 0;
- int nx_fifo_found = 0;
- int uninitialized_var(ct);
-
- chip_id = of_get_ibm_chip_id(pn);
- if (chip_id < 0) {
- pr_err("ibm,chip-id missing\n");
- return -EINVAL;
- }
-
- vasid = chip_to_vas_id(chip_id);
- if (vasid < 0) {
- pr_err("Unable to map chip_id %d to vasid\n", chip_id);
- return -EINVAL;
- }
-
- for_each_child_of_node(pn, dn) {
- if (of_device_is_compatible(dn, "ibm,p9-nx-842")) {
- ret = vas_cfg_coproc_info(dn, chip_id, vasid, &ct);
- if (ret) {
- of_node_put(dn);
- return ret;
- }
- nx_fifo_found++;
- }
- }
-
- if (!nx_fifo_found) {
- pr_err("NX842 FIFO nodes are missing\n");
- return -EINVAL;
- }
-
- /*
- * Initialize NX instance for both high and normal priority FIFOs.
- */
- if (opal_check_token(OPAL_NX_COPROC_INIT)) {
- ret = opal_nx_coproc_init(chip_id, ct);
- if (ret) {
- pr_err("Failed to initialize NX for chip(%d): %d\n",
- chip_id, ret);
- ret = opal_error_code(ret);
- }
- } else
- pr_warn("Firmware doesn't support NX initialization\n");
-
- return ret;
-}
-
-static int __init nx842_powernv_probe(struct device_node *dn)
-{
- struct nx842_coproc *coproc;
- unsigned int ct, ci;
- int chip_id;
-
- chip_id = of_get_ibm_chip_id(dn);
- if (chip_id < 0) {
- pr_err("ibm,chip-id missing\n");
- return -EINVAL;
- }
-
- if (of_property_read_u32(dn, "ibm,842-coprocessor-type", &ct)) {
- pr_err("ibm,842-coprocessor-type missing\n");
- return -EINVAL;
- }
-
- if (of_property_read_u32(dn, "ibm,842-coprocessor-instance", &ci)) {
- pr_err("ibm,842-coprocessor-instance missing\n");
- return -EINVAL;
- }
-
- coproc = kzalloc(sizeof(*coproc), GFP_KERNEL);
- if (!coproc)
- return -ENOMEM;
-
- coproc->ct = ct;
- coproc->ci = ci;
- nx842_add_coprocs_list(coproc, chip_id);
-
- pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci);
-
- if (!nx842_ct)
- nx842_ct = ct;
- else if (nx842_ct != ct)
- pr_err("NX842 chip %d, CT %d != first found CT %d\n",
- chip_id, ct, nx842_ct);
-
- return 0;
-}
-
-static void nx842_delete_coprocs(void)
-{
- struct nx842_coproc *coproc, *n;
- struct vas_window *txwin;
- int i;
-
- /*
- * close percpu txwins that are opened for the corresponding coproc.
- */
- for_each_possible_cpu(i) {
- txwin = per_cpu(cpu_txwin, i);
- if (txwin)
- vas_win_close(txwin);
-
- per_cpu(cpu_txwin, i) = 0;
- }
-
- list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
- if (coproc->vas.rxwin)
- vas_win_close(coproc->vas.rxwin);
-
- list_del(&coproc->list);
- kfree(coproc);
- }
-}
-
-static struct nx842_constraints nx842_powernv_constraints = {
- .alignment = DDE_BUFFER_ALIGN,
- .multiple = DDE_BUFFER_LAST_MULT,
- .minimum = DDE_BUFFER_LAST_MULT,
- .maximum = (DDL_LEN_MAX - 1) * PAGE_SIZE,
-};
-
-static struct nx842_driver nx842_powernv_driver = {
- .name = KBUILD_MODNAME,
- .owner = THIS_MODULE,
- .workmem_size = sizeof(struct nx842_workmem),
- .constraints = &nx842_powernv_constraints,
- .compress = nx842_powernv_compress,
- .decompress = nx842_powernv_decompress,
-};
-
-static int nx842_powernv_crypto_init(struct crypto_tfm *tfm)
-{
- return nx842_crypto_init(tfm, &nx842_powernv_driver);
-}
-
-static struct crypto_alg nx842_powernv_alg = {
- .cra_name = "842",
- .cra_driver_name = "842-nx",
- .cra_priority = 300,
- .cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
- .cra_ctxsize = sizeof(struct nx842_crypto_ctx),
- .cra_module = THIS_MODULE,
- .cra_init = nx842_powernv_crypto_init,
- .cra_exit = nx842_crypto_exit,
- .cra_u = { .compress = {
- .coa_compress = nx842_crypto_compress,
- .coa_decompress = nx842_crypto_decompress } }
-};
-
-static __init int nx842_powernv_init(void)
-{
- struct device_node *dn;
- int ret;
-
- /* verify workmem size/align restrictions */
- BUILD_BUG_ON(WORKMEM_ALIGN % CRB_ALIGN);
- BUILD_BUG_ON(CRB_ALIGN % DDE_ALIGN);
- BUILD_BUG_ON(CRB_SIZE % DDE_ALIGN);
- /* verify buffer size/align restrictions */
- BUILD_BUG_ON(PAGE_SIZE % DDE_BUFFER_ALIGN);
- BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT);
- BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT);
-
- for_each_compatible_node(dn, NULL, "ibm,power9-nx") {
- ret = nx842_powernv_probe_vas(dn);
- if (ret) {
- nx842_delete_coprocs();
- of_node_put(dn);
- return ret;
- }
- }
-
- if (list_empty(&nx842_coprocs)) {
- for_each_compatible_node(dn, NULL, "ibm,power-nx")
- nx842_powernv_probe(dn);
-
- if (!nx842_ct)
- return -ENODEV;
-
- nx842_powernv_exec = nx842_exec_icswx;
- } else {
- ret = nx842_open_percpu_txwins();
- if (ret) {
- nx842_delete_coprocs();
- return ret;
- }
-
- nx842_powernv_exec = nx842_exec_vas;
- }
-
- ret = crypto_register_alg(&nx842_powernv_alg);
- if (ret) {
- nx842_delete_coprocs();
- return ret;
- }
-
- return 0;
-}
-module_init(nx842_powernv_init);
-
-static void __exit nx842_powernv_exit(void)
-{
- crypto_unregister_alg(&nx842_powernv_alg);
-
- nx842_delete_coprocs();
-}
-module_exit(nx842_powernv_exit);
diff --git a/drivers/crypto/nx/nx-common-powernv.c b/drivers/crypto/nx/nx-common-powernv.c
new file mode 100644
index 0000000..f42881f
--- /dev/null
+++ b/drivers/crypto/nx/nx-common-powernv.c
@@ -0,0 +1,1062 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for IBM PowerNV compression accelerator
+ *
+ * Copyright (C) 2015 Dan Streetman, IBM Corp
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include "nx-842.h"
+
+#include <linux/timer.h>
+
+#include <asm/prom.h>
+#include <asm/icswx.h>
+#include <asm/vas.h>
+#include <asm/reg.h>
+#include <asm/opal-api.h>
+#include <asm/opal.h>
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
+MODULE_DESCRIPTION("H/W Compression driver for IBM PowerNV processors");
+MODULE_ALIAS_CRYPTO("842");
+MODULE_ALIAS_CRYPTO("842-nx");
+
+#define WORKMEM_ALIGN (CRB_ALIGN)
+#define CSB_WAIT_MAX (5000) /* ms */
+#define VAS_RETRIES (10)
+
+struct nx842_workmem {
+ /* Below fields must be properly aligned */
+ struct coprocessor_request_block crb; /* CRB_ALIGN align */
+ struct data_descriptor_entry ddl_in[DDL_LEN_MAX]; /* DDE_ALIGN align */
+ struct data_descriptor_entry ddl_out[DDL_LEN_MAX]; /* DDE_ALIGN align */
+ /* Above fields must be properly aligned */
+
+ ktime_t start;
+
+ char padding[WORKMEM_ALIGN]; /* unused, to allow alignment */
+} __packed __aligned(WORKMEM_ALIGN);
+
+struct nx842_coproc {
+ unsigned int chip_id;
+ unsigned int ct;
+ unsigned int ci; /* Coprocessor instance, used with icswx */
+ struct {
+ struct vas_window *rxwin;
+ int id;
+ } vas;
+ struct list_head list;
+};
+
+/*
+ * Send the request to NX engine on the chip for the corresponding CPU
+ * where the process is executing. Use with VAS function.
+ */
+static DEFINE_PER_CPU(struct vas_window *, cpu_txwin);
+
+/* no cpu hotplug on powernv, so this list never changes after init */
+static LIST_HEAD(nx842_coprocs);
+static unsigned int nx842_ct; /* used in icswx function */
+
+static int (*nx842_powernv_exec)(const unsigned char *in,
+ unsigned int inlen, unsigned char *out,
+ unsigned int *outlenp, void *workmem, int fc);
+
+/**
+ * setup_indirect_dde - Setup an indirect DDE
+ *
+ * The DDE is setup with the the DDE count, byte count, and address of
+ * first direct DDE in the list.
+ */
+static void setup_indirect_dde(struct data_descriptor_entry *dde,
+ struct data_descriptor_entry *ddl,
+ unsigned int dde_count, unsigned int byte_count)
+{
+ dde->flags = 0;
+ dde->count = dde_count;
+ dde->index = 0;
+ dde->length = cpu_to_be32(byte_count);
+ dde->address = cpu_to_be64(nx842_get_pa(ddl));
+}
+
+/**
+ * setup_direct_dde - Setup single DDE from buffer
+ *
+ * The DDE is setup with the buffer and length. The buffer must be properly
+ * aligned. The used length is returned.
+ * Returns:
+ * N Successfully set up DDE with N bytes
+ */
+static unsigned int setup_direct_dde(struct data_descriptor_entry *dde,
+ unsigned long pa, unsigned int len)
+{
+ unsigned int l = min_t(unsigned int, len, LEN_ON_PAGE(pa));
+
+ dde->flags = 0;
+ dde->count = 0;
+ dde->index = 0;
+ dde->length = cpu_to_be32(l);
+ dde->address = cpu_to_be64(pa);
+
+ return l;
+}
+
+/**
+ * setup_ddl - Setup DDL from buffer
+ *
+ * Returns:
+ * 0 Successfully set up DDL
+ */
+static int setup_ddl(struct data_descriptor_entry *dde,
+ struct data_descriptor_entry *ddl,
+ unsigned char *buf, unsigned int len,
+ bool in)
+{
+ unsigned long pa = nx842_get_pa(buf);
+ int i, ret, total_len = len;
+
+ if (!IS_ALIGNED(pa, DDE_BUFFER_ALIGN)) {
+ pr_debug("%s buffer pa 0x%lx not 0x%x-byte aligned\n",
+ in ? "input" : "output", pa, DDE_BUFFER_ALIGN);
+ return -EINVAL;
+ }
+
+ /* only need to check last mult; since buffer must be
+ * DDE_BUFFER_ALIGN aligned, and that is a multiple of
+ * DDE_BUFFER_SIZE_MULT, and pre-last page DDE buffers
+ * are guaranteed a multiple of DDE_BUFFER_SIZE_MULT.
+ */
+ if (len % DDE_BUFFER_LAST_MULT) {
+ pr_debug("%s buffer len 0x%x not a multiple of 0x%x\n",
+ in ? "input" : "output", len, DDE_BUFFER_LAST_MULT);
+ if (in)
+ return -EINVAL;
+ len = round_down(len, DDE_BUFFER_LAST_MULT);
+ }
+
+ /* use a single direct DDE */
+ if (len <= LEN_ON_PAGE(pa)) {
+ ret = setup_direct_dde(dde, pa, len);
+ WARN_ON(ret < len);
+ return 0;
+ }
+
+ /* use the DDL */
+ for (i = 0; i < DDL_LEN_MAX && len > 0; i++) {
+ ret = setup_direct_dde(&ddl[i], pa, len);
+ buf += ret;
+ len -= ret;
+ pa = nx842_get_pa(buf);
+ }
+
+ if (len > 0) {
+ pr_debug("0x%x total %s bytes 0x%x too many for DDL.\n",
+ total_len, in ? "input" : "output", len);
+ if (in)
+ return -EMSGSIZE;
+ total_len -= len;
+ }
+ setup_indirect_dde(dde, ddl, i, total_len);
+
+ return 0;
+}
+
+#define CSB_ERR(csb, msg, ...) \
+ pr_err("ERROR: " msg " : %02x %02x %02x %02x %08x\n", \
+ ##__VA_ARGS__, (csb)->flags, \
+ (csb)->cs, (csb)->cc, (csb)->ce, \
+ be32_to_cpu((csb)->count))
+
+#define CSB_ERR_ADDR(csb, msg, ...) \
+ CSB_ERR(csb, msg " at %lx", ##__VA_ARGS__, \
+ (unsigned long)be64_to_cpu((csb)->address))
+
+/**
+ * wait_for_csb
+ */
+static int wait_for_csb(struct nx842_workmem *wmem,
+ struct coprocessor_status_block *csb)
+{
+ ktime_t start = wmem->start, now = ktime_get();
+ ktime_t timeout = ktime_add_ms(start, CSB_WAIT_MAX);
+
+ while (!(READ_ONCE(csb->flags) & CSB_V)) {
+ cpu_relax();
+ now = ktime_get();
+ if (ktime_after(now, timeout))
+ break;
+ }
+
+ /* hw has updated csb and output buffer */
+ barrier();
+
+ /* check CSB flags */
+ if (!(csb->flags & CSB_V)) {
+ CSB_ERR(csb, "CSB still not valid after %ld us, giving up",
+ (long)ktime_us_delta(now, start));
+ return -ETIMEDOUT;
+ }
+ if (csb->flags & CSB_F) {
+ CSB_ERR(csb, "Invalid CSB format");
+ return -EPROTO;
+ }
+ if (csb->flags & CSB_CH) {
+ CSB_ERR(csb, "Invalid CSB chaining state");
+ return -EPROTO;
+ }
+
+ /* verify CSB completion sequence is 0 */
+ if (csb->cs) {
+ CSB_ERR(csb, "Invalid CSB completion sequence");
+ return -EPROTO;
+ }
+
+ /* check CSB Completion Code */
+ switch (csb->cc) {
+ /* no error */
+ case CSB_CC_SUCCESS:
+ break;
+ case CSB_CC_TPBC_GT_SPBC:
+ /* not an error, but the compressed data is
+ * larger than the uncompressed data :(
+ */
+ break;
+
+ /* input data errors */
+ case CSB_CC_OPERAND_OVERLAP:
+ /* input and output buffers overlap */
+ CSB_ERR(csb, "Operand Overlap error");
+ return -EINVAL;
+ case CSB_CC_INVALID_OPERAND:
+ CSB_ERR(csb, "Invalid operand");
+ return -EINVAL;
+ case CSB_CC_NOSPC:
+ /* output buffer too small */
+ return -ENOSPC;
+ case CSB_CC_ABORT:
+ CSB_ERR(csb, "Function aborted");
+ return -EINTR;
+ case CSB_CC_CRC_MISMATCH:
+ CSB_ERR(csb, "CRC mismatch");
+ return -EINVAL;
+ case CSB_CC_TEMPL_INVALID:
+ CSB_ERR(csb, "Compressed data template invalid");
+ return -EINVAL;
+ case CSB_CC_TEMPL_OVERFLOW:
+ CSB_ERR(csb, "Compressed data template shows data past end");
+ return -EINVAL;
+ case CSB_CC_EXCEED_BYTE_COUNT: /* P9 or later */
+ /*
+ * DDE byte count exceeds the limit specified in Maximum
+ * byte count register.
+ */
+ CSB_ERR(csb, "DDE byte count exceeds the limit");
+ return -EINVAL;
+
+ /* these should not happen */
+ case CSB_CC_INVALID_ALIGN:
+ /* setup_ddl should have detected this */
+ CSB_ERR_ADDR(csb, "Invalid alignment");
+ return -EINVAL;
+ case CSB_CC_DATA_LENGTH:
+ /* setup_ddl should have detected this */
+ CSB_ERR(csb, "Invalid data length");
+ return -EINVAL;
+ case CSB_CC_WR_TRANSLATION:
+ case CSB_CC_TRANSLATION:
+ case CSB_CC_TRANSLATION_DUP1:
+ case CSB_CC_TRANSLATION_DUP2:
+ case CSB_CC_TRANSLATION_DUP3:
+ case CSB_CC_TRANSLATION_DUP4:
+ case CSB_CC_TRANSLATION_DUP5:
+ case CSB_CC_TRANSLATION_DUP6:
+ /* should not happen, we use physical addrs */
+ CSB_ERR_ADDR(csb, "Translation error");
+ return -EPROTO;
+ case CSB_CC_WR_PROTECTION:
+ case CSB_CC_PROTECTION:
+ case CSB_CC_PROTECTION_DUP1:
+ case CSB_CC_PROTECTION_DUP2:
+ case CSB_CC_PROTECTION_DUP3:
+ case CSB_CC_PROTECTION_DUP4:
+ case CSB_CC_PROTECTION_DUP5:
+ case CSB_CC_PROTECTION_DUP6:
+ /* should not happen, we use physical addrs */
+ CSB_ERR_ADDR(csb, "Protection error");
+ return -EPROTO;
+ case CSB_CC_PRIVILEGE:
+ /* shouldn't happen, we're in HYP mode */
+ CSB_ERR(csb, "Insufficient Privilege error");
+ return -EPROTO;
+ case CSB_CC_EXCESSIVE_DDE:
+ /* shouldn't happen, setup_ddl doesn't use many dde's */
+ CSB_ERR(csb, "Too many DDEs in DDL");
+ return -EINVAL;
+ case CSB_CC_TRANSPORT:
+ case CSB_CC_INVALID_CRB: /* P9 or later */
+ /* shouldn't happen, we setup CRB correctly */
+ CSB_ERR(csb, "Invalid CRB");
+ return -EINVAL;
+ case CSB_CC_INVALID_DDE: /* P9 or later */
+ /*
+ * shouldn't happen, setup_direct/indirect_dde creates
+ * DDE right
+ */
+ CSB_ERR(csb, "Invalid DDE");
+ return -EINVAL;
+ case CSB_CC_SEGMENTED_DDL:
+ /* shouldn't happen, setup_ddl creates DDL right */
+ CSB_ERR(csb, "Segmented DDL error");
+ return -EINVAL;
+ case CSB_CC_DDE_OVERFLOW:
+ /* shouldn't happen, setup_ddl creates DDL right */
+ CSB_ERR(csb, "DDE overflow error");
+ return -EINVAL;
+ case CSB_CC_SESSION:
+ /* should not happen with ICSWX */
+ CSB_ERR(csb, "Session violation error");
+ return -EPROTO;
+ case CSB_CC_CHAIN:
+ /* should not happen, we don't use chained CRBs */
+ CSB_ERR(csb, "Chained CRB error");
+ return -EPROTO;
+ case CSB_CC_SEQUENCE:
+ /* should not happen, we don't use chained CRBs */
+ CSB_ERR(csb, "CRB sequence number error");
+ return -EPROTO;
+ case CSB_CC_UNKNOWN_CODE:
+ CSB_ERR(csb, "Unknown subfunction code");
+ return -EPROTO;
+
+ /* hardware errors */
+ case CSB_CC_RD_EXTERNAL:
+ case CSB_CC_RD_EXTERNAL_DUP1:
+ case CSB_CC_RD_EXTERNAL_DUP2:
+ case CSB_CC_RD_EXTERNAL_DUP3:
+ CSB_ERR_ADDR(csb, "Read error outside coprocessor");
+ return -EPROTO;
+ case CSB_CC_WR_EXTERNAL:
+ CSB_ERR_ADDR(csb, "Write error outside coprocessor");
+ return -EPROTO;
+ case CSB_CC_INTERNAL:
+ CSB_ERR(csb, "Internal error in coprocessor");
+ return -EPROTO;
+ case CSB_CC_PROVISION:
+ CSB_ERR(csb, "Storage provision error");
+ return -EPROTO;
+ case CSB_CC_HW:
+ CSB_ERR(csb, "Correctable hardware error");
+ return -EPROTO;
+ case CSB_CC_HW_EXPIRED_TIMER: /* P9 or later */
+ CSB_ERR(csb, "Job did not finish within allowed time");
+ return -EPROTO;
+
+ default:
+ CSB_ERR(csb, "Invalid CC %d", csb->cc);
+ return -EPROTO;
+ }
+
+ /* check Completion Extension state */
+ if (csb->ce & CSB_CE_TERMINATION) {
+ CSB_ERR(csb, "CSB request was terminated");
+ return -EPROTO;
+ }
+ if (csb->ce & CSB_CE_INCOMPLETE) {
+ CSB_ERR(csb, "CSB request not complete");
+ return -EPROTO;
+ }
+ if (!(csb->ce & CSB_CE_TPBC)) {
+ CSB_ERR(csb, "TPBC not provided, unknown target length");
+ return -EPROTO;
+ }
+
+ /* successful completion */
+ pr_debug_ratelimited("Processed %u bytes in %lu us\n",
+ be32_to_cpu(csb->count),
+ (unsigned long)ktime_us_delta(now, start));
+
+ return 0;
+}
+
+static int nx842_config_crb(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int outlen,
+ struct nx842_workmem *wmem)
+{
+ struct coprocessor_request_block *crb;
+ struct coprocessor_status_block *csb;
+ u64 csb_addr;
+ int ret;
+
+ crb = &wmem->crb;
+ csb = &crb->csb;
+
+ /* Clear any previous values */
+ memset(crb, 0, sizeof(*crb));
+
+ /* set up DDLs */
+ ret = setup_ddl(&crb->source, wmem->ddl_in,
+ (unsigned char *)in, inlen, true);
+ if (ret)
+ return ret;
+
+ ret = setup_ddl(&crb->target, wmem->ddl_out,
+ out, outlen, false);
+ if (ret)
+ return ret;
+
+ /* set up CRB's CSB addr */
+ csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS;
+ csb_addr |= CRB_CSB_AT; /* Addrs are phys */
+ crb->csb_addr = cpu_to_be64(csb_addr);
+
+ return 0;
+}
+
+/**
+ * nx842_exec_icswx - compress/decompress data using the 842 algorithm
+ *
+ * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
+ * This compresses or decompresses the provided input buffer into the provided
+ * output buffer.
+ *
+ * Upon return from this function @outlen contains the length of the
+ * output data. If there is an error then @outlen will be 0 and an
+ * error will be specified by the return code from this function.
+ *
+ * The @workmem buffer should only be used by one function call at a time.
+ *
+ * @in: input buffer pointer
+ * @inlen: input buffer size
+ * @out: output buffer pointer
+ * @outlenp: output buffer size pointer
+ * @workmem: working memory buffer pointer, size determined by
+ * nx842_powernv_driver.workmem_size
+ * @fc: function code, see CCW Function Codes in nx-842.h
+ *
+ * Returns:
+ * 0 Success, output of length @outlenp stored in the buffer at @out
+ * -ENODEV Hardware unavailable
+ * -ENOSPC Output buffer is to small
+ * -EMSGSIZE Input buffer too large
+ * -EINVAL buffer constraints do not fix nx842_constraints
+ * -EPROTO hardware error during operation
+ * -ETIMEDOUT hardware did not complete operation in reasonable time
+ * -EINTR operation was aborted
+ */
+static int nx842_exec_icswx(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int *outlenp,
+ void *workmem, int fc)
+{
+ struct coprocessor_request_block *crb;
+ struct coprocessor_status_block *csb;
+ struct nx842_workmem *wmem;
+ int ret;
+ u32 ccw;
+ unsigned int outlen = *outlenp;
+
+ wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);
+
+ *outlenp = 0;
+
+ /* shoudn't happen, we don't load without a coproc */
+ if (!nx842_ct) {
+ pr_err_ratelimited("coprocessor CT is 0");
+ return -ENODEV;
+ }
+
+ ret = nx842_config_crb(in, inlen, out, outlen, wmem);
+ if (ret)
+ return ret;
+
+ crb = &wmem->crb;
+ csb = &crb->csb;
+
+ /* set up CCW */
+ ccw = 0;
+ ccw = SET_FIELD(CCW_CT, ccw, nx842_ct);
+ ccw = SET_FIELD(CCW_CI_842, ccw, 0); /* use 0 for hw auto-selection */
+ ccw = SET_FIELD(CCW_FC_842, ccw, fc);
+
+ wmem->start = ktime_get();
+
+ /* do ICSWX */
+ ret = icswx(cpu_to_be32(ccw), crb);
+
+ pr_debug_ratelimited("icswx CR %x ccw %x crb->ccw %x\n", ret,
+ (unsigned int)ccw,
+ (unsigned int)be32_to_cpu(crb->ccw));
+
+ /*
+ * NX842 coprocessor sets 3rd bit in CR register with XER[S0].
+ * XER[S0] is the integer summary overflow bit which is nothing
+ * to do NX. Since this bit can be set with other return values,
+ * mask this bit.
+ */
+ ret &= ~ICSWX_XERS0;
+
+ switch (ret) {
+ case ICSWX_INITIATED:
+ ret = wait_for_csb(wmem, csb);
+ break;
+ case ICSWX_BUSY:
+ pr_debug_ratelimited("842 Coprocessor busy\n");
+ ret = -EBUSY;
+ break;
+ case ICSWX_REJECTED:
+ pr_err_ratelimited("ICSWX rejected\n");
+ ret = -EPROTO;
+ break;
+ }
+
+ if (!ret)
+ *outlenp = be32_to_cpu(csb->count);
+
+ return ret;
+}
+
+/**
+ * nx842_exec_vas - compress/decompress data using the 842 algorithm
+ *
+ * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
+ * This compresses or decompresses the provided input buffer into the provided
+ * output buffer.
+ *
+ * Upon return from this function @outlen contains the length of the
+ * output data. If there is an error then @outlen will be 0 and an
+ * error will be specified by the return code from this function.
+ *
+ * The @workmem buffer should only be used by one function call at a time.
+ *
+ * @in: input buffer pointer
+ * @inlen: input buffer size
+ * @out: output buffer pointer
+ * @outlenp: output buffer size pointer
+ * @workmem: working memory buffer pointer, size determined by
+ * nx842_powernv_driver.workmem_size
+ * @fc: function code, see CCW Function Codes in nx-842.h
+ *
+ * Returns:
+ * 0 Success, output of length @outlenp stored in the buffer
+ * at @out
+ * -ENODEV Hardware unavailable
+ * -ENOSPC Output buffer is to small
+ * -EMSGSIZE Input buffer too large
+ * -EINVAL buffer constraints do not fix nx842_constraints
+ * -EPROTO hardware error during operation
+ * -ETIMEDOUT hardware did not complete operation in reasonable time
+ * -EINTR operation was aborted
+ */
+static int nx842_exec_vas(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int *outlenp,
+ void *workmem, int fc)
+{
+ struct coprocessor_request_block *crb;
+ struct coprocessor_status_block *csb;
+ struct nx842_workmem *wmem;
+ struct vas_window *txwin;
+ int ret, i = 0;
+ u32 ccw;
+ unsigned int outlen = *outlenp;
+
+ wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);
+
+ *outlenp = 0;
+
+ crb = &wmem->crb;
+ csb = &crb->csb;
+
+ ret = nx842_config_crb(in, inlen, out, outlen, wmem);
+ if (ret)
+ return ret;
+
+ ccw = 0;
+ ccw = SET_FIELD(CCW_FC_842, ccw, fc);
+ crb->ccw = cpu_to_be32(ccw);
+
+ do {
+ wmem->start = ktime_get();
+ preempt_disable();
+ txwin = this_cpu_read(cpu_txwin);
+
+ /*
+ * VAS copy CRB into L2 cache. Refer <asm/vas.h>.
+ * @crb and @offset.
+ */
+ vas_copy_crb(crb, 0);
+
+ /*
+ * VAS paste previously copied CRB to NX.
+ * @txwin, @offset and @last (must be true).
+ */
+ ret = vas_paste_crb(txwin, 0, 1);
+ preempt_enable();
+ /*
+ * Retry copy/paste function for VAS failures.
+ */
+ } while (ret && (i++ < VAS_RETRIES));
+
+ if (ret) {
+ pr_err_ratelimited("VAS copy/paste failed\n");
+ return ret;
+ }
+
+ ret = wait_for_csb(wmem, csb);
+ if (!ret)
+ *outlenp = be32_to_cpu(csb->count);
+
+ return ret;
+}
+
+/**
+ * nx842_powernv_compress - Compress data using the 842 algorithm
+ *
+ * Compression provided by the NX842 coprocessor on IBM PowerNV systems.
+ * The input buffer is compressed and the result is stored in the
+ * provided output buffer.
+ *
+ * Upon return from this function @outlen contains the length of the
+ * compressed data. If there is an error then @outlen will be 0 and an
+ * error will be specified by the return code from this function.
+ *
+ * @in: input buffer pointer
+ * @inlen: input buffer size
+ * @out: output buffer pointer
+ * @outlenp: output buffer size pointer
+ * @workmem: working memory buffer pointer, size determined by
+ * nx842_powernv_driver.workmem_size
+ *
+ * Returns: see @nx842_powernv_exec()
+ */
+static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int *outlenp,
+ void *wmem)
+{
+ return nx842_powernv_exec(in, inlen, out, outlenp,
+ wmem, CCW_FC_842_COMP_CRC);
+}
+
+/**
+ * nx842_powernv_decompress - Decompress data using the 842 algorithm
+ *
+ * Decompression provided by the NX842 coprocessor on IBM PowerNV systems.
+ * The input buffer is decompressed and the result is stored in the
+ * provided output buffer.
+ *
+ * Upon return from this function @outlen contains the length of the
+ * decompressed data. If there is an error then @outlen will be 0 and an
+ * error will be specified by the return code from this function.
+ *
+ * @in: input buffer pointer
+ * @inlen: input buffer size
+ * @out: output buffer pointer
+ * @outlenp: output buffer size pointer
+ * @workmem: working memory buffer pointer, size determined by
+ * nx842_powernv_driver.workmem_size
+ *
+ * Returns: see @nx842_powernv_exec()
+ */
+static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen,
+ unsigned char *out, unsigned int *outlenp,
+ void *wmem)
+{
+ return nx842_powernv_exec(in, inlen, out, outlenp,
+ wmem, CCW_FC_842_DECOMP_CRC);
+}
+
+static inline void nx842_add_coprocs_list(struct nx842_coproc *coproc,
+ int chipid)
+{
+ coproc->chip_id = chipid;
+ INIT_LIST_HEAD(&coproc->list);
+ list_add(&coproc->list, &nx842_coprocs);
+}
+
+static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc)
+{
+ struct vas_window *txwin = NULL;
+ struct vas_tx_win_attr txattr;
+
+ /*
+ * Kernel requests will be high priority. So open send
+ * windows only for high priority RxFIFO entries.
+ */
+ vas_init_tx_win_attr(&txattr, coproc->ct);
+ txattr.lpid = 0; /* lpid is 0 for kernel requests */
+ txattr.pid = 0; /* pid is 0 for kernel requests */
+
+ /*
+ * Open a VAS send window which is used to send request to NX.
+ */
+ txwin = vas_tx_win_open(coproc->vas.id, coproc->ct, &txattr);
+ if (IS_ERR(txwin))
+ pr_err("ibm,nx-842: Can not open TX window: %ld\n",
+ PTR_ERR(txwin));
+
+ return txwin;
+}
+
+/*
+ * Identify chip ID for each CPU, open send wndow for the corresponding NX
+ * engine and save txwin in percpu cpu_txwin.
+ * cpu_txwin is used in copy/paste operation for each compression /
+ * decompression request.
+ */
+static int nx842_open_percpu_txwins(void)
+{
+ struct nx842_coproc *coproc, *n;
+ unsigned int i, chip_id;
+
+ for_each_possible_cpu(i) {
+ struct vas_window *txwin = NULL;
+
+ chip_id = cpu_to_chip_id(i);
+
+ list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
+ /*
+ * Kernel requests use only high priority FIFOs. So
+ * open send windows for these FIFOs.
+ */
+
+ if (coproc->ct != VAS_COP_TYPE_842_HIPRI)
+ continue;
+
+ if (coproc->chip_id == chip_id) {
+ txwin = nx842_alloc_txwin(coproc);
+ if (IS_ERR(txwin))
+ return PTR_ERR(txwin);
+
+ per_cpu(cpu_txwin, i) = txwin;
+ break;
+ }
+ }
+
+ if (!per_cpu(cpu_txwin, i)) {
+ /* shouldn't happen, Each chip will have NX engine */
+ pr_err("NX engine is not available for CPU %d\n", i);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
+ int vasid, int *ct)
+{
+ struct vas_window *rxwin = NULL;
+ struct vas_rx_win_attr rxattr;
+ struct nx842_coproc *coproc;
+ u32 lpid, pid, tid, fifo_size;
+ u64 rx_fifo;
+ const char *priority;
+ int ret;
+
+ ret = of_property_read_u64(dn, "rx-fifo-address", &rx_fifo);
+ if (ret) {
+ pr_err("Missing rx-fifo-address property\n");
+ return ret;
+ }
+
+ ret = of_property_read_u32(dn, "rx-fifo-size", &fifo_size);
+ if (ret) {
+ pr_err("Missing rx-fifo-size property\n");
+ return ret;
+ }
+
+ ret = of_property_read_u32(dn, "lpid", &lpid);
+ if (ret) {
+ pr_err("Missing lpid property\n");
+ return ret;
+ }
+
+ ret = of_property_read_u32(dn, "pid", &pid);
+ if (ret) {
+ pr_err("Missing pid property\n");
+ return ret;
+ }
+
+ ret = of_property_read_u32(dn, "tid", &tid);
+ if (ret) {
+ pr_err("Missing tid property\n");
+ return ret;
+ }
+
+ ret = of_property_read_string(dn, "priority", &priority);
+ if (ret) {
+ pr_err("Missing priority property\n");
+ return ret;
+ }
+
+ coproc = kzalloc(sizeof(*coproc), GFP_KERNEL);
+ if (!coproc)
+ return -ENOMEM;
+
+ if (!strcmp(priority, "High"))
+ coproc->ct = VAS_COP_TYPE_842_HIPRI;
+ else if (!strcmp(priority, "Normal"))
+ coproc->ct = VAS_COP_TYPE_842;
+ else {
+ pr_err("Invalid RxFIFO priority value\n");
+ ret = -EINVAL;
+ goto err_out;
+ }
+
+ vas_init_rx_win_attr(&rxattr, coproc->ct);
+ rxattr.rx_fifo = (void *)rx_fifo;
+ rxattr.rx_fifo_size = fifo_size;
+ rxattr.lnotify_lpid = lpid;
+ rxattr.lnotify_pid = pid;
+ rxattr.lnotify_tid = tid;
+ /*
+ * Maximum RX window credits can not be more than #CRBs in
+ * RxFIFO. Otherwise, can get checkstop if RxFIFO overruns.
+ */
+ rxattr.wcreds_max = fifo_size / CRB_SIZE;
+
+ /*
+ * Open a VAS receice window which is used to configure RxFIFO
+ * for NX.
+ */
+ rxwin = vas_rx_win_open(vasid, coproc->ct, &rxattr);
+ if (IS_ERR(rxwin)) {
+ ret = PTR_ERR(rxwin);
+ pr_err("setting RxFIFO with VAS failed: %d\n",
+ ret);
+ goto err_out;
+ }
+
+ coproc->vas.rxwin = rxwin;
+ coproc->vas.id = vasid;
+ nx842_add_coprocs_list(coproc, chip_id);
+
+ /*
+ * (lpid, pid, tid) combination has to be unique for each
+ * coprocessor instance in the system. So to make it
+ * unique, skiboot uses coprocessor type such as 842 or
+ * GZIP for pid and provides this value to kernel in pid
+ * device-tree property.
+ */
+ *ct = pid;
+
+ return 0;
+
+err_out:
+ kfree(coproc);
+ return ret;
+}
+
+
+static int __init nx842_powernv_probe_vas(struct device_node *pn)
+{
+ struct device_node *dn;
+ int chip_id, vasid, ret = 0;
+ int nx_fifo_found = 0;
+ int uninitialized_var(ct);
+
+ chip_id = of_get_ibm_chip_id(pn);
+ if (chip_id < 0) {
+ pr_err("ibm,chip-id missing\n");
+ return -EINVAL;
+ }
+
+ vasid = chip_to_vas_id(chip_id);
+ if (vasid < 0) {
+ pr_err("Unable to map chip_id %d to vasid\n", chip_id);
+ return -EINVAL;
+ }
+
+ for_each_child_of_node(pn, dn) {
+ if (of_device_is_compatible(dn, "ibm,p9-nx-842")) {
+ ret = vas_cfg_coproc_info(dn, chip_id, vasid, &ct);
+ if (ret) {
+ of_node_put(dn);
+ return ret;
+ }
+ nx_fifo_found++;
+ }
+ }
+
+ if (!nx_fifo_found) {
+ pr_err("NX842 FIFO nodes are missing\n");
+ return -EINVAL;
+ }
+
+ /*
+ * Initialize NX instance for both high and normal priority FIFOs.
+ */
+ if (opal_check_token(OPAL_NX_COPROC_INIT)) {
+ ret = opal_nx_coproc_init(chip_id, ct);
+ if (ret) {
+ pr_err("Failed to initialize NX for chip(%d): %d\n",
+ chip_id, ret);
+ ret = opal_error_code(ret);
+ }
+ } else
+ pr_warn("Firmware doesn't support NX initialization\n");
+
+ return ret;
+}
+
+static int __init nx842_powernv_probe(struct device_node *dn)
+{
+ struct nx842_coproc *coproc;
+ unsigned int ct, ci;
+ int chip_id;
+
+ chip_id = of_get_ibm_chip_id(dn);
+ if (chip_id < 0) {
+ pr_err("ibm,chip-id missing\n");
+ return -EINVAL;
+ }
+
+ if (of_property_read_u32(dn, "ibm,842-coprocessor-type", &ct)) {
+ pr_err("ibm,842-coprocessor-type missing\n");
+ return -EINVAL;
+ }
+
+ if (of_property_read_u32(dn, "ibm,842-coprocessor-instance", &ci)) {
+ pr_err("ibm,842-coprocessor-instance missing\n");
+ return -EINVAL;
+ }
+
+ coproc = kzalloc(sizeof(*coproc), GFP_KERNEL);
+ if (!coproc)
+ return -ENOMEM;
+
+ coproc->ct = ct;
+ coproc->ci = ci;
+ nx842_add_coprocs_list(coproc, chip_id);
+
+ pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci);
+
+ if (!nx842_ct)
+ nx842_ct = ct;
+ else if (nx842_ct != ct)
+ pr_err("NX842 chip %d, CT %d != first found CT %d\n",
+ chip_id, ct, nx842_ct);
+
+ return 0;
+}
+
+static void nx842_delete_coprocs(void)
+{
+ struct nx842_coproc *coproc, *n;
+ struct vas_window *txwin;
+ int i;
+
+ /*
+ * close percpu txwins that are opened for the corresponding coproc.
+ */
+ for_each_possible_cpu(i) {
+ txwin = per_cpu(cpu_txwin, i);
+ if (txwin)
+ vas_win_close(txwin);
+
+ per_cpu(cpu_txwin, i) = 0;
+ }
+
+ list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
+ if (coproc->vas.rxwin)
+ vas_win_close(coproc->vas.rxwin);
+
+ list_del(&coproc->list);
+ kfree(coproc);
+ }
+}
+
+static struct nx842_constraints nx842_powernv_constraints = {
+ .alignment = DDE_BUFFER_ALIGN,
+ .multiple = DDE_BUFFER_LAST_MULT,
+ .minimum = DDE_BUFFER_LAST_MULT,
+ .maximum = (DDL_LEN_MAX - 1) * PAGE_SIZE,
+};
+
+static struct nx842_driver nx842_powernv_driver = {
+ .name = KBUILD_MODNAME,
+ .owner = THIS_MODULE,
+ .workmem_size = sizeof(struct nx842_workmem),
+ .constraints = &nx842_powernv_constraints,
+ .compress = nx842_powernv_compress,
+ .decompress = nx842_powernv_decompress,
+};
+
+static int nx842_powernv_crypto_init(struct crypto_tfm *tfm)
+{
+ return nx842_crypto_init(tfm, &nx842_powernv_driver);
+}
+
+static struct crypto_alg nx842_powernv_alg = {
+ .cra_name = "842",
+ .cra_driver_name = "842-nx",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_COMPRESS,
+ .cra_ctxsize = sizeof(struct nx842_crypto_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_init = nx842_powernv_crypto_init,
+ .cra_exit = nx842_crypto_exit,
+ .cra_u = { .compress = {
+ .coa_compress = nx842_crypto_compress,
+ .coa_decompress = nx842_crypto_decompress } }
+};
+
+static __init int nx842_powernv_init(void)
+{
+ struct device_node *dn;
+ int ret;
+
+ /* verify workmem size/align restrictions */
+ BUILD_BUG_ON(WORKMEM_ALIGN % CRB_ALIGN);
+ BUILD_BUG_ON(CRB_ALIGN % DDE_ALIGN);
+ BUILD_BUG_ON(CRB_SIZE % DDE_ALIGN);
+ /* verify buffer size/align restrictions */
+ BUILD_BUG_ON(PAGE_SIZE % DDE_BUFFER_ALIGN);
+ BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT);
+ BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT);
+
+ for_each_compatible_node(dn, NULL, "ibm,power9-nx") {
+ ret = nx842_powernv_probe_vas(dn);
+ if (ret) {
+ nx842_delete_coprocs();
+ of_node_put(dn);
+ return ret;
+ }
+ }
+
+ if (list_empty(&nx842_coprocs)) {
+ for_each_compatible_node(dn, NULL, "ibm,power-nx")
+ nx842_powernv_probe(dn);
+
+ if (!nx842_ct)
+ return -ENODEV;
+
+ nx842_powernv_exec = nx842_exec_icswx;
+ } else {
+ ret = nx842_open_percpu_txwins();
+ if (ret) {
+ nx842_delete_coprocs();
+ return ret;
+ }
+
+ nx842_powernv_exec = nx842_exec_vas;
+ }
+
+ ret = crypto_register_alg(&nx842_powernv_alg);
+ if (ret) {
+ nx842_delete_coprocs();
+ return ret;
+ }
+
+ return 0;
+}
+module_init(nx842_powernv_init);
+
+static void __exit nx842_powernv_exit(void)
+{
+ crypto_unregister_alg(&nx842_powernv_alg);
+
+ nx842_delete_coprocs();
+}
+module_exit(nx842_powernv_exit);
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 4/9] crypto/nx: Initialize coproc entry with kzalloc
From: Haren Myneni @ 2020-03-23 4:03 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
coproc entry is initialized during NX probe on power9, but not on P8.
nx842_delete_coprocs() is used for both and frees receive window if it
is allocated. Getting crash for rmmod on P8 since coproc->vas.rxwin
is not initialized.
This patch replaces kmalloc with kzalloc in nx842_powernv_probe()
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
drivers/crypto/nx/nx-842-powernv.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c
index c037a24..8e63326 100644
--- a/drivers/crypto/nx/nx-842-powernv.c
+++ b/drivers/crypto/nx/nx-842-powernv.c
@@ -922,7 +922,7 @@ static int __init nx842_powernv_probe(struct device_node *dn)
return -EINVAL;
}
- coproc = kmalloc(sizeof(*coproc), GFP_KERNEL);
+ coproc = kzalloc(sizeof(*coproc), GFP_KERNEL);
if (!coproc)
return -ENOMEM;
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 3/9] powerpc/vas: Add VAS user space API
From: Haren Myneni @ 2020-03-23 4:02 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
On power9, userspace can send GZIP compression requests directly to NX
once kernel establishes NX channel / window with VAS. This patch provides
user space API which allows user space to establish channel using open
VAS_TX_WIN_OPEN ioctl, mmap and close operations.
Each window corresponds to file descriptor and application can open
multiple windows. After the window is opened, VAS_TX_WIN_OPEN icoctl to
open a window on specific VAS instance, mmap() system call to map
the hardware address of engine's request queue into the application's
virtual address space.
Then the application can then submit one or more requests to the the
engine by using the copy/paste instructions and pasting the CRBs to
the virtual address (aka paste_address) returned by mmap().
Only NX GZIP coprocessor type is supported right now and allow GZIP
engine access via /dev/crypto/nx-gzip device node.
Signed-off-by: Sukadev Bhattiprolu <sukadev@linux.vnet.ibm.com>
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
arch/powerpc/include/asm/vas.h | 11 ++
arch/powerpc/platforms/powernv/Makefile | 2 +-
arch/powerpc/platforms/powernv/vas-api.c | 257 ++++++++++++++++++++++++++++
arch/powerpc/platforms/powernv/vas-window.c | 6 +-
arch/powerpc/platforms/powernv/vas.h | 2 +
5 files changed, 274 insertions(+), 4 deletions(-)
create mode 100644 arch/powerpc/platforms/powernv/vas-api.c
diff --git a/arch/powerpc/include/asm/vas.h b/arch/powerpc/include/asm/vas.h
index f93e6b0..e064953 100644
--- a/arch/powerpc/include/asm/vas.h
+++ b/arch/powerpc/include/asm/vas.h
@@ -163,4 +163,15 @@ struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
*/
int vas_paste_crb(struct vas_window *win, int offset, bool re);
+/*
+ * Register / unregister coprocessor type to VAS API which will be exported
+ * to user space. Applications can use this API to open / close window
+ * which can be used to send / receive requests directly to cooprcessor.
+ *
+ * Only NX GZIP coprocessor type is supported now, but this API can be
+ * used for others in future.
+ */
+int vas_register_coproc_api(struct module *mod);
+void vas_unregister_coproc_api(void);
+
#endif /* __ASM_POWERPC_VAS_H */
diff --git a/arch/powerpc/platforms/powernv/Makefile b/arch/powerpc/platforms/powernv/Makefile
index 395789f..fe3f0fb 100644
--- a/arch/powerpc/platforms/powernv/Makefile
+++ b/arch/powerpc/platforms/powernv/Makefile
@@ -17,7 +17,7 @@ obj-$(CONFIG_MEMORY_FAILURE) += opal-memory-errors.o
obj-$(CONFIG_OPAL_PRD) += opal-prd.o
obj-$(CONFIG_PERF_EVENTS) += opal-imc.o
obj-$(CONFIG_PPC_MEMTRACE) += memtrace.o
-obj-$(CONFIG_PPC_VAS) += vas.o vas-window.o vas-debug.o vas-fault.o
+obj-$(CONFIG_PPC_VAS) += vas.o vas-window.o vas-debug.o vas-fault.o vas-api.o
obj-$(CONFIG_OCXL_BASE) += ocxl.o
obj-$(CONFIG_SCOM_DEBUGFS) += opal-xscom.o
obj-$(CONFIG_PPC_SECURE_BOOT) += opal-secvar.o
diff --git a/arch/powerpc/platforms/powernv/vas-api.c b/arch/powerpc/platforms/powernv/vas-api.c
new file mode 100644
index 0000000..7d049af
--- /dev/null
+++ b/arch/powerpc/platforms/powernv/vas-api.c
@@ -0,0 +1,257 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * VAS user space API for its accelerators (Only NX-GZIP is supported now)
+ * Copyright (C) 2019 Haren Myneni, IBM Corp
+ */
+
+#include <linux/kernel.h>
+#include <linux/device.h>
+#include <linux/cdev.h>
+#include <linux/fs.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <asm/vas.h>
+#include <uapi/asm/vas-api.h>
+#include "vas.h"
+
+/*
+ * The driver creates the device node that can be used as follows:
+ * For NX-GZIP
+ *
+ * fd = open("/dev/crypto/nx-gzip", O_RDWR);
+ * rc = ioctl(fd, VAS_TX_WIN_OPEN, &attr);
+ * paste_addr = mmap(NULL, PAGE_SIZE, prot, MAP_SHARED, fd, 0ULL).
+ * vas_copy(&crb, 0, 1);
+ * vas_paste(paste_addr, 0, 1);
+ * close(fd) or exit process to close window.
+ *
+ * where "vas_copy" and "vas_paste" are defined in copy-paste.h.
+ * copy/paste returns to the user space directly. So refer NX hardware
+ * documententation for exact copy/paste usage and completion / error
+ * conditions.
+ */
+
+static char *coproc_dev_name = "nx-gzip";
+
+/*
+ * Wrapper object for the nx-gzip device - there is just one instance of
+ * this node for the whole system.
+ */
+static struct coproc_dev {
+ struct cdev cdev;
+ struct device *device;
+ char *name;
+ dev_t devt;
+ struct class *class;
+} coproc_device;
+
+static char *coproc_devnode(struct device *dev, umode_t *mode)
+{
+ return kasprintf(GFP_KERNEL, "crypto/%s", dev_name(dev));
+}
+
+static int coproc_open(struct inode *inode, struct file *fp)
+{
+ /*
+ * vas_window is allocated and assigned to fp->private_data
+ * in ioctl. Nothing to do here for NX GZIP.
+ */
+ return 0;
+}
+
+static int coproc_ioc_tx_win_open(struct file *fp, unsigned long arg)
+{
+ void __user *uptr = (void __user *)arg;
+ struct vas_tx_win_attr txattr = {};
+ struct vas_tx_win_open_attr uattr;
+ struct vas_window *txwin;
+ int rc, vasid;
+
+ /*
+ * One window for file descriptor
+ */
+ if (fp->private_data)
+ return -EEXIST;
+
+ rc = copy_from_user(&uattr, uptr, sizeof(uattr));
+ if (rc) {
+ pr_err("%s(): copy_from_user() returns %d\n", __func__, rc);
+ return -EFAULT;
+ }
+
+ if (uattr.version != 1) {
+ pr_err("Invalid version\n");
+ return -EINVAL;
+ }
+
+ vasid = uattr.vas_id;
+
+ vas_init_tx_win_attr(&txattr, VAS_COP_TYPE_GZIP);
+
+ txattr.lpid = mfspr(SPRN_LPID);
+ txattr.pidr = mfspr(SPRN_PID);
+ txattr.user_win = true;
+ txattr.rsvd_txbuf_count = false;
+ txattr.pswid = false;
+
+ pr_devel("Pid %d: Opening txwin, PIDR %ld\n", txattr.pidr,
+ mfspr(SPRN_PID));
+
+ txwin = vas_tx_win_open(vasid, VAS_COP_TYPE_GZIP, &txattr);
+ if (IS_ERR(txwin)) {
+ pr_err("%s() vas_tx_win_open() failed, %ld\n", __func__,
+ PTR_ERR(txwin));
+ return PTR_ERR(txwin);
+ }
+
+ fp->private_data = txwin;
+
+ return 0;
+}
+
+static int coproc_release(struct inode *inode, struct file *fp)
+{
+ struct vas_window *txwin = fp->private_data;
+
+ if (txwin) {
+ vas_win_close(txwin);
+ fp->private_data = NULL;
+ }
+
+ /*
+ * We don't know here if user has other receive windows
+ * open, so we can't really call clear_thread_tidr().
+ * So, once the process calls set_thread_tidr(), the
+ * TIDR value sticks around until process exits, resulting
+ * in an extra copy in restore_sprs().
+ */
+
+ return 0;
+}
+
+static int coproc_mmap(struct file *fp, struct vm_area_struct *vma)
+{
+ struct vas_window *txwin = fp->private_data;
+ unsigned long pfn;
+ u64 paste_addr;
+ pgprot_t prot;
+ int rc;
+
+ if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) {
+ pr_debug("%s(): size 0x%zx, PAGE_SIZE 0x%zx\n", __func__,
+ (vma->vm_end - vma->vm_start), PAGE_SIZE);
+ return -EINVAL;
+ }
+
+ /* Ensure instance has an open send window */
+ if (!txwin) {
+ pr_err("%s(): No send window open?\n", __func__);
+ return -EINVAL;
+ }
+
+ vas_win_paste_addr(txwin, &paste_addr, NULL);
+ pfn = paste_addr >> PAGE_SHIFT;
+
+ /* flags, page_prot from cxl_mmap(), except we want cachable */
+ vma->vm_flags |= VM_IO | VM_PFNMAP;
+ vma->vm_page_prot = pgprot_cached(vma->vm_page_prot);
+
+ prot = __pgprot(pgprot_val(vma->vm_page_prot) | _PAGE_DIRTY);
+
+ rc = remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,
+ vma->vm_end - vma->vm_start, prot);
+
+ pr_devel("%s(): paste addr %llx at %lx, rc %d\n", __func__,
+ paste_addr, vma->vm_start, rc);
+
+ return rc;
+}
+
+static long coproc_ioctl(struct file *fp, unsigned int cmd, unsigned long arg)
+{
+ switch (cmd) {
+ case VAS_TX_WIN_OPEN:
+ return coproc_ioc_tx_win_open(fp, arg);
+ default:
+ return -EINVAL;
+ }
+}
+
+static struct file_operations coproc_fops = {
+ .open = coproc_open,
+ .release = coproc_release,
+ .mmap = coproc_mmap,
+ .unlocked_ioctl = coproc_ioctl,
+};
+
+/*
+ * Supporting only nx-gzip coprocessor type now, but this API code
+ * extended to other coprocessor types later.
+ */
+int vas_register_coproc_api(struct module *mod)
+{
+ int rc = -EINVAL;
+ dev_t devno;
+
+ rc = alloc_chrdev_region(&coproc_device.devt, 1, 1, coproc_dev_name);
+ if (rc) {
+ pr_err("Unable to allocate coproc major number: %i\n", rc);
+ return rc;
+ }
+
+ pr_devel("%s device allocated, dev [%i,%i]\n", coproc_dev_name,
+ MAJOR(coproc_device.devt), MINOR(coproc_device.devt));
+
+ coproc_device.class = class_create(mod, coproc_dev_name);
+ if (IS_ERR(coproc_device.class)) {
+ rc = PTR_ERR(coproc_device.class);
+ pr_err("Unable to create %s class %d\n", coproc_dev_name, rc);
+ goto err_class;
+ }
+ coproc_device.class->devnode = coproc_devnode;
+
+ coproc_fops.owner = mod;
+ cdev_init(&coproc_device.cdev, &coproc_fops);
+
+ devno = MKDEV(MAJOR(coproc_device.devt), 0);
+ rc = cdev_add(&coproc_device.cdev, devno, 1);
+ if (rc) {
+ pr_err("cdev_add() failed %d\n", rc);
+ goto err_cdev;
+ }
+
+ coproc_device.device = device_create(coproc_device.class, NULL,
+ devno, NULL, coproc_dev_name, MINOR(devno));
+ if (IS_ERR(coproc_device.device)) {
+ rc = PTR_ERR(coproc_device.device);
+ pr_err("Unable to create coproc-%d %d\n", MINOR(devno), rc);
+ goto err;
+ }
+
+ pr_devel("%s: Added dev [%d,%d]\n", __func__, MAJOR(devno),
+ MINOR(devno));
+
+ return 0;
+
+err:
+ cdev_del(&coproc_device.cdev);
+err_cdev:
+ class_destroy(coproc_device.class);
+err_class:
+ unregister_chrdev_region(coproc_device.devt, 1);
+ return rc;
+}
+EXPORT_SYMBOL_GPL(vas_register_coproc_api);
+
+void vas_unregister_coproc_api(void)
+{
+ dev_t devno;
+
+ cdev_del(&coproc_device.cdev);
+ devno = MKDEV(MAJOR(coproc_device.devt), 0);
+ device_destroy(coproc_device.class, devno);
+
+ class_destroy(coproc_device.class);
+ unregister_chrdev_region(coproc_device.devt, 1);
+}
+EXPORT_SYMBOL_GPL(vas_unregister_coproc_api);
diff --git a/arch/powerpc/platforms/powernv/vas-window.c b/arch/powerpc/platforms/powernv/vas-window.c
index 4ce421f..7b5187b 100644
--- a/arch/powerpc/platforms/powernv/vas-window.c
+++ b/arch/powerpc/platforms/powernv/vas-window.c
@@ -26,7 +26,7 @@
* Compute the paste address region for the window @window using the
* ->paste_base_addr and ->paste_win_id_shift we got from device tree.
*/
-static void compute_paste_address(struct vas_window *window, u64 *addr, int *len)
+void vas_win_paste_addr(struct vas_window *window, u64 *addr, int *len)
{
int winid;
u64 base, shift;
@@ -80,7 +80,7 @@ static void *map_paste_region(struct vas_window *txwin)
goto free_name;
txwin->paste_addr_name = name;
- compute_paste_address(txwin, &start, &len);
+ vas_win_paste_addr(txwin, &start, &len);
if (!request_mem_region(start, len, name)) {
pr_devel("%s(): request_mem_region(0x%llx, %d) failed\n",
@@ -138,7 +138,7 @@ static void unmap_paste_region(struct vas_window *window)
u64 busaddr_start;
if (window->paste_kaddr) {
- compute_paste_address(window, &busaddr_start, &len);
+ vas_win_paste_addr(window, &busaddr_start, &len);
unmap_region(window->paste_kaddr, busaddr_start, len);
window->paste_kaddr = NULL;
kfree(window->paste_addr_name);
diff --git a/arch/powerpc/platforms/powernv/vas.h b/arch/powerpc/platforms/powernv/vas.h
index 8c39a7d..a10abed 100644
--- a/arch/powerpc/platforms/powernv/vas.h
+++ b/arch/powerpc/platforms/powernv/vas.h
@@ -431,6 +431,8 @@ struct vas_winctx {
extern void vas_return_credit(struct vas_window *window, bool tx);
extern struct vas_window *vas_pswid_to_window(struct vas_instance *vinst,
uint32_t pswid);
+extern void vas_win_paste_addr(struct vas_window *window, u64 *addr,
+ int *len);
static inline int vas_window_pid(struct vas_window *window)
{
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 2/9] powerpc/vas: Define VAS_TX_WIN_OPEN ioctl API
From: Haren Myneni @ 2020-03-23 4:01 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
Define the VAS_TX_WIN_OPEN ioctl interface for NX GZIP access
from user space. This interface is used to open GZIP send window and
mmap region which can be used by userspace to send requests to NX
directly with copy/paste instructions.
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
Documentation/userspace-api/ioctl/ioctl-number.rst | 1 +
arch/powerpc/include/uapi/asm/vas-api.h | 22 ++++++++++++++++++++++
2 files changed, 23 insertions(+)
create mode 100644 arch/powerpc/include/uapi/asm/vas-api.h
diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst
index 2e91370..deabc73 100644
--- a/Documentation/userspace-api/ioctl/ioctl-number.rst
+++ b/Documentation/userspace-api/ioctl/ioctl-number.rst
@@ -287,6 +287,7 @@ Code Seq# Include File Comments
'v' 00-1F linux/fs.h conflict!
'v' 00-0F linux/sonypi.h conflict!
'v' 00-0F media/v4l2-subdev.h conflict!
+'v' 20-27 arch/powerpc/include/uapi/asm/vas-api.h VAS API
'v' C0-FF linux/meye.h conflict!
'w' all CERN SCI driver
'y' 00-1F packet based user level communications
diff --git a/arch/powerpc/include/uapi/asm/vas-api.h b/arch/powerpc/include/uapi/asm/vas-api.h
new file mode 100644
index 0000000..fe95d67
--- /dev/null
+++ b/arch/powerpc/include/uapi/asm/vas-api.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */
+/*
+ * Copyright 2019 IBM Corp.
+ */
+
+#ifndef _UAPI_MISC_VAS_H
+#define _UAPI_MISC_VAS_H
+
+#include <asm/ioctl.h>
+
+#define VAS_MAGIC 'v'
+#define VAS_TX_WIN_OPEN _IOW(VAS_MAGIC, 0x20, struct vas_tx_win_open_attr)
+
+struct vas_tx_win_open_attr {
+ __u32 version;
+ __s16 vas_id; /* specific instance of vas or -1 for default */
+ __u16 reserved1;
+ __u64 flags; /* Future use */
+ __u64 reserved2[6];
+};
+
+#endif /* _UAPI_MISC_VAS_H */
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 1/9] powerpc/vas: Initialize window attributes for GZIP coprocessor type
From: Haren Myneni @ 2020-03-23 4:00 UTC (permalink / raw)
To: herbert; +Cc: mikey, npiggin, linux-crypto, sukadev, linuxppc-dev, dja
In-Reply-To: <1584934879.9256.15321.camel@hbabu-laptop>
Initialize send and receive window attributes for GZIP high and
normal priority types.
Signed-off-by: Haren Myneni <haren@linux.ibm.com>
---
arch/powerpc/platforms/powernv/vas-window.c | 17 ++++++++++++-----
1 file changed, 12 insertions(+), 5 deletions(-)
diff --git a/arch/powerpc/platforms/powernv/vas-window.c b/arch/powerpc/platforms/powernv/vas-window.c
index a1d3fb1..4ce421f 100644
--- a/arch/powerpc/platforms/powernv/vas-window.c
+++ b/arch/powerpc/platforms/powernv/vas-window.c
@@ -817,7 +817,8 @@ void vas_init_rx_win_attr(struct vas_rx_win_attr *rxattr, enum vas_cop_type cop)
{
memset(rxattr, 0, sizeof(*rxattr));
- if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) {
+ if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
+ cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
rxattr->pin_win = true;
rxattr->nx_win = true;
rxattr->fault_win = false;
@@ -892,7 +893,8 @@ void vas_init_tx_win_attr(struct vas_tx_win_attr *txattr, enum vas_cop_type cop)
{
memset(txattr, 0, sizeof(*txattr));
- if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI) {
+ if (cop == VAS_COP_TYPE_842 || cop == VAS_COP_TYPE_842_HIPRI ||
+ cop == VAS_COP_TYPE_GZIP || cop == VAS_COP_TYPE_GZIP_HIPRI) {
txattr->rej_no_credit = false;
txattr->rx_wcred_mode = true;
txattr->tx_wcred_mode = true;
@@ -976,9 +978,14 @@ static bool tx_win_args_valid(enum vas_cop_type cop,
if (attr->wcreds_max > VAS_TX_WCREDS_MAX)
return false;
- if (attr->user_win &&
- (cop != VAS_COP_TYPE_FTW || attr->rsvd_txbuf_count))
- return false;
+ if (attr->user_win) {
+ if (attr->rsvd_txbuf_count)
+ return false;
+
+ if (cop != VAS_COP_TYPE_FTW && cop != VAS_COP_TYPE_GZIP &&
+ cop != VAS_COP_TYPE_GZIP_HIPRI)
+ return false;
+ }
return true;
}
--
1.8.3.1
^ permalink raw reply related
* [PATCH v4 0/9] crypto/nx: Enable GZIP engine and provide userpace API
From: Haren Myneni @ 2020-03-23 3:41 UTC (permalink / raw)
To: herbert, mpe, dja; +Cc: mikey, sukadev, linuxppc-dev, linux-crypto, npiggin
Power9 processor supports Virtual Accelerator Switchboard (VAS) which
allows kernel and userspace to send compression requests to Nest
Accelerator (NX) directly. The NX unit comprises of 2 842 compression
engines and 1 GZIP engine. Linux kernel already has 842 compression
support on kernel. This patch series adds GZIP compression support
from user space. The GZIP Compression engine implements the ZLIB and
GZIP compression algorithms. No plans of adding NX-GZIP compression
support in kernel right now.
Applications can send requests to NX directly with COPY/PASTE
instructions. But kernel has to establish channel / window on NX-GZIP
device for the userspace. So userspace access to the GZIP engine is
provided through /dev/crypto/nx-gzip device with several operations.
An application must open the this device to obtain a file descriptor (fd).
Using the fd, application should issue the VAS_TX_WIN_OPEN ioctl to
establish a connection to the engine. Once window is opened, should use
mmap() system call to map the hardware address of engine's request queue
into the application's virtual address space. Then user space forms the
request as co-processor Request Block (CRB) and paste this CRB on the
mapped HW address using COPY/PASTE instructions. Application can poll
on status flags (part of CRB) with timeout for request completion.
For VAS_TX_WIN_OPEN ioctl, if user space passes vas_id = -1 (struct
vas_tx_win_open_attr), kernel determines the VAS instance on the
corresponding chip based on the CPU on which the process is executing.
Otherwise, the specified VAS instance is used if application passes the
proper VAS instance (vas_id listed in /proc/device-tree/vas@*/ibm,vas_id).
Process can open multiple windows with different FDs or can send several
requests to NX on the same window at the same time.
A userspace library libnxz is available:
https://github.com/abalib/power-gzip
Applications that use inflate/deflate calls can link with libNXz and use
NX GZIP compression without any modification.
Tested the available 842 compression on power8 and power9 system to make
sure no regression and tested GZIP compression on power9 with tests
available in the above link.
Thanks to Bulent Abali for nxz library and tests development.
Changelog:
V2:
- Move user space API code to powerpc as suggested. Also this API
can be extended to any other coprocessor type that VAS can support
in future. Example: Fast thread wakeup feature from VAS
- Rebased to 5.6-rc3
V3:
- Fix sparse warnings (patches 3&6)
V4:
- Remove unused coproc_instid and add only window address in
fp->private_data.
- Add NX User's manual and Copy/paste links in VAS API documentation
in patch and other changes as Daniel Axtens suggested
Haren Myneni (9):
powerpc/vas: Initialize window attributes for GZIP coprocessor type
powerpc/vas: Define VAS_TX_WIN_OPEN ioctl API
powerpc/vas: Add VAS user space API
crypto/nx: Initialize coproc entry with kzalloc
crypto/nx: Rename nx-842-powernv file name to nx-common-powernv
crypto/NX: Make enable code generic to add new GZIP compression type
crypto/nx: Enable and setup GZIP compresstion type
crypto/nx: Remove 'pid' in vas_tx_win_attr struct
Documentation/powerpc: VAS API
Documentation/powerpc/index.rst | 1 +
Documentation/powerpc/vas-api.rst | 249 +++++
Documentation/userspace-api/ioctl/ioctl-number.rst | 1 +
arch/powerpc/include/asm/vas.h | 12 +-
arch/powerpc/include/uapi/asm/vas-api.h | 22 +
arch/powerpc/platforms/powernv/Makefile | 2 +-
arch/powerpc/platforms/powernv/vas-api.c | 257 +++++
arch/powerpc/platforms/powernv/vas-window.c | 23 +-
arch/powerpc/platforms/powernv/vas.h | 2 +
drivers/crypto/nx/Makefile | 2 +-
drivers/crypto/nx/nx-842-powernv.c | 1062 ------------------
drivers/crypto/nx/nx-common-powernv.c | 1133 ++++++++++++++++++++
12 files changed, 1693 insertions(+), 1073 deletions(-)
create mode 100644 Documentation/powerpc/vas-api.rst
create mode 100644 arch/powerpc/include/uapi/asm/vas-api.h
create mode 100644 arch/powerpc/platforms/powernv/vas-api.c
delete mode 100644 drivers/crypto/nx/nx-842-powernv.c
create mode 100644 drivers/crypto/nx/nx-common-powernv.c
--
1.8.3.1
^ permalink raw reply
* Re: [patch V3 13/20] Documentation: Add lock ordering and nesting documentation
From: Paul E. McKenney @ 2020-03-23 2:55 UTC (permalink / raw)
To: Thomas Gleixner
Cc: linux-usb, linux-ia64, Peter Zijlstra, linux-pci,
Sebastian Siewior, Oleg Nesterov, Guo Ren, Joel Fernandes,
Vincent Chen, Ingo Molnar, Davidlohr Bueso, linux-acpi,
Brian Cain, Jonathan Corbet, linux-hexagon, Rafael J. Wysocki,
linux-csky, Linus Torvalds, Darren Hart, Zhang Rui, Len Brown,
Fenghua Yu, Arnd Bergmann, linux-pm, linuxppc-dev, Greentime Hu,
Bjorn Helgaas, Kurt Schwemmer, platform-driver-x86, Kalle Valo,
kbuild test robot, Felipe Balbi, Michal Simek, Tony Luck, Nick Hu,
Geoff Levand, netdev, Randy Dunlap, linux-wireless, LKML,
Davidlohr Bueso, Greg Kroah-Hartman, Logan Gunthorpe,
David S. Miller, Andy Shevchenko
In-Reply-To: <20200321113242.026561244@linutronix.de>
On Sat, Mar 21, 2020 at 12:25:57PM +0100, Thomas Gleixner wrote:
> From: Thomas Gleixner <tglx@linutronix.de>
>
> The kernel provides a variety of locking primitives. The nesting of these
> lock types and the implications of them on RT enabled kernels is nowhere
> documented.
>
> Add initial documentation.
>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> Cc: "Paul E . McKenney" <paulmck@kernel.org>
> Cc: Jonathan Corbet <corbet@lwn.net>
> Cc: Davidlohr Bueso <dave@stgolabs.net>
> Cc: Randy Dunlap <rdunlap@infradead.org>
> ---
> V3: Addressed review comments from Paul, Jonathan, Davidlohr
> V2: Addressed review comments from Randy
> ---
> Documentation/locking/index.rst | 1
> Documentation/locking/locktypes.rst | 299 ++++++++++++++++++++++++++++++++++++
> 2 files changed, 300 insertions(+)
> create mode 100644 Documentation/locking/locktypes.rst
>
> --- a/Documentation/locking/index.rst
> +++ b/Documentation/locking/index.rst
> @@ -7,6 +7,7 @@ locking
> .. toctree::
> :maxdepth: 1
>
> + locktypes
> lockdep-design
> lockstat
> locktorture
> --- /dev/null
> +++ b/Documentation/locking/locktypes.rst
> @@ -0,0 +1,299 @@
[ . . . Adding your example execution sequences . . . ]
> +PREEMPT_RT kernels preserve all other spinlock_t semantics:
> +
> + - Tasks holding a spinlock_t do not migrate. Non-PREEMPT_RT kernels
> + avoid migration by disabling preemption. PREEMPT_RT kernels instead
> + disable migration, which ensures that pointers to per-CPU variables
> + remain valid even if the task is preempted.
> +
> + - Task state is preserved across spinlock acquisition, ensuring that the
> + task-state rules apply to all kernel configurations. Non-PREEMPT_RT
> + kernels leave task state untouched. However, PREEMPT_RT must change
> + task state if the task blocks during acquisition. Therefore, it saves
> + the current task state before blocking and the corresponding lock wakeup
> + restores it.
> +
> + Other types of wakeups would normally unconditionally set the task state
> + to RUNNING, but that does not work here because the task must remain
> + blocked until the lock becomes available. Therefore, when a non-lock
> + wakeup attempts to awaken a task blocked waiting for a spinlock, it
> + instead sets the saved state to RUNNING. Then, when the lock
> + acquisition completes, the lock wakeup sets the task state to the saved
> + state, in this case setting it to RUNNING.
In the normal case where the task sleeps through the entire lock
acquisition, the sequence of events is as follows:
state = UNINTERRUPTIBLE
lock()
block()
real_state = state
state = SLEEPONLOCK
lock wakeup
state = real_state == UNINTERRUPTIBLE
This sequence of events can occur when the task acquires spinlocks
on its way to sleeping, for example, in a call to wait_event().
The non-lock wakeup can occur when a wakeup races with this wait_event(),
which can result in the following sequence of events:
state = UNINTERRUPTIBLE
lock()
block()
real_state = state
state = SLEEPONLOCK
non lock wakeup
real_state = RUNNING
lock wakeup
state = real_state == RUNNING
Without this real_state subterfuge, the wakeup might be lost.
[ . . . and continuing where I left off earlier . . . ]
> +bit spinlocks
> +-------------
> +
> +Bit spinlocks are problematic for PREEMPT_RT as they cannot be easily
> +substituted by an RT-mutex based implementation for obvious reasons.
> +
> +The semantics of bit spinlocks are preserved on PREEMPT_RT kernels and the
> +caveats vs. raw_spinlock_t apply.
> +
> +Some bit spinlocks are substituted by regular spinlock_t for PREEMPT_RT but
> +this requires conditional (#ifdef'ed) code changes at the usage site while
> +the spinlock_t substitution is simply done by the compiler and the
> +conditionals are restricted to header files and core implementation of the
> +locking primitives and the usage sites do not require any changes.
PREEMPT_RT cannot substitute bit spinlocks because a single bit is
too small to accommodate an RT-mutex. Therefore, the semantics of bit
spinlocks are preserved on PREEMPT_RT kernels, so that the raw_spinlock_t
caveats also apply to bit spinlocks.
Some bit spinlocks are replaced with regular spinlock_t for PREEMPT_RT
using conditional (#ifdef'ed) code changes at the usage site.
In contrast, usage-site changes are not needed for the spinlock_t
substitution. Instead, conditionals in header files and the core locking
implemementation enable the compiler to do the substitution transparently.
> +Lock type nesting rules
> +=======================
> +
> +The most basic rules are:
> +
> + - Lock types of the same lock category (sleeping, spinning) can nest
> + arbitrarily as long as they respect the general lock ordering rules to
> + prevent deadlocks.
- Lock types in the same category (sleeping, spinning) can nest
arbitrarily as long as they respect the general deadlock-avoidance
ordering rules.
[ Give or take lockdep eventually complaining about too-deep nesting,
but that is probably not worth mentioning here. Leave that caveat
to the lockdep documentation. ]
> + - Sleeping lock types cannot nest inside spinning lock types.
> +
> + - Spinning lock types can nest inside sleeping lock types.
> +
> +These rules apply in general independent of CONFIG_PREEMPT_RT.
These constraints apply both in CONFIG_PREEMPT_RT and otherwise.
> +As PREEMPT_RT changes the lock category of spinlock_t and rwlock_t from
> +spinning to sleeping this has obviously restrictions how they can nest with
> +raw_spinlock_t.
> +
> +This results in the following nest ordering:
The fact that PREEMPT_RT changes the lock category of spinlock_t and
rwlock_t from spinning to sleeping means that they cannot be acquired
while holding a raw spinlock. This results in the following nesting
ordering:
> + 1) Sleeping locks
> + 2) spinlock_t and rwlock_t
> + 3) raw_spinlock_t and bit spinlocks
> +
> +Lockdep is aware of these constraints to ensure that they are respected.
Lockdep will complain if these constraints are violated, both in
CONFIG_PREEMPT_RT and otherwise.
> +Owner semantics
> +===============
> +
> +Most lock types in the Linux kernel have strict owner semantics, i.e. the
> +context (task) which acquires a lock has to release it.
The aforementioned lock types have strict owner semantics: The context
(task) that acquired the lock must release it.
> +There are two exceptions:
> +
> + - semaphores
> + - rwsems
> +
> +semaphores have no owner semantics for historical reason, and as such
> +trylock and release operations can be called from any context. They are
> +often used for both serialization and waiting purposes. That's generally
> +discouraged and should be replaced by separate serialization and wait
> +mechanisms, such as mutexes and completions.
semaphores lack owner semantics for historical reasons, so their trylock
and release operations may be called from any context. They are often
used for both serialization and waiting, but new use cases should
instead use separate serialization and wait mechanisms, such as mutexes
and completions.
> +rwsems have grown interfaces which allow non owner release for special
> +purposes. This usage is problematic on PREEMPT_RT because PREEMPT_RT
> +substitutes all locking primitives except semaphores with RT-mutex based
> +implementations to provide priority inheritance for all lock types except
> +the truly spinning ones. Priority inheritance on ownerless locks is
> +obviously impossible.
> +
> +For now the rwsem non-owner release excludes code which utilizes it from
> +being used on PREEMPT_RT enabled kernels. In same cases this can be
> +mitigated by disabling portions of the code, in other cases the complete
> +functionality has to be disabled until a workable solution has been found.
rwsems have grown special-purpose interfaces that allow non-owner release.
This non-owner release prevents PREEMPT_RT from substituting RT-mutex
implementations, for example, by defeating priority inheritance.
After all, if the lock has no owner, whose priority should be boosted?
As a result, PREEMPT_RT does not currently support rwsem, which in turn
means that code using it must therefore be disabled until a workable
solution presents itself.
[ Note: Not as confident as I would like to be in the above. ]
Thanx, Paul
^ permalink raw reply
* Re: [PATCH v8 12/14] powerpc/vas: Return credits after handling fault
From: Nicholas Piggin @ 2020-03-23 2:44 UTC (permalink / raw)
To: Haren Myneni, mpe; +Cc: mikey, ajd, hch, oohall, sukadev, linuxppc-dev, herbert
In-Reply-To: <1584598755.9256.15264.camel@hbabu-laptop>
Haren Myneni's on March 19, 2020 4:19 pm:
>
> NX expects OS to return credit for send window after processing each
> fault. Also credit has to be returned even for fault window.
And this should be merged in the fault handler function.
>
> Signed-off-by: Sukadev Bhattiprolu <sukadev@linux.vnet.ibm.com>
> Signed-off-by: Haren Myneni <haren@linux.ibm.com>
> ---
> arch/powerpc/platforms/powernv/vas-fault.c | 9 +++++++++
> arch/powerpc/platforms/powernv/vas-window.c | 17 +++++++++++++++++
> arch/powerpc/platforms/powernv/vas.h | 1 +
> 3 files changed, 27 insertions(+)
>
> diff --git a/arch/powerpc/platforms/powernv/vas-fault.c b/arch/powerpc/platforms/powernv/vas-fault.c
> index 40e1de4..292f7ba 100644
> --- a/arch/powerpc/platforms/powernv/vas-fault.c
> +++ b/arch/powerpc/platforms/powernv/vas-fault.c
> @@ -238,6 +238,10 @@ irqreturn_t vas_fault_thread_fn(int irq, void *data)
> memcpy(crb, fifo, CRB_SIZE);
> entry->stamp.nx.pswid = cpu_to_be32(FIFO_INVALID_ENTRY);
> entry->ccw |= cpu_to_be32(CCW0_INVALID);
> + /*
> + * Return credit for the fault window.
> + */
None of the comments in this patch are useful.
> + vas_return_credit(vinst->fault_win, 0);
Can you use true/false for bools?
> mutex_unlock(&vinst->mutex);
>
> pr_devel("VAS[%d] fault_fifo %p, fifo %p, fault_crbs %d\n",
> @@ -267,6 +271,11 @@ irqreturn_t vas_fault_thread_fn(int irq, void *data)
> }
>
> update_csb(window, crb);
> + /*
> + * Return credit for send window after processing
> + * fault CRB.
> + */
Any chance of a little bit of explanation how the credit system works?
Or is it in the code somewhere already?
I don't suppose there is a chance to batch credit updates with multiple
faults? (maybe the MMIO is insignificant)
> + vas_return_credit(window, 1);
> } while (true);
> }
>
Thanks,
Nick
^ permalink raw reply
* Re: [PATCH v8 11/14] powerpc/vas: Do not use default credits for receive window
From: Nicholas Piggin @ 2020-03-23 2:40 UTC (permalink / raw)
To: Haren Myneni, mpe; +Cc: mikey, ajd, hch, oohall, sukadev, linuxppc-dev, herbert
In-Reply-To: <1584598722.9256.15262.camel@hbabu-laptop>
Haren Myneni's on March 19, 2020 4:18 pm:
>
> System checkstops if RxFIFO overruns with more requests than the
> maximum possible number of CRBs allowed in FIFO at any time. So
> max credits value (rxattr.wcreds_max) is set and is passed to
> vas_rx_win_open() by the the driver.
This seems like it should be a bug fix or merged in the NX fault
window register patch or something.
Thanks,
Nick
^ permalink raw reply
* Re: [PATCH v8 09/14] powerpc/vas: Update CSB and notify process for fault CRBs
From: Nicholas Piggin @ 2020-03-23 2:37 UTC (permalink / raw)
To: Haren Myneni, mpe; +Cc: mikey, ajd, hch, oohall, sukadev, linuxppc-dev, herbert
In-Reply-To: <1584598648.9256.15258.camel@hbabu-laptop>
Haren Myneni's on March 19, 2020 4:17 pm:
>
> For each fault CRB, update fault address in CRB (fault_storage_addr)
> and translation error status in CSB so that user space can touch the
> fault address and resend the request. If the user space passed invalid
> CSB address send signal to process with SIGSEGV.
This is where the actual fault handling is done? Does this need to be
split from the other patch? Why not merge them and put it after the
reference counting one?
I'll wait until comments and questions on the first fault handling patch
are resolved before I look at this one.
Thanks,
Nick
>
> Signed-off-by: Sukadev Bhattiprolu <sukadev@linux.vnet.ibm.com>
> Signed-off-by: Haren Myneni <haren@linux.ibm.com>
> ---
> arch/powerpc/platforms/powernv/vas-fault.c | 115 +++++++++++++++++++++++++++++
> 1 file changed, 115 insertions(+)
>
> diff --git a/arch/powerpc/platforms/powernv/vas-fault.c b/arch/powerpc/platforms/powernv/vas-fault.c
> index 1c6d5cc..6eceac5d 100644
> --- a/arch/powerpc/platforms/powernv/vas-fault.c
> +++ b/arch/powerpc/platforms/powernv/vas-fault.c
> @@ -11,6 +11,7 @@
> #include <linux/slab.h>
> #include <linux/uaccess.h>
> #include <linux/kthread.h>
> +#include <linux/sched/signal.h>
> #include <linux/mmu_context.h>
> #include <asm/icswx.h>
>
> @@ -26,6 +27,119 @@
> #define VAS_FAULT_WIN_FIFO_SIZE (4 << 20)
>
> /*
> + * Update the CSB to indicate a translation error.
> + *
> + * If we are unable to update the CSB means copy_to_user failed due to
> + * invalid csb_addr, send a signal to the process.
> + *
> + * Remaining settings in the CSB are based on wait_for_csb() of
> + * NX-GZIP.
> + */
> +static void update_csb(struct vas_window *window,
> + struct coprocessor_request_block *crb)
> +{
> + struct coprocessor_status_block csb;
> + struct kernel_siginfo info;
> + struct task_struct *tsk;
> + void __user *csb_addr;
> + struct pid *pid;
> + int rc;
> +
> + /*
> + * NX user space windows can not be opened for task->mm=NULL
> + * and faults will not be generated for kernel requests.
> + */
> + if (WARN_ON_ONCE(!window->mm || !window->user_win))
> + return;
> +
> + csb_addr = (void __user *)be64_to_cpu(crb->csb_addr);
> +
> + memset(&csb, 0, sizeof(csb));
> + csb.cc = CSB_CC_TRANSLATION;
> + csb.ce = CSB_CE_TERMINATION;
> + csb.cs = 0;
> + csb.count = 0;
> +
> + /*
> + * NX operates and returns in BE format as defined CRB struct.
> + * So return fault_storage_addr in BE as NX pastes in FIFO and
> + * expects user space to convert to CPU format.
> + */
> + csb.address = crb->stamp.nx.fault_storage_addr;
> + csb.flags = 0;
> +
> + pid = window->pid;
> + tsk = get_pid_task(pid, PIDTYPE_PID);
> + /*
> + * Send window will be closed after processing all NX requests
> + * and process exits after closing all windows. In multi-thread
> + * applications, thread may not exists, but does not close FD
> + * (means send window) upon exit. Parent thread (tgid) can use
> + * and close the window later.
> + * pid and mm references are taken when window is opened by
> + * process (pid). So tgid is used only when child thread opens
> + * a window and exits without closing it in multithread tasks.
> + */
> + if (!tsk) {
> + pid = window->tgid;
> + tsk = get_pid_task(pid, PIDTYPE_PID);
> + /*
> + * Parent thread will be closing window during its exit.
> + * So should not get here.
> + */
> + if (WARN_ON_ONCE(!tsk))
> + return;
> + }
> +
> + /* Return if the task is exiting. */
> + if (tsk->flags & PF_EXITING) {
> + put_task_struct(tsk);
> + return;
> + }
> +
> + use_mm(window->mm);
> + rc = copy_to_user(csb_addr, &csb, sizeof(csb));
> + /*
> + * User space polls on csb.flags (first byte). So add barrier
> + * then copy first byte with csb flags update.
> + */
> + if (!rc) {
> + csb.flags = CSB_V;
> + smp_mb();
> + rc = copy_to_user(csb_addr, &csb, sizeof(u8));
> + }
> + unuse_mm(window->mm);
> + put_task_struct(tsk);
> +
> + /* Success */
> + if (!rc)
> + return;
> +
> + pr_debug("Invalid CSB address 0x%p signalling pid(%d)\n",
> + csb_addr, pid_vnr(pid));
> +
> + clear_siginfo(&info);
> + info.si_signo = SIGSEGV;
> + info.si_errno = EFAULT;
> + info.si_code = SEGV_MAPERR;
> + info.si_addr = csb_addr;
> +
> + /*
> + * process will be polling on csb.flags after request is sent to
> + * NX. So generally CSB update should not fail except when an
> + * application does not follow the process properly. So an error
> + * message will be displayed and leave it to user space whether
> + * to ignore or handle this signal.
> + */
> + rcu_read_lock();
> + rc = kill_pid_info(SIGSEGV, &info, pid);
> + rcu_read_unlock();
> +
> + pr_devel("%s(): pid %d kill_proc_info() rc %d\n", __func__,
> + pid_vnr(pid), rc);
> +}
> +
> +/*
> * Process valid CRBs in fault FIFO.
> */
> irqreturn_t vas_fault_thread_fn(int irq, void *data)
> @@ -111,6 +225,7 @@ irqreturn_t vas_fault_thread_fn(int irq, void *data)
> return IRQ_HANDLED;
> }
>
> + update_csb(window, crb);
> } while (true);
> }
>
> --
> 1.8.3.1
>
>
>
>
^ permalink raw reply
* Re: [PATCH v8 08/14] powerpc/vas: Take reference to PID and mm for user space windows
From: Nicholas Piggin @ 2020-03-23 2:34 UTC (permalink / raw)
To: Haren Myneni, mpe; +Cc: mikey, ajd, hch, oohall, sukadev, linuxppc-dev, herbert
In-Reply-To: <1584598613.9256.15257.camel@hbabu-laptop>
Haren Myneni's on March 19, 2020 4:16 pm:
>
> When process opens a window, its pid and tgid will be saved in vas_window
> struct. This window will be closed when the process exits. Kernel handles
> NX faults by updating CSB or send SEGV signal to pid if user space csb_addr
> is invalid.
Bit of a nitpick, but can you use articles consistently ("the", "a")? I
won't keep nitpicking changelogs but I think they could be made easier
to read. I'm happy to help proof read and suggest things offline when
you're happy with the technical content of them, let me know.
>
> In multi-thread applications, a window can be opened by child thread, but
> it will not be closed when this thread exits. Expects parent to clean up
> all resources including NX windows. Child thread can send requests using
> this window and can be killed before they are completed. But the pid
> assigned to this thread can be reused for other task while requests are
> pending. If the csb_addr passed in these requests is invalid, kernel will
> end up sending signal to the wrong task.
>
> To prevent reusing the pid, take references to pid and mm when the window
> is opened and release them during window close.
We went over this together a while back, but task management isn't
something I look at every day and it's complicated and easy to introduce
bugs. I suggest if we can get the changelog and comments written well
and understandable for someone who does not know or care about vas,
then cc linux-kernel and the maintainers, and hopefully someone will
take a look. It's not a large patch so if assumptions and concurrency
etc is documented, then it shouldn't be too much work.
Thanks,
Nick
>
> Signed-off-by: Haren Myneni <haren@linux.ibm.com>
> ---
> arch/powerpc/platforms/powernv/vas-debug.c | 2 +-
> arch/powerpc/platforms/powernv/vas-window.c | 53 ++++++++++++++++++++++++++---
> arch/powerpc/platforms/powernv/vas.h | 9 ++++-
> 3 files changed, 57 insertions(+), 7 deletions(-)
>
> diff --git a/arch/powerpc/platforms/powernv/vas-debug.c b/arch/powerpc/platforms/powernv/vas-debug.c
> index 09e63df..ef9a717 100644
> --- a/arch/powerpc/platforms/powernv/vas-debug.c
> +++ b/arch/powerpc/platforms/powernv/vas-debug.c
> @@ -38,7 +38,7 @@ static int info_show(struct seq_file *s, void *private)
>
> seq_printf(s, "Type: %s, %s\n", cop_to_str(window->cop),
> window->tx_win ? "Send" : "Receive");
> - seq_printf(s, "Pid : %d\n", window->pid);
> + seq_printf(s, "Pid : %d\n", vas_window_pid(window));
>
> unlock:
> mutex_unlock(&vas_mutex);
> diff --git a/arch/powerpc/platforms/powernv/vas-window.c b/arch/powerpc/platforms/powernv/vas-window.c
> index acb6a22..e7641a5 100644
> --- a/arch/powerpc/platforms/powernv/vas-window.c
> +++ b/arch/powerpc/platforms/powernv/vas-window.c
> @@ -12,6 +12,8 @@
> #include <linux/log2.h>
> #include <linux/rcupdate.h>
> #include <linux/cred.h>
> +#include <linux/sched/mm.h>
> +#include <linux/mmu_context.h>
> #include <asm/switch_to.h>
> #include <asm/ppc-opcode.h>
> #include "vas.h"
> @@ -876,8 +878,6 @@ struct vas_window *vas_rx_win_open(int vasid, enum vas_cop_type cop,
> rxwin->user_win = rxattr->user_win;
> rxwin->cop = cop;
> rxwin->wcreds_max = rxattr->wcreds_max ?: VAS_WCREDS_DEFAULT;
> - if (rxattr->user_win)
> - rxwin->pid = task_pid_vnr(current);
>
> init_winctx_for_rxwin(rxwin, rxattr, &winctx);
> init_winctx_regs(rxwin, &winctx);
> @@ -1027,7 +1027,6 @@ struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
> txwin->tx_win = 1;
> txwin->rxwin = rxwin;
> txwin->nx_win = txwin->rxwin->nx_win;
> - txwin->pid = attr->pid;
> txwin->user_win = attr->user_win;
> txwin->wcreds_max = attr->wcreds_max ?: VAS_WCREDS_DEFAULT;
>
> @@ -1068,8 +1067,43 @@ struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
> goto free_window;
> }
>
> - set_vinst_win(vinst, txwin);
> + if (txwin->user_win) {
> + /*
> + * Window opened by child thread may not be closed when
> + * it exits. So take reference to its pid and release it
> + * when the window is free by parent thread.
> + * Acquire a reference to the task's pid to make sure
> + * pid will not be re-used - needed only for multithread
> + * applications.
> + */
> + txwin->pid = get_task_pid(current, PIDTYPE_PID);
> + /*
> + * Acquire a reference to the task's mm.
> + */
> + txwin->mm = get_task_mm(current);
>
> + if (!txwin->mm) {
> + put_pid(txwin->pid);
> + pr_err("VAS: pid(%d): mm_struct is not found\n",
> + current->pid);
> + rc = -EPERM;
> + goto free_window;
> + }
> +
> + mmgrab(txwin->mm);
> + mmput(txwin->mm);
> + mm_context_add_copro(txwin->mm);
> + /*
> + * Process closes window during exit. In the case of
> + * multithread application, child can open window and
> + * can exit without closing it. Expects parent thread
> + * to use and close the window. So do not need to take
> + * pid reference for parent thread.
> + */
> + txwin->tgid = find_get_pid(task_tgid_vnr(current));
> + }
> +
> + set_vinst_win(vinst, txwin);
> return txwin;
>
> free_window:
> @@ -1266,8 +1300,17 @@ int vas_win_close(struct vas_window *window)
> poll_window_castout(window);
>
> /* if send window, drop reference to matching receive window */
> - if (window->tx_win)
> + if (window->tx_win) {
> + if (window->user_win) {
> + /* Drop references to pid and mm */
> + put_pid(window->pid);
> + if (window->mm) {
> + mm_context_remove_copro(window->mm);
> + mmdrop(window->mm);
> + }
> + }
> put_rx_win(window->rxwin);
> + }
>
> vas_window_free(window);
>
> diff --git a/arch/powerpc/platforms/powernv/vas.h b/arch/powerpc/platforms/powernv/vas.h
> index 310b8a0..16aa8ec 100644
> --- a/arch/powerpc/platforms/powernv/vas.h
> +++ b/arch/powerpc/platforms/powernv/vas.h
> @@ -353,7 +353,9 @@ struct vas_window {
> bool user_win; /* True if user space window */
> void *hvwc_map; /* HV window context */
> void *uwc_map; /* OS/User window context */
> - pid_t pid; /* Linux process id of owner */
> + struct pid *pid; /* Linux process id of owner */
> + struct pid *tgid; /* Thread group ID of owner */
> + struct mm_struct *mm; /* Linux process mm_struct */
> int wcreds_max; /* Window credits */
>
> char *dbgname;
> @@ -431,6 +433,11 @@ struct vas_winctx {
> extern struct vas_window *vas_pswid_to_window(struct vas_instance *vinst,
> uint32_t pswid);
>
> +static inline int vas_window_pid(struct vas_window *window)
> +{
> + return pid_vnr(window->pid);
> +}
> +
> static inline void vas_log_write(struct vas_window *win, char *name,
> void *regptr, u64 val)
> {
> --
> 1.8.3.1
>
>
>
>
^ permalink raw reply
* Argh, can't find dcache properties !
From: Chris Packham @ 2020-03-23 2:25 UTC (permalink / raw)
To: christophe.leroy@c-s.fr, paulus@samba.org, mpe@ellerman.id.au,
benh@kernel.crashing.org, tglx@linutronix.de, cai@lca.pw
Cc: Hamish Martin, linuxppc-dev@lists.ozlabs.org,
linux-kernel@vger.kernel.org
Hi All,
Just booting up v5.5.11 on a Freescale T2080RDB and I'm seeing the
following mesage.
kern.warning linuxbox kernel: Argh, can't find dcache properties !
kern.warning linuxbox kernel: Argh, can't find icache properties !
This was changed from DBG() to pr_warn() in commit 3b9176e9a874
("powerpc/setup_64: fix -Wempty-body warnings") but the message seems
to be much older than that. So it's probably been an issue on the T2080
(and other QorIQ SoCs) for a while.
Looking at the code the t208x doesn't specifiy any of the d-cache-
size/i-cache-size properties. Should I add them to silence the warning
or switch it to pr_debug()/pr_info()?
Thanks,
Chris
^ permalink raw reply
* Re: [PATCH v8 06/14] powerpc/vas: Setup thread IRQ handler per VAS instance
From: Nicholas Piggin @ 2020-03-23 2:23 UTC (permalink / raw)
To: Haren Myneni, mpe; +Cc: mikey, ajd, hch, oohall, sukadev, linuxppc-dev, herbert
In-Reply-To: <1584598540.9256.15252.camel@hbabu-laptop>
Haren Myneni's on March 19, 2020 4:15 pm:
>
> Setup thread IRQ handler per each VAS instance. When NX sees a fault
> on CRB, kernel gets an interrupt and vas_fault_handler will be
> executed to process fault CRBs. Read all valid CRBs from fault FIFO,
> determine the corresponding send window from CRB and process fault
> requests.
Perhaps some more overview/why.
"If NX encounters a translation error when accessing the CRB or one
of addresses in the request, it raises an interrupt on the CPU to
handle the fault.
>
> Signed-off-by: Sukadev Bhattiprolu <sukadev@linux.vnet.ibm.com>
> Signed-off-by: Haren Myneni <haren@linux.ibm.com>
> ---
> arch/powerpc/platforms/powernv/vas-fault.c | 90 +++++++++++++++++++++++++++++
> arch/powerpc/platforms/powernv/vas-window.c | 60 +++++++++++++++++++
> arch/powerpc/platforms/powernv/vas.c | 49 +++++++++++++++-
> arch/powerpc/platforms/powernv/vas.h | 6 ++
> 4 files changed, 204 insertions(+), 1 deletion(-)
>
> diff --git a/arch/powerpc/platforms/powernv/vas-fault.c b/arch/powerpc/platforms/powernv/vas-fault.c
> index 4044998..1c6d5cc 100644
> --- a/arch/powerpc/platforms/powernv/vas-fault.c
> +++ b/arch/powerpc/platforms/powernv/vas-fault.c
> @@ -11,6 +11,7 @@
> #include <linux/slab.h>
> #include <linux/uaccess.h>
> #include <linux/kthread.h>
> +#include <linux/mmu_context.h>
> #include <asm/icswx.h>
>
> #include "vas.h"
> @@ -25,6 +26,95 @@
> #define VAS_FAULT_WIN_FIFO_SIZE (4 << 20)
>
> /*
> + * Process valid CRBs in fault FIFO.
> + */
> +irqreturn_t vas_fault_thread_fn(int irq, void *data)
Are page faults the only reason why VAS would raise this interrupt? Is
NX really the only possible user of this, so you can have NX specifics
in here?
> +{
> + struct vas_instance *vinst = data;
> + struct coprocessor_request_block *crb, *entry;
> + struct coprocessor_request_block buf;
> + struct vas_window *window;
> + unsigned long flags;
> + void *fifo;
> +
> + crb = &buf;
The below comment could just be moved to replace the one at the top of the
function. Can you explain slightly more about how the faults work, and
be more clear about what the coprocessor does versus what the host does? The
use of VAS and NX is a bit confusing too. VAS doesn't interrupt with
page faults, does it? NX has the page fault(s), and it requests VAS to
interrupt the host?
> +
> + /*
> + * VAS can interrupt with multiple page faults. So process all
> + * valid CRBs within fault FIFO until reaches invalid CRB.
When NX encounters a fault accessing a memory address for a particular
CRB, it updates the nx_fault_stamp field in the CRB (to what?), and
copies the CRB to the fault FIFO memory, then raises an interrupt on the
CPU (memory ordering on the store and load sides are provided how?). NX
can store multiple faults into the FIFO per interrupt (does it proceed
asynchronously after the interrupt? what's the stopping condition?).
When the CPU takes this interrupt, it reads the faulting CRBs from the
FIFO and processes them in order until it reaches an invalid entry, FIFO
empty (memory ordering how?). After each FIFO entry is processed, store
to mark them as invalid. (How does NX resume after this?)
How is the fault actually even "handled" here? Nothing seems to be
actually done for them.
> + * NX updates nx_fault_stamp in CRB and pastes in fault FIFO.
> + * kernel retrives send window from parition send window ID
> + * (pswid) in nx_fault_stamp. So pswid should be valid and
> + * ccw[0] (in be) should be zero since this bit is reserved.
> + * If user space touches this bit, NX returns with "CRB format
> + * error".
> + *
> + * After reading CRB entry, invalidate it with pswid (set
> + * 0xffffffff) and ccw[0] (set to 1).
Al this is very busy and hard to decipher unambiguously. It should read
more like a spec, a precise sequence of things happening.
> + *
> + * In case kernel receives another interrupt with different page
> + * fault, CRBs are already processed by the previous handling. So
> + * will be returned from this function when it sees invalid CRB.
> + */
Ambiguous at best. Assuming the NX continues running asynchronously and
it's a usual kind of FIFO, I assume this means if the kernel gets
another interrupt for a page fault corresponding to a FIFO entry that
has already been processed by this fault.
> + do {
Can you make this 'while (true)' or 'for (;;)' so you don't need to go
to the bottom to see it's an infinite loop.
> + mutex_lock(&vinst->mutex);
What does this protect? Threaded handlers don't run concurrently for the
same request_threaded_irq?
> +
> + spin_lock_irqsave(&vinst->fault_lock, flags);
> + /*
> + * Advance the fault fifo pointer to next CRB.
The code below the comment isn't advancing the fault fifo pointer, it's
grabbing the current one. The pointer (fault_crbs) is advanced later.
You presumabl don't want to advance over an invalid entry.
> + * Use CRB_SIZE rather than sizeof(*crb) since the latter is
> + * aligned to CRB_ALIGN (256) but the CRB written to by VAS is
> + * only CRB_SIZE in len.
> + */
> + fifo = vinst->fault_fifo + (vinst->fault_crbs * CRB_SIZE);
> + entry = fifo;
Don't think you should really do this. It may be harmless in this case,
but the compiler expects the type to be aligned. Make it another type,
like coprocessor_fault_block or something?
> +
> + if ((entry->stamp.nx.pswid == cpu_to_be32(FIFO_INVALID_ENTRY))
> + || (entry->ccw & cpu_to_be32(CCW0_INVALID))) {
> + atomic_set(&vinst->faults_in_progress, 0);
> + spin_unlock_irqrestore(&vinst->fault_lock, flags);
So what does the fault_lock protect? The only data it protects is
faults_in_progress (vs the hard interrupt handler), which doesn't
achieve anything by itself, so I guess it also prevents the hard irq
handler from returning until the handler here has checked that the
fault FIFO is empty then returns IRQ_HANDLED? That seems fine (so long
as memory ordering details are okay), but it should be documented
that way.
Also why is the hard handler in a different file? Makes it harder to
see how this works at a glance.
faults_in_progress does not have to be atomic because it's always
accessed under the lock. And IMO it should have a better name. If the
NX can be causing more faults as we go, it really doesn't indicate
anything about faults. It's whether or not the threaded handler is
currently woken and processing faults.
> + mutex_unlock(&vinst->mutex);
> + return IRQ_HANDLED;
> + }
> +
> + spin_unlock_irqrestore(&vinst->fault_lock, flags);
> + vinst->fault_crbs++;
> + if (vinst->fault_crbs == (vinst->fault_fifo_size / CRB_SIZE))
> + vinst->fault_crbs = 0;
> +
> + memcpy(crb, fifo, CRB_SIZE);
> + entry->stamp.nx.pswid = cpu_to_be32(FIFO_INVALID_ENTRY);
> + entry->ccw |= cpu_to_be32(CCW0_INVALID);
> + mutex_unlock(&vinst->mutex);
> +
> + pr_devel("VAS[%d] fault_fifo %p, fifo %p, fault_crbs %d\n",
> + vinst->vas_id, vinst->fault_fifo, fifo,
> + vinst->fault_crbs);
> +
> + window = vas_pswid_to_window(vinst,
> + be32_to_cpu(crb->stamp.nx.pswid));
> +
> + if (IS_ERR(window)) {
> + /*
> + * We got an interrupt about a specific send
> + * window but we can't find that window and we can't
> + * even clean it up (return credit).
> + * But we should not get here.
> + */
> + pr_err("VAS[%d] fault_fifo %p, fifo %p, pswid 0x%x, fault_crbs %d bad CRB?\n",
> + vinst->vas_id, vinst->fault_fifo, fifo,
> + be32_to_cpu(crb->stamp.nx.pswid),
> + vinst->fault_crbs);
> +
> + WARN_ON_ONCE(1);
> + atomic_set(&vinst->faults_in_progress, 0);
> + return IRQ_HANDLED;
Shouldn't get here but you have a handler for it, so it should try to
be graceful. Keep processing the rest of the FIFO until it's empty
otherwise you have a missed wakeup here? Probably less code too, just
delete the last 2 lines.
Thanks,
Nick
> + }
> +
> + } while (true);
> +}
> +
> +/*
> * Fault window is opened per VAS instance. NX pastes fault CRB in fault
> * FIFO upon page faults.
> */
> diff --git a/arch/powerpc/platforms/powernv/vas-window.c b/arch/powerpc/platforms/powernv/vas-window.c
> index 1783fa9..1f31c18 100644
> --- a/arch/powerpc/platforms/powernv/vas-window.c
> +++ b/arch/powerpc/platforms/powernv/vas-window.c
> @@ -1040,6 +1040,15 @@ struct vas_window *vas_tx_win_open(int vasid, enum vas_cop_type cop,
> }
> } else {
> /*
> + * Interrupt hanlder or fault window setup failed. Means
> + * NX can not generate fault for page fault. So not
> + * opening for user space tx window.
> + */
> + if (!vinst->virq) {
> + rc = -ENODEV;
> + goto free_window;
> + }
> + /*
> * A user mapping must ensure that context switch issues
> * CP_ABORT for this thread.
> */
> @@ -1254,3 +1263,54 @@ int vas_win_close(struct vas_window *window)
> return 0;
> }
> EXPORT_SYMBOL_GPL(vas_win_close);
> +
> +struct vas_window *vas_pswid_to_window(struct vas_instance *vinst,
> + uint32_t pswid)
> +{
> + struct vas_window *window;
> + int winid;
> +
> + if (!pswid) {
> + pr_devel("%s: called for pswid 0!\n", __func__);
> + return ERR_PTR(-ESRCH);
> + }
> +
> + decode_pswid(pswid, NULL, &winid);
> +
> + if (winid >= VAS_WINDOWS_PER_CHIP)
> + return ERR_PTR(-ESRCH);
> +
> + /*
> + * If application closes the window before the hardware
> + * returns the fault CRB, we should wait in vas_win_close()
> + * for the pending requests. so the window must be active
> + * and the process alive.
> + *
> + * If its a kernel process, we should not get any faults and
> + * should not get here.
> + */
> + window = vinst->windows[winid];
> +
> + if (!window) {
> + pr_err("PSWID decode: Could not find window for winid %d pswid %d vinst 0x%p\n",
> + winid, pswid, vinst);
> + return NULL;
> + }
> +
> + /*
> + * Do some sanity checks on the decoded window. Window should be
> + * NX GZIP user send window. FTW windows should not incur faults
> + * since their CRBs are ignored (not queued on FIFO or processed
> + * by NX).
> + */
> + if (!window->tx_win || !window->user_win || !window->nx_win ||
> + window->cop == VAS_COP_TYPE_FAULT ||
> + window->cop == VAS_COP_TYPE_FTW) {
> + pr_err("PSWID decode: id %d, tx %d, user %d, nx %d, cop %d\n",
> + winid, window->tx_win, window->user_win,
> + window->nx_win, window->cop);
> + WARN_ON(1);
> + }
> +
> + return window;
> +}
> diff --git a/arch/powerpc/platforms/powernv/vas.c b/arch/powerpc/platforms/powernv/vas.c
> index 557c8e4..3d9ba58 100644
> --- a/arch/powerpc/platforms/powernv/vas.c
> +++ b/arch/powerpc/platforms/powernv/vas.c
> @@ -14,6 +14,8 @@
> #include <linux/of_platform.h>
> #include <linux/of_address.h>
> #include <linux/of.h>
> +#include <linux/irqdomain.h>
> +#include <linux/interrupt.h>
> #include <asm/prom.h>
> #include <asm/xive.h>
>
> @@ -24,9 +26,53 @@
>
> static DEFINE_PER_CPU(int, cpu_vas_id);
>
> +static irqreturn_t vas_fault_handler(int irq, void *dev_id)
> +{
> + struct vas_instance *vinst = dev_id;
> + irqreturn_t ret = IRQ_WAKE_THREAD;
> + unsigned long flags;
> +
> + /*
> + * NX can generate an interrupt for multiple faults. So the
> + * fault handler thread process all CRBs until finds invalid
> + * entry. In case if NX sees continuous faults, it is possible
> + * that the thread function entered with the first interrupt
> + * can execute and process all valid CRBs.
> + * So wake up thread only if the fault thread is not in progress.
> + */
> + spin_lock_irqsave(&vinst->fault_lock, flags);
> +
> + if (atomic_read(&vinst->faults_in_progress))
> + ret = IRQ_HANDLED;
> + else
> + atomic_set(&vinst->faults_in_progress, 1);
> +
> + spin_unlock_irqrestore(&vinst->fault_lock, flags);
> +
> + return ret;
> +}
> +
> static int vas_irq_fault_window_setup(struct vas_instance *vinst)
> {
> - return vas_setup_fault_window(vinst);
> + char devname[64];
> + int rc = 0;
> +
> + snprintf(devname, sizeof(devname), "vas-%d", vinst->vas_id);
> + rc = request_threaded_irq(vinst->virq, vas_fault_handler,
> + vas_fault_thread_fn, 0, devname, vinst);
> +
> + if (rc) {
> + pr_err("VAS[%d]: Request IRQ(%d) failed with %d\n",
> + vinst->vas_id, vinst->virq, rc);
> + goto out;
> + }
> +
> + rc = vas_setup_fault_window(vinst);
> + if (rc)
> + free_irq(vinst->virq, vinst);
> +
> +out:
> + return rc;
> }
>
> static int init_vas_instance(struct platform_device *pdev)
> @@ -109,6 +155,7 @@ static int init_vas_instance(struct platform_device *pdev)
> list_add(&vinst->node, &vas_instances);
> mutex_unlock(&vas_mutex);
>
> + spin_lock_init(&vinst->fault_lock);
> /*
> * IRQ and fault handling setup is needed only for user space
> * send windows.
> diff --git a/arch/powerpc/platforms/powernv/vas.h b/arch/powerpc/platforms/powernv/vas.h
> index 6c4baf5..ecae7cd 100644
> --- a/arch/powerpc/platforms/powernv/vas.h
> +++ b/arch/powerpc/platforms/powernv/vas.h
> @@ -326,7 +326,10 @@ struct vas_instance {
>
> u64 irq_port;
> int virq;
> + int fault_crbs;
> int fault_fifo_size;
> + atomic_t faults_in_progress;
> + spinlock_t fault_lock;
> void *fault_fifo;
> struct vas_window *fault_win; /* Fault window */
>
> @@ -424,6 +427,9 @@ struct vas_winctx {
> extern void vas_window_init_dbgdir(struct vas_window *win);
> extern void vas_window_free_dbgdir(struct vas_window *win);
> extern int vas_setup_fault_window(struct vas_instance *vinst);
> +extern irqreturn_t vas_fault_thread_fn(int irq, void *data);
> +extern struct vas_window *vas_pswid_to_window(struct vas_instance *vinst,
> + uint32_t pswid);
>
> static inline void vas_log_write(struct vas_window *win, char *name,
> void *regptr, u64 val)
> --
> 1.8.3.1
>
>
>
>
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox