* [PATCH v3 14/20] arm64: erratum: Work around Falkor erratum #E1003 in trampoline code
From: Will Deacon @ 2017-12-06 12:35 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512563739-25239-1-git-send-email-will.deacon@arm.com>
We rely on an atomic swizzling of TTBR1 when transitioning from the entry
trampoline to the kernel proper on an exception. We can't rely on this
atomicity in the face of Falkor erratum #E1003, so on affected cores we
can issue a TLB invalidation to invalidate the walk cache prior to
jumping into the kernel. There is still the possibility of a TLB conflict
here due to conflicting walk cache entries prior to the invalidation, but
this doesn't appear to be the case on these CPUs in practice.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
arch/arm64/Kconfig | 17 +++++------------
arch/arm64/kernel/entry.S | 12 ++++++++++++
2 files changed, 17 insertions(+), 12 deletions(-)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index a93339f5178f..fdcc7b9bb15d 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -522,20 +522,13 @@ config CAVIUM_ERRATUM_30115
config QCOM_FALKOR_ERRATUM_1003
bool "Falkor E1003: Incorrect translation due to ASID change"
default y
- select ARM64_PAN if ARM64_SW_TTBR0_PAN
help
On Falkor v1, an incorrect ASID may be cached in the TLB when ASID
- and BADDR are changed together in TTBRx_EL1. The workaround for this
- issue is to use a reserved ASID in cpu_do_switch_mm() before
- switching to the new ASID. Saying Y here selects ARM64_PAN if
- ARM64_SW_TTBR0_PAN is selected. This is done because implementing and
- maintaining the E1003 workaround in the software PAN emulation code
- would be an unnecessary complication. The affected Falkor v1 CPU
- implements ARMv8.1 hardware PAN support and using hardware PAN
- support versus software PAN emulation is mutually exclusive at
- runtime.
-
- If unsure, say Y.
+ and BADDR are changed together in TTBRx_EL1. Since we keep the ASID
+ in TTBR1_EL1, this situation only occurs in the entry trampoline and
+ then only for entries in the walk cache, since the leaf translation
+ is unchanged. Work around the erratum by invalidating the walk cache
+ entries for the trampoline before entering the kernel proper.
config QCOM_FALKOR_ERRATUM_1009
bool "Falkor E1009: Prematurely complete a DSB after a TLBI"
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 39e3873b8d5a..ce56592b5f70 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -989,6 +989,18 @@ __ni_sys_trace:
sub \tmp, \tmp, #(SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
bic \tmp, \tmp, #USER_ASID_FLAG
msr ttbr1_el1, \tmp
+#ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
+alternative_if ARM64_WORKAROUND_QCOM_FALKOR_E1003
+ /* ASID already in \tmp[63:48] */
+ movk \tmp, #:abs_g2_nc:(TRAMP_VALIAS >> 12)
+ movk \tmp, #:abs_g1_nc:(TRAMP_VALIAS >> 12)
+ /* 2MB boundary containing the vectors, so we nobble the walk cache */
+ movk \tmp, #:abs_g0_nc:((TRAMP_VALIAS & ~(SZ_2M - 1)) >> 12)
+ isb
+ tlbi vae1, \tmp
+ dsb nsh
+alternative_else_nop_endif
+#endif /* CONFIG_QCOM_FALKOR_ERRATUM_1003 */
.endm
.macro tramp_unmap_kernel, tmp
--
2.1.4
^ permalink raw reply related
* [PATCH v3 13/20] arm64: entry: Hook up entry trampoline to exception vectors
From: Will Deacon @ 2017-12-06 12:35 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512563739-25239-1-git-send-email-will.deacon@arm.com>
Hook up the entry trampoline to our exception vectors so that all
exceptions from and returns to EL0 go via the trampoline, which swizzles
the vector base register accordingly. Transitioning to and from the
kernel clobbers x30, so we use tpidrro_el0 and far_el1 as scratch
registers for native tasks.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
arch/arm64/kernel/entry.S | 39 ++++++++++++++++++++++++++++++++++++---
1 file changed, 36 insertions(+), 3 deletions(-)
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index b99fc928119c..39e3873b8d5a 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -73,6 +73,17 @@
.macro kernel_ventry, el, label, regsize = 64
.align 7
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+ .if \el == 0
+ .if \regsize == 64
+ mrs x30, tpidrro_el0
+ msr tpidrro_el0, xzr
+ .else
+ mov x30, xzr
+ .endif
+ .endif
+#endif
+
sub sp, sp, #S_FRAME_SIZE
#ifdef CONFIG_VMAP_STACK
/*
@@ -119,6 +130,11 @@
b el\()\el\()_\label
.endm
+ .macro tramp_alias, dst, sym
+ mov_q \dst, TRAMP_VALIAS
+ add \dst, \dst, #(\sym - .entry.tramp.text)
+ .endm
+
.macro kernel_entry, el, regsize = 64
.if \regsize == 32
mov w0, w0 // zero upper 32 bits of x0
@@ -271,18 +287,20 @@ alternative_else_nop_endif
.if \el == 0
ldr x23, [sp, #S_SP] // load return stack pointer
msr sp_el0, x23
+ tst x22, #PSR_MODE32_BIT // native task?
+ b.eq 3f
+
#ifdef CONFIG_ARM64_ERRATUM_845719
alternative_if ARM64_WORKAROUND_845719
- tbz x22, #4, 1f
#ifdef CONFIG_PID_IN_CONTEXTIDR
mrs x29, contextidr_el1
msr contextidr_el1, x29
#else
msr contextidr_el1, xzr
#endif
-1:
alternative_else_nop_endif
#endif
+3:
.endif
msr elr_el1, x21 // set up the return data
@@ -304,7 +322,22 @@ alternative_else_nop_endif
ldp x28, x29, [sp, #16 * 14]
ldr lr, [sp, #S_LR]
add sp, sp, #S_FRAME_SIZE // restore sp
- eret // return to kernel
+
+#ifndef CONFIG_UNMAP_KERNEL_AT_EL0
+ eret
+#else
+ .if \el == 0
+ bne 4f
+ msr far_el1, x30
+ tramp_alias x30, tramp_exit_native
+ br x30
+4:
+ tramp_alias x30, tramp_exit_compat
+ br x30
+ .else
+ eret
+ .endif
+#endif
.endm
.macro irq_stack_entry
--
2.1.4
^ permalink raw reply related
* [PATCH v3 12/20] arm64: entry: Explicitly pass exception level to kernel_ventry macro
From: Will Deacon @ 2017-12-06 12:35 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512563739-25239-1-git-send-email-will.deacon@arm.com>
We will need to treat exceptions from EL0 differently in kernel_ventry,
so rework the macro to take the exception level as an argument and
construct the branch target using that.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
arch/arm64/kernel/entry.S | 46 +++++++++++++++++++++++-----------------------
1 file changed, 23 insertions(+), 23 deletions(-)
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 716b5ef42e29..b99fc928119c 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -71,7 +71,7 @@
#define BAD_FIQ 2
#define BAD_ERROR 3
- .macro kernel_ventry label
+ .macro kernel_ventry, el, label, regsize = 64
.align 7
sub sp, sp, #S_FRAME_SIZE
#ifdef CONFIG_VMAP_STACK
@@ -84,7 +84,7 @@
tbnz x0, #THREAD_SHIFT, 0f
sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0
sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp
- b \label
+ b el\()\el\()_\label
0:
/*
@@ -116,7 +116,7 @@
sub sp, sp, x0
mrs x0, tpidrro_el0
#endif
- b \label
+ b el\()\el\()_\label
.endm
.macro kernel_entry, el, regsize = 64
@@ -369,31 +369,31 @@ tsk .req x28 // current thread_info
.align 11
ENTRY(vectors)
- kernel_ventry el1_sync_invalid // Synchronous EL1t
- kernel_ventry el1_irq_invalid // IRQ EL1t
- kernel_ventry el1_fiq_invalid // FIQ EL1t
- kernel_ventry el1_error_invalid // Error EL1t
+ kernel_ventry 1, sync_invalid // Synchronous EL1t
+ kernel_ventry 1, irq_invalid // IRQ EL1t
+ kernel_ventry 1, fiq_invalid // FIQ EL1t
+ kernel_ventry 1, error_invalid // Error EL1t
- kernel_ventry el1_sync // Synchronous EL1h
- kernel_ventry el1_irq // IRQ EL1h
- kernel_ventry el1_fiq_invalid // FIQ EL1h
- kernel_ventry el1_error // Error EL1h
+ kernel_ventry 1, sync // Synchronous EL1h
+ kernel_ventry 1, irq // IRQ EL1h
+ kernel_ventry 1, fiq_invalid // FIQ EL1h
+ kernel_ventry 1, error // Error EL1h
- kernel_ventry el0_sync // Synchronous 64-bit EL0
- kernel_ventry el0_irq // IRQ 64-bit EL0
- kernel_ventry el0_fiq_invalid // FIQ 64-bit EL0
- kernel_ventry el0_error // Error 64-bit EL0
+ kernel_ventry 0, sync // Synchronous 64-bit EL0
+ kernel_ventry 0, irq // IRQ 64-bit EL0
+ kernel_ventry 0, fiq_invalid // FIQ 64-bit EL0
+ kernel_ventry 0, error // Error 64-bit EL0
#ifdef CONFIG_COMPAT
- kernel_ventry el0_sync_compat // Synchronous 32-bit EL0
- kernel_ventry el0_irq_compat // IRQ 32-bit EL0
- kernel_ventry el0_fiq_invalid_compat // FIQ 32-bit EL0
- kernel_ventry el0_error_compat // Error 32-bit EL0
+ kernel_ventry 0, sync_compat, 32 // Synchronous 32-bit EL0
+ kernel_ventry 0, irq_compat, 32 // IRQ 32-bit EL0
+ kernel_ventry 0, fiq_invalid_compat, 32 // FIQ 32-bit EL0
+ kernel_ventry 0, error_compat, 32 // Error 32-bit EL0
#else
- kernel_ventry el0_sync_invalid // Synchronous 32-bit EL0
- kernel_ventry el0_irq_invalid // IRQ 32-bit EL0
- kernel_ventry el0_fiq_invalid // FIQ 32-bit EL0
- kernel_ventry el0_error_invalid // Error 32-bit EL0
+ kernel_ventry 0, sync_invalid, 32 // Synchronous 32-bit EL0
+ kernel_ventry 0, irq_invalid, 32 // IRQ 32-bit EL0
+ kernel_ventry 0, fiq_invalid, 32 // FIQ 32-bit EL0
+ kernel_ventry 0, error_invalid, 32 // Error 32-bit EL0
#endif
END(vectors)
--
2.1.4
^ permalink raw reply related
* [PATCH v3 11/20] arm64: mm: Map entry trampoline into trampoline and kernel page tables
From: Will Deacon @ 2017-12-06 12:35 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512563739-25239-1-git-send-email-will.deacon@arm.com>
The exception entry trampoline needs to be mapped at the same virtual
address in both the trampoline page table (which maps nothing else)
and also the kernel page table, so that we can swizzle TTBR1_EL1 on
exceptions from and return to EL0.
This patch maps the trampoline at a fixed virtual address in the fixmap
area of the kernel virtual address space, which allows the kernel proper
to be randomized with respect to the trampoline when KASLR is enabled.
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
arch/arm64/include/asm/fixmap.h | 4 ++++
arch/arm64/include/asm/pgtable.h | 1 +
arch/arm64/kernel/asm-offsets.c | 6 +++++-
arch/arm64/mm/mmu.c | 23 +++++++++++++++++++++++
4 files changed, 33 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h
index 4052ec39e8db..8119b49be98d 100644
--- a/arch/arm64/include/asm/fixmap.h
+++ b/arch/arm64/include/asm/fixmap.h
@@ -58,6 +58,10 @@ enum fixed_addresses {
FIX_APEI_GHES_NMI,
#endif /* CONFIG_ACPI_APEI_GHES */
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+ FIX_ENTRY_TRAMP_TEXT,
+#define TRAMP_VALIAS (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT))
+#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
__end_of_permanent_fixed_addresses,
/*
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 149d05fb9421..774003b247ad 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -680,6 +680,7 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm,
extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
+extern pgd_t tramp_pg_dir[PTRS_PER_PGD];
/*
* Encode and decode a swap entry:
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 71bf088f1e4b..af247d10252f 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -24,6 +24,7 @@
#include <linux/kvm_host.h>
#include <linux/suspend.h>
#include <asm/cpufeature.h>
+#include <asm/fixmap.h>
#include <asm/thread_info.h>
#include <asm/memory.h>
#include <asm/smp_plat.h>
@@ -148,11 +149,14 @@ int main(void)
DEFINE(ARM_SMCCC_RES_X2_OFFS, offsetof(struct arm_smccc_res, a2));
DEFINE(ARM_SMCCC_QUIRK_ID_OFFS, offsetof(struct arm_smccc_quirk, id));
DEFINE(ARM_SMCCC_QUIRK_STATE_OFFS, offsetof(struct arm_smccc_quirk, state));
-
BLANK();
DEFINE(HIBERN_PBE_ORIG, offsetof(struct pbe, orig_address));
DEFINE(HIBERN_PBE_ADDR, offsetof(struct pbe, address));
DEFINE(HIBERN_PBE_NEXT, offsetof(struct pbe, next));
DEFINE(ARM64_FTR_SYSVAL, offsetof(struct arm64_ftr_reg, sys_val));
+ BLANK();
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+ DEFINE(TRAMP_VALIAS, TRAMP_VALIAS);
+#endif
return 0;
}
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 267d2b79d52d..fe68a48c64cb 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -525,6 +525,29 @@ static int __init parse_rodata(char *arg)
}
early_param("rodata", parse_rodata);
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+static int __init map_entry_trampoline(void)
+{
+ extern char __entry_tramp_text_start[];
+
+ pgprot_t prot = rodata_enabled ? PAGE_KERNEL_ROX : PAGE_KERNEL_EXEC;
+ phys_addr_t pa_start = __pa_symbol(__entry_tramp_text_start);
+
+ /* The trampoline is always mapped and can therefore be global */
+ pgprot_val(prot) &= ~PTE_NG;
+
+ /* Map only the text into the trampoline page table */
+ memset(tramp_pg_dir, 0, PGD_SIZE);
+ __create_pgd_mapping(tramp_pg_dir, pa_start, TRAMP_VALIAS, PAGE_SIZE,
+ prot, pgd_pgtable_alloc, 0);
+
+ /* ...as well as the kernel page table */
+ __set_fixmap(FIX_ENTRY_TRAMP_TEXT, pa_start, prot);
+ return 0;
+}
+core_initcall(map_entry_trampoline);
+#endif
+
/*
* Create fine-grained mappings for the kernel.
*/
--
2.1.4
^ permalink raw reply related
* [PATCH v3 10/20] arm64: entry: Add exception trampoline page for exceptions from EL0
From: Will Deacon @ 2017-12-06 12:35 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512563739-25239-1-git-send-email-will.deacon@arm.com>
To allow unmapping of the kernel whilst running at EL0, we need to
point the exception vectors at an entry trampoline that can map/unmap
the kernel on entry/exit respectively.
This patch adds the trampoline page, although it is not yet plugged
into the vector table and is therefore unused.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
arch/arm64/kernel/entry.S | 86 +++++++++++++++++++++++++++++++++++++++++
arch/arm64/kernel/vmlinux.lds.S | 17 ++++++++
2 files changed, 103 insertions(+)
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index d454d8ed45e4..716b5ef42e29 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -28,6 +28,8 @@
#include <asm/errno.h>
#include <asm/esr.h>
#include <asm/irq.h>
+#include <asm/memory.h>
+#include <asm/mmu.h>
#include <asm/processor.h>
#include <asm/ptrace.h>
#include <asm/thread_info.h>
@@ -943,6 +945,90 @@ __ni_sys_trace:
.popsection // .entry.text
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+/*
+ * Exception vectors trampoline.
+ */
+ .pushsection ".entry.tramp.text", "ax"
+
+ .macro tramp_map_kernel, tmp
+ mrs \tmp, ttbr1_el1
+ sub \tmp, \tmp, #(SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
+ bic \tmp, \tmp, #USER_ASID_FLAG
+ msr ttbr1_el1, \tmp
+ .endm
+
+ .macro tramp_unmap_kernel, tmp
+ mrs \tmp, ttbr1_el1
+ add \tmp, \tmp, #(SWAPPER_DIR_SIZE + RESERVED_TTBR0_SIZE)
+ orr \tmp, \tmp, #USER_ASID_FLAG
+ msr ttbr1_el1, \tmp
+ /*
+ * We avoid running the post_ttbr_update_workaround here because the
+ * user and kernel ASIDs don't have conflicting mappings, so any
+ * "blessing" as described in:
+ *
+ * http://lkml.kernel.org/r/56BB848A.6060603 at caviumnetworks.com
+ *
+ * will not hurt correctness. Whilst this may partially defeat the
+ * point of using split ASIDs in the first place, it avoids
+ * the hit of invalidating the entire I-cache on every return to
+ * userspace.
+ */
+ .endm
+
+ .macro tramp_ventry, regsize = 64
+ .align 7
+1:
+ .if \regsize == 64
+ msr tpidrro_el0, x30 // Restored in kernel_ventry
+ .endif
+ tramp_map_kernel x30
+ ldr x30, =vectors
+ prfm plil1strm, [x30, #(1b - tramp_vectors)]
+ msr vbar_el1, x30
+ add x30, x30, #(1b - tramp_vectors)
+ isb
+ br x30
+ .endm
+
+ .macro tramp_exit, regsize = 64
+ adr x30, tramp_vectors
+ msr vbar_el1, x30
+ tramp_unmap_kernel x30
+ .if \regsize == 64
+ mrs x30, far_el1
+ .endif
+ eret
+ .endm
+
+ .align 11
+ENTRY(tramp_vectors)
+ .space 0x400
+
+ tramp_ventry
+ tramp_ventry
+ tramp_ventry
+ tramp_ventry
+
+ tramp_ventry 32
+ tramp_ventry 32
+ tramp_ventry 32
+ tramp_ventry 32
+END(tramp_vectors)
+
+ENTRY(tramp_exit_native)
+ tramp_exit
+END(tramp_exit_native)
+
+ENTRY(tramp_exit_compat)
+ tramp_exit 32
+END(tramp_exit_compat)
+
+ .ltorg
+ .popsection // .entry.tramp.text
+#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
+
/*
* Special system call wrappers.
*/
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 7da3e5c366a0..6b4260f22aab 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -57,6 +57,17 @@ jiffies = jiffies_64;
#define HIBERNATE_TEXT
#endif
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+#define TRAMP_TEXT \
+ . = ALIGN(PAGE_SIZE); \
+ VMLINUX_SYMBOL(__entry_tramp_text_start) = .; \
+ *(.entry.tramp.text) \
+ . = ALIGN(PAGE_SIZE); \
+ VMLINUX_SYMBOL(__entry_tramp_text_end) = .;
+#else
+#define TRAMP_TEXT
+#endif
+
/*
* The size of the PE/COFF section that covers the kernel image, which
* runs from stext to _edata, must be a round multiple of the PE/COFF
@@ -113,6 +124,7 @@ SECTIONS
HYPERVISOR_TEXT
IDMAP_TEXT
HIBERNATE_TEXT
+ TRAMP_TEXT
*(.fixup)
*(.gnu.warning)
. = ALIGN(16);
@@ -214,6 +226,11 @@ SECTIONS
. += RESERVED_TTBR0_SIZE;
#endif
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+ tramp_pg_dir = .;
+ . += PAGE_SIZE;
+#endif
+
__pecoff_data_size = ABSOLUTE(. - __initdata_begin);
_end = .;
--
2.1.4
^ permalink raw reply related
* [PATCH v3 09/20] arm64: mm: Invalidate both kernel and user ASIDs when performing TLBI
From: Will Deacon @ 2017-12-06 12:35 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512563739-25239-1-git-send-email-will.deacon@arm.com>
Since an mm has both a kernel and a user ASID, we need to ensure that
broadcast TLB maintenance targets both address spaces so that things
like CoW continue to work with the uaccess primitives in the kernel.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
arch/arm64/include/asm/tlbflush.h | 16 ++++++++++++++--
1 file changed, 14 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index af1c76981911..9e82dd79c7db 100644
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -23,6 +23,7 @@
#include <linux/sched.h>
#include <asm/cputype.h>
+#include <asm/mmu.h>
/*
* Raw TLBI operations.
@@ -54,6 +55,11 @@
#define __tlbi(op, ...) __TLBI_N(op, ##__VA_ARGS__, 1, 0)
+#define __tlbi_user(op, arg) do { \
+ if (arm64_kernel_unmapped_at_el0()) \
+ __tlbi(op, (arg) | USER_ASID_FLAG); \
+} while (0)
+
/*
* TLB Management
* ==============
@@ -115,6 +121,7 @@ static inline void flush_tlb_mm(struct mm_struct *mm)
dsb(ishst);
__tlbi(aside1is, asid);
+ __tlbi_user(aside1is, asid);
dsb(ish);
}
@@ -125,6 +132,7 @@ static inline void flush_tlb_page(struct vm_area_struct *vma,
dsb(ishst);
__tlbi(vale1is, addr);
+ __tlbi_user(vale1is, addr);
dsb(ish);
}
@@ -151,10 +159,13 @@ static inline void __flush_tlb_range(struct vm_area_struct *vma,
dsb(ishst);
for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) {
- if (last_level)
+ if (last_level) {
__tlbi(vale1is, addr);
- else
+ __tlbi_user(vale1is, addr);
+ } else {
__tlbi(vae1is, addr);
+ __tlbi_user(vae1is, addr);
+ }
}
dsb(ish);
}
@@ -194,6 +205,7 @@ static inline void __flush_tlb_pgtable(struct mm_struct *mm,
unsigned long addr = uaddr >> 12 | (ASID(mm) << 48);
__tlbi(vae1is, addr);
+ __tlbi_user(vae1is, addr);
dsb(ish);
}
--
2.1.4
^ permalink raw reply related
* [PATCH v3 08/20] arm64: mm: Add arm64_kernel_unmapped_at_el0 helper
From: Will Deacon @ 2017-12-06 12:35 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512563739-25239-1-git-send-email-will.deacon@arm.com>
In order for code such as TLB invalidation to operate efficiently when
the decision to map the kernel at EL0 is determined at runtime, this
patch introduces a helper function, arm64_kernel_unmapped_at_el0, to
determine whether or not the kernel is mapped whilst running in userspace.
Currently, this just reports the value of CONFIG_UNMAP_KERNEL_AT_EL0,
but will later be hooked up to a fake CPU capability using a static key.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
arch/arm64/include/asm/mmu.h | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h
index 01bfb184f2a8..c07954638658 100644
--- a/arch/arm64/include/asm/mmu.h
+++ b/arch/arm64/include/asm/mmu.h
@@ -19,6 +19,8 @@
#define MMCF_AARCH32 0x1 /* mm context flag for AArch32 executables */
#define USER_ASID_FLAG (UL(1) << 48)
+#ifndef __ASSEMBLY__
+
typedef struct {
atomic64_t id;
void *vdso;
@@ -32,6 +34,11 @@ typedef struct {
*/
#define ASID(mm) ((mm)->context.id.counter & 0xffff)
+static inline bool arm64_kernel_unmapped_at_el0(void)
+{
+ return IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0);
+}
+
extern void paging_init(void);
extern void bootmem_init(void);
extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt);
@@ -42,4 +49,5 @@ extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
extern void *fixmap_remap_fdt(phys_addr_t dt_phys);
extern void mark_linear_text_alias_ro(void);
+#endif /* !__ASSEMBLY__ */
#endif
--
2.1.4
^ permalink raw reply related
* [PATCH v3 07/20] arm64: mm: Allocate ASIDs in pairs
From: Will Deacon @ 2017-12-06 12:35 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512563739-25239-1-git-send-email-will.deacon@arm.com>
In preparation for separate kernel/user ASIDs, allocate them in pairs
for each mm_struct. The bottom bit distinguishes the two: if it is set,
then the ASID will map only userspace.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
arch/arm64/include/asm/mmu.h | 1 +
arch/arm64/mm/context.c | 25 +++++++++++++++++--------
2 files changed, 18 insertions(+), 8 deletions(-)
diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h
index 0d34bf0a89c7..01bfb184f2a8 100644
--- a/arch/arm64/include/asm/mmu.h
+++ b/arch/arm64/include/asm/mmu.h
@@ -17,6 +17,7 @@
#define __ASM_MMU_H
#define MMCF_AARCH32 0x1 /* mm context flag for AArch32 executables */
+#define USER_ASID_FLAG (UL(1) << 48)
typedef struct {
atomic64_t id;
diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c
index 78a2dc596fee..1cb3bc92ae5c 100644
--- a/arch/arm64/mm/context.c
+++ b/arch/arm64/mm/context.c
@@ -39,7 +39,16 @@ static cpumask_t tlb_flush_pending;
#define ASID_MASK (~GENMASK(asid_bits - 1, 0))
#define ASID_FIRST_VERSION (1UL << asid_bits)
-#define NUM_USER_ASIDS ASID_FIRST_VERSION
+
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+#define NUM_USER_ASIDS (ASID_FIRST_VERSION >> 1)
+#define asid2idx(asid) (((asid) & ~ASID_MASK) >> 1)
+#define idx2asid(idx) (((idx) << 1) & ~ASID_MASK)
+#else
+#define NUM_USER_ASIDS (ASID_FIRST_VERSION)
+#define asid2idx(asid) ((asid) & ~ASID_MASK)
+#define idx2asid(idx) asid2idx(idx)
+#endif
/* Get the ASIDBits supported by the current CPU */
static u32 get_cpu_asid_bits(void)
@@ -98,7 +107,7 @@ static void flush_context(unsigned int cpu)
*/
if (asid == 0)
asid = per_cpu(reserved_asids, i);
- __set_bit(asid & ~ASID_MASK, asid_map);
+ __set_bit(asid2idx(asid), asid_map);
per_cpu(reserved_asids, i) = asid;
}
@@ -153,16 +162,16 @@ static u64 new_context(struct mm_struct *mm, unsigned int cpu)
* We had a valid ASID in a previous life, so try to re-use
* it if possible.
*/
- asid &= ~ASID_MASK;
- if (!__test_and_set_bit(asid, asid_map))
+ if (!__test_and_set_bit(asid2idx(asid), asid_map))
return newasid;
}
/*
* Allocate a free ASID. If we can't find one, take a note of the
- * currently active ASIDs and mark the TLBs as requiring flushes.
- * We always count from ASID #1, as we use ASID #0 when setting a
- * reserved TTBR0 for the init_mm.
+ * currently active ASIDs and mark the TLBs as requiring flushes. We
+ * always count from ASID #2 (index 1), as we use ASID #0 when setting
+ * a reserved TTBR0 for the init_mm and we allocate ASIDs in even/odd
+ * pairs.
*/
asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, cur_idx);
if (asid != NUM_USER_ASIDS)
@@ -179,7 +188,7 @@ static u64 new_context(struct mm_struct *mm, unsigned int cpu)
set_asid:
__set_bit(asid, asid_map);
cur_idx = asid;
- return asid | generation;
+ return idx2asid(asid) | generation;
}
void check_and_switch_context(struct mm_struct *mm, unsigned int cpu)
--
2.1.4
^ permalink raw reply related
* [PATCH v3 06/20] arm64: mm: Fix and re-enable ARM64_SW_TTBR0_PAN
From: Will Deacon @ 2017-12-06 12:35 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512563739-25239-1-git-send-email-will.deacon@arm.com>
With the ASID now installed in TTBR1, we can re-enable ARM64_SW_TTBR0_PAN
by ensuring that we switch to a reserved ASID of zero when disabling
user access and restore the active user ASID on the uaccess enable path.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
arch/arm64/Kconfig | 1 -
arch/arm64/include/asm/asm-uaccess.h | 25 +++++++++++++++++--------
arch/arm64/include/asm/uaccess.h | 21 +++++++++++++++++----
arch/arm64/kernel/entry.S | 4 ++--
arch/arm64/lib/clear_user.S | 2 +-
arch/arm64/lib/copy_from_user.S | 2 +-
arch/arm64/lib/copy_in_user.S | 2 +-
arch/arm64/lib/copy_to_user.S | 2 +-
arch/arm64/mm/cache.S | 2 +-
arch/arm64/xen/hypercall.S | 2 +-
10 files changed, 42 insertions(+), 21 deletions(-)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 7e7d7fd152c4..a93339f5178f 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -910,7 +910,6 @@ endif
config ARM64_SW_TTBR0_PAN
bool "Emulate Privileged Access Never using TTBR0_EL1 switching"
- depends on BROKEN # Temporary while switch_mm is reworked
help
Enabling this option prevents the kernel from accessing
user-space memory directly by pointing TTBR0_EL1 to a reserved
diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h
index b3da6c886835..21b8cf304028 100644
--- a/arch/arm64/include/asm/asm-uaccess.h
+++ b/arch/arm64/include/asm/asm-uaccess.h
@@ -16,11 +16,20 @@
add \tmp1, \tmp1, #SWAPPER_DIR_SIZE // reserved_ttbr0 at the end of swapper_pg_dir
msr ttbr0_el1, \tmp1 // set reserved TTBR0_EL1
isb
+ sub \tmp1, \tmp1, #SWAPPER_DIR_SIZE
+ bic \tmp1, \tmp1, #(0xffff << 48)
+ msr ttbr1_el1, \tmp1 // set reserved ASID
+ isb
.endm
- .macro __uaccess_ttbr0_enable, tmp1
+ .macro __uaccess_ttbr0_enable, tmp1, tmp2
get_thread_info \tmp1
ldr \tmp1, [\tmp1, #TSK_TI_TTBR0] // load saved TTBR0_EL1
+ mrs \tmp2, ttbr1_el1
+ extr \tmp2, \tmp2, \tmp1, #48
+ ror \tmp2, \tmp2, #16
+ msr ttbr1_el1, \tmp2 // set the active ASID
+ isb
msr ttbr0_el1, \tmp1 // set the non-PAN TTBR0_EL1
isb
.endm
@@ -31,18 +40,18 @@ alternative_if_not ARM64_HAS_PAN
alternative_else_nop_endif
.endm
- .macro uaccess_ttbr0_enable, tmp1, tmp2
+ .macro uaccess_ttbr0_enable, tmp1, tmp2, tmp3
alternative_if_not ARM64_HAS_PAN
- save_and_disable_irq \tmp2 // avoid preemption
- __uaccess_ttbr0_enable \tmp1
- restore_irq \tmp2
+ save_and_disable_irq \tmp3 // avoid preemption
+ __uaccess_ttbr0_enable \tmp1, \tmp2
+ restore_irq \tmp3
alternative_else_nop_endif
.endm
#else
.macro uaccess_ttbr0_disable, tmp1
.endm
- .macro uaccess_ttbr0_enable, tmp1, tmp2
+ .macro uaccess_ttbr0_enable, tmp1, tmp2, tmp3
.endm
#endif
@@ -56,8 +65,8 @@ alternative_if ARM64_ALT_PAN_NOT_UAO
alternative_else_nop_endif
.endm
- .macro uaccess_enable_not_uao, tmp1, tmp2
- uaccess_ttbr0_enable \tmp1, \tmp2
+ .macro uaccess_enable_not_uao, tmp1, tmp2, tmp3
+ uaccess_ttbr0_enable \tmp1, \tmp2, \tmp3
alternative_if ARM64_ALT_PAN_NOT_UAO
SET_PSTATE_PAN(0)
alternative_else_nop_endif
diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index fc0f9eb66039..750a3b76a01c 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -107,15 +107,19 @@ static inline void __uaccess_ttbr0_disable(void)
{
unsigned long ttbr;
+ ttbr = read_sysreg(ttbr1_el1);
/* reserved_ttbr0 placed@the end of swapper_pg_dir */
- ttbr = read_sysreg(ttbr1_el1) + SWAPPER_DIR_SIZE;
- write_sysreg(ttbr, ttbr0_el1);
+ write_sysreg(ttbr + SWAPPER_DIR_SIZE, ttbr0_el1);
+ isb();
+ /* Set reserved ASID */
+ ttbr &= ~(0xffffUL << 48);
+ write_sysreg(ttbr, ttbr1_el1);
isb();
}
static inline void __uaccess_ttbr0_enable(void)
{
- unsigned long flags;
+ unsigned long flags, ttbr0, ttbr1;
/*
* Disable interrupts to avoid preemption between reading the 'ttbr0'
@@ -123,7 +127,16 @@ static inline void __uaccess_ttbr0_enable(void)
* roll-over and an update of 'ttbr0'.
*/
local_irq_save(flags);
- write_sysreg(current_thread_info()->ttbr0, ttbr0_el1);
+ ttbr0 = current_thread_info()->ttbr0;
+
+ /* Restore active ASID */
+ ttbr1 = read_sysreg(ttbr1_el1);
+ ttbr1 |= ttbr0 & (0xffffUL << 48);
+ write_sysreg(ttbr1, ttbr1_el1);
+ isb();
+
+ /* Restore user page table */
+ write_sysreg(ttbr0, ttbr0_el1);
isb();
local_irq_restore(flags);
}
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 804e43c9cb0b..d454d8ed45e4 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -184,7 +184,7 @@ alternative_if ARM64_HAS_PAN
alternative_else_nop_endif
.if \el != 0
- mrs x21, ttbr0_el1
+ mrs x21, ttbr1_el1
tst x21, #0xffff << 48 // Check for the reserved ASID
orr x23, x23, #PSR_PAN_BIT // Set the emulated PAN in the saved SPSR
b.eq 1f // TTBR0 access already disabled
@@ -248,7 +248,7 @@ alternative_else_nop_endif
tbnz x22, #22, 1f // Skip re-enabling TTBR0 access if the PSR_PAN_BIT is set
.endif
- __uaccess_ttbr0_enable x0
+ __uaccess_ttbr0_enable x0, x1
.if \el == 0
/*
diff --git a/arch/arm64/lib/clear_user.S b/arch/arm64/lib/clear_user.S
index e88fb99c1561..8f9c4641e706 100644
--- a/arch/arm64/lib/clear_user.S
+++ b/arch/arm64/lib/clear_user.S
@@ -30,7 +30,7 @@
* Alignment fixed up by hardware.
*/
ENTRY(__clear_user)
- uaccess_enable_not_uao x2, x3
+ uaccess_enable_not_uao x2, x3, x4
mov x2, x1 // save the size for fixup return
subs x1, x1, #8
b.mi 2f
diff --git a/arch/arm64/lib/copy_from_user.S b/arch/arm64/lib/copy_from_user.S
index 4b5d826895ff..69d86a80f3e2 100644
--- a/arch/arm64/lib/copy_from_user.S
+++ b/arch/arm64/lib/copy_from_user.S
@@ -64,7 +64,7 @@
end .req x5
ENTRY(__arch_copy_from_user)
- uaccess_enable_not_uao x3, x4
+ uaccess_enable_not_uao x3, x4, x5
add end, x0, x2
#include "copy_template.S"
uaccess_disable_not_uao x3
diff --git a/arch/arm64/lib/copy_in_user.S b/arch/arm64/lib/copy_in_user.S
index b24a830419ad..e442b531252a 100644
--- a/arch/arm64/lib/copy_in_user.S
+++ b/arch/arm64/lib/copy_in_user.S
@@ -65,7 +65,7 @@
end .req x5
ENTRY(raw_copy_in_user)
- uaccess_enable_not_uao x3, x4
+ uaccess_enable_not_uao x3, x4, x5
add end, x0, x2
#include "copy_template.S"
uaccess_disable_not_uao x3
diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S
index 351f0766f7a6..318f15d5c336 100644
--- a/arch/arm64/lib/copy_to_user.S
+++ b/arch/arm64/lib/copy_to_user.S
@@ -63,7 +63,7 @@
end .req x5
ENTRY(__arch_copy_to_user)
- uaccess_enable_not_uao x3, x4
+ uaccess_enable_not_uao x3, x4, x5
add end, x0, x2
#include "copy_template.S"
uaccess_disable_not_uao x3
diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index 7f1dbe962cf5..6cd20a8c0952 100644
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -49,7 +49,7 @@ ENTRY(flush_icache_range)
* - end - virtual end address of region
*/
ENTRY(__flush_cache_user_range)
- uaccess_ttbr0_enable x2, x3
+ uaccess_ttbr0_enable x2, x3, x4
dcache_line_size x2, x3
sub x3, x2, #1
bic x4, x0, x3
diff --git a/arch/arm64/xen/hypercall.S b/arch/arm64/xen/hypercall.S
index 401ceb71540c..acdbd2c9e899 100644
--- a/arch/arm64/xen/hypercall.S
+++ b/arch/arm64/xen/hypercall.S
@@ -101,7 +101,7 @@ ENTRY(privcmd_call)
* need the explicit uaccess_enable/disable if the TTBR0 PAN emulation
* is enabled (it implies that hardware UAO and PAN disabled).
*/
- uaccess_ttbr0_enable x6, x7
+ uaccess_ttbr0_enable x6, x7, x8
hvc XEN_IMM
/*
--
2.1.4
^ permalink raw reply related
* [PATCH v3 05/20] arm64: mm: Rename post_ttbr0_update_workaround
From: Will Deacon @ 2017-12-06 12:35 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512563739-25239-1-git-send-email-will.deacon@arm.com>
The post_ttbr0_update_workaround hook applies to any change to TTBRx_EL1.
Since we're using TTBR1 for the ASID, rename the hook to make it clearer
as to what it's doing.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
arch/arm64/include/asm/assembler.h | 5 ++---
arch/arm64/kernel/entry.S | 2 +-
arch/arm64/mm/proc.S | 2 +-
3 files changed, 4 insertions(+), 5 deletions(-)
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index e1fa5db858b7..c45bc94f15d0 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -477,10 +477,9 @@ alternative_endif
.endm
/*
-/*
- * Errata workaround post TTBR0_EL1 update.
+ * Errata workaround post TTBRx_EL1 update.
*/
- .macro post_ttbr0_update_workaround
+ .macro post_ttbr_update_workaround
#ifdef CONFIG_CAVIUM_ERRATUM_27456
alternative_if ARM64_WORKAROUND_CAVIUM_27456
ic iallu
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 6d14b8f29b5f..804e43c9cb0b 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -257,7 +257,7 @@ alternative_else_nop_endif
* Cavium erratum 27456 (broadcast TLBI instructions may cause I-cache
* corruption).
*/
- post_ttbr0_update_workaround
+ post_ttbr_update_workaround
.endif
1:
.if \el != 0
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index f2ff0837577c..3146dc96f05b 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -145,7 +145,7 @@ ENTRY(cpu_do_switch_mm)
isb
msr ttbr0_el1, x0 // now update TTBR0
isb
- post_ttbr0_update_workaround
+ post_ttbr_update_workaround
ret
ENDPROC(cpu_do_switch_mm)
--
2.1.4
^ permalink raw reply related
* [PATCH v3 04/20] arm64: mm: Remove pre_ttbr0_update_workaround for Falkor erratum #E1003
From: Will Deacon @ 2017-12-06 12:35 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512563739-25239-1-git-send-email-will.deacon@arm.com>
The pre_ttbr0_update_workaround hook is called prior to context-switching
TTBR0 because Falkor erratum E1003 can cause TLB allocation with the wrong
ASID if both the ASID and the base address of the TTBR are updated at
the same time.
With the ASID sitting safely in TTBR1, we no longer update things
atomically, so we can remove the pre_ttbr0_update_workaround macro as
it's no longer required. The erratum infrastructure and documentation
is left around for #E1003, as it will be required by the entry
trampoline code in a future patch.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
arch/arm64/include/asm/assembler.h | 22 ----------------------
arch/arm64/include/asm/mmu_context.h | 2 --
arch/arm64/mm/context.c | 11 -----------
arch/arm64/mm/proc.S | 1 -
4 files changed, 36 deletions(-)
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index aef72d886677..e1fa5db858b7 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -26,7 +26,6 @@
#include <asm/asm-offsets.h>
#include <asm/cpufeature.h>
#include <asm/debug-monitors.h>
-#include <asm/mmu_context.h>
#include <asm/page.h>
#include <asm/pgtable-hwdef.h>
#include <asm/ptrace.h>
@@ -478,27 +477,6 @@ alternative_endif
.endm
/*
- * Errata workaround prior to TTBR0_EL1 update
- *
- * val: TTBR value with new BADDR, preserved
- * tmp0: temporary register, clobbered
- * tmp1: other temporary register, clobbered
- */
- .macro pre_ttbr0_update_workaround, val, tmp0, tmp1
-#ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
-alternative_if ARM64_WORKAROUND_QCOM_FALKOR_E1003
- mrs \tmp0, ttbr0_el1
- mov \tmp1, #FALKOR_RESERVED_ASID
- bfi \tmp0, \tmp1, #48, #16 // reserved ASID + old BADDR
- msr ttbr0_el1, \tmp0
- isb
- bfi \tmp0, \val, #0, #48 // reserved ASID + new BADDR
- msr ttbr0_el1, \tmp0
- isb
-alternative_else_nop_endif
-#endif
- .endm
-
/*
* Errata workaround post TTBR0_EL1 update.
*/
diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h
index 2d63611e4311..f9744944cf12 100644
--- a/arch/arm64/include/asm/mmu_context.h
+++ b/arch/arm64/include/asm/mmu_context.h
@@ -19,8 +19,6 @@
#ifndef __ASM_MMU_CONTEXT_H
#define __ASM_MMU_CONTEXT_H
-#define FALKOR_RESERVED_ASID 1
-
#ifndef __ASSEMBLY__
#include <linux/compiler.h>
diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c
index 6f4017046323..78a2dc596fee 100644
--- a/arch/arm64/mm/context.c
+++ b/arch/arm64/mm/context.c
@@ -79,13 +79,6 @@ void verify_cpu_asid_bits(void)
}
}
-static void set_reserved_asid_bits(void)
-{
- if (IS_ENABLED(CONFIG_QCOM_FALKOR_ERRATUM_1003) &&
- cpus_have_const_cap(ARM64_WORKAROUND_QCOM_FALKOR_E1003))
- __set_bit(FALKOR_RESERVED_ASID, asid_map);
-}
-
static void flush_context(unsigned int cpu)
{
int i;
@@ -94,8 +87,6 @@ static void flush_context(unsigned int cpu)
/* Update the list of reserved ASIDs and the ASID bitmap. */
bitmap_clear(asid_map, 0, NUM_USER_ASIDS);
- set_reserved_asid_bits();
-
for_each_possible_cpu(i) {
asid = atomic64_xchg_relaxed(&per_cpu(active_asids, i), 0);
/*
@@ -254,8 +245,6 @@ static int asids_init(void)
panic("Failed to allocate bitmap for %lu ASIDs\n",
NUM_USER_ASIDS);
- set_reserved_asid_bits();
-
pr_info("ASID allocator initialised with %lu entries\n", NUM_USER_ASIDS);
return 0;
}
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index a8a64898a2aa..f2ff0837577c 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -138,7 +138,6 @@ ENDPROC(cpu_do_resume)
* - pgd_phys - physical address of new TTB
*/
ENTRY(cpu_do_switch_mm)
- pre_ttbr0_update_workaround x0, x2, x3
mrs x2, ttbr1_el1
mmid x1, x1 // get mm->context.id
bfi x2, x1, #48, #16 // set the ASID
--
2.1.4
^ permalink raw reply related
* [PATCH v3 03/20] arm64: mm: Move ASID from TTBR0 to TTBR1
From: Will Deacon @ 2017-12-06 12:35 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512563739-25239-1-git-send-email-will.deacon@arm.com>
In preparation for mapping kernelspace and userspace with different
ASIDs, move the ASID to TTBR1 and update switch_mm to context-switch
TTBR0 via an invalid mapping (the zero page).
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
arch/arm64/include/asm/mmu_context.h | 7 +++++++
arch/arm64/include/asm/pgtable-hwdef.h | 1 +
arch/arm64/include/asm/proc-fns.h | 6 ------
arch/arm64/mm/proc.S | 9 ++++++---
4 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h
index 3257895a9b5e..2d63611e4311 100644
--- a/arch/arm64/include/asm/mmu_context.h
+++ b/arch/arm64/include/asm/mmu_context.h
@@ -57,6 +57,13 @@ static inline void cpu_set_reserved_ttbr0(void)
isb();
}
+static inline void cpu_switch_mm(pgd_t *pgd, struct mm_struct *mm)
+{
+ BUG_ON(pgd == swapper_pg_dir);
+ cpu_set_reserved_ttbr0();
+ cpu_do_switch_mm(virt_to_phys(pgd),mm);
+}
+
/*
* TCR.T0SZ value to use when the ID map is active. Usually equals
* TCR_T0SZ(VA_BITS), unless system RAM is positioned very high in
diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h
index eb0c2bd90de9..8df4cb6ac6f7 100644
--- a/arch/arm64/include/asm/pgtable-hwdef.h
+++ b/arch/arm64/include/asm/pgtable-hwdef.h
@@ -272,6 +272,7 @@
#define TCR_TG1_4K (UL(2) << TCR_TG1_SHIFT)
#define TCR_TG1_64K (UL(3) << TCR_TG1_SHIFT)
+#define TCR_A1 (UL(1) << 22)
#define TCR_ASID16 (UL(1) << 36)
#define TCR_TBI0 (UL(1) << 37)
#define TCR_HA (UL(1) << 39)
diff --git a/arch/arm64/include/asm/proc-fns.h b/arch/arm64/include/asm/proc-fns.h
index 14ad6e4e87d1..16cef2e8449e 100644
--- a/arch/arm64/include/asm/proc-fns.h
+++ b/arch/arm64/include/asm/proc-fns.h
@@ -35,12 +35,6 @@ extern u64 cpu_do_resume(phys_addr_t ptr, u64 idmap_ttbr);
#include <asm/memory.h>
-#define cpu_switch_mm(pgd,mm) \
-do { \
- BUG_ON(pgd == swapper_pg_dir); \
- cpu_do_switch_mm(virt_to_phys(pgd),mm); \
-} while (0)
-
#endif /* __ASSEMBLY__ */
#endif /* __KERNEL__ */
#endif /* __ASM_PROCFNS_H */
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 95233dfc4c39..a8a64898a2aa 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -139,9 +139,12 @@ ENDPROC(cpu_do_resume)
*/
ENTRY(cpu_do_switch_mm)
pre_ttbr0_update_workaround x0, x2, x3
+ mrs x2, ttbr1_el1
mmid x1, x1 // get mm->context.id
- bfi x0, x1, #48, #16 // set the ASID
- msr ttbr0_el1, x0 // set TTBR0
+ bfi x2, x1, #48, #16 // set the ASID
+ msr ttbr1_el1, x2 // in TTBR1 (since TCR.A1 is set)
+ isb
+ msr ttbr0_el1, x0 // now update TTBR0
isb
post_ttbr0_update_workaround
ret
@@ -224,7 +227,7 @@ ENTRY(__cpu_setup)
* both user and kernel.
*/
ldr x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
- TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0
+ TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0 | TCR_A1
tcr_set_idmap_t0sz x10, x9
/*
--
2.1.4
^ permalink raw reply related
* [PATCH v3 02/20] arm64: mm: Temporarily disable ARM64_SW_TTBR0_PAN
From: Will Deacon @ 2017-12-06 12:35 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512563739-25239-1-git-send-email-will.deacon@arm.com>
We're about to rework the way ASIDs are allocated, switch_mm is
implemented and low-level kernel entry/exit is handled, so keep the
ARM64_SW_TTBR0_PAN code out of the way whilst we do the heavy lifting.
It will be re-enabled in a subsequent patch.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
arch/arm64/Kconfig | 1 +
1 file changed, 1 insertion(+)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index a93339f5178f..7e7d7fd152c4 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -910,6 +910,7 @@ endif
config ARM64_SW_TTBR0_PAN
bool "Emulate Privileged Access Never using TTBR0_EL1 switching"
+ depends on BROKEN # Temporary while switch_mm is reworked
help
Enabling this option prevents the kernel from accessing
user-space memory directly by pointing TTBR0_EL1 to a reserved
--
2.1.4
^ permalink raw reply related
* [PATCH v3 01/20] arm64: mm: Use non-global mappings for kernel space
From: Will Deacon @ 2017-12-06 12:35 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512563739-25239-1-git-send-email-will.deacon@arm.com>
In preparation for unmapping the kernel whilst running in userspace,
make the kernel mappings non-global so we can avoid expensive TLB
invalidation on kernel exit to userspace.
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
arch/arm64/include/asm/kernel-pgtable.h | 12 ++++++++++--
arch/arm64/include/asm/pgtable-prot.h | 21 +++++++++++++++------
2 files changed, 25 insertions(+), 8 deletions(-)
diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h
index 7803343e5881..77a27af01371 100644
--- a/arch/arm64/include/asm/kernel-pgtable.h
+++ b/arch/arm64/include/asm/kernel-pgtable.h
@@ -78,8 +78,16 @@
/*
* Initial memory map attributes.
*/
-#define SWAPPER_PTE_FLAGS (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
-#define SWAPPER_PMD_FLAGS (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
+#define _SWAPPER_PTE_FLAGS (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
+#define _SWAPPER_PMD_FLAGS (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
+
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+#define SWAPPER_PTE_FLAGS (_SWAPPER_PTE_FLAGS | PTE_NG)
+#define SWAPPER_PMD_FLAGS (_SWAPPER_PMD_FLAGS | PMD_SECT_NG)
+#else
+#define SWAPPER_PTE_FLAGS _SWAPPER_PTE_FLAGS
+#define SWAPPER_PMD_FLAGS _SWAPPER_PMD_FLAGS
+#endif
#if ARM64_SWAPPER_USES_SECTION_MAPS
#define SWAPPER_MM_MMUFLAGS (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS)
diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h
index 0a5635fb0ef9..22a926825e3f 100644
--- a/arch/arm64/include/asm/pgtable-prot.h
+++ b/arch/arm64/include/asm/pgtable-prot.h
@@ -34,8 +34,16 @@
#include <asm/pgtable-types.h>
-#define PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
-#define PROT_SECT_DEFAULT (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
+#define _PROT_DEFAULT (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
+#define _PROT_SECT_DEFAULT (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
+
+#ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+#define PROT_DEFAULT (_PROT_DEFAULT | PTE_NG)
+#define PROT_SECT_DEFAULT (_PROT_SECT_DEFAULT | PMD_SECT_NG)
+#else
+#define PROT_DEFAULT _PROT_DEFAULT
+#define PROT_SECT_DEFAULT _PROT_SECT_DEFAULT
+#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
#define PROT_DEVICE_nGnRnE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRnE))
#define PROT_DEVICE_nGnRE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRE))
@@ -48,6 +56,7 @@
#define PROT_SECT_NORMAL_EXEC (PROT_SECT_DEFAULT | PMD_SECT_UXN | PMD_ATTRINDX(MT_NORMAL))
#define _PAGE_DEFAULT (PROT_DEFAULT | PTE_ATTRINDX(MT_NORMAL))
+#define _HYP_PAGE_DEFAULT (_PAGE_DEFAULT & ~PTE_NG)
#define PAGE_KERNEL __pgprot(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_WRITE)
#define PAGE_KERNEL_RO __pgprot(_PAGE_DEFAULT | PTE_PXN | PTE_UXN | PTE_DIRTY | PTE_RDONLY)
@@ -55,15 +64,15 @@
#define PAGE_KERNEL_EXEC __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE)
#define PAGE_KERNEL_EXEC_CONT __pgprot(_PAGE_DEFAULT | PTE_UXN | PTE_DIRTY | PTE_WRITE | PTE_CONT)
-#define PAGE_HYP __pgprot(_PAGE_DEFAULT | PTE_HYP | PTE_HYP_XN)
-#define PAGE_HYP_EXEC __pgprot(_PAGE_DEFAULT | PTE_HYP | PTE_RDONLY)
-#define PAGE_HYP_RO __pgprot(_PAGE_DEFAULT | PTE_HYP | PTE_RDONLY | PTE_HYP_XN)
+#define PAGE_HYP __pgprot(_HYP_PAGE_DEFAULT | PTE_HYP | PTE_HYP_XN)
+#define PAGE_HYP_EXEC __pgprot(_HYP_PAGE_DEFAULT | PTE_HYP | PTE_RDONLY)
+#define PAGE_HYP_RO __pgprot(_HYP_PAGE_DEFAULT | PTE_HYP | PTE_RDONLY | PTE_HYP_XN)
#define PAGE_HYP_DEVICE __pgprot(PROT_DEVICE_nGnRE | PTE_HYP)
#define PAGE_S2 __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_NORMAL) | PTE_S2_RDONLY)
#define PAGE_S2_DEVICE __pgprot(PROT_DEFAULT | PTE_S2_MEMATTR(MT_S2_DEVICE_nGnRE) | PTE_S2_RDONLY | PTE_UXN)
-#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_RDONLY | PTE_PXN | PTE_UXN)
+#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_RDONLY | PTE_NG | PTE_PXN | PTE_UXN)
#define PAGE_SHARED __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE)
#define PAGE_SHARED_EXEC __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_WRITE)
#define PAGE_READONLY __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_RDONLY | PTE_NG | PTE_PXN | PTE_UXN)
--
2.1.4
^ permalink raw reply related
* [PATCH v3 00/20] arm64: Unmap the kernel whilst running in userspace (KPTI)
From: Will Deacon @ 2017-12-06 12:35 UTC (permalink / raw)
To: linux-arm-kernel
Hi everybody,
This is version three of the patches formerly known as KAISER (?).
v1: http://lists.infradead.org/pipermail/linux-arm-kernel/2017-November/542751.html
v2: http://lists.infradead.org/pipermail/linux-arm-kernel/2017-November/544817.html
Changes since v2 include:
* Rename command-line option from "kaiser=" to "kpti=" for parity with x86
* Fixed Falkor erratum workaround (missing '~')
* Moved vectors base from literal pool into separate data page
* Added TTBR_ASID_MASK instead of open-coded constants
* Added missing newline to error message
* Fail to probe SPE if KPTI is enabled
* Addressed minor review comments
* Added tags
* Based on -rc2
Patches are also pushed here:
git://git.kernel.org/pub/scm/linux/kernel/git/will/linux.git kpti
Feedback and testing welcome. At this point, I'd like to start thinking
about getting this merged for 4.16.
Cheers,
Will
--->8
Will Deacon (20):
arm64: mm: Use non-global mappings for kernel space
arm64: mm: Temporarily disable ARM64_SW_TTBR0_PAN
arm64: mm: Move ASID from TTBR0 to TTBR1
arm64: mm: Remove pre_ttbr0_update_workaround for Falkor erratum
#E1003
arm64: mm: Rename post_ttbr0_update_workaround
arm64: mm: Fix and re-enable ARM64_SW_TTBR0_PAN
arm64: mm: Allocate ASIDs in pairs
arm64: mm: Add arm64_kernel_unmapped_at_el0 helper
arm64: mm: Invalidate both kernel and user ASIDs when performing TLBI
arm64: entry: Add exception trampoline page for exceptions from EL0
arm64: mm: Map entry trampoline into trampoline and kernel page tables
arm64: entry: Explicitly pass exception level to kernel_ventry macro
arm64: entry: Hook up entry trampoline to exception vectors
arm64: erratum: Work around Falkor erratum #E1003 in trampoline code
arm64: tls: Avoid unconditional zeroing of tpidrro_el0 for native
tasks
arm64: entry: Add fake CPU feature for unmapping the kernel at EL0
arm64: Kconfig: Add CONFIG_UNMAP_KERNEL_AT_EL0
perf: arm_spe: Fail device probe when arm64_kernel_unmapped_at_el0()
arm64: mm: Introduce TTBR_ASID_MASK for getting at the ASID in the
TTBR
arm64: kaslr: Put kernel vectors address in separate data page
arch/arm64/Kconfig | 30 +++--
arch/arm64/include/asm/asm-uaccess.h | 26 ++--
arch/arm64/include/asm/assembler.h | 27 +----
arch/arm64/include/asm/cpucaps.h | 3 +-
arch/arm64/include/asm/fixmap.h | 5 +
arch/arm64/include/asm/kernel-pgtable.h | 12 +-
arch/arm64/include/asm/mmu.h | 11 ++
arch/arm64/include/asm/mmu_context.h | 9 +-
arch/arm64/include/asm/pgtable-hwdef.h | 1 +
arch/arm64/include/asm/pgtable-prot.h | 21 +++-
arch/arm64/include/asm/pgtable.h | 1 +
arch/arm64/include/asm/proc-fns.h | 6 -
arch/arm64/include/asm/tlbflush.h | 16 ++-
arch/arm64/include/asm/uaccess.h | 21 +++-
arch/arm64/kernel/asm-offsets.c | 6 +-
arch/arm64/kernel/cpufeature.c | 41 +++++++
arch/arm64/kernel/entry.S | 203 +++++++++++++++++++++++++++-----
arch/arm64/kernel/process.c | 12 +-
arch/arm64/kernel/vmlinux.lds.S | 40 ++++++-
arch/arm64/lib/clear_user.S | 2 +-
arch/arm64/lib/copy_from_user.S | 2 +-
arch/arm64/lib/copy_in_user.S | 2 +-
arch/arm64/lib/copy_to_user.S | 2 +-
arch/arm64/mm/cache.S | 2 +-
arch/arm64/mm/context.c | 36 +++---
arch/arm64/mm/mmu.c | 31 +++++
arch/arm64/mm/proc.S | 12 +-
arch/arm64/xen/hypercall.S | 2 +-
drivers/perf/arm_spe_pmu.c | 9 ++
29 files changed, 454 insertions(+), 137 deletions(-)
--
2.1.4
^ permalink raw reply
* [PATCH v2 11/19] arm64: assembler: add macro to conditionally yield the NEON under PREEMPT
From: Ard Biesheuvel @ 2017-12-06 12:25 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20171206121251.GM12608@e103592.cambridge.arm.com>
On 6 December 2017 at 12:12, Dave P Martin <Dave.Martin@arm.com> wrote:
> On Wed, Dec 06, 2017 at 11:57:12AM +0000, Ard Biesheuvel wrote:
>> On 6 December 2017 at 11:51, Dave Martin <Dave.Martin@arm.com> wrote:
>> > On Tue, Dec 05, 2017 at 06:04:34PM +0000, Ard Biesheuvel wrote:
>> >> On 5 December 2017 at 12:45, Ard Biesheuvel <ard.biesheuvel@linaro.org> wrote:
>> >> >
>> >> >
>> >> >> On 5 Dec 2017, at 12:28, Dave Martin <Dave.Martin@arm.com> wrote:
>> >> >>
>> >> >>> On Mon, Dec 04, 2017 at 12:26:37PM +0000, Ard Biesheuvel wrote:
>> >> >>> Add a support macro to conditionally yield the NEON (and thus the CPU)
>> >> >>> that may be called from the assembler code. Given that especially the
>> >> >>> instruction based accelerated crypto code may use very tight loops, add
>> >> >>> some parametrization so that the TIF_NEED_RESCHED flag test is only
>> >> >>> executed every so many loop iterations.
>> >> >>>
>> >> >>> In some cases, yielding the NEON involves saving and restoring a non
>> >> >>> trivial amount of context (especially in the CRC folding algorithms),
>> >> >>> and so the macro is split into two, and the code in between is only
>> >> >>> executed when the yield path is taken, allowing the contex to be preserved.
>> >> >>> The second macro takes a label argument that marks the resume-from-yield
>> >> >>> path, which should restore the preserved context again.
>> >> >>>
>> >> >>> Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
>> >> >>> ---
>> >> >>> arch/arm64/include/asm/assembler.h | 50 ++++++++++++++++++++
>> >> >>> 1 file changed, 50 insertions(+)
>> >> >>>
>> >> >>> diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
>> >> >>> index aef72d886677..917b026d3e00 100644
>> >> >>> --- a/arch/arm64/include/asm/assembler.h
>> >> >>> +++ b/arch/arm64/include/asm/assembler.h
>> >> >>> @@ -512,4 +512,54 @@ alternative_else_nop_endif
>> >> >>> #endif
>> >> >>> .endm
>> >> >>>
>> >> >>> +/*
>> >> >>> + * yield_neon - check whether to yield to another runnable task from
>> >> >>> + * kernel mode NEON code (running with preemption disabled)
>> >> >>> + *
>> >> >>> + * - Check whether the preempt count is exactly 1, in which case disabling
>> >> >>> + * preemption once will make the task preemptible. If this is not the case,
>> >> >>> + * yielding is pointless.
>> >> >>> + * - Check whether TIF_NEED_RESCHED is set, and if so, disable and re-enable
>> >> >>> + * kernel mode NEON (which will trigger a reschedule), and branch to the
>> >> >>> + * yield fixup code at @lbl.
>> >> >>> + */
>> >> >>> + .macro yield_neon, lbl:req, ctr, order, stride, loop
>> >> >>> + yield_neon_pre \ctr, \order, \stride, \loop
>> >> >>> + yield_neon_post \lbl
>> >> >>> + .endm
>> >> >>> +
>> >> >>> + .macro yield_neon_pre, ctr, order=0, stride, loop=4444f
>> >> >>> +#ifdef CONFIG_PREEMPT
>> >> >>> + /*
>> >> >>> + * With some algorithms, it makes little sense to poll the
>> >> >>> + * TIF_NEED_RESCHED flag after every iteration, so only perform
>> >> >>> + * the check every 2^order strides.
>> >> >>> + */
>> >> >>> + .if \order > 1
>> >> >>> + .if (\stride & (\stride - 1)) != 0
>> >> >>> + .error "stride should be a power of 2"
>> >> >>> + .endif
>> >> >>> + tst \ctr, #((1 << \order) * \stride - 1) & ~(\stride - 1)
>> >> >>> + b.ne \loop
>> >> >>> + .endif
>> >> >>
>> >> >> I'm not sure what baking in this check gives us, and this seems to
>> >> >> conflate two rather different things: yielding and defining a
>> >> >> "heartbeat" frequency for the calling code.
>> >> >>
>> >> >> Can we separate out the crypto-loop-helper semantics from the yield-
>> >> >> neon stuff?
>> >> >>
>> >> >
>> >> > Fair enough. I incorporated the check here so it disappears from the code entirely when !CONFIG_PREEMPT, because otherwise, you end up with a sequence that is mispredicted every # iterations without any benefit.
>> >> > I guess i could macroise that separately though.
>> >> >
>> >> >> If we had
>> >> >>
>> >> >> if_cond_yield_neon
>> >> >> // patchup code
>> >> >> endif_yield_neon
>> >> >>
>> >>
>> >> I like this, but ...
>> >>
>> >> >> then the caller is free to conditionally branch over that as appropriate
>> >> >> like
>> >> >>
>> >> >> loop:
>> >> >> // crypto stuff
>> >> >> tst x0, #0xf
>> >> >> b.ne loop
>> >> >>
>> >> >> if_cond_yield_neon
>> >> >> // patchup code
>> >> >> endif_cond_yield_neon
>> >> >>
>> >>
>> >> I need to put the post patchup code somewhere too. Please refer to the
>> >> CRC patches for the best examples of this.
>> >
>> > I'm not sure I follow. If you need to do something different after
>> > patchup, can't you either pull that code into the if...endif too, or
>> > otherwise put a branch before the endif?
>> >
>>
>> No, not really.
>>
>> What I currently have is
>>
>> > if_cond_yield_neon
>> > // patchup code
>> > endif_cond_yield_neon
>>
>> which is being turned into
>>
>> get_thread_info x0
>> ldr w1, [x0, #TSK_TI_PREEMPT]
>> ldr x0, [x0, #TSK_TI_FLAGS]
>> cmp w1, #1 // == PREEMPT_OFFSET
>> csel x0, x0, xzr, eq
>> tbnz x0, #TIF_NEED_RESCHED, 5555f // needs rescheduling?
>> 4444:
>>
>> .subsection 1
>> 5555:
>> // patchup code
>> bl kernel_neon_end
>> bl kernel_neon_begin
>> // post patchup code goes here
>> b 4444b
>> .subsection
>>
>> so what I basically need is a place to put the post patchup code,
>> unless I open code it and branch to a different label right after
>> kernel_neon_begin
>
> Ah, I get you. I had convinced myself that there was only post-
> patchup code.
>
> So you conceptually want something like
>
> if_will_cond_yield_neon
> // pre-yield patchup code
> do_cond_yield_neon
> // post-yield patchup code
> endif_yield_neon
>
> .macro if_will_cond_yield_neon
> // do preempt_count == 1 && TIF_NEED_RESCHED check
> b.wont_yield 3432f
> .endm
>
> .macro do_cond_yield_neon
> bl kernel_neon_end
> bl kernel_neon_begin
> .endm
>
> .macro endif_yield_neon
> 3432:
> .endm
>
> The main difference I see is that this doesn't get the slow-case
> code out of line.
>
> Or have I still missed something?
>
No, that seems accurate. I think the idiom looks much better, but I'd
still like to get the code out of line, so that everything disappears
completely for !CONFIG_PREEMPT.
I'll try to rework my stuff using this as a starting point.
^ permalink raw reply
* [PATCH 5/6] cpufreq: Add DVFS support for Armada 37xx
From: Gregory CLEMENT @ 2017-12-06 12:24 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20171205055414.yghqdxg2but34n4i@vireshk-mac-ubuntu>
Hi Viresh,
On mar., d?c. 05 2017, Viresh Kumar <viresh.kumar@linaro.org> wrote:
> On 01-12-17, 12:25, Gregory CLEMENT wrote:
>> This patch adds DVFS support for the Armada 37xx SoCs
>>
>> There are up to four CPU frequency loads for Armada 37xx controlled by
>> the hardware.
>>
>> This driver associates the CPU load level to a frequency, then the
>> hardware will switch while selecting a load level.
>>
>> The hardware also can associate a voltage for each level (AVS support)
>> but it is not yet supported
>>
>> Signed-off-by: Gregory CLEMENT <gregory.clement@free-electrons.com>
>> ---
>> drivers/cpufreq/Kconfig.arm | 7 +
>> drivers/cpufreq/Makefile | 1 +
>> drivers/cpufreq/armada-37xx-cpufreq.c | 241 ++++++++++++++++++++++++++++++++++
>> 3 files changed, 249 insertions(+)
>> create mode 100644 drivers/cpufreq/armada-37xx-cpufreq.c
>>
>> diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm
>> index 17625115c67f..3018ff0d068f 100644
>> --- a/drivers/cpufreq/Kconfig.arm
>> +++ b/drivers/cpufreq/Kconfig.arm
>> @@ -19,6 +19,13 @@ config ACPI_CPPC_CPUFREQ
>>
>> If in doubt, say N.
>>
>> +config ARM_ARMADA_37XX_CPUFREQ
>> + tristate "Armada 37xx CPUFreq support"
>> + depends on ARCH_MVEBU
>> + help
>> + This adds the CPUFreq driver support for Marvell Armada 37xx SoCs.
>> + The Armada 37xx PMU supports 4 frequency and VDD levels.
>> +
>> # big LITTLE core layer and glue drivers
>> config ARM_BIG_LITTLE_CPUFREQ
>> tristate "Generic ARM big LITTLE CPUfreq driver"
>> diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile
>> index d762e76887e7..e07715ce8844 100644
>> --- a/drivers/cpufreq/Makefile
>> +++ b/drivers/cpufreq/Makefile
>> @@ -52,6 +52,7 @@ obj-$(CONFIG_ARM_BIG_LITTLE_CPUFREQ) += arm_big_little.o
>> # LITTLE drivers, so that it is probed last.
>> obj-$(CONFIG_ARM_DT_BL_CPUFREQ) += arm_big_little_dt.o
>>
>> +obj-$(CONFIG_ARM_ARMADA_37XX_CPUFREQ) += armada-37xx-cpufreq.o
>> obj-$(CONFIG_ARM_BRCMSTB_AVS_CPUFREQ) += brcmstb-avs-cpufreq.o
>> obj-$(CONFIG_ACPI_CPPC_CPUFREQ) += cppc_cpufreq.o
>> obj-$(CONFIG_ARCH_DAVINCI) += davinci-cpufreq.o
>> diff --git a/drivers/cpufreq/armada-37xx-cpufreq.c b/drivers/cpufreq/armada-37xx-cpufreq.c
>> new file mode 100644
>> index 000000000000..40c9a744cc6e
>> --- /dev/null
>> +++ b/drivers/cpufreq/armada-37xx-cpufreq.c
>> @@ -0,0 +1,241 @@
>> +// SPDX-License-Identifier: GPL-2.0+
>> +/*
>> + * CPU frequency scaling support for Armada 37xx platform.
>> + *
>> + * Copyright (C) 2017 Marvell
>> + *
>> + * Gregory CLEMENT <gregory.clement@free-electrons.com>
>> + */
>> +
>> +#include <linux/clk.h>
>> +#include <linux/cpu.h>
>> +#include <linux/cpufreq.h>
>> +#include <linux/err.h>
>> +#include <linux/interrupt.h>
>> +#include <linux/io.h>
>> +#include <linux/mfd/syscon.h>
>> +#include <linux/module.h>
>> +#include <linux/of_address.h>
>> +#include <linux/of_device.h>
>> +#include <linux/of_irq.h>
>> +#include <linux/platform_device.h>
>> +#include <linux/pm_opp.h>
>> +#include <linux/regmap.h>
>> +#include <linux/slab.h>
>> +
>> +/* Power management in North Bridge register set */
>> +#define ARMADA_37XX_NB_L0L1 0x18
>> +#define ARMADA_37XX_NB_L2L3 0x1C
>> +#define ARMADA_37XX_NB_TBG_DIV_OFF 13
>> +#define ARMADA_37XX_NB_TBG_DIV_MASK 0x7
>> +#define ARMADA_37XX_NB_CLK_SEL_OFF 11
>> +#define ARMADA_37XX_NB_CLK_SEL_MASK 0x1
>> +#define ARMADA_37XX_NB_CLK_SEL_TBG 0x1
>> +#define ARMADA_37XX_NB_TBG_SEL_OFF 9
>> +#define ARMADA_37XX_NB_TBG_SEL_MASK 0x3
>> +#define ARMADA_37XX_NB_VDD_SEL_OFF 6
>> +#define ARMADA_37XX_NB_VDD_SEL_MASK 0x3
>> +#define ARMADA_37XX_NB_CONFIG_SHIFT 16
>
> Looks like you have added tabs after #define as well? Perhaps a space
> is good enough there.
OK
>
>> +#define ARMADA_37XX_NB_DYN_MOD 0x24
>> +#define ARMADA_37XX_NB_CLK_SEL_EN BIT(26)
>> +#define ARMADA_37XX_NB_TBG_EN BIT(28)
>> +#define ARMADA_37XX_NB_DIV_EN BIT(29)
>> +#define ARMADA_37XX_NB_VDD_EN BIT(30)
>> +#define ARMADA_37XX_NB_DFS_EN BIT(31)
>> +#define ARMADA_37XX_NB_CPU_LOAD 0x30
>> +#define ARMADA_37XX_NB_CPU_LOAD_MASK 0x3
>> +#define ARMADA_37XX_DVFS_LOAD_0 0
>> +#define ARMADA_37XX_DVFS_LOAD_1 1
>> +#define ARMADA_37XX_DVFS_LOAD_2 2
>> +#define ARMADA_37XX_DVFS_LOAD_3 3
>> +
>> +/*
>> + * On Armada 37xx the Power management manages 4 level of CPU load,
>> + * each level can be associated with a CPU clock source, a CPU
>> + * divider, a VDD level, etc...
>> + */
>> +#define LOAD_LEVEL_NR 4
>> +
>> +struct armada_37xx_dvfs {
>> + u32 cpu_freq_max;
>> + u8 divider[LOAD_LEVEL_NR];
>> +};
>> +
>> +static struct armada_37xx_dvfs armada_37xx_dvfs[] = {
>> + {.cpu_freq_max = 1200*1000*1000, .divider = {1, 2, 4, 6} },
>> + {.cpu_freq_max = 1000*1000*1000, .divider = {1, 2, 4, 5} },
>> + {.cpu_freq_max = 800*1000*1000, .divider = {1, 2, 3, 4} },
>> + {.cpu_freq_max = 600*1000*1000, .divider = {2, 4, 5, 6} },
>> +};
>> +
>> +static struct armada_37xx_dvfs *armada_37xx_cpu_freq_info_get(u32 freq)
>> +{
>> + int i;
>> +
>> + for (i = 0; i < ARRAY_SIZE(armada_37xx_dvfs); i++) {
>> + if (freq == armada_37xx_dvfs[i].cpu_freq_max)
>> + return &armada_37xx_dvfs[i];
>> + }
>> +
>> + pr_err("Unsupported CPU frequency %d MHz\n", freq/1000000);
>> + return NULL;
>> +}
>> +
>> +/*
>> + * Setup the four level managed by the hardware. Once the four level
>> + * will be configured then the DVFS will be enabled.
>> + */
>> +static void __init armada37xx_cpufreq_dvfs_setup(struct regmap *base,
>> + struct clk *clk, u8 *divider)
>> +{
>> + int load_level;
>> + struct clk *parent;
>> +
>> + for (load_level = 0; load_level < LOAD_LEVEL_NR; load_level++) {
>> + unsigned int reg, mask, val, offset = 0;
>> +
>> + if (load_level <= ARMADA_37XX_DVFS_LOAD_1)
>> + reg = ARMADA_37XX_NB_L0L1;
>> + else
>> + reg = ARMADA_37XX_NB_L2L3;
>> +
>> + if (load_level == ARMADA_37XX_DVFS_LOAD_0 ||
>> + load_level == ARMADA_37XX_DVFS_LOAD_2)
>> + offset += ARMADA_37XX_NB_CONFIG_SHIFT;
>> +
>> + /* Set cpu clock source, for all the level we use TBG */
>> + val = ARMADA_37XX_NB_CLK_SEL_TBG << ARMADA_37XX_NB_CLK_SEL_OFF;
>> + mask = (ARMADA_37XX_NB_CLK_SEL_MASK
>> + << ARMADA_37XX_NB_CLK_SEL_OFF);
>> +
>> + /*
>> + * Set cpu divider based on the pre-computed array in
>> + * order to have balanced step.
>> + */
>> + val |= divider[load_level] << ARMADA_37XX_NB_TBG_DIV_OFF;
>> + mask |= (ARMADA_37XX_NB_TBG_DIV_MASK
>> + << ARMADA_37XX_NB_TBG_DIV_OFF);
>> +
>> + /* Set VDD divider which is actually the load level. */
>> + val |= load_level << ARMADA_37XX_NB_VDD_SEL_OFF;
>> + mask |= (ARMADA_37XX_NB_VDD_SEL_MASK
>> + << ARMADA_37XX_NB_VDD_SEL_OFF);
>> +
>> + val <<= offset;
>> + mask <<= offset;
>> +
>> + regmap_update_bits(base, reg, mask, val);
>> + }
>> +
>> + /*
>> + * Set cpu clock source, for all the level we keep the same
>> + * clock source that the one already configured. For this one
>> + * we need to use the clock framework
>> + */
>> + parent = clk_get_parent(clk);
>> + clk_set_parent(clk, parent);
>> +}
>> +
>> +static void __init armada37xx_cpufreq_disable_dvfs(struct regmap *base)
>> +{
>> + unsigned int reg = ARMADA_37XX_NB_DYN_MOD,
>> + mask = ARMADA_37XX_NB_DFS_EN;
>> +
>> + regmap_update_bits(base, reg, mask, 0);
>> +}
>> +
>> +static void __init armada37xx_cpufreq_enable_dvfs(struct regmap *base)
>> +{
>> + unsigned int val, reg = ARMADA_37XX_NB_CPU_LOAD,
>> + mask = ARMADA_37XX_NB_CPU_LOAD_MASK;
>> +
>> + /* Start with the highest load (0) */
>> + val = ARMADA_37XX_DVFS_LOAD_0;
>> + regmap_update_bits(base, reg, mask, val);
>> +
>> + /* Now enable DVFS for the CPUs */
>> + reg = ARMADA_37XX_NB_DYN_MOD;
>> + mask = ARMADA_37XX_NB_CLK_SEL_EN | ARMADA_37XX_NB_TBG_EN |
>> + ARMADA_37XX_NB_DIV_EN | ARMADA_37XX_NB_VDD_EN |
>> + ARMADA_37XX_NB_DFS_EN;
>> +
>> + regmap_update_bits(base, reg, mask, mask);
>> +}
>> +
>> +static int __init armada37xx_cpufreq_driver_init(void)
>> +{
>> + struct armada_37xx_dvfs *dvfs;
>> + struct platform_device *pdev;
>> + unsigned int cur_frequency;
>> + struct regmap *nb_pm_base;
>> + struct device *cpu_dev;
>> + int load_level, ret;
>> + struct clk *clk;
>> +
>> + nb_pm_base =
>> + syscon_regmap_lookup_by_compatible("marvell,armada-3700-nb-pm");
>> +
>> + if (IS_ERR(nb_pm_base))
>> + return -ENODEV;
>> +
>> + /* Before doing any configuration on the DVFS first, disable it */
>> + armada37xx_cpufreq_disable_dvfs(nb_pm_base);
>> +
>> + /*
>> + * On CPU 0 register the operating points supported (which are
>> + * the nominal CPU frequency and full integer divisions of
>> + * it).
>> + */
>> + cpu_dev = get_cpu_device(0);
>> + if (!cpu_dev) {
>> + dev_err(cpu_dev, "Cannot get CPU\n");
>> + return -ENODEV;
>> + }
>> +
>> + clk = clk_get(cpu_dev, 0);
>> + if (IS_ERR(clk)) {
>> + dev_err(cpu_dev, "Cannot get clock for CPU0\n");
>> + return PTR_ERR(clk);
>> + }
>> +
>> + /* Get nominal (current) CPU frequency */
>> + cur_frequency = clk_get_rate(clk);
>> + if (!cur_frequency) {
>> + dev_err(cpu_dev, "Failed to get clock rate for CPU\n");
>> + return -EINVAL;
>> + }
>> +
>> + dvfs = armada_37xx_cpu_freq_info_get(cur_frequency);
>> + if (!dvfs)
>> + return -EINVAL;
>> +
>> + armada37xx_cpufreq_dvfs_setup(nb_pm_base, clk, dvfs->divider);
>> +
>> + /*
>> + * In case of a failure of dev_pm_opp_add(), we don't bother
>> + * with cleaning up the registered OPP (there's no function to
>> + * do so),
>
> What do you mean by the comment within ()? We do have
> dev_pm_opp_remove() helper.
Actually it was a copy and paste from an other driver, I didn't check it
this was still true.
So the original comment was added in July 2014 and the
dev_pm_opp_remove() helpers was added few months later
in Nov 2014.
I will fix it and also in the other driver mvebu-cpufreq.c.
Thanks,
Gregory
>
>> and simply cancel the registration of the cpufreq
>> + * device.
>> + */
>> + for (load_level = ARMADA_37XX_DVFS_LOAD_0; load_level < LOAD_LEVEL_NR;
>> + load_level++) {
>> + unsigned long freq = dvfs->divider[load_level];
>> +
>> + ret = dev_pm_opp_add(cpu_dev, freq, 0);
>> + if (ret)
>> + return ret;
>> + }
>> +
>> + /* Now that everything is setup, enable the DVFS at hardware level */
>> + armada37xx_cpufreq_enable_dvfs(nb_pm_base);
>> +
>> + pdev = platform_device_register_simple("cpufreq-dt", -1, NULL, 0);
>> +
>> + return PTR_ERR_OR_ZERO(pdev);
>> +}
>> +/* late_initcall, to guarantee the driver is loaded after A37xx clock driver */
>> +late_initcall(armada37xx_cpufreq_driver_init);
>> +
>> +MODULE_AUTHOR("Gregory CLEMENT <gregory.clement@free-electrons.com>");
>> +MODULE_DESCRIPTION("Armada 37xx cpufreq driver");
>> +MODULE_LICENSE("GPL");
>> --
>> 2.15.0
>
> --
> viresh
--
Gregory Clement, Free Electrons
Kernel, drivers, real-time and embedded Linux
development, consulting, training and support.
http://free-electrons.com
^ permalink raw reply
* [PATCH 0/2] Fixes for SW PAN
From: Mark Rutland @ 2017-12-06 12:19 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512558968-28980-1-git-send-email-will.deacon@arm.com>
On Wed, Dec 06, 2017 at 11:16:06AM +0000, Will Deacon wrote:
> Hi all,
>
> After lots of collective head scratching in response to Vinayak's mail
> here:
>
> http://lists.infradead.org/pipermail/linux-arm-kernel/2017-December/545641.html
>
> It turns out that we have a problem with SW PAN and kernel threads, where
> the saved ttbr0 value for a kernel thread can be stale and subsequently
> inherited by other kernel threads over a fork.
>
> These two patches attempt to fix that. We've not be able to reproduce
> the exact failure reported above, but I added some assertions to the
> uaccess routines to check for discrepancies between the active_mm pgd
> and the saved ttbr0 value (ignoring the zero page) and these no longer
> fire with these changes, but do fire without them if EFI runtime services
> are enabled on my Seattle board.
Both patches look good to me, per my understanding of the problem, so
FWIW, for the series:
Reviewed-by: Mark Rutland <mark.rutland@arm.com>
However, I think we have another issue in this area. We have sequences
where we update the HW TTBR0 before the shadow, e.g. in efi_set_pgd(),
where we do:
cpu_set_reserved_ttbr0();
update_saved_ttbr0(current, current->active_mm);
... so if an exception comes in between after HW TTBR0 update but before
the shadow update, the stale shadow value will be restored upon the
exception return, leaving the HW TTBR0 stale.
That's easy enough to fix in the efi code, e.g.
update_saved_ttbr0(current, current->active_mm);
barrier();
cpu_set_reserved_ttbr0();
... but I haven't yet figured out a nice way to sort out switch_mm(),
__switch_mm(), and check_and_switch_context().
Thanks,
Mark.
^ permalink raw reply
* [PATCH 4/7 v2] net: ethernet: i825xx: Fix platform_get_irq's error checking
From: Sergei Shtylyov @ 2017-12-06 12:19 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20171205.104947.121188625730183933.davem@davemloft.net>
On 12/05/2017 06:49 PM, David Miller wrote:
>>> From: Arvind Yadav <arvind.yadav.cs@gmail.com>
>>> Date: Mon, 4 Dec 2017 23:18:20 +0530
>>>
>>>> @@ -120,9 +120,10 @@ static int sni_82596_probe(struct platform_device
>>>> *dev)
>>>> netdevice->dev_addr[5] = readb(eth_addr + 0x06);
>>>> iounmap(eth_addr);
>>>> - if (!netdevice->irq) {
>>>> + if (netdevice->irq <= 0) {
>>>> printk(KERN_ERR "%s: IRQ not found for i82596 at 0x%lx\n",
>>>> __FILE__, netdevice->base_addr);
>>>> + retval = netdevice->irq ? netdevice->irq : -ENODEV;
>>>> goto probe_failed;
>>>> }
>>> Ok, thinking about this some more...
>>>
>>> It is impossible to use platform_get_irq() without every single call
>>> site having this funny:
>>>
>>> ret = val ? val : -ENODEV;
>>>
>>> sequence.
>>>
>>> This is unnecessary duplication and it is also error prone, so I
>>> really think this logic belongs in platform_get_irq() itself. It can
>>> convert '0' to -ENODEV and that way we need no special logic in the
>>> callers at all.
>> platform_get_irq() will return 0 only for sparc, If sparc initialize
>> platform
>> data irq[PROMINTR_MAX] as zero. Otherwise platform_get_irq() will
>> never return
>> 0. It will return either IRQ number or error (as negative number). But
>> I am getting
>> review comment by reviewer/maintainer in other subsystem to add check
>> for
>> zero. So I have done same changes here. Please correct me if i am
>> wrong.
>
> If you make the change that I suggest, you instead can check for
I assume such change is needed only for the SPARC-specific section of
platform_get_irq()?
> '-ENODEV' to mean no IRQ.
No specific error check is needed, just irq < 0 check should be enough...
Also, looking@platform_get_irq(), -ENXIO should be returned in this case.
MBR, Sergei
^ permalink raw reply
* [PATCH 2/2] arm64: SW PAN: Update saved ttbr0 value on enter_lazy_tlb
From: Catalin Marinas @ 2017-12-06 12:17 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512558968-28980-3-git-send-email-will.deacon@arm.com>
On Wed, Dec 06, 2017 at 11:16:08AM +0000, Will Deacon wrote:
> enter_lazy_tlb is called when a kernel thread rides on the back of
> another mm, due to a context switch or an explicit call to unuse_mm
> where a call to switch_mm is elided.
>
> In these cases, it's important to keep the saved ttbr value up to date
> with the active mm, otherwise we can end up with a stale value which
> points to a potentially freed page table.
>
> This patch implements enter_lazy_tlb for arm64, so that the saved ttbr0
> is kept up-to-date with the active mm for kernel threads.
>
> Cc: Mark Rutland <mark.rutland@arm.com>
> Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
> Cc: Vinayak Menon <vinmenon@codeaurora.org>
> Reported-by: Vinayak Menon <vinmenon@codeaurora.org>
> Signed-off-by: Will Deacon <will.deacon@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
^ permalink raw reply
* [PATCH 1/2] arm64: SW PAN: Point saved ttbr0 at the zero page when switching to init_mm
From: Catalin Marinas @ 2017-12-06 12:15 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512558968-28980-2-git-send-email-will.deacon@arm.com>
On Wed, Dec 06, 2017 at 11:16:07AM +0000, Will Deacon wrote:
> update_saved_ttbr0 mandates that mm->pgd is not swapper, since swapper
> contains kernel mappings and should never be installed into ttbr0. However,
> this means that callers must avoid passing the init_mm to update_saved_ttbr0
> which in turn can cause the saved ttbr0 value to be out-of-date in the context
> of the idle thread. For example, EFI runtime services may leave the saved ttbr0
> pointing at the EFI page table, and kernel threads may end up with stale
> references to freed page tables.
>
> This patch changes update_saved_ttbr0 so that the init_mm points the saved
> ttbr0 value to the empty zero page, which always exists and never contains
> valid translations. EFI and switch can then call into update_saved_ttbr0
> unconditionally.
>
> Cc: Mark Rutland <mark.rutland@arm.com>
> Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
> Cc: Vinayak Menon <vinmenon@codeaurora.org>
> Reported-by: Vinayak Menon <vinmenon@codeaurora.org>
> Signed-off-by: Will Deacon <will.deacon@arm.com>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
^ permalink raw reply
* [PATCH v2 11/19] arm64: assembler: add macro to conditionally yield the NEON under PREEMPT
From: Dave P Martin @ 2017-12-06 12:12 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <CAKv+Gu83CR=KGawz5WRQygULcqX+DjhVjAeR-+QOGYx=2oPTzA@mail.gmail.com>
On Wed, Dec 06, 2017 at 11:57:12AM +0000, Ard Biesheuvel wrote:
> On 6 December 2017 at 11:51, Dave Martin <Dave.Martin@arm.com> wrote:
> > On Tue, Dec 05, 2017 at 06:04:34PM +0000, Ard Biesheuvel wrote:
> >> On 5 December 2017 at 12:45, Ard Biesheuvel <ard.biesheuvel@linaro.org> wrote:
> >> >
> >> >
> >> >> On 5 Dec 2017, at 12:28, Dave Martin <Dave.Martin@arm.com> wrote:
> >> >>
> >> >>> On Mon, Dec 04, 2017 at 12:26:37PM +0000, Ard Biesheuvel wrote:
> >> >>> Add a support macro to conditionally yield the NEON (and thus the CPU)
> >> >>> that may be called from the assembler code. Given that especially the
> >> >>> instruction based accelerated crypto code may use very tight loops, add
> >> >>> some parametrization so that the TIF_NEED_RESCHED flag test is only
> >> >>> executed every so many loop iterations.
> >> >>>
> >> >>> In some cases, yielding the NEON involves saving and restoring a non
> >> >>> trivial amount of context (especially in the CRC folding algorithms),
> >> >>> and so the macro is split into two, and the code in between is only
> >> >>> executed when the yield path is taken, allowing the contex to be preserved.
> >> >>> The second macro takes a label argument that marks the resume-from-yield
> >> >>> path, which should restore the preserved context again.
> >> >>>
> >> >>> Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
> >> >>> ---
> >> >>> arch/arm64/include/asm/assembler.h | 50 ++++++++++++++++++++
> >> >>> 1 file changed, 50 insertions(+)
> >> >>>
> >> >>> diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
> >> >>> index aef72d886677..917b026d3e00 100644
> >> >>> --- a/arch/arm64/include/asm/assembler.h
> >> >>> +++ b/arch/arm64/include/asm/assembler.h
> >> >>> @@ -512,4 +512,54 @@ alternative_else_nop_endif
> >> >>> #endif
> >> >>> .endm
> >> >>>
> >> >>> +/*
> >> >>> + * yield_neon - check whether to yield to another runnable task from
> >> >>> + * kernel mode NEON code (running with preemption disabled)
> >> >>> + *
> >> >>> + * - Check whether the preempt count is exactly 1, in which case disabling
> >> >>> + * preemption once will make the task preemptible. If this is not the case,
> >> >>> + * yielding is pointless.
> >> >>> + * - Check whether TIF_NEED_RESCHED is set, and if so, disable and re-enable
> >> >>> + * kernel mode NEON (which will trigger a reschedule), and branch to the
> >> >>> + * yield fixup code at @lbl.
> >> >>> + */
> >> >>> + .macro yield_neon, lbl:req, ctr, order, stride, loop
> >> >>> + yield_neon_pre \ctr, \order, \stride, \loop
> >> >>> + yield_neon_post \lbl
> >> >>> + .endm
> >> >>> +
> >> >>> + .macro yield_neon_pre, ctr, order=0, stride, loop=4444f
> >> >>> +#ifdef CONFIG_PREEMPT
> >> >>> + /*
> >> >>> + * With some algorithms, it makes little sense to poll the
> >> >>> + * TIF_NEED_RESCHED flag after every iteration, so only perform
> >> >>> + * the check every 2^order strides.
> >> >>> + */
> >> >>> + .if \order > 1
> >> >>> + .if (\stride & (\stride - 1)) != 0
> >> >>> + .error "stride should be a power of 2"
> >> >>> + .endif
> >> >>> + tst \ctr, #((1 << \order) * \stride - 1) & ~(\stride - 1)
> >> >>> + b.ne \loop
> >> >>> + .endif
> >> >>
> >> >> I'm not sure what baking in this check gives us, and this seems to
> >> >> conflate two rather different things: yielding and defining a
> >> >> "heartbeat" frequency for the calling code.
> >> >>
> >> >> Can we separate out the crypto-loop-helper semantics from the yield-
> >> >> neon stuff?
> >> >>
> >> >
> >> > Fair enough. I incorporated the check here so it disappears from the code entirely when !CONFIG_PREEMPT, because otherwise, you end up with a sequence that is mispredicted every # iterations without any benefit.
> >> > I guess i could macroise that separately though.
> >> >
> >> >> If we had
> >> >>
> >> >> if_cond_yield_neon
> >> >> // patchup code
> >> >> endif_yield_neon
> >> >>
> >>
> >> I like this, but ...
> >>
> >> >> then the caller is free to conditionally branch over that as appropriate
> >> >> like
> >> >>
> >> >> loop:
> >> >> // crypto stuff
> >> >> tst x0, #0xf
> >> >> b.ne loop
> >> >>
> >> >> if_cond_yield_neon
> >> >> // patchup code
> >> >> endif_cond_yield_neon
> >> >>
> >>
> >> I need to put the post patchup code somewhere too. Please refer to the
> >> CRC patches for the best examples of this.
> >
> > I'm not sure I follow. If you need to do something different after
> > patchup, can't you either pull that code into the if...endif too, or
> > otherwise put a branch before the endif?
> >
>
> No, not really.
>
> What I currently have is
>
> > if_cond_yield_neon
> > // patchup code
> > endif_cond_yield_neon
>
> which is being turned into
>
> get_thread_info x0
> ldr w1, [x0, #TSK_TI_PREEMPT]
> ldr x0, [x0, #TSK_TI_FLAGS]
> cmp w1, #1 // == PREEMPT_OFFSET
> csel x0, x0, xzr, eq
> tbnz x0, #TIF_NEED_RESCHED, 5555f // needs rescheduling?
> 4444:
>
> .subsection 1
> 5555:
> // patchup code
> bl kernel_neon_end
> bl kernel_neon_begin
> // post patchup code goes here
> b 4444b
> .subsection
>
> so what I basically need is a place to put the post patchup code,
> unless I open code it and branch to a different label right after
> kernel_neon_begin
Ah, I get you. I had convinced myself that there was only post-
patchup code.
So you conceptually want something like
if_will_cond_yield_neon
// pre-yield patchup code
do_cond_yield_neon
// post-yield patchup code
endif_yield_neon
.macro if_will_cond_yield_neon
// do preempt_count == 1 && TIF_NEED_RESCHED check
b.wont_yield 3432f
.endm
.macro do_cond_yield_neon
bl kernel_neon_end
bl kernel_neon_begin
.endm
.macro endif_yield_neon
3432:
.endm
The main difference I see is that this doesn't get the slow-case
code out of line.
Or have I still missed something?
Cheers
---Dave
IMPORTANT NOTICE: The contents of this email and any attachments are confidential and may also be privileged. If you are not the intended recipient, please notify the sender immediately and do not disclose the contents to any other person, use it for any purpose, or store or copy the information in any medium. Thank you.
^ permalink raw reply
* [PATCH v2] ARM: dts: r8a7745: Add APMU node and second CPU core
From: Geert Uytterhoeven @ 2017-12-06 12:12 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512561929-16540-1-git-send-email-fabrizio.castro@bp.renesas.com>
On Wed, Dec 6, 2017 at 1:05 PM, Fabrizio Castro
<fabrizio.castro@bp.renesas.com> wrote:
>
> Add DT node for the Advanced Power Management Unit (APMU), add the
> second CPU core, and use "renesas,apmu" as "enable-method".
>
> Signed-off-by: Fabrizio Castro <fabrizio.castro@bp.renesas.com>
> Signed-off-by: Chris Paterson <chris.paterson2@renesas.com>
> Reviewed-by: Biju Das <biju.das@bp.renesas.com>
> v2:
> - rebased against renesas-devel-20171205-v4.15-rc2
Seems my
Reviewed-by: Geert Uytterhoeven <geert+renesas@glider.be>
was not added.
Gr{oetje,eeting}s,
Geert
--
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- geert at linux-m68k.org
In personal conversations with technical people, I call myself a hacker. But
when I'm talking to journalists I just say "programmer" or something like that.
-- Linus Torvalds
^ permalink raw reply
* [PATCH 2/2] arm64: SW PAN: Update saved ttbr0 value on enter_lazy_tlb
From: Mark Rutland @ 2017-12-06 12:10 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1512558968-28980-3-git-send-email-will.deacon@arm.com>
On Wed, Dec 06, 2017 at 11:16:08AM +0000, Will Deacon wrote:
> enter_lazy_tlb is called when a kernel thread rides on the back of
> another mm, due to a context switch or an explicit call to unuse_mm
> where a call to switch_mm is elided.
>
> In these cases, it's important to keep the saved ttbr value up to date
> with the active mm, otherwise we can end up with a stale value which
> points to a potentially freed page table.
>
> This patch implements enter_lazy_tlb for arm64, so that the saved ttbr0
> is kept up-to-date with the active mm for kernel threads.
>
> Cc: Mark Rutland <mark.rutland@arm.com>
> Cc: Ard Biesheuvel <ard.biesheuvel@linaro.org>
> Cc: Vinayak Menon <vinmenon@codeaurora.org>
> Reported-by: Vinayak Menon <vinmenon@codeaurora.org>
> Signed-off-by: Will Deacon <will.deacon@arm.com>
As with the prior patch, I think this needs:
Fixes: 39bc88e5e38e9b21 ("arm64: Disable TTBR0_EL1 during normal kernel execution")
Other than that, looks good to me.
Mark.
> ---
> arch/arm64/include/asm/mmu_context.h | 24 ++++++++++--------------
> 1 file changed, 10 insertions(+), 14 deletions(-)
>
> diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h
> index f7773f90546e..9d155fa9a507 100644
> --- a/arch/arm64/include/asm/mmu_context.h
> +++ b/arch/arm64/include/asm/mmu_context.h
> @@ -156,20 +156,6 @@ void check_and_switch_context(struct mm_struct *mm, unsigned int cpu);
>
> #define init_new_context(tsk,mm) ({ atomic64_set(&(mm)->context.id, 0); 0; })
>
> -/*
> - * This is called when "tsk" is about to enter lazy TLB mode.
> - *
> - * mm: describes the currently active mm context
> - * tsk: task which is entering lazy tlb
> - * cpu: cpu number which is entering lazy tlb
> - *
> - * tsk->mm will be NULL
> - */
> -static inline void
> -enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
> -{
> -}
> -
> #ifdef CONFIG_ARM64_SW_TTBR0_PAN
> static inline void update_saved_ttbr0(struct task_struct *tsk,
> struct mm_struct *mm)
> @@ -193,6 +179,16 @@ static inline void update_saved_ttbr0(struct task_struct *tsk,
> }
> #endif
>
> +static inline void
> +enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
> +{
> + /*
> + * We don't actually care about the ttbr0 mapping, so point it at the
> + * zero page.
> + */
> + update_saved_ttbr0(tsk, &init_mm);
> +}
> +
> static inline void __switch_mm(struct mm_struct *next)
> {
> unsigned int cpu = smp_processor_id();
> --
> 2.1.4
>
^ permalink raw reply
* [RESEND PATCH 4/4] drm/meson: Add missing VPU init
From: Chris Wilson @ 2017-12-06 12:09 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <50101688-72c0-565e-bc15-9c229fd1a167@baylibre.com>
Quoting Neil Armstrong (2017-12-06 12:03:59)
> On 06/12/2017 13:02, Chris Wilson wrote:
> > Quoting Neil Armstrong (2017-12-06 11:54:28)
> >> The VPU init misses these configurations values.
> >>
> >> Signed-off-by: Neil Armstrong <narmstrong@baylibre.com>
> >> ---
> >> drivers/gpu/drm/meson/meson_drv.c | 9 +++++++++
> >> drivers/gpu/drm/meson/meson_registers.h | 4 ++++
> >> 2 files changed, 13 insertions(+)
> >>
> >> diff --git a/drivers/gpu/drm/meson/meson_drv.c b/drivers/gpu/drm/meson/meson_drv.c
> >> index 3b804fd..f9ad0e9 100644
> >> --- a/drivers/gpu/drm/meson/meson_drv.c
> >> +++ b/drivers/gpu/drm/meson/meson_drv.c
> >> @@ -151,6 +151,14 @@ static struct regmap_config meson_regmap_config = {
> >> .max_register = 0x1000,
> >> };
> >>
> >> +static void meson_vpu_init(struct meson_drm *priv)
> >> +{
> >> + writel_relaxed(0x210000, priv->io_base + _REG(VPU_RDARB_MODE_L1C1));
> >> + writel_relaxed(0x10000, priv->io_base + _REG(VPU_RDARB_MODE_L1C2));
> >> + writel_relaxed(0x900000, priv->io_base + _REG(VPU_RDARB_MODE_L2C1));
> >> + writel_relaxed(0x20000, priv->io_base + _REG(VPU_WRARB_MODE_L2C1));
> >> +}
> >
> >> diff --git a/drivers/gpu/drm/meson/meson_registers.h b/drivers/gpu/drm/meson/meson_registers.h
> >> index 2847381..bca8714 100644
> >> --- a/drivers/gpu/drm/meson/meson_registers.h
> >> +++ b/drivers/gpu/drm/meson/meson_registers.h
> >> @@ -1363,6 +1363,10 @@
> >> #define VPU_PROT3_STAT_1 0x277a
> >> #define VPU_PROT3_STAT_2 0x277b
> >> #define VPU_PROT3_REQ_ONOFF 0x277c
> >> +#define VPU_RDARB_MODE_L1C1 0x2790
> >> +#define VPU_RDARB_MODE_L1C2 0x2799
> >
> > Hmm, not naturally aligned for writel. Is the register width correct,
> > address correct, or this really is an unaligned iowrite?
>
> The registers are aligned with the documentation, then are used with the _REG() macro
> to align them with the bus (on top of the file).
Oh, that surprised me.
Acked-by: Chris Wilson <chris@chris-wilson.co.uk>
-Chris
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox