* [RFC PATCH 25/29] arm64/sve: Avoid preempt_disable() during sigreturn
From: Dave Martin @ 2016-11-25 19:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
Currently the sigreturn implementation for SVE relies on
preempt_disable() to avoid an intervening context switch from
corrupting the SVE state in the task_struct.
Unforunately, __get_user() and friends are not safe under
preempt_disable().
As an alternative, this patch removes preempt_disable() and sets
TIF_FOREIGN_FPSTATE instead: this will inform the context switch
code that the current CPU registers don't contain the SVE/FPSIMD
state of the current task, preventing writeback to the task_struct
during context switch.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/kernel/signal.c | 12 +++++-------
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 2528ec1..942d66f 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -256,10 +256,8 @@ static int __restore_sve_fpsimd_context(struct user_ctxs *user,
if (vl != sve_get_vl())
return -EINVAL;
- preempt_disable();
-
set_thread_flag(TIF_FOREIGN_FPSTATE);
- set_thread_flag(TIF_SVE);
+ barrier();
BUG_ON(SVE_SIG_REGS_SIZE(vq) > sizeof(*task_sve_regs));
BUG_ON(round_up(SVE_SIG_REGS_SIZE(vq), 16) < sizeof(*task_sve_regs));
@@ -270,7 +268,7 @@ static int __restore_sve_fpsimd_context(struct user_ctxs *user,
SVE_SIG_REGS_OFFSET,
SVE_SIG_REGS_SIZE(vq));
if (err)
- goto out_preempt;
+ return err;
/* copy the FP and status/control registers */
/* restore_sigframe() already checked that user->fpsimd != NULL. */
@@ -279,13 +277,13 @@ static int __restore_sve_fpsimd_context(struct user_ctxs *user,
__get_user_error(fpsimd.fpsr, &user->fpsimd->fpsr, err);
__get_user_error(fpsimd.fpcr, &user->fpsimd->fpcr, err);
+ barrier();
+ set_thread_flag(TIF_SVE);
+
/* load the hardware registers from the fpsimd_state structure */
if (!err)
fpsimd_update_current_state(&fpsimd);
-out_preempt:
- preempt_enable();
-
return err;
}
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 24/29] arm64/sve: Discard SVE state on system call
From: Dave Martin @ 2016-11-25 19:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
The base procedure call standard for the Scalable Vector Extension
defines all of the SVE programmer's model state (Z0-31, P0-15, FFR)
as caller-save, except for that subset of the state that aliases
FPSIMD state.
System calls from userspace will almost always be made through C
library wrappers -- as a consequence of the PCS there will thus
rarely if ever be any live SVE state at syscall entry in practice.
This gives us an opportinity to make SVE explicitly caller-save
around SVC and so stop carrying around the SVE state for tasks that
use SVE only occasionally (say, by calling a library).
Note that FPSIMD state will still be preserved around SVC.
As a crude heuristic to avoid pathological cases where a thread
that uses SVE frequently has to fault back into the kernel again to
re-enable SVE after a syscall, we switch the thread back to
FPSIMD-only context tracking only if the context is actually
switched out before returning to userspace.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/kernel/fpsimd.c | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 5834f81..2e1056e 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -203,6 +203,23 @@ static void task_fpsimd_load(struct task_struct *task)
static void task_fpsimd_save(struct task_struct *task)
{
if (IS_ENABLED(CONFIG_ARM64_SVE) &&
+ task_pt_regs(task)->syscallno != ~0UL &&
+ test_tsk_thread_flag(task, TIF_SVE)) {
+ unsigned long tmp;
+
+ clear_tsk_thread_flag(task, TIF_SVE);
+
+ /* Trap if the task tries to use SVE again: */
+ asm volatile (
+ "mrs %[tmp], cpacr_el1\n\t"
+ "bic %[tmp], %[tmp], %[mask]\n\t"
+ "msr cpacr_el1, %[tmp]"
+ : [tmp] "=r" (tmp)
+ : [mask] "i" (CPACR_EL1_ZEN_EL0EN)
+ );
+ }
+
+ if (IS_ENABLED(CONFIG_ARM64_SVE) &&
test_tsk_thread_flag(task, TIF_SVE))
sve_save_state(__task_pffr(task),
&task->thread.fpsimd_state.fpsr);
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 23/29] arm64/sve: Move ZEN handling to the common task_fpsimd_load() path
From: Dave Martin @ 2016-11-25 19:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
Currently, ZEN is handled only in fpsimd_restore_current_state(),
which is not sufficient since it applies only in certain
situations.
Since all the relevant paths call task_fpsimd_load(), this patch
moves the ZEN handling there.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/include/asm/thread_info.h | 1 +
arch/arm64/kernel/fpsimd.c | 48 +++++++++++++++++++-----------------
2 files changed, 27 insertions(+), 22 deletions(-)
diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index 2deac86..6819d08 100644
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -133,6 +133,7 @@ static inline struct thread_info *current_thread_info(void)
#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
#define _TIF_SYSCALL_TRACEPOINT (1 << TIF_SYSCALL_TRACEPOINT)
#define _TIF_SECCOMP (1 << TIF_SECCOMP)
+#define _TIF_SVE (1 << TIF_SVE)
#define _TIF_32BIT (1 << TIF_32BIT)
#define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index cad86e5..5834f81 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -102,21 +102,24 @@ void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
#ifdef CONFIG_ARM64_SVE
static void task_fpsimd_to_sve(struct task_struct *task);
+static void task_fpsimd_load(struct task_struct *task);
void do_sve_acc(unsigned int esr, struct pt_regs *regs)
{
- unsigned long tmp;
+ if (test_and_set_thread_flag(TIF_SVE)) {
+ unsigned long tmp;
+
+ asm ("mrs %0, cpacr_el1" : "=r" (tmp));
- if (test_and_set_thread_flag(TIF_SVE))
+ printk(KERN_INFO "%s: Strange, ZEN=%u\n",
+ __func__, (unsigned int)((tmp >> 16) & 3));
BUG();
+ }
BUG_ON(is_compat_task());
task_fpsimd_to_sve(current);
-
- asm ("mrs %0, cpacr_el1" : "=r" (tmp));
- asm volatile ("msr cpacr_el1, %0" :: "r" (tmp | (1 << 17)));
- /* Serialised by exception return to user */
+ task_fpsimd_load(current);
}
#endif /* CONFIG_ARM64_SVE */
@@ -178,6 +181,23 @@ static void task_fpsimd_load(struct task_struct *task)
&task->thread.fpsimd_state.fpsr);
else
fpsimd_load_state(&task->thread.fpsimd_state);
+
+ /*
+ * Flip SVE enable for userspace if it doesn't match the
+ * current_task.
+ */
+ if (IS_ENABLED(CONFIG_ARM64_SVE) && (elf_hwcap & HWCAP_SVE)) {
+ unsigned int tmp, flags;
+
+ asm ("mrs %0, cpacr_el1" : "=r" (tmp));
+ flags = task_thread_info(task)->flags;
+ BUILD_BUG_ON(_TIF_SVE != CPACR_EL1_ZEN_EL0EN);
+ if ((tmp ^ (unsigned long)flags) & _TIF_SVE) {
+ tmp ^= _TIF_SVE;
+ asm volatile ("msr cpacr_el1, %0" :: "r" (tmp));
+ /* Serialised by exception return to user */
+ }
+ }
}
static void task_fpsimd_save(struct task_struct *task)
@@ -306,27 +326,11 @@ void fpsimd_restore_current_state(void)
{
preempt_disable();
if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
- unsigned long tmp;
- unsigned long flags;
-
struct fpsimd_state *st = ¤t->thread.fpsimd_state;
task_fpsimd_load(current);
this_cpu_write(fpsimd_last_state, st);
st->cpu = smp_processor_id();
-
- if (IS_ENABLED(CONFIG_ARM64_SVE)) {
- /*
- * Flip SVE enable for userspace if it doesn't
- * match the current_task.
- */
- asm ("mrs %0, cpacr_el1" : "=r" (tmp));
- flags = current_thread_info()->flags;
- if ((tmp ^ (unsigned long)flags) & (1 << 17)) {
- tmp ^= 1 << 17;
- asm volatile ("msr cpacr_el1, %0" :: "r" (tmp));
- }
- }
}
preempt_enable();
}
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 22/29] arm64/sve: Implement FPSIMD-only context for tasks not using SVE
From: Dave Martin @ 2016-11-25 19:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
To reduce unnecessary context switch overhead, we don't need to
switch the whole SVE state for tasks that are not using it.
This patch restores the FPSIMD-only behaviour for tasks that have
never used SVE.
Note that coredumps and ptrace may see FPSIMD/SVE out of sync at
present -- this will be fixed later.
SVE state is saved on signal delivery only for tasks that have
used SVE. However, it should be possible to add SVE state on
return from a signal handler when the task didn't have any SVE
state previously. The caller may need to add its own SVE record
to the signal frame in this case.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/kernel/fpsimd.c | 34 +++++++++++++++++++++++-----------
arch/arm64/kernel/signal.c | 5 ++++-
2 files changed, 27 insertions(+), 12 deletions(-)
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 40566a9..cad86e5 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -100,6 +100,9 @@ void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
}
#ifdef CONFIG_ARM64_SVE
+
+static void task_fpsimd_to_sve(struct task_struct *task);
+
void do_sve_acc(unsigned int esr, struct pt_regs *regs)
{
unsigned long tmp;
@@ -107,11 +110,16 @@ void do_sve_acc(unsigned int esr, struct pt_regs *regs)
if (test_and_set_thread_flag(TIF_SVE))
BUG();
+ BUG_ON(is_compat_task());
+
+ task_fpsimd_to_sve(current);
+
asm ("mrs %0, cpacr_el1" : "=r" (tmp));
asm volatile ("msr cpacr_el1, %0" :: "r" (tmp | (1 << 17)));
/* Serialised by exception return to user */
}
-#endif
+
+#endif /* CONFIG_ARM64_SVE */
/*
* Raise a SIGFPE for the current process.
@@ -164,7 +172,8 @@ extern void *__task_pffr(struct task_struct *task);
static void task_fpsimd_load(struct task_struct *task)
{
- if (IS_ENABLED(CONFIG_ARM64_SVE) && (elf_hwcap & HWCAP_SVE))
+ if (IS_ENABLED(CONFIG_ARM64_SVE) &&
+ test_tsk_thread_flag(task, TIF_SVE))
sve_load_state(__task_pffr(task),
&task->thread.fpsimd_state.fpsr);
else
@@ -173,7 +182,8 @@ static void task_fpsimd_load(struct task_struct *task)
static void task_fpsimd_save(struct task_struct *task)
{
- if (IS_ENABLED(CONFIG_ARM64_SVE) && (elf_hwcap & HWCAP_SVE))
+ if (IS_ENABLED(CONFIG_ARM64_SVE) &&
+ test_tsk_thread_flag(task, TIF_SVE))
sve_save_state(__task_pffr(task),
&task->thread.fpsimd_state.fpsr);
else
@@ -202,11 +212,9 @@ void fpsimd_thread_switch(struct task_struct *next)
if (__this_cpu_read(fpsimd_last_state) == st
&& st->cpu == smp_processor_id())
- clear_ti_thread_flag(task_thread_info(next),
- TIF_FOREIGN_FPSTATE);
+ clear_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE);
else
- set_ti_thread_flag(task_thread_info(next),
- TIF_FOREIGN_FPSTATE);
+ set_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE);
}
}
@@ -285,7 +293,8 @@ static void task_sve_to_fpsimd(struct task_struct *task __always_unused) { }
void fpsimd_signal_preserve_current_state(void)
{
fpsimd_preserve_current_state();
- task_sve_to_fpsimd(current);
+ if (test_thread_flag(TIF_SVE))
+ task_sve_to_fpsimd(current);
}
/*
@@ -367,7 +376,7 @@ void fpsimd_update_current_state(struct fpsimd_state *state)
{
preempt_disable();
- if (IS_ENABLED(CONFIG_ARM64_SVE)) {
+ if (IS_ENABLED(CONFIG_ARM64_SVE) && test_thread_flag(TIF_SVE)) {
current->thread.fpsimd_state = *state;
task_fpsimd_to_sve(current);
}
@@ -408,8 +417,8 @@ void kernel_neon_begin_partial(u32 num_regs)
* interrupt context, so always save the userland SVE state
* if there is any, even for interrupts.
*/
- if (IS_ENABLED(CONFIG_ARM64_SVE) && (elf_hwcap & HWCAP_SVE) &&
- current->mm &&
+ if (IS_ENABLED(CONFIG_ARM64_SVE) &&
+ test_thread_flag(TIF_SVE) && current->mm &&
!test_and_set_thread_flag(TIF_FOREIGN_FPSTATE)) {
fpsimd_save_state(¤t->thread.fpsimd_state);
this_cpu_write(fpsimd_last_state, NULL);
@@ -532,6 +541,9 @@ static int __init fpsimd_init(void)
if (!(elf_hwcap & HWCAP_ASIMD))
pr_notice("Advanced SIMD is not implemented\n");
+ if (!(elf_hwcap & HWCAP_SVE))
+ pr_info("Scalable Vector Extension available\n");
+
return 0;
}
late_initcall(fpsimd_init);
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 129b016..2528ec1 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -259,6 +259,7 @@ static int __restore_sve_fpsimd_context(struct user_ctxs *user,
preempt_disable();
set_thread_flag(TIF_FOREIGN_FPSTATE);
+ set_thread_flag(TIF_SVE);
BUG_ON(SVE_SIG_REGS_SIZE(vq) > sizeof(*task_sve_regs));
BUG_ON(round_up(SVE_SIG_REGS_SIZE(vq), 16) < sizeof(*task_sve_regs));
@@ -543,9 +544,11 @@ static int setup_sigframe_layout(struct rt_sigframe_user_layout *user)
return err;
}
- if (IS_ENABLED(CONFIG_ARM64_SVE) && (elf_hwcap & HWCAP_SVE)) {
+ if (IS_ENABLED(CONFIG_ARM64_SVE) && test_thread_flag(TIF_SVE)) {
unsigned int vq = sve_vq_from_vl(sve_get_vl());
+ BUG_ON(!(elf_hwcap & HWCAP_SVE));
+
err = sigframe_alloc(user, &user->sve_offset,
SVE_SIG_CONTEXT_SIZE(vq));
if (err)
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 21/29] arm64/sve: Enable SVE on demand for userspace
From: Dave Martin @ 2016-11-25 19:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
This patch tracks whether a task has ever attempted to use the
Scalable Vector Extension. If and only if SVE is in use by a task,
it will be enabled for userspace when scheduling the task in. For
other tasks, SVE is disabled when scheduling in.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/include/asm/thread_info.h | 1 +
arch/arm64/kernel/entry.S | 18 +++++++++++++++++-
arch/arm64/kernel/fpsimd.c | 30 ++++++++++++++++++++++++++++++
3 files changed, 48 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index e9ea5a6..2deac86 100644
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -117,6 +117,7 @@ static inline struct thread_info *current_thread_info(void)
#define TIF_SYSCALL_AUDIT 9
#define TIF_SYSCALL_TRACEPOINT 10
#define TIF_SECCOMP 11
+#define TIF_SVE 17 /* Scalable Vector Extension in use */
#define TIF_MEMDIE 18 /* is terminating due to OOM killer */
#define TIF_FREEZE 19
#define TIF_RESTORE_SIGMASK 20
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 223d54a..fe20560 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -465,6 +465,10 @@ el0_sync:
b.eq el0_ia
cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access
b.eq el0_fpsimd_acc
+#ifdef CONFIG_ARM64_SVE
+ cmp x24, #ESR_ELx_EC_SVE // SVE access
+ b.eq el0_sve_acc
+#endif
cmp x24, #ESR_ELx_EC_FP_EXC64 // FP/ASIMD exception
b.eq el0_fpsimd_exc
cmp x24, #ESR_ELx_EC_SYS64 // configurable trap
@@ -563,9 +567,21 @@ el0_fpsimd_acc:
mov x1, sp
bl do_fpsimd_acc
b ret_to_user
+#ifdef CONFIG_ARM64_SVE
+ /*
+ * Scalable Vector Extension access
+ */
+el0_sve_acc:
+ enable_dbg
+ ct_user_exit
+ mov x0, x25
+ mov x1, sp
+ bl do_sve_acc
+ b ret_to_user
+#endif
el0_fpsimd_exc:
/*
- * Floating Point or Advanced SIMD exception
+ * Floating Point, Advanced SIMD or SVE exception
*/
enable_dbg
ct_user_exit
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index cda079e..40566a9 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -99,6 +99,20 @@ void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
WARN_ON(1);
}
+#ifdef CONFIG_ARM64_SVE
+void do_sve_acc(unsigned int esr, struct pt_regs *regs)
+{
+ unsigned long tmp;
+
+ if (test_and_set_thread_flag(TIF_SVE))
+ BUG();
+
+ asm ("mrs %0, cpacr_el1" : "=r" (tmp));
+ asm volatile ("msr cpacr_el1, %0" :: "r" (tmp | (1 << 17)));
+ /* Serialised by exception return to user */
+}
+#endif
+
/*
* Raise a SIGFPE for the current process.
*/
@@ -283,11 +297,27 @@ void fpsimd_restore_current_state(void)
{
preempt_disable();
if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
+ unsigned long tmp;
+ unsigned long flags;
+
struct fpsimd_state *st = ¤t->thread.fpsimd_state;
task_fpsimd_load(current);
this_cpu_write(fpsimd_last_state, st);
st->cpu = smp_processor_id();
+
+ if (IS_ENABLED(CONFIG_ARM64_SVE)) {
+ /*
+ * Flip SVE enable for userspace if it doesn't
+ * match the current_task.
+ */
+ asm ("mrs %0, cpacr_el1" : "=r" (tmp));
+ flags = current_thread_info()->flags;
+ if ((tmp ^ (unsigned long)flags) & (1 << 17)) {
+ tmp ^= 1 << 17;
+ asm volatile ("msr cpacr_el1, %0" :: "r" (tmp));
+ }
+ }
}
preempt_enable();
}
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 20/29] arm64/sve: traps: Add descriptive string for SVE exceptions
From: Dave Martin @ 2016-11-25 19:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
In preparation for SVE trapping in userspace, let's print something
relevant instead of "UNREGOCNIZED EC" when an unhandled SVE
exception occurs.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/include/asm/esr.h | 3 ++-
arch/arm64/kernel/traps.c | 1 +
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h
index d14c478..87729f3 100644
--- a/arch/arm64/include/asm/esr.h
+++ b/arch/arm64/include/asm/esr.h
@@ -42,7 +42,8 @@
#define ESR_ELx_EC_HVC64 (0x16)
#define ESR_ELx_EC_SMC64 (0x17)
#define ESR_ELx_EC_SYS64 (0x18)
-/* Unallocated EC: 0x19 - 0x1E */
+#define ESR_ELx_EC_SVE (0x19)
+/* Unallocated EC: 0x1A - 0x1E */
#define ESR_ELx_EC_IMP_DEF (0x1f)
#define ESR_ELx_EC_IABT_LOW (0x20)
#define ESR_ELx_EC_IABT_CUR (0x21)
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index c9986b3..aaab1dd 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -569,6 +569,7 @@ static const char *esr_class_str[] = {
[ESR_ELx_EC_HVC64] = "HVC (AArch64)",
[ESR_ELx_EC_SMC64] = "SMC (AArch64)",
[ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)",
+ [ESR_ELx_EC_SVE] = "SVE",
[ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF",
[ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)",
[ESR_ELx_EC_IABT_CUR] = "IABT (current EL)",
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 19/29] arm64/sve: Avoid corruption when replacing the SVE state
From: Dave Martin @ 2016-11-25 19:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
If preemption occurs during replacement of the whole SVE state,
as occurs during execve() or rt_sigreturn(), then some or all of
the new state for the thread can be lost, due to erroneous saving
of the pre-existing state over the new data.
This patch disables preemption around the affected operations to
avoid this failure mode.
This should be reexamined later if the impact on preemption latency
proves to be excessive.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/kernel/fpsimd.c | 4 ++++
arch/arm64/kernel/signal.c | 9 ++++++++-
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index b1a8d3e..cda079e 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -198,6 +198,8 @@ void fpsimd_thread_switch(struct task_struct *next)
void fpsimd_flush_thread(void)
{
+ preempt_disable();
+
fpsimd_flush_task_state(current);
memset(¤t->thread.fpsimd_state, 0, sizeof(struct fpsimd_state));
@@ -213,6 +215,8 @@ void fpsimd_flush_thread(void)
}
set_thread_flag(TIF_FOREIGN_FPSTATE);
+
+ preempt_enable();
}
/*
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 2697d09..129b016 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -256,6 +256,10 @@ static int __restore_sve_fpsimd_context(struct user_ctxs *user,
if (vl != sve_get_vl())
return -EINVAL;
+ preempt_disable();
+
+ set_thread_flag(TIF_FOREIGN_FPSTATE);
+
BUG_ON(SVE_SIG_REGS_SIZE(vq) > sizeof(*task_sve_regs));
BUG_ON(round_up(SVE_SIG_REGS_SIZE(vq), 16) < sizeof(*task_sve_regs));
BUG_ON(SVE_SIG_FFR_OFFSET(vq) - SVE_SIG_REGS_OFFSET !=
@@ -265,7 +269,7 @@ static int __restore_sve_fpsimd_context(struct user_ctxs *user,
SVE_SIG_REGS_OFFSET,
SVE_SIG_REGS_SIZE(vq));
if (err)
- return err;
+ goto out_preempt;
/* copy the FP and status/control registers */
/* restore_sigframe() already checked that user->fpsimd != NULL. */
@@ -278,6 +282,9 @@ static int __restore_sve_fpsimd_context(struct user_ctxs *user,
if (!err)
fpsimd_update_current_state(&fpsimd);
+out_preempt:
+ preempt_enable();
+
return err;
}
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 18/29] arm64/sve: signal: Restore FPSIMD/SVE state in rt_sigreturn
From: Dave Martin @ 2016-11-25 19:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
This patch adds the missing logic to restore the SVE state in
rt_sigreturn.
Because the FPSIMD and SVE state alias, this code replaces the
existing fpsimd restore code when there is SVE state to restore.
For Zn[127:0], the saved FPSIMD state in Vn takes precedence.
Since __task_fpsimd_to_sve() is used to merge the FPSIMD and SVE
state back together, and only for this purpose, we don't want it to
zero out the SVE state -- hence delete the memset() from there.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/kernel/fpsimd.c | 4 ---
arch/arm64/kernel/signal.c | 87 ++++++++++++++++++++++++++++++++++++++++------
2 files changed, 76 insertions(+), 15 deletions(-)
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 4ef2e37..b1a8d3e 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -266,9 +266,6 @@ static void task_sve_to_fpsimd(struct task_struct *task __always_unused) { }
void fpsimd_signal_preserve_current_state(void)
{
- WARN_ONCE(elf_hwcap & HWCAP_SVE,
- "SVE state save/restore around signals doesn't work properly, expect userspace corruption!\n");
-
fpsimd_preserve_current_state();
task_sve_to_fpsimd(current);
}
@@ -301,7 +298,6 @@ static void __task_fpsimd_to_sve(struct task_struct *task, unsigned int vq)
struct fpsimd_state *fst = &task->thread.fpsimd_state;
unsigned int i;
- memset(sst, 0, sizeof(*sst));
for (i = 0; i < 32; ++i)
sst->zregs[i][0] = fst->vregs[i];
}
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 038e7338..2697d09 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -211,6 +211,11 @@ static int restore_fpsimd_context(struct fpsimd_context __user *ctx)
}
+struct user_ctxs {
+ struct fpsimd_context __user *fpsimd;
+ struct sve_context __user *sve;
+};
+
#ifdef CONFIG_ARM64_SVE
static int preserve_sve_context(struct sve_context __user *ctx)
@@ -240,19 +245,68 @@ static int preserve_sve_context(struct sve_context __user *ctx)
return err ? -EFAULT : 0;
}
+static int __restore_sve_fpsimd_context(struct user_ctxs *user,
+ unsigned int vl, unsigned int vq)
+{
+ int err;
+ struct fpsimd_sve_state(vq) *task_sve_regs =
+ __task_sve_state(current);
+ struct fpsimd_state fpsimd;
+
+ if (vl != sve_get_vl())
+ return -EINVAL;
+
+ BUG_ON(SVE_SIG_REGS_SIZE(vq) > sizeof(*task_sve_regs));
+ BUG_ON(round_up(SVE_SIG_REGS_SIZE(vq), 16) < sizeof(*task_sve_regs));
+ BUG_ON(SVE_SIG_FFR_OFFSET(vq) - SVE_SIG_REGS_OFFSET !=
+ (char *)&task_sve_regs->ffr - (char *)task_sve_regs);
+ err = __copy_from_user(task_sve_regs,
+ (char __user const *)user->sve +
+ SVE_SIG_REGS_OFFSET,
+ SVE_SIG_REGS_SIZE(vq));
+ if (err)
+ return err;
+
+ /* copy the FP and status/control registers */
+ /* restore_sigframe() already checked that user->fpsimd != NULL. */
+ err = __copy_from_user(fpsimd.vregs, user->fpsimd->vregs,
+ sizeof(fpsimd.vregs));
+ __get_user_error(fpsimd.fpsr, &user->fpsimd->fpsr, err);
+ __get_user_error(fpsimd.fpcr, &user->fpsimd->fpcr, err);
+
+ /* load the hardware registers from the fpsimd_state structure */
+ if (!err)
+ fpsimd_update_current_state(&fpsimd);
+
+ return err;
+}
+
+static int restore_sve_fpsimd_context(struct user_ctxs *user)
+{
+ int err;
+ u16 vl, vq;
+
+ err = __get_user(vl, &user->sve->vl);
+ if (err)
+ return err;
+
+ if (!sve_vl_valid(vl))
+ return -EINVAL;
+
+ vq = sve_vq_from_vl(vl);
+
+ return __restore_sve_fpsimd_context(user, vl, vq);
+}
+
#else /* ! CONFIG_ARM64_SVE */
-/* Turn any non-optimised out attempt to use this into a link error: */
+/* Turn any non-optimised out attempts to use these into a link error: */
extern int preserve_sve_context(void __user *ctx);
+extern int restore_sve_fpsimd_context(struct user_ctxs *user);
#endif /* ! CONFIG_ARM64_SVE */
-struct user_ctxs {
- struct fpsimd_context __user *fpsimd;
- struct sve_context __user *sve;
-};
-
static int parse_user_sigframe(struct user_ctxs *user,
struct rt_sigframe __user *sf)
{
@@ -316,6 +370,9 @@ static int parse_user_sigframe(struct user_ctxs *user,
if (!IS_ENABLED(CONFIG_ARM64_SVE))
goto invalid;
+ if (!(elf_hwcap & HWCAP_SVE))
+ goto invalid;
+
if (user->sve)
goto invalid;
@@ -375,9 +432,6 @@ static int parse_user_sigframe(struct user_ctxs *user,
}
done:
- if (!user->fpsimd)
- goto invalid;
-
return 0;
invalid:
@@ -411,8 +465,19 @@ static int restore_sigframe(struct pt_regs *regs,
if (err == 0)
err = parse_user_sigframe(&user, sf);
- if (err == 0)
- err = restore_fpsimd_context(user.fpsimd);
+ if (err == 0) {
+ if (!user.fpsimd)
+ return -EINVAL;
+
+ if (user.sve) {
+ if (!IS_ENABLED(CONFIG_ARM64_SVE) ||
+ !(elf_hwcap & HWCAP_SVE))
+ return -EINVAL;
+
+ err = restore_sve_fpsimd_context(&user);
+ } else
+ err = restore_fpsimd_context(user.fpsimd);
+ }
return err;
}
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 17/29] arm64/sve: signal: Dump Scalable Vector Extension registers to user stack
From: Dave Martin @ 2016-11-25 19:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
This patch populates the sve_regs() area reserved on the user stack
with the actual register context.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/include/asm/fpsimd.h | 1 +
arch/arm64/kernel/fpsimd.c | 5 ++---
arch/arm64/kernel/signal.c | 8 ++++++++
3 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index aa82b38..e39066a 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -93,6 +93,7 @@ extern void fpsimd_load_partial_state(struct fpsimd_partial_state *state);
extern void __init fpsimd_init_task_struct_size(void);
+extern void *__task_sve_state(struct task_struct *task);
extern void sve_save_state(void *state, u32 *pfpsr);
extern void sve_load_state(void const *state, u32 const *pfpsr);
extern unsigned int sve_get_vl(void);
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 9a90921..4ef2e37 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -128,7 +128,7 @@ void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
#ifdef CONFIG_ARM64_SVE
-static void *__task_sve_state(struct task_struct *task)
+void *__task_sve_state(struct task_struct *task)
{
return (char *)task + ALIGN(sizeof(*task), 16);
}
@@ -143,8 +143,7 @@ static void *__task_pffr(struct task_struct *task)
#else /* !CONFIG_ARM64_SVE */
-/* Turn any non-optimised out attempts to use these into a link error: */
-extern void *__task_sve_state(struct task_struct *task);
+/* Turn any non-optimised out attempts to use this into a link error: */
extern void *__task_pffr(struct task_struct *task);
#endif /* !CONFIG_ARM64_SVE */
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 7418237..038e7338 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -229,6 +229,14 @@ static int preserve_sve_context(struct sve_context __user *ctx)
BUILD_BUG_ON(sizeof(ctx->__reserved) != sizeof(reserved));
err |= copy_to_user(&ctx->__reserved, reserved, sizeof(reserved));
+ /*
+ * This assumes that the SVE state has already been saved to
+ * the task struct by calling preserve_fpsimd_context().
+ */
+ err |= copy_to_user((char __user *)ctx + SVE_SIG_REGS_OFFSET,
+ __task_sve_state(current),
+ SVE_SIG_REGS_SIZE(vq));
+
return err ? -EFAULT : 0;
}
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 16/29] arm64/sve: signal: Add SVE state record to sigcontext
From: Dave Martin @ 2016-11-25 19:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
This patch adds a record to sigcontext that will contain the SVE
state.
Subsequent patches will implement the actual register dumping.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/include/uapi/asm/sigcontext.h | 86 ++++++++++++++++++++++++++++++++
arch/arm64/kernel/signal.c | 62 +++++++++++++++++++++++
2 files changed, 148 insertions(+)
diff --git a/arch/arm64/include/uapi/asm/sigcontext.h b/arch/arm64/include/uapi/asm/sigcontext.h
index 1af8437..11c915d 100644
--- a/arch/arm64/include/uapi/asm/sigcontext.h
+++ b/arch/arm64/include/uapi/asm/sigcontext.h
@@ -88,4 +88,90 @@ struct extra_context {
__u32 size; /* size in bytes of the extra space */
};
+#define SVE_MAGIC 0x53564501
+
+struct sve_context {
+ struct _aarch64_ctx head;
+ __u16 vl;
+ __u16 __reserved[3];
+};
+
+/*
+ * The SVE architecture leaves space for future expansion of the
+ * vector length beyond its initial architectural limit of 2048 bits
+ * (16 quadwords).
+ */
+#define SVE_VQ_MIN 1
+#define SVE_VQ_MAX 0x200
+
+#define SVE_VL_MIN (SVE_VQ_MIN * 0x10)
+#define SVE_VL_MAX (SVE_VQ_MAX * 0x10)
+
+#define SVE_NUM_ZREGS 32
+#define SVE_NUM_PREGS 16
+
+#define sve_vl_valid(vl) \
+ ((vl) % 0x10 == 0 && (vl) >= SVE_VL_MIN && (vl) <= SVE_VL_MAX)
+#define sve_vq_from_vl(vl) ((vl) / 0x10)
+
+/*
+ * The total size of meaningful data in the SVE context in bytes,
+ * including the header, is given by SVE_SIG_CONTEXT_SIZE(vq).
+ *
+ * Note: for all these macros, the "vq" argument denotes the SVE
+ * vector length in quadwords (i.e., units of 128 bits).
+ *
+ * The correct way to obtain vq is to use sve_vq_from_vl(vl). The
+ * result is valid if and only if sve_vl_valid(vl) is true. This is
+ * guaranteed for a struct sve_context written by the kernel.
+ *
+ *
+ * Additional macros describe the contents and layout of the payload.
+ * For each, SVE_SIG_x_OFFSET(args) is the start offset relative to
+ * the start of struct sve_context, and SVE_SIG_x_SIZE(args) is the
+ * size in bytes:
+ *
+ * x type description
+ * - ---- -----------
+ * REGS the entire SVE context
+ *
+ * ZREGS __uint128_t[SVE_NUM_ZREGS][vq] all Z-registers
+ * ZREG __uint128_t[vq] individual Z-register Zn
+ *
+ * PREGS uint16_t[SVE_NUM_PREGS][vq] all P-registers
+ * PREG uint16_t[vq] individual P-register Pn
+ *
+ * FFR uint16_t[vq] first-fault status register
+ *
+ * Additional data might be appended in the future.
+ */
+
+#define SVE_SIG_ZREG_SIZE(vq) ((__u32)(vq) * 16)
+#define SVE_SIG_PREG_SIZE(vq) ((__u32)(vq) * 2)
+#define SVE_SIG_FFR_SIZE(vq) SVE_SIG_PREG_SIZE(vq)
+
+#define SVE_SIG_REGS_OFFSET ((sizeof(struct sve_context) + 15) / 16 * 16)
+
+#define SVE_SIG_ZREGS_OFFSET SVE_SIG_REGS_OFFSET
+#define SVE_SIG_ZREG_OFFSET(vq, n) \
+ (SVE_SIG_ZREGS_OFFSET + SVE_SIG_ZREG_SIZE(vq) * (n))
+#define SVE_SIG_ZREGS_SIZE(vq) \
+ (SVE_SIG_ZREG_OFFSET(vq, SVE_NUM_ZREGS) - SVE_SIG_ZREGS_OFFSET)
+
+#define SVE_SIG_PREGS_OFFSET(vq) \
+ (SVE_SIG_ZREGS_OFFSET + SVE_SIG_ZREGS_SIZE(vq))
+#define SVE_SIG_PREG_OFFSET(vq, n) \
+ (SVE_SIG_PREGS_OFFSET(vq) + SVE_SIG_PREG_SIZE(vq) * (n))
+#define SVE_SIG_PREGS_SIZE(vq) \
+ (SVE_SIG_PREG_OFFSET(vq, SVE_NUM_PREGS) - SVE_SIG_PREGS_OFFSET(vq))
+
+#define SVE_SIG_FFR_OFFSET(vq) \
+ (SVE_SIG_PREGS_OFFSET(vq) + SVE_SIG_PREGS_SIZE(vq))
+
+#define SVE_SIG_REGS_SIZE(vq) \
+ (SVE_SIG_FFR_OFFSET(vq) + SVE_SIG_FFR_SIZE(vq) - SVE_SIG_REGS_OFFSET)
+
+#define SVE_SIG_CONTEXT_SIZE(vq) (SVE_SIG_REGS_OFFSET + SVE_SIG_REGS_SIZE(vq))
+
+
#endif /* _UAPI__ASM_SIGCONTEXT_H */
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 1e430b4..7418237 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -57,6 +57,7 @@ struct rt_sigframe_user_layout {
unsigned long fpsimd_offset;
unsigned long esr_offset;
+ unsigned long sve_offset;
unsigned long extra_offset;
unsigned long end_offset;
};
@@ -209,8 +210,39 @@ static int restore_fpsimd_context(struct fpsimd_context __user *ctx)
return err ? -EFAULT : 0;
}
+
+#ifdef CONFIG_ARM64_SVE
+
+static int preserve_sve_context(struct sve_context __user *ctx)
+{
+ int err = 0;
+ u16 reserved[ARRAY_SIZE(ctx->__reserved)];
+ unsigned int vl = sve_get_vl();
+ unsigned int vq = sve_vq_from_vl(vl);
+
+ memset(reserved, 0, sizeof(reserved));
+
+ __put_user_error(SVE_MAGIC, &ctx->head.magic, err);
+ __put_user_error(round_up(SVE_SIG_CONTEXT_SIZE(vq), 16),
+ &ctx->head.size, err);
+ __put_user_error(vl, &ctx->vl, err);
+ BUILD_BUG_ON(sizeof(ctx->__reserved) != sizeof(reserved));
+ err |= copy_to_user(&ctx->__reserved, reserved, sizeof(reserved));
+
+ return err ? -EFAULT : 0;
+}
+
+#else /* ! CONFIG_ARM64_SVE */
+
+/* Turn any non-optimised out attempt to use this into a link error: */
+extern int preserve_sve_context(void __user *ctx);
+
+#endif /* ! CONFIG_ARM64_SVE */
+
+
struct user_ctxs {
struct fpsimd_context __user *fpsimd;
+ struct sve_context __user *sve;
};
static int parse_user_sigframe(struct user_ctxs *user,
@@ -224,6 +256,7 @@ static int parse_user_sigframe(struct user_ctxs *user,
bool have_extra_context = false;
user->fpsimd = NULL;
+ user->sve = NULL;
if (!IS_ALIGNED((unsigned long)base, 16))
goto invalid;
@@ -271,6 +304,19 @@ static int parse_user_sigframe(struct user_ctxs *user,
/* ignore */
break;
+ case SVE_MAGIC:
+ if (!IS_ENABLED(CONFIG_ARM64_SVE))
+ goto invalid;
+
+ if (user->sve)
+ goto invalid;
+
+ if (size < sizeof(*user->sve))
+ goto invalid;
+
+ user->sve = (struct sve_context __user *)head;
+ break;
+
case EXTRA_MAGIC:
if (have_extra_context)
goto invalid;
@@ -417,6 +463,15 @@ static int setup_sigframe_layout(struct rt_sigframe_user_layout *user)
return err;
}
+ if (IS_ENABLED(CONFIG_ARM64_SVE) && (elf_hwcap & HWCAP_SVE)) {
+ unsigned int vq = sve_vq_from_vl(sve_get_vl());
+
+ err = sigframe_alloc(user, &user->sve_offset,
+ SVE_SIG_CONTEXT_SIZE(vq));
+ if (err)
+ return err;
+ }
+
return sigframe_alloc_end(user);
}
@@ -458,6 +513,13 @@ static int setup_sigframe(struct rt_sigframe_user_layout *user,
__put_user_error(current->thread.fault_code, &esr_ctx->esr, err);
}
+ /* Scalable Vector Extension state, if present */
+ if (IS_ENABLED(CONFIG_ARM64_SVE) && err == 0 && user->sve_offset) {
+ struct sve_context __user *sve_ctx =
+ apply_user_offset(user, user->sve_offset);
+ err |= preserve_sve_context(sve_ctx);
+ }
+
if (err == 0 && user->extra_offset) {
struct extra_context __user *extra =
apply_user_offset(user, user->extra_offset);
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 15/29] arm64/sve: Restore working FPSIMD save/restore around signals
From: Dave Martin @ 2016-11-25 19:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
Because fpsimd_state and the SVE state are not magically
synchronised in the task_struct, stale FPSIMD data may be saved on
signal handler entry, and restored data my be lost on sigreturn.
This patch converts between SVE and FPSIMD views around the signal,
restoring working FPSIMD save/restore.
This will not save/restore the SVE state properly, but it should
restore a working FPSIMD ABI.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/include/asm/fpsimd.h | 1 +
arch/arm64/kernel/fpsimd.c | 92 ++++++++++++++++++++++++++++++++++++++++-
arch/arm64/kernel/signal.c | 2 +-
arch/arm64/kernel/signal32.c | 2 +-
4 files changed, 94 insertions(+), 3 deletions(-)
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index 1c41259..aa82b38 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -80,6 +80,7 @@ extern void fpsimd_load_state(struct fpsimd_state *state);
extern void fpsimd_thread_switch(struct task_struct *next);
extern void fpsimd_flush_thread(void);
+extern void fpsimd_signal_preserve_current_state(void);
extern void fpsimd_preserve_current_state(void);
extern void fpsimd_restore_current_state(void);
extern void fpsimd_update_current_state(struct fpsimd_state *state);
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index cb947dd..9a90921 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -228,6 +228,52 @@ void fpsimd_preserve_current_state(void)
preempt_enable();
}
+#ifdef CONFIG_ARM64_SVE
+
+/* Helpers to sync task FPSIMD and SVE register views */
+
+static void __task_sve_to_fpsimd(struct task_struct *task, unsigned int vq)
+{
+ struct sve_struct fpsimd_sve_state(vq) *sst =
+ __task_sve_state(task);
+ struct fpsimd_state *fst = &task->thread.fpsimd_state;
+ unsigned int i;
+
+ for (i = 0; i < 32; ++i)
+ fst->vregs[i] = sst->zregs[i][0];
+}
+
+static void task_sve_to_fpsimd(struct task_struct *task)
+{
+ unsigned int vl = sve_get_vl();
+ unsigned int vq;
+
+ if (!(elf_hwcap & HWCAP_SVE))
+ return;
+
+ BUG_ON(vl % 16);
+ vq = vl / 16;
+ BUG_ON(vq < 1 || vq > 16);
+
+ __task_sve_to_fpsimd(task, vq);
+}
+
+#else /* ! CONFIG_ARM64_SVE */
+
+static void task_sve_to_fpsimd(struct task_struct *task __always_unused) { }
+
+#endif /* ! CONFIG_ARM64_SVE */
+
+
+void fpsimd_signal_preserve_current_state(void)
+{
+ WARN_ONCE(elf_hwcap & HWCAP_SVE,
+ "SVE state save/restore around signals doesn't work properly, expect userspace corruption!\n");
+
+ fpsimd_preserve_current_state();
+ task_sve_to_fpsimd(current);
+}
+
/*
* Load the userland FPSIMD state of 'current' from memory, but only if the
* FPSIMD state already held in the registers is /not/ the most recent FPSIMD
@@ -246,6 +292,43 @@ void fpsimd_restore_current_state(void)
preempt_enable();
}
+
+#ifdef CONFIG_ARM64_SVE
+
+static void __task_fpsimd_to_sve(struct task_struct *task, unsigned int vq)
+{
+ struct sve_struct fpsimd_sve_state(vq) *sst =
+ __task_sve_state(task);
+ struct fpsimd_state *fst = &task->thread.fpsimd_state;
+ unsigned int i;
+
+ memset(sst, 0, sizeof(*sst));
+ for (i = 0; i < 32; ++i)
+ sst->zregs[i][0] = fst->vregs[i];
+}
+
+static void task_fpsimd_to_sve(struct task_struct *task)
+{
+ unsigned int vl = sve_get_vl();
+ unsigned int vq;
+
+ if (!(elf_hwcap & HWCAP_SVE))
+ return;
+
+ BUG_ON(vl % 16);
+ vq = vl / 16;
+ BUG_ON(vq < 1 || vq > 16);
+
+ __task_fpsimd_to_sve(task, vq);
+}
+
+#else /* ! CONFIG_ARM64_SVE */
+
+/* Turn any non-optimised out attempts to use this into a link error: */
+extern void task_fpsimd_to_sve(struct task_struct *task);
+
+#endif /* ! CONFIG_ARM64_SVE */
+
/*
* Load an updated userland FPSIMD state for 'current' from memory and set the
* flag that indicates that the FPSIMD register contents are the most recent
@@ -254,13 +337,20 @@ void fpsimd_restore_current_state(void)
void fpsimd_update_current_state(struct fpsimd_state *state)
{
preempt_disable();
- fpsimd_load_state(state);
+
+ if (IS_ENABLED(CONFIG_ARM64_SVE)) {
+ current->thread.fpsimd_state = *state;
+ task_fpsimd_to_sve(current);
+ }
+ task_fpsimd_load(current);
+
if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
struct fpsimd_state *st = ¤t->thread.fpsimd_state;
this_cpu_write(fpsimd_last_state, st);
st->cpu = smp_processor_id();
}
+
preempt_enable();
}
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index c7175a3..1e430b4 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -168,7 +168,7 @@ static int preserve_fpsimd_context(struct fpsimd_context __user *ctx)
int err;
/* dump the hardware registers to the fpsimd_state structure */
- fpsimd_preserve_current_state();
+ fpsimd_signal_preserve_current_state();
/* copy the FP and status/control registers */
err = __copy_to_user(ctx->vregs, fpsimd->vregs, sizeof(fpsimd->vregs));
diff --git a/arch/arm64/kernel/signal32.c b/arch/arm64/kernel/signal32.c
index b7063de..08ca0dd 100644
--- a/arch/arm64/kernel/signal32.c
+++ b/arch/arm64/kernel/signal32.c
@@ -244,7 +244,7 @@ static int compat_preserve_vfp_context(struct compat_vfp_sigframe __user *frame)
* Note that this also saves V16-31, which aren't visible
* in AArch32.
*/
- fpsimd_preserve_current_state();
+ fpsimd_signal_preserve_current_state();
/* Place structure header on the stack */
__put_user_error(magic, &frame->magic, err);
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 14/29] Revert "arm64/sve: Allow kernel-mode NEON to be disabled in Kconfig"
From: Dave Martin @ 2016-11-25 19:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
Now that KERNEL_MODE_NEON works for SVE, we can just default it
back to y.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/Kconfig | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 7266761..bf9915cb 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -224,6 +224,9 @@ config SWIOTLB
config IOMMU_HELPER
def_bool SWIOTLB
+config KERNEL_MODE_NEON
+ def_bool y
+
config FIX_EARLYCON_MEM
def_bool y
@@ -266,10 +269,6 @@ endmenu
menu "Kernel Features"
-config KERNEL_MODE_NEON
- bool "Support NEON/FPSIMD code in the kernel"
- default y
-
menu "ARM errata workarounds via the alternatives framework"
config ARM64_ERRATUM_826319
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 13/29] arm64/sve: Basic support for KERNEL_MODE_NEON
From: Dave Martin @ 2016-11-25 19:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
In order to enable CONFIG_KERNEL_MODE_NEON and things that rely on
it to be configured together with Scalable Vector Extension support
in the same kernel, this patch implements basic support for
saving/restoring the SVE state around kernel_neon_begin()...
kernel_neon_end().
This patch is not optimal and will generally save more state than
necessary, more often than necessary. Further optimisations can be
implemented in future patches.
This patch is not intended to allow general-purpose _SVE_ code to
execute in the kernel safely. That functionality may also follow
in later patches.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/Kconfig | 1 -
arch/arm64/kernel/fpsimd.c | 22 ++++++++++++++++++----
2 files changed, 18 insertions(+), 5 deletions(-)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index e8d04dd..7266761 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -880,7 +880,6 @@ endmenu
config ARM64_SVE
bool "ARM Scalable Vector Extension support"
default y
- depends on !KERNEL_MODE_NEON # until it works with SVE
help
The Scalable Vector Extension (SVE) is an extension to the AArch64
execution state which complements and extends the SIMD functionality
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 81cfdb5..cb947dd 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -282,11 +282,26 @@ static DEFINE_PER_CPU(struct fpsimd_partial_state, softirq_fpsimdstate);
*/
void kernel_neon_begin_partial(u32 num_regs)
{
+ preempt_disable();
+
+ /*
+ * For now, we have no special storage for SVE registers in
+ * interrupt context, so always save the userland SVE state
+ * if there is any, even for interrupts.
+ */
+ if (IS_ENABLED(CONFIG_ARM64_SVE) && (elf_hwcap & HWCAP_SVE) &&
+ current->mm &&
+ !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE)) {
+ fpsimd_save_state(¤t->thread.fpsimd_state);
+ this_cpu_write(fpsimd_last_state, NULL);
+ }
+
if (in_interrupt()) {
struct fpsimd_partial_state *s = this_cpu_ptr(
in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate);
-
BUG_ON(num_regs > 32);
+
+ /* Save partial state for interrupted kernel-mode NEON code: */
fpsimd_save_partial_state(s, roundup(num_regs, 2));
} else {
/*
@@ -295,7 +310,6 @@ void kernel_neon_begin_partial(u32 num_regs)
* that there is no longer userland FPSIMD state in the
* registers.
*/
- preempt_disable();
if (current->mm &&
!test_and_set_thread_flag(TIF_FOREIGN_FPSTATE))
fpsimd_save_state(¤t->thread.fpsimd_state);
@@ -310,9 +324,9 @@ void kernel_neon_end(void)
struct fpsimd_partial_state *s = this_cpu_ptr(
in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate);
fpsimd_load_partial_state(s);
- } else {
- preempt_enable();
}
+
+ preempt_enable();
}
EXPORT_SYMBOL(kernel_neon_end);
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 12/29] arm64/sve: Save/restore SVE state on context switch paths
From: Dave Martin @ 2016-11-25 19:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
This patch implements basic handling of the Scalable Vector
Extension state on the primary context switch paths.
This does *not* (correctly) handle the signal path, and doesn't do
save/restore for SVE-only accesses that don't affect the FPSIMD
state (i.e., FFR).
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/kernel/fpsimd.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 05eca45..81cfdb5 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -175,7 +175,7 @@ void fpsimd_thread_switch(struct task_struct *next)
* 'current'.
*/
if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE))
- fpsimd_save_state(¤t->thread.fpsimd_state);
+ task_fpsimd_save(current);
if (next->mm) {
/*
@@ -224,7 +224,7 @@ void fpsimd_preserve_current_state(void)
{
preempt_disable();
if (!test_thread_flag(TIF_FOREIGN_FPSTATE))
- fpsimd_save_state(¤t->thread.fpsimd_state);
+ task_fpsimd_save(current);
preempt_enable();
}
@@ -239,7 +239,7 @@ void fpsimd_restore_current_state(void)
if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
struct fpsimd_state *st = ¤t->thread.fpsimd_state;
- fpsimd_load_state(st);
+ task_fpsimd_load(current);
this_cpu_write(fpsimd_last_state, st);
st->cpu = smp_processor_id();
}
@@ -325,7 +325,7 @@ static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
switch (cmd) {
case CPU_PM_ENTER:
if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE))
- fpsimd_save_state(¤t->thread.fpsimd_state);
+ task_fpsimd_save(current);
this_cpu_write(fpsimd_last_state, NULL);
break;
case CPU_PM_EXIT:
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 11/29] arm64/sve: Expand task_struct for Scalable Vector Extension state
From: Dave Martin @ 2016-11-25 19:38 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
This patch expands task_struct to accommodate the Scalable Vector
Extension state.
The extra space is not used for anything yet.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/Kconfig | 1 +
arch/arm64/include/asm/fpsimd.h | 12 +++++++
arch/arm64/kernel/fpsimd.c | 71 ++++++++++++++++++++++++++++++++++++++++-
arch/arm64/kernel/process.c | 2 +-
arch/arm64/kernel/setup.c | 3 ++
5 files changed, 87 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index cd6c846..e8d04dd 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -19,6 +19,7 @@ config ARM64
select ARCH_SUPPORTS_NUMA_BALANCING
select ARCH_WANT_COMPAT_IPC_PARSE_VERSION
select ARCH_WANT_FRAME_POINTERS
+ select ARCH_WANTS_DYNAMIC_TASK_STRUCT
select ARCH_HAS_UBSAN_SANITIZE_ALL
select ARM_AMBA
select ARM_ARCH_TIMER
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index 92f45ee..1c41259 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -51,6 +51,15 @@ struct fpsimd_partial_state {
__uint128_t vregs[32];
};
+/*
+ * Scalable Vector Extension state structure template.
+ * The layout is vector length dependent, with vector length = vl * 16 bytes.
+ */
+#define fpsimd_sve_state(vl) { \
+ __uint128_t zregs[32][vl]; \
+ u16 pregs[16][vl]; \
+ u16 ffr[vl]; \
+}
#if defined(__KERNEL__) && defined(CONFIG_COMPAT)
/* Masks for extracting the FPSR and FPCR from the FPSCR */
@@ -81,8 +90,11 @@ extern void fpsimd_save_partial_state(struct fpsimd_partial_state *state,
u32 num_regs);
extern void fpsimd_load_partial_state(struct fpsimd_partial_state *state);
+extern void __init fpsimd_init_task_struct_size(void);
+
extern void sve_save_state(void *state, u32 *pfpsr);
extern void sve_load_state(void const *state, u32 const *pfpsr);
+extern unsigned int sve_get_vl(void);
#endif
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index 394c61d..05eca45 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -26,6 +26,7 @@
#include <linux/hardirq.h>
#include <asm/fpsimd.h>
+#include <asm/cpufeature.h>
#include <asm/cputype.h>
#define FPEXC_IOF (1 << 0)
@@ -125,6 +126,47 @@ void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
send_sig_info(SIGFPE, &info, current);
}
+#ifdef CONFIG_ARM64_SVE
+
+static void *__task_sve_state(struct task_struct *task)
+{
+ return (char *)task + ALIGN(sizeof(*task), 16);
+}
+
+static void *__task_pffr(struct task_struct *task)
+{
+ unsigned int vl = sve_get_vl();
+
+ BUG_ON(vl % 16);
+ return (char *)__task_sve_state(task) + 34 * vl;
+}
+
+#else /* !CONFIG_ARM64_SVE */
+
+/* Turn any non-optimised out attempts to use these into a link error: */
+extern void *__task_sve_state(struct task_struct *task);
+extern void *__task_pffr(struct task_struct *task);
+
+#endif /* !CONFIG_ARM64_SVE */
+
+static void task_fpsimd_load(struct task_struct *task)
+{
+ if (IS_ENABLED(CONFIG_ARM64_SVE) && (elf_hwcap & HWCAP_SVE))
+ sve_load_state(__task_pffr(task),
+ &task->thread.fpsimd_state.fpsr);
+ else
+ fpsimd_load_state(&task->thread.fpsimd_state);
+}
+
+static void task_fpsimd_save(struct task_struct *task)
+{
+ if (IS_ENABLED(CONFIG_ARM64_SVE) && (elf_hwcap & HWCAP_SVE))
+ sve_save_state(__task_pffr(task),
+ &task->thread.fpsimd_state.fpsr);
+ else
+ fpsimd_save_state(&task->thread.fpsimd_state);
+}
+
void fpsimd_thread_switch(struct task_struct *next)
{
/*
@@ -157,8 +199,20 @@ void fpsimd_thread_switch(struct task_struct *next)
void fpsimd_flush_thread(void)
{
- memset(¤t->thread.fpsimd_state, 0, sizeof(struct fpsimd_state));
fpsimd_flush_task_state(current);
+
+ memset(¤t->thread.fpsimd_state, 0, sizeof(struct fpsimd_state));
+
+ if (IS_ENABLED(CONFIG_ARM64_SVE) && (elf_hwcap & HWCAP_SVE)) {
+ BUG_ON((char *)__task_sve_state(current) < (char *)current);
+ BUG_ON(arch_task_struct_size <
+ ((char *)__task_sve_state(current) - (char *)current));
+
+ memset(__task_sve_state(current), 0,
+ arch_task_struct_size -
+ ((char *)__task_sve_state(current) - (char *)current));
+ }
+
set_thread_flag(TIF_FOREIGN_FPSTATE);
}
@@ -315,6 +369,21 @@ static inline void fpsimd_hotplug_init(void)
static inline void fpsimd_hotplug_init(void) { }
#endif
+void __init fpsimd_init_task_struct_size(void)
+{
+ arch_task_struct_size = sizeof(struct task_struct);
+
+ if (IS_ENABLED(CONFIG_ARM64_SVE) &&
+ ((read_cpuid(ID_AA64PFR0_EL1) >> ID_AA64PFR0_SVE_SHIFT)
+ & 0xf) == 1) {
+ arch_task_struct_size = sizeof(struct task_struct) +
+ 35 * sve_get_vl();
+
+ pr_info("SVE: enabled with maximum %u bits per vector\n",
+ sve_get_vl() * 8);
+ }
+}
+
/*
* FP/SIMD support code initialisation.
*/
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 01753cd..7e19c3c 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -242,7 +242,7 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
{
if (current->mm)
fpsimd_preserve_current_state();
- *dst = *src;
+ memcpy(dst, src, arch_task_struct_size);
return 0;
}
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index f534f49..f0f551e 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -234,6 +234,9 @@ void __init setup_arch(char **cmdline_p)
pr_info("Boot CPU: AArch64 Processor [%08x]\n", read_cpuid_id());
sprintf(init_utsname()->machine, UTS_MACHINE);
+
+ fpsimd_init_task_struct_size();
+
init_mm.start_code = (unsigned long) _text;
init_mm.end_code = (unsigned long) _etext;
init_mm.end_data = (unsigned long) _edata;
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 10/29] arm64/sve: Boot-time feature enablement
From: Dave Martin @ 2016-11-25 19:38 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
This patch enables Scalable Vector Extension access for the kernel
on boot.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/include/asm/kvm_arm.h | 1 +
arch/arm64/include/asm/sysreg.h | 10 ++++++++++
arch/arm64/kernel/head.S | 16 +++++++++++++++-
arch/arm64/mm/proc.S | 22 +++++++++++++++++++++-
4 files changed, 47 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h
index 2a2752b..ae7afb2 100644
--- a/arch/arm64/include/asm/kvm_arm.h
+++ b/arch/arm64/include/asm/kvm_arm.h
@@ -185,6 +185,7 @@
#define CPTR_EL2_TCPAC (1 << 31)
#define CPTR_EL2_TTA (1 << 20)
#define CPTR_EL2_TFP (1 << CPTR_EL2_TFP_SHIFT)
+#define CPTR_EL2_TZ (1 << 8)
#define CPTR_EL2_DEFAULT 0x000033ff
/* Hyp Debug Configuration Register bits */
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index ccce9ad..09a44b3 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -302,4 +302,14 @@ static inline void config_sctlr_el1(u32 clear, u32 set)
#endif
+#define ZIDR_EL1 sys_reg(3, 0, 0, 0, 7)
+#define ZCR_EL1 sys_reg(3, 0, 1, 2, 0)
+#define ZCR_EL2 sys_reg(3, 4, 1, 2, 0)
+
+#define ZCR_EL1_LEN_MASK 0x1ff
+
+#define CPACR_EL1_ZEN_EL1EN (1 << 16)
+#define CPACR_EL1_ZEN_EL0EN (1 << 17)
+#define CPACR_EL1_ZEN (CPACR_EL1_ZEN_EL1EN | CPACR_EL1_ZEN_EL0EN)
+
#endif /* __ASM_SYSREG_H */
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 332e331..ae4448f 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -572,9 +572,23 @@ CPU_LE( movk x0, #0x30d0, lsl #16 ) // Clear EE and E0E on LE systems
/* Coprocessor traps. */
mov x0, #0x33ff
+
+ /* SVE register access */
+ mrs x1, id_aa64pfr0_el1
+ ubfx x1, x1, #ID_AA64PFR0_SVE_SHIFT, #4
+ cbz x1, 4f
+
+ bic x0, x0, #CPTR_EL2_TZ // Disable SVE traps to EL2
msr cptr_el2, x0 // Disable copro. traps to EL2
-1:
+ isb
+
+ mrs_s x1, ZIDR_EL1 // Scalable Vector Extension:
+ and x1, x1, #ZCR_EL1_LEN_MASK // Enable full vector length
+ msr_s ZCR_EL2, x1 // for EL1.
+ b 1f
+4: msr cptr_el2, x0 // Disable copro. traps to EL2
+1:
#ifdef CONFIG_COMPAT
msr hstr_el2, xzr // Disable CP15 traps to EL2
#endif
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 352c73b..1da8160 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -27,6 +27,7 @@
#include <asm/pgtable-hwdef.h>
#include <asm/cpufeature.h>
#include <asm/alternative.h>
+#include <asm/sysreg.h>
#ifdef CONFIG_ARM64_64K_PAGES
#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K
@@ -184,12 +185,31 @@ ENTRY(__cpu_setup)
dsb nsh
mov x0, #3 << 20
+
+ /* SVE */
+ mrs x5, id_aa64pfr0_el1
+ ubfx x5, x5, #ID_AA64PFR0_SVE_SHIFT, #4
+ cbz x5, 1f
+
+ bic x0, x0, #CPACR_EL1_ZEN
+ orr x0, x0, #CPACR_EL1_ZEN_EL1EN // SVE: trap for EL0, not EL1
msr cpacr_el1, x0 // Enable FP/ASIMD
- mov x0, #1 << 12 // Reset mdscr_el1 and disable
+ isb
+
+ mrs_s x5, ZIDR_EL1 // SVE: Enable full vector len
+ and x5, x5, #ZCR_EL1_LEN_MASK // initially
+ msr_s ZCR_EL1, x5
+
+ b 2f
+
+1: msr cpacr_el1, x0 // Enable FP/ASIMD
+
+2: mov x0, #1 << 12 // Reset mdscr_el1 and disable
msr mdscr_el1, x0 // access to the DCC from EL0
isb // Unmask debug exceptions now,
enable_dbg // since this is per-cpu
reset_pmuserenr_el0 x0 // Disable PMU access from EL0
+
/*
* Memory region attributes for LPAE:
*
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 09/29] arm64/sve: Boot-time feature detection and reporting
From: Dave Martin @ 2016-11-25 19:38 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
This patch adds feature detection for the ARM Scalable Vector
Extension, and adds basic informative feature reporting via
/proc/cpuinfo.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/include/asm/sysreg.h | 1 +
arch/arm64/include/uapi/asm/hwcap.h | 1 +
arch/arm64/kernel/cpufeature.c | 3 +++
arch/arm64/kernel/cpuinfo.c | 1 +
4 files changed, 6 insertions(+)
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 6c80b36..ccce9ad 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -113,6 +113,7 @@
#define ID_AA64ISAR0_AES_SHIFT 4
/* id_aa64pfr0 */
+#define ID_AA64PFR0_SVE_SHIFT 32
#define ID_AA64PFR0_GIC_SHIFT 24
#define ID_AA64PFR0_ASIMD_SHIFT 20
#define ID_AA64PFR0_FP_SHIFT 16
diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h
index a739287..f0de828 100644
--- a/arch/arm64/include/uapi/asm/hwcap.h
+++ b/arch/arm64/include/uapi/asm/hwcap.h
@@ -30,5 +30,6 @@
#define HWCAP_ATOMICS (1 << 8)
#define HWCAP_FPHP (1 << 9)
#define HWCAP_ASIMDHP (1 << 10)
+#define HWCAP_SVE (1 << 11)
#endif /* _UAPI__ASM_HWCAP_H */
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index c02504e..5126288 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -856,6 +856,9 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_FPHP),
HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 0, CAP_HWCAP, HWCAP_ASIMD),
HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, FTR_SIGNED, 1, CAP_HWCAP, HWCAP_ASIMDHP),
+#ifdef CONFIG_ARM64_SVE
+ HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, FTR_UNSIGNED, 1, CAP_HWCAP, HWCAP_SVE),
+#endif
{},
};
diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
index b3d5b3e..53e9fe2 100644
--- a/arch/arm64/kernel/cpuinfo.c
+++ b/arch/arm64/kernel/cpuinfo.c
@@ -63,6 +63,7 @@ static const char *const hwcap_str[] = {
"atomics",
"fphp",
"asimdhp",
+ "sve",
NULL
};
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 08/29] arm64/sve: Low-level save/restore code
From: Dave Martin @ 2016-11-25 19:38 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
This patch adds low-level save/restore for the Scalable Vector
Extension.
This is helper code only, and is not used for anything yet.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/Kconfig | 12 +++
arch/arm64/include/asm/fpsimd.h | 3 +
arch/arm64/include/asm/fpsimdmacros.h | 145 ++++++++++++++++++++++++++++++++++
arch/arm64/kernel/entry-fpsimd.S | 17 ++++
4 files changed, 177 insertions(+)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 1bdcaf1..cd6c846 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -876,6 +876,18 @@ config ARM64_UAO
endmenu
+config ARM64_SVE
+ bool "ARM Scalable Vector Extension support"
+ default y
+ depends on !KERNEL_MODE_NEON # until it works with SVE
+ help
+ The Scalable Vector Extension (SVE) is an extension to the AArch64
+ execution state which complements and extends the SIMD functionality
+ of the base architecture to support much larger vectors and to enable
+ additional vectorisation opportunities.
+
+ To enable use of this extension on CPUs that implement it, say Y.
+
config ARM64_MODULE_CMODEL_LARGE
bool
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index 50f559f..92f45ee 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -81,6 +81,9 @@ extern void fpsimd_save_partial_state(struct fpsimd_partial_state *state,
u32 num_regs);
extern void fpsimd_load_partial_state(struct fpsimd_partial_state *state);
+extern void sve_save_state(void *state, u32 *pfpsr);
+extern void sve_load_state(void const *state, u32 const *pfpsr);
+
#endif
#endif
diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h
index a2daf12..e2bb032 100644
--- a/arch/arm64/include/asm/fpsimdmacros.h
+++ b/arch/arm64/include/asm/fpsimdmacros.h
@@ -131,3 +131,148 @@
ldp q0, q1, [\state, #-16 * 0 - 16]
0:
.endm
+
+.macro _check_reg nr
+ .if (\nr) < 0 || (\nr) > 31
+ .error "Bad register number \nr."
+ .endif
+.endm
+
+.macro _check_zreg znr
+ .if (\znr) < 0 || (\znr) > 31
+ .error "Bad Scalable Vector Extension vector register number \znr."
+ .endif
+.endm
+
+.macro _check_preg pnr
+ .if (\pnr) < 0 || (\pnr) > 15
+ .error "Bad Scalable Vector Extension predicate register number \pnr."
+ .endif
+.endm
+
+.macro _check_num n, min, max
+ .if (\n) < (\min) || (\n) > (\max)
+ .error "Number \n out of range [\min,\max]"
+ .endif
+.endm
+
+.macro _zstrv znt, nspb, ioff=0
+ _check_zreg \znt
+ _check_reg \nspb
+ _check_num (\ioff), -0x100, 0xff
+ .inst 0xe5804000 \
+ | (\znt) \
+ | ((\nspb) << 5) \
+ | (((\ioff) & 7) << 10) \
+ | (((\ioff) & 0x1f8) << 13)
+.endm
+
+.macro _zldrv znt, nspb, ioff=0
+ _check_zreg \znt
+ _check_reg \nspb
+ _check_num (\ioff), -0x100, 0xff
+ .inst 0x85804000 \
+ | (\znt) \
+ | ((\nspb) << 5) \
+ | (((\ioff) & 7) << 10) \
+ | (((\ioff) & 0x1f8) << 13)
+.endm
+
+.macro _zstrp pnt, nspb, ioff=0
+ _check_preg \pnt
+ _check_reg \nspb
+ _check_num (\ioff), -0x100, 0xff
+ .inst 0xe5800000 \
+ | (\pnt) \
+ | ((\nspb) << 5) \
+ | (((\ioff) & 7) << 10) \
+ | (((\ioff) & 0x1f8) << 13)
+.endm
+
+.macro _zldrp pnt, nspb, ioff=0
+ _check_preg \pnt
+ _check_reg \nspb
+ _check_num (\ioff), -0x100, 0xff
+ .inst 0x85800000 \
+ | (\pnt) \
+ | ((\nspb) << 5) \
+ | (((\ioff) & 7) << 10) \
+ | (((\ioff) & 0x1f8) << 13)
+.endm
+
+.macro _zrdvl nspd, is1
+ _check_reg \nspd
+ _check_num (\is1), -0x20, 0x1f
+ .inst 0x04bf5000 \
+ | (\nspd) \
+ | (((\is1) & 0x3f) << 5)
+.endm
+
+.macro _zrdffr pnd
+ _check_preg \pnd
+ .inst 0x2519f000 \
+ | (\pnd)
+.endm
+
+.macro _zwrffr pnd
+ _check_preg \pnd
+ .inst 0x25289000 \
+ | ((\pnd) << 5)
+.endm
+
+.macro for from, to, insn
+ .if (\from) >= (\to)
+ \insn (\from)
+ .exitm
+ .endif
+
+ for \from, ((\from) + (\to)) / 2, \insn
+ for ((\from) + (\to)) / 2 + 1, \to, \insn
+.endm
+
+.macro sve_save nb, xpfpsr, ntmp
+ .macro savez n
+ _zstrv \n, \nb, (\n) - 34
+ .endm
+
+ .macro savep n
+ _zstrp \n, \nb, (\n) - 16
+ .endm
+
+ for 0, 31, savez
+ for 0, 15, savep
+ _zrdffr 0
+ _zstrp 0, \nb
+ _zldrp 0, \nb, -16
+
+ mrs x\ntmp, fpsr
+ str w\ntmp, [\xpfpsr]
+ mrs x\ntmp, fpcr
+ str w\ntmp, [\xpfpsr, #4]
+
+ .purgem savez
+ .purgem savep
+.endm
+
+.macro sve_load nb, xpfpsr, ntmp
+ .macro loadz n
+ _zldrv \n, \nb, (\n) - 34
+ .endm
+
+ .macro loadp n
+ _zldrp \n, \nb, (\n) - 16
+ .endm
+
+ for 0, 31, loadz
+ _zldrp 0, \nb
+ _zwrffr 0
+ for 0, 15, loadp
+
+ ldr w\ntmp, [\xpfpsr]
+ msr fpsr, x\ntmp
+ ldr w\ntmp, [\xpfpsr, #4]
+ msr fpcr, x\ntmp
+
+ .purgem loadz
+ .purgem loadp
+.endm
diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S
index c44a82f..5dcec55 100644
--- a/arch/arm64/kernel/entry-fpsimd.S
+++ b/arch/arm64/kernel/entry-fpsimd.S
@@ -65,3 +65,20 @@ ENTRY(fpsimd_load_partial_state)
ENDPROC(fpsimd_load_partial_state)
#endif
+
+#ifdef CONFIG_ARM64_SVE
+ENTRY(sve_save_state)
+ sve_save 0, x1, 2
+ ret
+ENDPROC(sve_save_state)
+
+ENTRY(sve_load_state)
+ sve_load 0, x1, 2
+ ret
+ENDPROC(sve_load_state)
+
+ENTRY(sve_get_vl)
+ _zrdvl 0, 1
+ ret
+ENDPROC(sve_get_vl)
+#endif /* CONFIG_ARM64_SVE */
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 07/29] arm64/sve: Allow kernel-mode NEON to be disabled in Kconfig
From: Dave Martin @ 2016-11-25 19:38 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
Currently, support for kernel-mode NEON alongside the Scalable
Vector Extension doesn't work, so allow KERNEL_MODE_NEON to be
disabled.
This is only needed for bisectability of the SVE patches and will
be removed later.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/Kconfig | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index d008bb6..1bdcaf1 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -223,9 +223,6 @@ config SWIOTLB
config IOMMU_HELPER
def_bool SWIOTLB
-config KERNEL_MODE_NEON
- def_bool y
-
config FIX_EARLYCON_MEM
def_bool y
@@ -268,6 +265,10 @@ endmenu
menu "Kernel Features"
+config KERNEL_MODE_NEON
+ bool "Support NEON/FPSIMD code in the kernel"
+ default y
+
menu "ARM errata workarounds via the alternatives framework"
config ARM64_ERRATUM_826319
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 06/29] arm64: efi: Add missing Kconfig dependency on KERNEL_MODE_NEON
From: Dave Martin @ 2016-11-25 19:38 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
The EFI runtime services ABI permits calls to EFI to clobber
certain FPSIMD/NEON registers, as per the AArch64 procedure call
standard.
Saving/restoring the clobbered registers around such calls needs
KERNEL_MODE_NEON, but the dependency is missing from Kconfig.
This patch adds the missing dependency.
This will aid bisection of the patches implementing support for the
ARM Scalable Vector Extension (SVE).
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/Kconfig | 1 +
1 file changed, 1 insertion(+)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 969ef88..d008bb6 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -963,6 +963,7 @@ config EFI_STUB
config EFI
bool "UEFI runtime support"
depends on OF && !CPU_BIG_ENDIAN
+ depends on KERNEL_MODE_NEON
select LIBFDT
select UCS2_STRING
select EFI_PARAMS_FROM_FDT
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 05/29] arm64: signal: Parse extra_context during sigreturn
From: Dave Martin @ 2016-11-25 19:38 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
If extra_context is present, parse it.
To avoid abuse by userspace, this patch attempts to ensure that:
* that no more than one extra_context is accepted;
* that the extra_context is a sensible size;
* that the extra context data is properly aligned.
This patch relies on the user accessors in order to ensure that the
user-supplied extra context data pointer is an honest userspace
address.
Other than that, the kernel doesn't care specially whether the
pointer supplied is sensible (e.g., not garbage, doesn't overlap
sigcontext.__reserved[], etc.) since this cannot harm the kernel.
More checks may be added later in order to aid debugging of
botched sigreturns from userspace.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/kernel/signal.c | 40 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 40 insertions(+)
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index ea3f6bf..c7175a3 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -221,6 +221,7 @@ static int parse_user_sigframe(struct user_ctxs *user,
char __user *base = (char __user *)&sc->__reserved;
size_t offset = 0;
size_t limit = sizeof(sc->__reserved);
+ bool have_extra_context = false;
user->fpsimd = NULL;
@@ -230,6 +231,9 @@ static int parse_user_sigframe(struct user_ctxs *user,
while (1) {
int err = 0;
u32 magic, size;
+ struct extra_context const __user *extra;
+ void __user *extra_data;
+ u32 extra_size;
if (limit - offset < sizeof(*head))
goto invalid;
@@ -267,6 +271,42 @@ static int parse_user_sigframe(struct user_ctxs *user,
/* ignore */
break;
+ case EXTRA_MAGIC:
+ if (have_extra_context)
+ goto invalid;
+
+ if (size < sizeof(*extra))
+ goto invalid;
+
+ extra = (struct extra_context const __user *)head;
+ __get_user_error(extra_data, &extra->data, err);
+ __get_user_error(extra_size, &extra->size, err);
+ if (err)
+ return err;
+
+ /* Prevent looping/repeated parsing of extra_conext */
+ have_extra_context = true;
+
+ /*
+ * Rely on the __user accessors to reject bogus
+ * pointers.
+ */
+ base = extra_data;
+ if (!IS_ALIGNED((unsigned long)base, 16))
+ goto invalid;
+
+ /* Reject "unreasonably large" frames: */
+ limit = extra_size;
+ if (limit > SIGFRAME_MAXSZ - sizeof(sc->__reserved))
+ goto invalid;
+
+ /*
+ * Ignore trailing terminator in __reserved[]
+ * and start parsing extra_data:
+ */
+ offset = 0;
+ continue;
+
default:
goto invalid;
}
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 04/29] arm64: signal: Allocate extra sigcontext space as needed
From: Dave Martin @ 2016-11-25 19:38 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
This patch modifies the context block allocator to create an
extra_context expansion block as necessary, and adds the necessary
code to populate, parse and decode this block.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/include/uapi/asm/sigcontext.h | 27 ++++++++
arch/arm64/kernel/signal.c | 112 +++++++++++++++++++++++++------
2 files changed, 120 insertions(+), 19 deletions(-)
diff --git a/arch/arm64/include/uapi/asm/sigcontext.h b/arch/arm64/include/uapi/asm/sigcontext.h
index ee469be..1af8437 100644
--- a/arch/arm64/include/uapi/asm/sigcontext.h
+++ b/arch/arm64/include/uapi/asm/sigcontext.h
@@ -61,4 +61,31 @@ struct esr_context {
__u64 esr;
};
+/*
+ * Pointer to extra space for additional structures that don't fit in
+ * sigcontext.__reserved[]. Note:
+ *
+ * 1) fpsimd_context, esr_context and extra_context must be placed in
+ * sigcontext.__reserved[] if present. They cannot be placed in the
+ * extra space. Any other record can be placed either in the extra
+ * space or in sigcontext.__reserved[].
+ *
+ * 2) There must not be more than one extra_context.
+ *
+ * 3) If extra_context is present, it must be followed immediately in
+ * sigcontext.__reserved[] by the terminating null _aarch64_ctx (i.e.,
+ * extra_context must be the last record in sigcontext.__reserved[]
+ * except for the terminator).
+ *
+ * 4) The extra space must itself be terminated with a null
+ * _aarch64_ctx.
+ */
+#define EXTRA_MAGIC 0x45585401
+
+struct extra_context {
+ struct _aarch64_ctx head;
+ void *data; /* 16-byte aligned pointer to the extra space */
+ __u32 size; /* size in bytes of the extra space */
+};
+
#endif /* _UAPI__ASM_SIGCONTEXT_H */
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 653b614..ea3f6bf 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -25,6 +25,7 @@
#include <linux/freezer.h>
#include <linux/stddef.h>
#include <linux/uaccess.h>
+#include <linux/sizes.h>
#include <linux/string.h>
#include <linux/tracehook.h>
#include <linux/ratelimit.h>
@@ -56,18 +57,22 @@ struct rt_sigframe_user_layout {
unsigned long fpsimd_offset;
unsigned long esr_offset;
+ unsigned long extra_offset;
unsigned long end_offset;
};
static void init_user_layout(struct rt_sigframe_user_layout *user)
{
+ const size_t __reserved_size =
+ sizeof(user->sigframe->uc.uc_mcontext.__reserved);
+ const size_t terminator_size =
+ round_up(sizeof(struct _aarch64_ctx), 16);
+
memset(user, 0, sizeof(*user));
user->size = offsetof(struct rt_sigframe, uc.uc_mcontext.__reserved);
-
- user->limit = user->size +
- sizeof(user->sigframe->uc.uc_mcontext.__reserved) -
- round_up(sizeof(struct _aarch64_ctx), 16);
- /* ^ reserve space for terminator */
+ user->limit = user->size + (__reserved_size - terminator_size -
+ sizeof(struct extra_context));
+ /* Reserve space for extension and terminator ^ */
}
static size_t sigframe_size(struct rt_sigframe_user_layout const *user)
@@ -75,6 +80,49 @@ static size_t sigframe_size(struct rt_sigframe_user_layout const *user)
return round_up(max(user->size, sizeof(struct rt_sigframe)), 16);
}
+/* Sanity limit on the maximum size of signal frame we'll try to generate. */
+/* This is NOT ABI. */
+#define SIGFRAME_MAXSZ SZ_64K
+
+static int __sigframe_alloc(struct rt_sigframe_user_layout *user,
+ unsigned long *offset, size_t size, bool extend)
+{
+ size_t padded_size = round_up(size, 16);
+
+ if (padded_size > user->limit - user->size &&
+ !user->extra_offset &&
+ extend) {
+ int ret;
+
+ ret = __sigframe_alloc(user, &user->extra_offset,
+ sizeof(struct extra_context), false);
+ if (ret)
+ return ret;
+
+ /*
+ * Further allocations must go after the fixed-size
+ * part of the signal frame:
+ */
+ user->size = round_up(sizeof(struct rt_sigframe), 16);
+
+ /*
+ * Allow expansion up to SIGFRAME_MAXSZ, ensuring space for
+ * the terminator:
+ */
+ user->limit = SIGFRAME_MAXSZ -
+ round_up(sizeof(struct _aarch64_ctx), 16);
+ }
+
+ /* Still not enough space? Bad luck! */
+ if (padded_size > user->limit - user->size)
+ return -ENOMEM;
+
+ *offset = user->size;
+ user->size += padded_size;
+
+ return 0;
+}
+
/*
* Allocate space for an optional record of <size> bytes in the user
* signal frame. The offset from the signal frame base address to the
@@ -83,11 +131,26 @@ static size_t sigframe_size(struct rt_sigframe_user_layout const *user)
static int sigframe_alloc(struct rt_sigframe_user_layout *user,
unsigned long *offset, size_t size)
{
- size_t padded_size = round_up(size, 16);
+ return __sigframe_alloc(user, offset, size, true);
+}
- *offset = user->size;
- user->size += padded_size;
+/* Allocate the null terminator record and prevent further allocations */
+static int sigframe_alloc_end(struct rt_sigframe_user_layout *user)
+{
+ int ret;
+ const size_t terminator_size =
+ round_up(sizeof(struct _aarch64_ctx), 16);
+
+ /* Un-reserve the space reserved for the terminator: */
+ user->limit += terminator_size;
+
+ ret = sigframe_alloc(user, &user->end_offset,
+ sizeof(struct _aarch64_ctx));
+ if (ret)
+ return ret;
+ /* Prevent further allocation: */
+ user->limit = user->size;
return 0;
}
@@ -314,17 +377,7 @@ static int setup_sigframe_layout(struct rt_sigframe_user_layout *user)
return err;
}
- /*
- * Allocate space for the terminator record.
- * HACK: here we undo the reservation of space for the end record.
- * This bodge should be replaced with a cleaner approach later on.
- */
- user->limit = offsetof(struct rt_sigframe, uc.uc_mcontext.__reserved) +
- sizeof(user->sigframe->uc.uc_mcontext.__reserved);
-
- err = sigframe_alloc(user, &user->end_offset,
- sizeof(struct _aarch64_ctx));
- return err;
+ return sigframe_alloc_end(user);
}
@@ -365,6 +418,27 @@ static int setup_sigframe(struct rt_sigframe_user_layout *user,
__put_user_error(current->thread.fault_code, &esr_ctx->esr, err);
}
+ if (err == 0 && user->extra_offset) {
+ struct extra_context __user *extra =
+ apply_user_offset(user, user->extra_offset);
+ struct _aarch64_ctx __user *end =
+ (struct _aarch64_ctx __user *)((char __user *)extra +
+ round_up(sizeof(*extra), 16));
+ void __user *extra_data = apply_user_offset(user,
+ round_up(sizeof(struct rt_sigframe), 16));
+ u32 extra_size = round_up(user->size, 16) -
+ round_up(sizeof(struct rt_sigframe), 16);
+
+ __put_user_error(EXTRA_MAGIC, &extra->head.magic, err);
+ __put_user_error(sizeof(*extra), &extra->head.size, err);
+ __put_user_error(extra_data, &extra->data, err);
+ __put_user_error(extra_size, &extra->size, err);
+
+ /* Add the terminator */
+ __put_user_error(0, &end->magic, err);
+ __put_user_error(0, &end->size, err);
+ }
+
/* set the "end" magic */
if (err == 0) {
struct _aarch64_ctx __user *end =
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 03/29] arm64: signal: factor out signal frame record allocation
From: Dave Martin @ 2016-11-25 19:38 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
Factor out the allocator for signal frame optional records into a
separate function, to ensure consistency and facilitate later
expansion of the signal frame.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/kernel/signal.c | 43 ++++++++++++++++++++++++++++++++++++-------
1 file changed, 36 insertions(+), 7 deletions(-)
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index fc08371..653b614 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -75,6 +75,22 @@ static size_t sigframe_size(struct rt_sigframe_user_layout const *user)
return round_up(max(user->size, sizeof(struct rt_sigframe)), 16);
}
+/*
+ * Allocate space for an optional record of <size> bytes in the user
+ * signal frame. The offset from the signal frame base address to the
+ * allocated block is assigned to *offset.
+ */
+static int sigframe_alloc(struct rt_sigframe_user_layout *user,
+ unsigned long *offset, size_t size)
+{
+ size_t padded_size = round_up(size, 16);
+
+ *offset = user->size;
+ user->size += padded_size;
+
+ return 0;
+}
+
static void __user *apply_user_offset(
struct rt_sigframe_user_layout const *user, unsigned long offset)
{
@@ -283,19 +299,32 @@ asmlinkage long sys_rt_sigreturn(struct pt_regs *regs)
/* Determine the layout of optional records in the signal frame */
static int setup_sigframe_layout(struct rt_sigframe_user_layout *user)
{
- user->fpsimd_offset = user->size;
- user->size += round_up(sizeof(struct fpsimd_context), 16);
+ int err;
+
+ err = sigframe_alloc(user, &user->fpsimd_offset,
+ sizeof(struct fpsimd_context));
+ if (err)
+ return err;
/* fault information, if valid */
if (current->thread.fault_code) {
- user->esr_offset = user->size;
- user->size += round_up(sizeof(struct esr_context), 16);
+ err = sigframe_alloc(user, &user->esr_offset,
+ sizeof(struct esr_context));
+ if (err)
+ return err;
}
- /* set the "end" magic */
- user->end_offset = user->size;
+ /*
+ * Allocate space for the terminator record.
+ * HACK: here we undo the reservation of space for the end record.
+ * This bodge should be replaced with a cleaner approach later on.
+ */
+ user->limit = offsetof(struct rt_sigframe, uc.uc_mcontext.__reserved) +
+ sizeof(user->sigframe->uc.uc_mcontext.__reserved);
- return 0;
+ err = sigframe_alloc(user, &user->end_offset,
+ sizeof(struct _aarch64_ctx));
+ return err;
}
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 02/29] arm64: signal: factor frame layout and population into separate passes
From: Dave Martin @ 2016-11-25 19:38 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
In preparation for expanding the signal frame, this patch refactors
the signal frame setup code in setup_sigframe() into two separate
passes.
The first pass, setup_sigframe_layout(), determines the sizeof the
signal frame and its internal layout, including the presence and
location of optional records. The resulting knowledge is used to
allocate and locate the user stack space required for the signal
frame and to determine which optional records to include.
The second pass, setup_sigframe(), is called once the stack frame
is allocated in order to populate it with the necessary context
information.
This change has no effect on the signal ABI, but will make it
easier to expand the signal frame in future patches.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/kernel/signal.c | 112 +++++++++++++++++++++++++++++++++++----------
1 file changed, 88 insertions(+), 24 deletions(-)
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 4f8dbe0..fc08371 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -25,6 +25,7 @@
#include <linux/freezer.h>
#include <linux/stddef.h>
#include <linux/uaccess.h>
+#include <linux/string.h>
#include <linux/tracehook.h>
#include <linux/ratelimit.h>
@@ -49,8 +50,39 @@ struct rt_sigframe {
struct rt_sigframe_user_layout {
struct rt_sigframe __user *sigframe;
+
+ unsigned long size; /* size of allocated sigframe data */
+ unsigned long limit; /* largest allowed size */
+
+ unsigned long fpsimd_offset;
+ unsigned long esr_offset;
+ unsigned long end_offset;
};
+static void init_user_layout(struct rt_sigframe_user_layout *user)
+{
+ memset(user, 0, sizeof(*user));
+ user->size = offsetof(struct rt_sigframe, uc.uc_mcontext.__reserved);
+
+ user->limit = user->size +
+ sizeof(user->sigframe->uc.uc_mcontext.__reserved) -
+ round_up(sizeof(struct _aarch64_ctx), 16);
+ /* ^ reserve space for terminator */
+}
+
+static size_t sigframe_size(struct rt_sigframe_user_layout const *user)
+{
+ return round_up(max(user->size, sizeof(struct rt_sigframe)), 16);
+}
+
+static void __user *apply_user_offset(
+ struct rt_sigframe_user_layout const *user, unsigned long offset)
+{
+ char __user *base = (char __user *)user->sigframe;
+
+ return base + offset;
+}
+
static int preserve_fpsimd_context(struct fpsimd_context __user *ctx)
{
struct fpsimd_state *fpsimd = ¤t->thread.fpsimd_state;
@@ -106,26 +138,35 @@ static int parse_user_sigframe(struct user_ctxs *user,
struct rt_sigframe __user *sf)
{
struct sigcontext __user *sc = &sf->uc.uc_mcontext;
- struct _aarch64_ctx __user *head =
- (struct _aarch64_ctx __user *)&sc->__reserved;
+ struct _aarch64_ctx __user *head;
+ char __user *base = (char __user *)&sc->__reserved;
size_t offset = 0;
+ size_t limit = sizeof(sc->__reserved);
user->fpsimd = NULL;
+ if (!IS_ALIGNED((unsigned long)base, 16))
+ goto invalid;
+
while (1) {
- int err;
+ int err = 0;
u32 magic, size;
- head = (struct _aarch64_ctx __user *)&sc->__reserved[offset];
- if (!IS_ALIGNED((unsigned long)head, 16))
+ if (limit - offset < sizeof(*head))
goto invalid;
- err = 0;
+ if (!IS_ALIGNED(offset, 16))
+ goto invalid;
+
+ head = (struct _aarch64_ctx __user *)(base + offset);
__get_user_error(magic, &head->magic, err);
__get_user_error(size, &head->size, err);
if (err)
return err;
+ if (limit - offset < size)
+ goto invalid;
+
switch (magic) {
case 0:
if (size)
@@ -137,9 +178,7 @@ static int parse_user_sigframe(struct user_ctxs *user,
if (user->fpsimd)
goto invalid;
- if (offset > sizeof(sc->__reserved) -
- sizeof(*user->fpsimd) ||
- size < sizeof(*user->fpsimd))
+ if (size < sizeof(*user->fpsimd))
goto invalid;
user->fpsimd = (struct fpsimd_context __user *)head;
@@ -156,7 +195,7 @@ static int parse_user_sigframe(struct user_ctxs *user,
if (size < sizeof(*head))
goto invalid;
- if (size > sizeof(sc->__reserved) - (sizeof(*head) + offset))
+ if (limit - offset < size)
goto invalid;
offset += size;
@@ -241,13 +280,30 @@ asmlinkage long sys_rt_sigreturn(struct pt_regs *regs)
return 0;
}
+/* Determine the layout of optional records in the signal frame */
+static int setup_sigframe_layout(struct rt_sigframe_user_layout *user)
+{
+ user->fpsimd_offset = user->size;
+ user->size += round_up(sizeof(struct fpsimd_context), 16);
+
+ /* fault information, if valid */
+ if (current->thread.fault_code) {
+ user->esr_offset = user->size;
+ user->size += round_up(sizeof(struct esr_context), 16);
+ }
+
+ /* set the "end" magic */
+ user->end_offset = user->size;
+
+ return 0;
+}
+
+
static int setup_sigframe(struct rt_sigframe_user_layout *user,
struct pt_regs *regs, sigset_t *set)
{
int i, err = 0;
struct rt_sigframe __user *sf = user->sigframe;
- void *aux = sf->uc.uc_mcontext.__reserved;
- struct _aarch64_ctx *end;
/* set up the stack frame for unwinding */
__put_user_error(regs->regs[29], &sf->fp, err);
@@ -265,26 +321,29 @@ static int setup_sigframe(struct rt_sigframe_user_layout *user,
err |= __copy_to_user(&sf->uc.uc_sigmask, set, sizeof(*set));
if (err == 0) {
- struct fpsimd_context *fpsimd_ctx =
- container_of(aux, struct fpsimd_context, head);
+ struct fpsimd_context __user *fpsimd_ctx =
+ apply_user_offset(user, user->fpsimd_offset);
err |= preserve_fpsimd_context(fpsimd_ctx);
- aux += sizeof(*fpsimd_ctx);
}
/* fault information, if valid */
- if (current->thread.fault_code) {
- struct esr_context *esr_ctx =
- container_of(aux, struct esr_context, head);
+ if (err == 0 && user->esr_offset) {
+ struct esr_context __user *esr_ctx =
+ apply_user_offset(user, user->esr_offset);
+
__put_user_error(ESR_MAGIC, &esr_ctx->head.magic, err);
__put_user_error(sizeof(*esr_ctx), &esr_ctx->head.size, err);
__put_user_error(current->thread.fault_code, &esr_ctx->esr, err);
- aux += sizeof(*esr_ctx);
}
/* set the "end" magic */
- end = aux;
- __put_user_error(0, &end->magic, err);
- __put_user_error(0, &end->size, err);
+ if (err == 0) {
+ struct _aarch64_ctx __user *end =
+ apply_user_offset(user, user->end_offset);
+
+ __put_user_error(0, &end->magic, err);
+ __put_user_error(0, &end->size, err);
+ }
return err;
}
@@ -293,10 +352,15 @@ static int get_sigframe(struct rt_sigframe_user_layout *user,
struct ksignal *ksig, struct pt_regs *regs)
{
unsigned long sp, sp_top;
+ int err;
- sp = sp_top = sigsp(regs->sp, ksig);
+ init_user_layout(user);
+ err = setup_sigframe_layout(user);
+ if (err)
+ return err;
- sp = (sp - sizeof(struct rt_sigframe)) & ~15;
+ sp = sp_top = sigsp(regs->sp, ksig);
+ sp = (sp & ~15) - sigframe_size(user);
user->sigframe = (struct rt_sigframe __user *)sp;
/*
--
2.1.4
^ permalink raw reply related
* [RFC PATCH 01/29] arm64: signal: Refactor sigcontext parsing in rt_sigreturn
From: Dave Martin @ 2016-11-25 19:38 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <1480102762-23647-1-git-send-email-Dave.Martin@arm.com>
Currently, rt_sigreturn does very limited checking on the
sigcontext coming from userspace.
Future additions of extra dynamic sigcontext data will increase the
potential for surprises. Also, it is not clear whether the
sigcontext extension records are supposed to occur in a particular
order.
This patch factors out the sigcontext parsing into a separate
function, and adds extra checks to validate the well-formedness of
the sigcontext structure.
To help with this, an abstraction for the signal frame layout is
also added, using offsets to track the location of different
records in the frame. Although trivial, this provides a base to
extend upon in order to track more complex layouts.
Signed-off-by: Dave Martin <Dave.Martin@arm.com>
---
arch/arm64/kernel/signal.c | 121 +++++++++++++++++++++++++++++++++++++--------
1 file changed, 101 insertions(+), 20 deletions(-)
diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c
index 404dd67..4f8dbe0 100644
--- a/arch/arm64/kernel/signal.c
+++ b/arch/arm64/kernel/signal.c
@@ -19,9 +19,11 @@
#include <linux/compat.h>
#include <linux/errno.h>
+#include <linux/kernel.h>
#include <linux/signal.h>
#include <linux/personality.h>
#include <linux/freezer.h>
+#include <linux/stddef.h>
#include <linux/uaccess.h>
#include <linux/tracehook.h>
#include <linux/ratelimit.h>
@@ -45,6 +47,10 @@ struct rt_sigframe {
u64 lr;
};
+struct rt_sigframe_user_layout {
+ struct rt_sigframe __user *sigframe;
+};
+
static int preserve_fpsimd_context(struct fpsimd_context __user *ctx)
{
struct fpsimd_state *fpsimd = ¤t->thread.fpsimd_state;
@@ -92,12 +98,86 @@ static int restore_fpsimd_context(struct fpsimd_context __user *ctx)
return err ? -EFAULT : 0;
}
+struct user_ctxs {
+ struct fpsimd_context __user *fpsimd;
+};
+
+static int parse_user_sigframe(struct user_ctxs *user,
+ struct rt_sigframe __user *sf)
+{
+ struct sigcontext __user *sc = &sf->uc.uc_mcontext;
+ struct _aarch64_ctx __user *head =
+ (struct _aarch64_ctx __user *)&sc->__reserved;
+ size_t offset = 0;
+
+ user->fpsimd = NULL;
+
+ while (1) {
+ int err;
+ u32 magic, size;
+
+ head = (struct _aarch64_ctx __user *)&sc->__reserved[offset];
+ if (!IS_ALIGNED((unsigned long)head, 16))
+ goto invalid;
+
+ err = 0;
+ __get_user_error(magic, &head->magic, err);
+ __get_user_error(size, &head->size, err);
+ if (err)
+ return err;
+
+ switch (magic) {
+ case 0:
+ if (size)
+ goto invalid;
+
+ goto done;
+
+ case FPSIMD_MAGIC:
+ if (user->fpsimd)
+ goto invalid;
+
+ if (offset > sizeof(sc->__reserved) -
+ sizeof(*user->fpsimd) ||
+ size < sizeof(*user->fpsimd))
+ goto invalid;
+
+ user->fpsimd = (struct fpsimd_context __user *)head;
+ break;
+
+ case ESR_MAGIC:
+ /* ignore */
+ break;
+
+ default:
+ goto invalid;
+ }
+
+ if (size < sizeof(*head))
+ goto invalid;
+
+ if (size > sizeof(sc->__reserved) - (sizeof(*head) + offset))
+ goto invalid;
+
+ offset += size;
+ }
+
+done:
+ if (!user->fpsimd)
+ goto invalid;
+
+ return 0;
+
+invalid:
+ return -EINVAL;
+}
+
static int restore_sigframe(struct pt_regs *regs,
struct rt_sigframe __user *sf)
{
sigset_t set;
int i, err;
- void *aux = sf->uc.uc_mcontext.__reserved;
+ struct user_ctxs user;
err = __copy_from_user(&set, &sf->uc.uc_sigmask, sizeof(set));
if (err == 0)
@@ -116,12 +196,11 @@ static int restore_sigframe(struct pt_regs *regs,
regs->syscallno = ~0UL;
err |= !valid_user_regs(®s->user_regs, current);
+ if (err == 0)
+ err = parse_user_sigframe(&user, sf);
- if (err == 0) {
- struct fpsimd_context *fpsimd_ctx =
- container_of(aux, struct fpsimd_context, head);
- err |= restore_fpsimd_context(fpsimd_ctx);
- }
+ if (err == 0)
+ err = restore_fpsimd_context(user.fpsimd);
return err;
}
@@ -162,10 +241,11 @@ asmlinkage long sys_rt_sigreturn(struct pt_regs *regs)
return 0;
}
-static int setup_sigframe(struct rt_sigframe __user *sf,
+static int setup_sigframe(struct rt_sigframe_user_layout *user,
struct pt_regs *regs, sigset_t *set)
{
int i, err = 0;
+ struct rt_sigframe __user *sf = user->sigframe;
void *aux = sf->uc.uc_mcontext.__reserved;
struct _aarch64_ctx *end;
@@ -209,33 +289,32 @@ static int setup_sigframe(struct rt_sigframe __user *sf,
return err;
}
-static struct rt_sigframe __user *get_sigframe(struct ksignal *ksig,
- struct pt_regs *regs)
+static int get_sigframe(struct rt_sigframe_user_layout *user,
+ struct ksignal *ksig, struct pt_regs *regs)
{
unsigned long sp, sp_top;
- struct rt_sigframe __user *frame;
sp = sp_top = sigsp(regs->sp, ksig);
sp = (sp - sizeof(struct rt_sigframe)) & ~15;
- frame = (struct rt_sigframe __user *)sp;
+ user->sigframe = (struct rt_sigframe __user *)sp;
/*
* Check that we can actually write to the signal frame.
*/
- if (!access_ok(VERIFY_WRITE, frame, sp_top - sp))
- frame = NULL;
+ if (!access_ok(VERIFY_WRITE, user->sigframe, sp_top - sp))
+ return -EFAULT;
- return frame;
+ return 0;
}
static void setup_return(struct pt_regs *regs, struct k_sigaction *ka,
- void __user *frame, int usig)
+ struct rt_sigframe_user_layout *user, int usig)
{
__sigrestore_t sigtramp;
regs->regs[0] = usig;
- regs->sp = (unsigned long)frame;
+ regs->sp = (unsigned long)user->sigframe;
regs->regs[29] = regs->sp + offsetof(struct rt_sigframe, fp);
regs->pc = (unsigned long)ka->sa.sa_handler;
@@ -250,20 +329,22 @@ static void setup_return(struct pt_regs *regs, struct k_sigaction *ka,
static int setup_rt_frame(int usig, struct ksignal *ksig, sigset_t *set,
struct pt_regs *regs)
{
+ struct rt_sigframe_user_layout user;
struct rt_sigframe __user *frame;
int err = 0;
- frame = get_sigframe(ksig, regs);
- if (!frame)
+ if (get_sigframe(&user, ksig, regs))
return 1;
+ frame = user.sigframe;
+
__put_user_error(0, &frame->uc.uc_flags, err);
__put_user_error(NULL, &frame->uc.uc_link, err);
err |= __save_altstack(&frame->uc.uc_stack, regs->sp);
- err |= setup_sigframe(frame, regs, set);
+ err |= setup_sigframe(&user, regs, set);
if (err == 0) {
- setup_return(regs, &ksig->ka, frame, usig);
+ setup_return(regs, &ksig->ka, &user, usig);
if (ksig->ka.sa.sa_flags & SA_SIGINFO) {
err |= copy_siginfo_to_user(&frame->info, &ksig->info);
regs->regs[1] = (unsigned long)&frame->info;
--
2.1.4
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox