* [PATCH 3/6] arm64: scs: Use 'scs_sp' register alias for x18
From: Will Deacon @ 2020-05-15 17:27 UTC (permalink / raw)
To: linux-kernel
Cc: kernel-team, Kees Cook, Jann Horn, Peter Zijlstra,
Catalin Marinas, Mark Rutland, Sami Tolvanen, Will Deacon,
Ard Biesheuvel, linux-arm-kernel
In-Reply-To: <20200515172756.27185-1-will@kernel.org>
x18 holds the SCS stack pointer value, so introduce a register alias to
make this easier to read in assembly code.
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/scs.h | 6 ++++--
arch/arm64/kernel/entry.S | 10 +++++-----
arch/arm64/kernel/head.S | 2 +-
3 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/arch/arm64/include/asm/scs.h b/arch/arm64/include/asm/scs.h
index 6b8cf4352fe3..d46efdd2060a 100644
--- a/arch/arm64/include/asm/scs.h
+++ b/arch/arm64/include/asm/scs.h
@@ -7,12 +7,14 @@
#include <asm/asm-offsets.h>
#ifdef CONFIG_SHADOW_CALL_STACK
+ scs_sp .req x18
+
.macro scs_load tsk, tmp
- ldr x18, [\tsk, #TSK_TI_SCS_SP]
+ ldr scs_sp, [\tsk, #TSK_TI_SCS_SP]
.endm
.macro scs_save tsk, tmp
- str x18, [\tsk, #TSK_TI_SCS_SP]
+ str scs_sp, [\tsk, #TSK_TI_SCS_SP]
.endm
#else
.macro scs_load tsk, tmp
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index cb0516e6f963..741faf0706f1 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -394,7 +394,7 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
.macro irq_stack_entry
mov x19, sp // preserve the original sp
#ifdef CONFIG_SHADOW_CALL_STACK
- mov x24, x18 // preserve the original shadow stack
+ mov x24, scs_sp // preserve the original shadow stack
#endif
/*
@@ -416,7 +416,7 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
#ifdef CONFIG_SHADOW_CALL_STACK
/* also switch to the irq shadow stack */
- adr_this_cpu x18, irq_shadow_call_stack, x26
+ adr_this_cpu scs_sp, irq_shadow_call_stack, x26
#endif
9998:
@@ -430,7 +430,7 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0
.macro irq_stack_exit
mov sp, x19
#ifdef CONFIG_SHADOW_CALL_STACK
- mov x18, x24
+ mov scs_sp, x24
#endif
.endm
@@ -1071,9 +1071,9 @@ SYM_CODE_START(__sdei_asm_handler)
#ifdef CONFIG_SHADOW_CALL_STACK
/* Use a separate shadow call stack for normal and critical events */
cbnz w4, 3f
- adr_this_cpu dst=x18, sym=sdei_shadow_call_stack_normal, tmp=x6
+ adr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_normal, tmp=x6
b 4f
-3: adr_this_cpu dst=x18, sym=sdei_shadow_call_stack_critical, tmp=x6
+3: adr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_critical, tmp=x6
4:
#endif
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 2b01c19c5483..1293baddfd20 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -426,7 +426,7 @@ SYM_FUNC_START_LOCAL(__primary_switched)
mov x29, sp
#ifdef CONFIG_SHADOW_CALL_STACK
- adr_l x18, init_shadow_call_stack // Set shadow call stack
+ adr_l scs_sp, init_shadow_call_stack // Set shadow call stack
#endif
str_l x21, __fdt_pointer, x5 // Save FDT pointer
--
2.26.2.761.g0e0b3e54be-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH 2/6] scs: Move accounting into alloc/free functions
From: Will Deacon @ 2020-05-15 17:27 UTC (permalink / raw)
To: linux-kernel
Cc: kernel-team, Kees Cook, Jann Horn, Peter Zijlstra,
Catalin Marinas, Mark Rutland, Sami Tolvanen, Will Deacon,
Ard Biesheuvel, linux-arm-kernel
In-Reply-To: <20200515172756.27185-1-will@kernel.org>
There's no need to perform the shadow stack page accounting independently
of the lifetime of the underlying allocation, so call the accounting code
from the {alloc,free}() functions and simplify the code in the process.
Signed-off-by: Will Deacon <will@kernel.org>
---
kernel/scs.c | 45 +++++++++++++++++++++------------------------
1 file changed, 21 insertions(+), 24 deletions(-)
diff --git a/kernel/scs.c b/kernel/scs.c
index 5ff8663e4a67..aea841cd7586 100644
--- a/kernel/scs.c
+++ b/kernel/scs.c
@@ -14,25 +14,35 @@
static struct kmem_cache *scs_cache;
+static void __scs_account(void *s, int account)
+{
+ struct page *scs_page = virt_to_page(s);
+
+ mod_zone_page_state(page_zone(scs_page), NR_KERNEL_SCS_KB,
+ account * (SCS_SIZE / SZ_1K));
+}
+
static void *scs_alloc(int node)
{
- void *s;
-
- s = kmem_cache_alloc_node(scs_cache, GFP_SCS, node);
- if (s) {
- *__scs_magic(s) = SCS_END_MAGIC;
- /*
- * Poison the allocation to catch unintentional accesses to
- * the shadow stack when KASAN is enabled.
- */
- kasan_poison_object_data(scs_cache, s);
- }
+ void *s = kmem_cache_alloc_node(scs_cache, GFP_SCS, node);
+
+ if (!s)
+ return NULL;
+ *__scs_magic(s) = SCS_END_MAGIC;
+
+ /*
+ * Poison the allocation to catch unintentional accesses to
+ * the shadow stack when KASAN is enabled.
+ */
+ kasan_poison_object_data(scs_cache, s);
+ __scs_account(s, 1);
return s;
}
static void scs_free(void *s)
{
+ __scs_account(s, -1);
kasan_unpoison_object_data(scs_cache, s);
kmem_cache_free(scs_cache, s);
}
@@ -42,17 +52,6 @@ void __init scs_init(void)
scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, 0, 0, NULL);
}
-static struct page *__scs_page(struct task_struct *tsk)
-{
- return virt_to_page(task_scs(tsk));
-}
-
-static void scs_account(struct task_struct *tsk, int account)
-{
- mod_zone_page_state(page_zone(__scs_page(tsk)), NR_KERNEL_SCS_KB,
- account * (SCS_SIZE / 1024));
-}
-
int scs_prepare(struct task_struct *tsk, int node)
{
void *s = scs_alloc(node);
@@ -61,7 +60,6 @@ int scs_prepare(struct task_struct *tsk, int node)
return -ENOMEM;
task_scs(tsk) = task_scs_sp(tsk) = s;
- scs_account(tsk, 1);
return 0;
}
@@ -102,6 +100,5 @@ void scs_release(struct task_struct *tsk)
WARN(scs_corrupted(tsk), "corrupted shadow stack detected when freeing task\n");
scs_check_usage(tsk);
- scs_account(tsk, -1);
scs_free(s);
}
--
2.26.2.761.g0e0b3e54be-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH 1/6] arm64: scs: Store absolute SCS stack pointer value in thread_info
From: Will Deacon @ 2020-05-15 17:27 UTC (permalink / raw)
To: linux-kernel
Cc: kernel-team, Kees Cook, Jann Horn, Peter Zijlstra,
Catalin Marinas, Mark Rutland, Sami Tolvanen, Will Deacon,
Ard Biesheuvel, linux-arm-kernel
In-Reply-To: <20200515172756.27185-1-will@kernel.org>
Storing the SCS information in thread_info as a {base,offset} pair
introduces an additional load instruction on the ret-to-user path,
since the SCS stack pointer in x18 has to be converted back to an offset
by subtracting the base.
Replace the offset with the absolute SCS stack pointer value instead
and avoid the redundant load.
Signed-off-by: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/scs.h | 9 ++++-----
arch/arm64/include/asm/thread_info.h | 4 ++--
arch/arm64/kernel/asm-offsets.c | 2 +-
include/linux/scs.h | 8 ++++----
kernel/scs.c | 3 +--
5 files changed, 12 insertions(+), 14 deletions(-)
diff --git a/arch/arm64/include/asm/scs.h b/arch/arm64/include/asm/scs.h
index 96549353b0cb..6b8cf4352fe3 100644
--- a/arch/arm64/include/asm/scs.h
+++ b/arch/arm64/include/asm/scs.h
@@ -4,16 +4,15 @@
#ifdef __ASSEMBLY__
+#include <asm/asm-offsets.h>
+
#ifdef CONFIG_SHADOW_CALL_STACK
.macro scs_load tsk, tmp
- ldp x18, \tmp, [\tsk, #TSK_TI_SCS_BASE]
- add x18, x18, \tmp
+ ldr x18, [\tsk, #TSK_TI_SCS_SP]
.endm
.macro scs_save tsk, tmp
- ldr \tmp, [\tsk, #TSK_TI_SCS_BASE]
- sub \tmp, x18, \tmp
- str \tmp, [\tsk, #TSK_TI_SCS_OFFSET]
+ str x18, [\tsk, #TSK_TI_SCS_SP]
.endm
#else
.macro scs_load tsk, tmp
diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index 9df79c0a4c43..6ea8b6a26ae9 100644
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -43,7 +43,7 @@ struct thread_info {
};
#ifdef CONFIG_SHADOW_CALL_STACK
void *scs_base;
- unsigned long scs_offset;
+ void *scs_sp;
#endif
};
@@ -107,7 +107,7 @@ void arch_release_task_struct(struct task_struct *tsk);
#ifdef CONFIG_SHADOW_CALL_STACK
#define INIT_SCS \
.scs_base = init_shadow_call_stack, \
- .scs_offset = 0,
+ .scs_sp = init_shadow_call_stack,
#else
#define INIT_SCS
#endif
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index d7934250b68c..a098a45f63d8 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -36,7 +36,7 @@ int main(void)
#endif
#ifdef CONFIG_SHADOW_CALL_STACK
DEFINE(TSK_TI_SCS_BASE, offsetof(struct task_struct, thread_info.scs_base));
- DEFINE(TSK_TI_SCS_OFFSET, offsetof(struct task_struct, thread_info.scs_offset));
+ DEFINE(TSK_TI_SCS_SP, offsetof(struct task_struct, thread_info.scs_sp));
#endif
DEFINE(TSK_STACK, offsetof(struct task_struct, stack));
#ifdef CONFIG_STACKPROTECTOR
diff --git a/include/linux/scs.h b/include/linux/scs.h
index 3f3662621a27..0eb2485ef832 100644
--- a/include/linux/scs.h
+++ b/include/linux/scs.h
@@ -27,7 +27,7 @@
#define SCS_END_MAGIC (0x5f6UL + POISON_POINTER_DELTA)
#define task_scs(tsk) (task_thread_info(tsk)->scs_base)
-#define task_scs_offset(tsk) (task_thread_info(tsk)->scs_offset)
+#define task_scs_sp(tsk) (task_thread_info(tsk)->scs_sp)
void scs_init(void);
int scs_prepare(struct task_struct *tsk, int node);
@@ -39,7 +39,7 @@ static inline void scs_task_reset(struct task_struct *tsk)
* Reset the shadow stack to the base address in case the task
* is reused.
*/
- task_scs_offset(tsk) = 0;
+ task_scs_sp(tsk) = task_scs(tsk);
}
static inline unsigned long *__scs_magic(void *s)
@@ -50,9 +50,9 @@ static inline unsigned long *__scs_magic(void *s)
static inline bool scs_corrupted(struct task_struct *tsk)
{
unsigned long *magic = __scs_magic(task_scs(tsk));
+ unsigned long sz = task_scs_sp(tsk) - task_scs(tsk);
- return (task_scs_offset(tsk) >= SCS_SIZE - 1 ||
- READ_ONCE_NOCHECK(*magic) != SCS_END_MAGIC);
+ return sz >= SCS_SIZE - 1 || READ_ONCE_NOCHECK(*magic) != SCS_END_MAGIC;
}
#else /* CONFIG_SHADOW_CALL_STACK */
diff --git a/kernel/scs.c b/kernel/scs.c
index 9389c28f0853..5ff8663e4a67 100644
--- a/kernel/scs.c
+++ b/kernel/scs.c
@@ -60,8 +60,7 @@ int scs_prepare(struct task_struct *tsk, int node)
if (!s)
return -ENOMEM;
- task_scs(tsk) = s;
- task_scs_offset(tsk) = 0;
+ task_scs(tsk) = task_scs_sp(tsk) = s;
scs_account(tsk, 1);
return 0;
}
--
2.26.2.761.g0e0b3e54be-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH 0/6] Clean up Shadow Call Stack patches for 5.8
From: Will Deacon @ 2020-05-15 17:27 UTC (permalink / raw)
To: linux-kernel
Cc: kernel-team, Kees Cook, Jann Horn, Peter Zijlstra,
Catalin Marinas, Mark Rutland, Sami Tolvanen, Will Deacon,
Ard Biesheuvel, linux-arm-kernel
Hi all,
Here's a series of cleanups I hacked together on top of a modified v13
of the Shadow Call Stack patches from Sami:
https://lore.kernel.org/r/20200515172355.GD23334@willie-the-truck
The main changes are:
* Move code out of arch/arm64 and into the core implementation
* Store the full SCS stack pointer instead of the offset
* Code simplification and general style things
I'd like to queue this on top early next week so that it can spend some
quality time in linux-next.
Cheers,
Will
Cc: Sami Tolvanen <samitolvanen@google.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Mark Rutland <mark.rutland@am.com>
Cc: Jann Horn <jannh@google.com>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: <kernel-team@android.com>
--->8
Will Deacon (6):
arm64: scs: Store absolute SCS stack pointer value in thread_info
scs: Move accounting into alloc/free functions
arm64: scs: Use 'scs_sp' register alias for x18
scs: Move scs_overflow_check() out of architecture code
scs: Remove references to asm/scs.h from core code
scs: Move DEFINE_SCS macro into core code
arch/Kconfig | 4 +--
arch/arm64/include/asm/scs.h | 29 ++++------------
arch/arm64/include/asm/thread_info.h | 4 +--
arch/arm64/kernel/asm-offsets.c | 2 +-
arch/arm64/kernel/entry.S | 10 +++---
arch/arm64/kernel/head.S | 2 +-
arch/arm64/kernel/process.c | 2 --
arch/arm64/kernel/scs.c | 6 +---
include/linux/scs.h | 16 +++++----
kernel/sched/core.c | 3 ++
kernel/scs.c | 52 +++++++++++++---------------
11 files changed, 55 insertions(+), 75 deletions(-)
--
2.26.2.761.g0e0b3e54be-goog
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply
* [GIT PULL] arm64 fixes for 5.7-rc6
From: Catalin Marinas @ 2020-05-15 17:24 UTC (permalink / raw)
To: Linus Torvalds; +Cc: Will Deacon, linux-kernel, linux-arm-kernel, hch
Hi Linus,
Please pull the arm64 fix below. Thanks.
The following changes since commit 027d0c7101f50cf03aeea9eebf484afd4920c8d3:
arm64: hugetlb: avoid potential NULL dereference (2020-05-07 09:24:15 +0100)
are available in the Git repository at:
git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux tags/arm64-fixes
for you to fetch changes up to d51c214541c5154dda3037289ee895ea3ded5ebd:
arm64: fix the flush_icache_range arguments in machine_kexec (2020-05-11 12:02:14 +0100)
----------------------------------------------------------------
Fix flush_icache_range() second argument in machine_kexec() to be an
address rather than size.
----------------------------------------------------------------
Christoph Hellwig (1):
arm64: fix the flush_icache_range arguments in machine_kexec
arch/arm64/kernel/machine_kexec.c | 1 +
1 file changed, 1 insertion(+)
--
Catalin
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply
* Re: [PATCH v13 00/12] add support for Clang's Shadow Call Stack
From: Will Deacon @ 2020-05-15 17:23 UTC (permalink / raw)
To: Sami Tolvanen
Cc: Mark Rutland, Juri Lelli, kernel-hardening, Peter Zijlstra,
Catalin Marinas, Marc Zyngier, Masahiro Yamada, clang-built-linux,
Ingo Molnar, Laura Abbott, Dave Martin, Kees Cook, Jann Horn,
Steven Rostedt, linux-arm-kernel, Michal Marek, Ard Biesheuvel,
Nick Desaulniers, linux-kernel, Miguel Ojeda, James Morse,
Masami Hiramatsu
In-Reply-To: <20200427160018.243569-1-samitolvanen@google.com>
Hi Sami,
On Mon, Apr 27, 2020 at 09:00:06AM -0700, Sami Tolvanen wrote:
> This patch series adds support for Clang's Shadow Call Stack
> (SCS) mitigation, which uses a separately allocated shadow stack
> to protect against return address overwrites. More information
> can be found here:
>
> https://clang.llvm.org/docs/ShadowCallStack.html
I'm planning to queue this with the (mostly cosmetic) diff below folded in.
I also have some extra patches on top which I'll send out shortly for
review.
However, I really think we need to get to the bottom of the size issue
since I'm highly sceptical about not being able to afford a full page
for the shadow stack allocation. We can change this later so it needn't
hold up the patchset, but given that Android is the only user, I'd like
to make sure that if we change to use a full page upstream then that is
also acceptable in AOSP.
Thanks,
Will
--->8
diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h
index 18fc4d29ef27..790c0c6b8552 100644
--- a/include/linux/compiler-clang.h
+++ b/include/linux/compiler-clang.h
@@ -45,6 +45,4 @@
#if __has_feature(shadow_call_stack)
# define __noscs __attribute__((__no_sanitize__("shadow-call-stack")))
-#else
-# define __noscs
#endif
diff --git a/include/linux/scs.h b/include/linux/scs.h
index 060eeb3d1390..3f3662621a27 100644
--- a/include/linux/scs.h
+++ b/include/linux/scs.h
@@ -11,7 +11,7 @@
#include <linux/gfp.h>
#include <linux/poison.h>
#include <linux/sched.h>
-#include <asm/page.h>
+#include <linux/sizes.h>
#ifdef CONFIG_SHADOW_CALL_STACK
@@ -20,7 +20,7 @@
* architecture) provided ~40% safety margin on stack usage while keeping
* memory allocation overhead reasonable.
*/
-#define SCS_SIZE 1024UL
+#define SCS_SIZE SZ_1K
#define GFP_SCS (GFP_KERNEL | __GFP_ZERO)
/* An illegal pointer value to mark the end of the shadow stack. */
@@ -29,7 +29,9 @@
#define task_scs(tsk) (task_thread_info(tsk)->scs_base)
#define task_scs_offset(tsk) (task_thread_info(tsk)->scs_offset)
-extern void scs_init(void);
+void scs_init(void);
+int scs_prepare(struct task_struct *tsk, int node);
+void scs_release(struct task_struct *tsk);
static inline void scs_task_reset(struct task_struct *tsk)
{
@@ -40,8 +42,6 @@ static inline void scs_task_reset(struct task_struct *tsk)
task_scs_offset(tsk) = 0;
}
-extern int scs_prepare(struct task_struct *tsk, int node);
-
static inline unsigned long *__scs_magic(void *s)
{
return (unsigned long *)(s + SCS_SIZE) - 1;
@@ -55,12 +55,8 @@ static inline bool scs_corrupted(struct task_struct *tsk)
READ_ONCE_NOCHECK(*magic) != SCS_END_MAGIC);
}
-extern void scs_release(struct task_struct *tsk);
-
#else /* CONFIG_SHADOW_CALL_STACK */
-#define task_scs(tsk) NULL
-
static inline void scs_init(void) {}
static inline void scs_task_reset(struct task_struct *tsk) {}
static inline int scs_prepare(struct task_struct *tsk, int node) { return 0; }
diff --git a/kernel/scs.c b/kernel/scs.c
index 2a96573f2b1b..9389c28f0853 100644
--- a/kernel/scs.c
+++ b/kernel/scs.c
@@ -55,45 +55,37 @@ static void scs_account(struct task_struct *tsk, int account)
int scs_prepare(struct task_struct *tsk, int node)
{
- void *s;
+ void *s = scs_alloc(node);
- s = scs_alloc(node);
if (!s)
return -ENOMEM;
task_scs(tsk) = s;
task_scs_offset(tsk) = 0;
scs_account(tsk, 1);
-
return 0;
}
-#ifdef CONFIG_DEBUG_STACK_USAGE
-static unsigned long __scs_used(struct task_struct *tsk)
+static void scs_check_usage(struct task_struct *tsk)
{
- unsigned long *p = task_scs(tsk);
- unsigned long *end = __scs_magic(p);
- unsigned long s = (unsigned long)p;
+ static unsigned long highest;
- while (p < end && READ_ONCE_NOCHECK(*p))
- p++;
+ unsigned long *p, prev, curr = highest, used = 0;
- return (unsigned long)p - s;
-}
+ if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE))
+ return;
-static void scs_check_usage(struct task_struct *tsk)
-{
- static unsigned long highest;
- unsigned long used = __scs_used(tsk);
- unsigned long prev;
- unsigned long curr = highest;
+ for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) {
+ if (!READ_ONCE_NOCHECK(*p))
+ break;
+ used++;
+ }
while (used > curr) {
prev = cmpxchg_relaxed(&highest, curr, used);
if (prev == curr) {
- pr_info("%s (%d): highest shadow stack usage: "
- "%lu bytes\n",
+ pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
tsk->comm, task_pid_nr(tsk), used);
break;
}
@@ -101,21 +93,16 @@ static void scs_check_usage(struct task_struct *tsk)
curr = prev;
}
}
-#else
-static inline void scs_check_usage(struct task_struct *tsk) {}
-#endif
void scs_release(struct task_struct *tsk)
{
- void *s;
+ void *s = task_scs(tsk);
- s = task_scs(tsk);
if (!s)
return;
- WARN_ON(scs_corrupted(tsk));
+ WARN(scs_corrupted(tsk), "corrupted shadow stack detected when freeing task\n");
scs_check_usage(tsk);
-
scs_account(tsk, -1);
scs_free(s);
}
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* Re: [PATCH 0/4] PCI, iommu: Factor 'untrusted' check for ATS enablement
From: Will Deacon @ 2020-05-15 17:21 UTC (permalink / raw)
To: Raj, Ashok
Cc: Jean-Philippe Brucker, linux-pci, joro, alex.williamson,
Christoph Hellwig, iommu, bhelgaas, robin.murphy, dwmw2,
linux-arm-kernel, baolu.lu
In-Reply-To: <20200515171948.GC75440@otc-nc-03>
Hi,
On Fri, May 15, 2020 at 10:19:49AM -0700, Raj, Ashok wrote:
> On Fri, May 15, 2020 at 08:43:51AM -0700, Christoph Hellwig wrote:
> > Can you please lift the untrusted flag into struct device? It really
> > isn't a PCI specific concept, and we should not have code poking into
> > pci_dev all over the iommu code.
>
> Just for clarification: All IOMMU's today mostly pertain to only PCI devices
> and for devices that aren't PCI like HPET for instance we give a PCI
> identifier. Facilities like ATS for instance require the platform to have
> an IOMMU.
>
> what additional problems does moving this to struct device solve?
ATS is PCI specific, but IOMMUs certainly aren't! The vast majority of
IOMMUs deployed in arm/arm64 SoCs are /not/ using any sort of PCI.
Will
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply
* Re: [PATCH 0/4] PCI, iommu: Factor 'untrusted' check for ATS enablement
From: Raj, Ashok @ 2020-05-15 17:19 UTC (permalink / raw)
To: Christoph Hellwig
Cc: Jean-Philippe Brucker, Ashok Raj, linux-pci, joro,
alex.williamson, iommu, bhelgaas, baolu.lu, will, dwmw2,
linux-arm-kernel, robin.murphy
In-Reply-To: <20200515154351.GA6546@infradead.org>
Hi Christoph
On Fri, May 15, 2020 at 08:43:51AM -0700, Christoph Hellwig wrote:
> Can you please lift the untrusted flag into struct device? It really
> isn't a PCI specific concept, and we should not have code poking into
> pci_dev all over the iommu code.
Just for clarification: All IOMMU's today mostly pertain to only PCI devices
and for devices that aren't PCI like HPET for instance we give a PCI
identifier. Facilities like ATS for instance require the platform to have
an IOMMU.
what additional problems does moving this to struct device solve?
Cheers,
Ashok
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply
* [PATCH v4 26/26] arm64: mte: Add Memory Tagging Extension documentation
From: Catalin Marinas @ 2020-05-15 17:16 UTC (permalink / raw)
To: linux-arm-kernel
Cc: linux-arch, Szabolcs Nagy, Andrey Konovalov, Kevin Brodsky,
Peter Collingbourne, linux-mm, Vincenzo Frascino, Will Deacon,
Dave P Martin
In-Reply-To: <20200515171612.1020-1-catalin.marinas@arm.com>
From: Vincenzo Frascino <vincenzo.frascino@arm.com>
Memory Tagging Extension (part of the ARMv8.5 Extensions) provides
a mechanism to detect the sources of memory related errors which
may be vulnerable to exploitation, including bounds violations,
use-after-free, use-after-return, use-out-of-scope and use before
initialization errors.
Add Memory Tagging Extension documentation for the arm64 linux
kernel support.
Signed-off-by: Vincenzo Frascino <vincenzo.frascino@arm.com>
Co-developed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
---
Notes:
v4:
- Document behaviour of madvise(MADV_DONTNEED/MADV_FREE).
- Document the initial process state on fork/execve.
- Clarify when the kernel uaccess checks the tags.
- Minor updates to the example code.
- A few other minor clean-ups following review.
v3:
- Modify the uaccess checking conditions: only when the sync mode is
selected by the user. In async mode, the kernel uaccesses are not
checked.
- Clarify that an include mask of 0 (exclude mask 0xffff) results in
always generating tag 0.
- Document the ptrace() interface.
v2:
- Documented the uaccess kernel tag checking mode.
- Removed the BTI definitions from cpu-feature-registers.rst.
- Removed the paragraph stating that MTE depends on the tagged address
ABI (while the Kconfig entry does, there is no requirement for the
user to enable both).
- Changed the GCR_EL1.Exclude handling description following the change
in the prctl() interface (include vs exclude mask).
- Updated the example code.
Documentation/arm64/cpu-feature-registers.rst | 2 +
Documentation/arm64/elf_hwcaps.rst | 5 +
Documentation/arm64/index.rst | 1 +
.../arm64/memory-tagging-extension.rst | 297 ++++++++++++++++++
4 files changed, 305 insertions(+)
create mode 100644 Documentation/arm64/memory-tagging-extension.rst
diff --git a/Documentation/arm64/cpu-feature-registers.rst b/Documentation/arm64/cpu-feature-registers.rst
index 41937a8091aa..b5679fa85ad9 100644
--- a/Documentation/arm64/cpu-feature-registers.rst
+++ b/Documentation/arm64/cpu-feature-registers.rst
@@ -174,6 +174,8 @@ infrastructure:
+------------------------------+---------+---------+
| Name | bits | visible |
+------------------------------+---------+---------+
+ | MTE | [11-8] | y |
+ +------------------------------+---------+---------+
| SSBS | [7-4] | y |
+------------------------------+---------+---------+
diff --git a/Documentation/arm64/elf_hwcaps.rst b/Documentation/arm64/elf_hwcaps.rst
index 7dfb97dfe416..ca7f90e99e3a 100644
--- a/Documentation/arm64/elf_hwcaps.rst
+++ b/Documentation/arm64/elf_hwcaps.rst
@@ -236,6 +236,11 @@ HWCAP2_RNG
Functionality implied by ID_AA64ISAR0_EL1.RNDR == 0b0001.
+HWCAP2_MTE
+
+ Functionality implied by ID_AA64PFR1_EL1.MTE == 0b0010, as described
+ by Documentation/arm64/memory-tagging-extension.rst.
+
4. Unused AT_HWCAP bits
-----------------------
diff --git a/Documentation/arm64/index.rst b/Documentation/arm64/index.rst
index 09cbb4ed2237..4cd0e696f064 100644
--- a/Documentation/arm64/index.rst
+++ b/Documentation/arm64/index.rst
@@ -14,6 +14,7 @@ ARM64 Architecture
hugetlbpage
legacy_instructions
memory
+ memory-tagging-extension
pointer-authentication
silicon-errata
sve
diff --git a/Documentation/arm64/memory-tagging-extension.rst b/Documentation/arm64/memory-tagging-extension.rst
new file mode 100644
index 000000000000..e7cdcecb656a
--- /dev/null
+++ b/Documentation/arm64/memory-tagging-extension.rst
@@ -0,0 +1,297 @@
+===============================================
+Memory Tagging Extension (MTE) in AArch64 Linux
+===============================================
+
+Authors: Vincenzo Frascino <vincenzo.frascino@arm.com>
+ Catalin Marinas <catalin.marinas@arm.com>
+
+Date: 2020-02-25
+
+This document describes the provision of the Memory Tagging Extension
+functionality in AArch64 Linux.
+
+Introduction
+============
+
+ARMv8.5 based processors introduce the Memory Tagging Extension (MTE)
+feature. MTE is built on top of the ARMv8.0 virtual address tagging TBI
+(Top Byte Ignore) feature and allows software to access a 4-bit
+allocation tag for each 16-byte granule in the physical address space.
+Such memory range must be mapped with the Normal-Tagged memory
+attribute. A logical tag is derived from bits 59-56 of the virtual
+address used for the memory access. A CPU with MTE enabled will compare
+the logical tag against the allocation tag and potentially raise an
+exception on mismatch, subject to system registers configuration.
+
+Userspace Support
+=================
+
+When ``CONFIG_ARM64_MTE`` is selected and Memory Tagging Extension is
+supported by the hardware, the kernel advertises the feature to
+userspace via ``HWCAP2_MTE``.
+
+PROT_MTE
+--------
+
+To access the allocation tags, a user process must enable the Tagged
+memory attribute on an address range using a new ``prot`` flag for
+``mmap()`` and ``mprotect()``:
+
+``PROT_MTE`` - Pages allow access to the MTE allocation tags.
+
+The allocation tag is set to 0 when such pages are first mapped in the
+user address space and preserved on copy-on-write. ``MAP_SHARED`` is
+supported and the allocation tags can be shared between processes.
+
+**Note**: ``PROT_MTE`` is only supported on ``MAP_ANONYMOUS`` and
+RAM-based file mappings (``tmpfs``, ``memfd``). Passing it to other
+types of mapping will result in ``-EINVAL`` returned by these system
+calls.
+
+**Note**: The ``PROT_MTE`` flag (and corresponding memory type) cannot
+be cleared by ``mprotect()``.
+
+**Note**: ``madvise()`` memory ranges with ``MADV_DONTNEED`` and
+``MADV_FREE`` may have the allocation tags cleared (set to 0) at any
+point after the system call.
+
+Tag Check Faults
+----------------
+
+When ``PROT_MTE`` is enabled on an address range and a mismatch between
+the logical and allocation tags occurs on access, there are three
+configurable behaviours:
+
+- *Ignore* - This is the default mode. The CPU (and kernel) ignores the
+ tag check fault.
+
+- *Synchronous* - The kernel raises a ``SIGSEGV`` synchronously, with
+ ``.si_code = SEGV_MTESERR`` and ``.si_addr = <fault-address>``. The
+ memory access is not performed. If ``SIGSEGV`` is ignored or blocked
+ by the offending thread, the containing process is terminated with a
+ ``coredump``.
+
+- *Asynchronous* - The kernel raises a ``SIGSEGV``, in the offending
+ thread, asynchronously following one or multiple tag check faults,
+ with ``.si_code = SEGV_MTEAERR`` and ``.si_addr = 0`` (the faulting
+ address is unknown).
+
+The user can select the above modes, per thread, using the
+``prctl(PR_SET_TAGGED_ADDR_CTRL, flags, 0, 0, 0)`` system call where
+``flags`` contain one of the following values in the ``PR_MTE_TCF_MASK``
+bit-field:
+
+- ``PR_MTE_TCF_NONE`` - *Ignore* tag check faults
+- ``PR_MTE_TCF_SYNC`` - *Synchronous* tag check fault mode
+- ``PR_MTE_TCF_ASYNC`` - *Asynchronous* tag check fault mode
+
+The current tag check fault mode can be read using the
+``prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)`` system call.
+
+Tag checking can also be disabled for a user thread by setting the
+``PSTATE.TCO`` bit with ``MSR TCO, #1``.
+
+**Note**: Signal handlers are always invoked with ``PSTATE.TCO = 0``,
+irrespective of the interrupted context. ``PSTATE.TCO`` is restored on
+``sigreturn()``.
+
+**Note**: There are no *match-all* logical tags available for user
+applications.
+
+**Note**: Kernel accesses to the user address space (e.g. ``read()``
+system call) are not checked if the user thread tag checking mode is
+``PR_MTE_TCF_NONE`` or ``PR_MTE_TCF_ASYNC``. If the tag checking mode is
+``PR_MTE_TCF_SYNC``, the kernel makes a best effort to check its user
+address accesses, however it cannot always guarantee it.
+
+Excluding Tags in the ``IRG``, ``ADDG`` and ``SUBG`` instructions
+-----------------------------------------------------------------
+
+The architecture allows excluding certain tags to be randomly generated
+via the ``GCR_EL1.Exclude`` register bit-field. By default, Linux
+excludes all tags other than 0. A user thread can enable specific tags
+in the randomly generated set using the ``prctl(PR_SET_TAGGED_ADDR_CTRL,
+flags, 0, 0, 0)`` system call where ``flags`` contains the tags bitmap
+in the ``PR_MTE_TAG_MASK`` bit-field.
+
+**Note**: The hardware uses an exclude mask but the ``prctl()``
+interface provides an include mask. An include mask of ``0`` (exclusion
+mask ``0xffff``) results in the CPU always generating tag ``0``.
+
+Initial process state
+---------------------
+
+On ``execve()``, the new process has the following configuration:
+
+- ``PR_TAGGED_ADDR_ENABLE`` set to 0 (disabled)
+- Tag checking mode set to ``PR_MTE_TCF_NONE``
+- ``PR_MTE_TAG_MASK`` set to 0 (all tags excluded)
+- ``PSTATE.TCO`` set to 0
+- ``PROT_MTE`` not set on any of the initial memory maps
+
+On ``fork()``, the new process inherits the parent's configuration and
+memory map attributes with the exception of the ``madvise()`` ranges
+with ``MADV_WIPEONFORK`` which will have the data and tags cleared (set
+to 0).
+
+The ``ptrace()`` interface
+--------------------------
+
+``PTRACE_PEEKMTETAGS`` and ``PTRACE_POKEMTETAGS`` allow a tracer to read
+the tags from or set the tags to a tracee's address space. The
+``ptrace()`` system call is invoked as ``ptrace(request, pid, addr,
+data)`` where:
+
+- ``request`` - one of ``PTRACE_PEEKMTETAGS`` or ``PTRACE_PEEKMTETAGS``.
+- ``pid`` - the tracee's PID.
+- ``addr`` - address in the tracee's address space.
+- ``data`` - pointer to a ``struct iovec`` where ``iov_base`` points to
+ a buffer of ``iov_len`` length in the tracer's address space.
+
+The tags in the tracer's ``iov_base`` buffer are represented as one
+4-bit tag per byte and correspond to a 16-byte MTE tag granule in the
+tracee's address space.
+
+**Note**: If ``addr`` is not aligned to a 16-byte granule, the kernel
+will use the corresponding aligned address.
+
+``ptrace()`` return value:
+
+- 0 - tags were copied, the tracer's ``iov_len`` was updated to the
+ number of tags transferred. This may be smaller than the requested
+ ``iov_len`` if the requested address range in the tracee's or the
+ tracer's space cannot be accessed or does not have valid tags.
+- ``-EPERM`` - the specified process cannot be traced.
+- ``-EIO`` - the tracee's address range cannot be accessed (e.g. invalid
+ address) and no tags copied. ``iov_len`` not updated.
+- ``-EFAULT`` - fault on accessing the tracer's memory (``struct iovec``
+ or ``iov_base`` buffer) and no tags copied. ``iov_len`` not updated.
+- ``-EOPNOTSUPP`` - the tracee's address does not have valid tags (never
+ mapped with the ``PROT_MTE`` flag). ``iov_len`` not updated.
+
+**Note**: There are no transient errors for the requests above, so user
+programs should not retry in case of a non-zero system call return.
+
+Example of correct usage
+========================
+
+*MTE Example code*
+
+.. code-block:: c
+
+ /*
+ * To be compiled with -march=armv8.5-a+memtag
+ */
+ #include <errno.h>
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <unistd.h>
+ #include <sys/auxv.h>
+ #include <sys/mman.h>
+ #include <sys/prctl.h>
+
+ /*
+ * From arch/arm64/include/uapi/asm/hwcap.h
+ */
+ #define HWCAP2_MTE (1 << 18)
+
+ /*
+ * From arch/arm64/include/uapi/asm/mman.h
+ */
+ #define PROT_MTE 0x20
+
+ /*
+ * From include/uapi/linux/prctl.h
+ */
+ #define PR_SET_TAGGED_ADDR_CTRL 55
+ #define PR_GET_TAGGED_ADDR_CTRL 56
+ # define PR_TAGGED_ADDR_ENABLE (1UL << 0)
+ # define PR_MTE_TCF_SHIFT 1
+ # define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT)
+ # define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT)
+ # define PR_MTE_TCF_ASYNC (2UL << PR_MTE_TCF_SHIFT)
+ # define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT)
+ # define PR_MTE_TAG_SHIFT 3
+ # define PR_MTE_TAG_MASK (0xffffUL << PR_MTE_TAG_SHIFT)
+
+ /*
+ * Insert a random logical tag into the given pointer.
+ */
+ #define insert_random_tag(ptr) ({ \
+ __u64 __val; \
+ asm("irg %0, %1" : "=r" (__val) : "r" (ptr)); \
+ __val; \
+ })
+
+ /*
+ * Set the allocation tag on the destination address.
+ */
+ #define set_tag(tagged_addr) do { \
+ asm volatile("stg %0, [%0]" : : "r" (tagged_addr) : "memory"); \
+ } while (0)
+
+ int main()
+ {
+ unsigned char *a;
+ unsigned long page_sz = sysconf(_SC_PAGESIZE);
+ unsigned long hwcap2 = getauxval(AT_HWCAP2);
+
+ /* check if MTE is present */
+ if (!(hwcap2 & HWCAP2_MTE))
+ return EXIT_FAILURE;
+
+ /*
+ * Enable the tagged address ABI, synchronous MTE tag check faults and
+ * allow all non-zero tags in the randomly generated set.
+ */
+ if (prctl(PR_SET_TAGGED_ADDR_CTRL,
+ PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC | (0xfffe << PR_MTE_TAG_SHIFT),
+ 0, 0, 0)) {
+ perror("prctl() failed");
+ return EXIT_FAILURE;
+ }
+
+ a = mmap(0, page_sz, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+ if (a == MAP_FAILED) {
+ perror("mmap() failed");
+ return EXIT_FAILURE;
+ }
+
+ /*
+ * Enable MTE on the above anonymous mmap. The flag could be passed
+ * directly to mmap() and skip this step.
+ */
+ if (mprotect(a, page_sz, PROT_READ | PROT_WRITE | PROT_MTE)) {
+ perror("mprotect() failed");
+ return EXIT_FAILURE;
+ }
+
+ /* access with the default tag (0) */
+ a[0] = 1;
+ a[1] = 2;
+
+ printf("a[0] = %hhu a[1] = %hhu\n", a[0], a[1]);
+
+ /* set the logical and allocation tags */
+ a = (unsigned char *)insert_random_tag(a);
+ set_tag(a);
+
+ printf("%p\n", a);
+
+ /* non-zero tag access */
+ a[0] = 3;
+ printf("a[0] = %hhu a[1] = %hhu\n", a[0], a[1]);
+
+ /*
+ * If MTE is enabled correctly the next instruction will generate an
+ * exception.
+ */
+ printf("Expecting SIGSEGV...\n");
+ a[16] = 0xdd;
+
+ /* this should not be printed in the PR_MTE_TCF_SYNC mode */
+ printf("...haven't got one\n");
+
+ return EXIT_FAILURE;
+ }
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH v4 24/26] arm64: mte: Introduce early param to disable MTE support
From: Catalin Marinas @ 2020-05-15 17:16 UTC (permalink / raw)
To: linux-arm-kernel
Cc: linux-arch, Szabolcs Nagy, Andrey Konovalov, Kevin Brodsky,
Peter Collingbourne, linux-mm, Vincenzo Frascino, Will Deacon,
Dave P Martin
In-Reply-To: <20200515171612.1020-1-catalin.marinas@arm.com>
For performance analysis it may be desirable to disable MTE altogether
via an early param. Introduce arm64.mte_disable and, if true, filter out
the sanitised ID_AA64PFR1_EL1.MTE field to avoid exposing the HWCAP to
user.
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
---
Notes:
New in v4.
Documentation/admin-guide/kernel-parameters.txt | 4 ++++
arch/arm64/kernel/cpufeature.c | 11 +++++++++++
2 files changed, 15 insertions(+)
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index f2a93c8679e8..7436e7462b85 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -373,6 +373,10 @@
arcrimi= [HW,NET] ARCnet - "RIM I" (entirely mem-mapped) cards
Format: <io>,<irq>,<nodeID>
+ arm64.mte_disable=
+ [ARM64] Disable Linux support for the Memory
+ Tagging Extension (both user and in-kernel).
+
ataflop= [HW,M68k]
atarimouse= [HW,MOUSE] Atari Mouse
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index aaadc1cbc006..f7596830694f 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -126,12 +126,23 @@ static void cpu_enable_cnp(struct arm64_cpu_capabilities const *cap);
static bool __system_matches_cap(unsigned int n);
#ifdef CONFIG_ARM64_MTE
+static bool mte_disable;
+
+static int __init arm64_mte_disable(char *buf)
+{
+ return strtobool(buf, &mte_disable);
+}
+early_param("arm64.mte_disable", arm64_mte_disable);
+
s64 mte_ftr_filter(const struct arm64_ftr_bits *ftrp, s64 val)
{
struct device_node *np;
static bool memory_checked = false;
static bool mte_capable = true;
+ if (mte_disable)
+ return ID_AA64PFR1_MTE_NI;
+
/* EL0-only MTE is not supported by Linux, don't expose it */
if (val < ID_AA64PFR1_MTE)
return ID_AA64PFR1_MTE_NI;
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH v4 22/26] arm64: mte: Save tags when hibernating
From: Catalin Marinas @ 2020-05-15 17:16 UTC (permalink / raw)
To: linux-arm-kernel
Cc: linux-arch, Szabolcs Nagy, Andrey Konovalov, Kevin Brodsky,
Steven Price, Peter Collingbourne, linux-mm, James Morse,
Vincenzo Frascino, Will Deacon, Dave P Martin
In-Reply-To: <20200515171612.1020-1-catalin.marinas@arm.com>
From: Steven Price <steven.price@arm.com>
When hibernating the contents of all pages in the system are written to
disk, however the MTE tags are not visible to the generic hibernation
code. So just before the hibernation image is created copy the tags out
of the physical tag storage into standard memory so they will be
included in the hibernation image. After hibernation apply the tags back
into the physical tag storage.
Signed-off-by: Steven Price <steven.price@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Will Deacon <will@kernel.org>
---
arch/arm64/kernel/hibernate.c | 118 ++++++++++++++++++++++++++++++++++
1 file changed, 118 insertions(+)
diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c
index 5b73e92c99e3..d042ea325417 100644
--- a/arch/arm64/kernel/hibernate.c
+++ b/arch/arm64/kernel/hibernate.c
@@ -31,6 +31,7 @@
#include <asm/kexec.h>
#include <asm/memory.h>
#include <asm/mmu_context.h>
+#include <asm/mte.h>
#include <asm/pgalloc.h>
#include <asm/pgtable.h>
#include <asm/pgtable-hwdef.h>
@@ -277,6 +278,117 @@ static int create_safe_exec_page(void *src_start, size_t length,
#define dcache_clean_range(start, end) __flush_dcache_area(start, (end - start))
+#ifdef CONFIG_ARM64_MTE
+
+static DEFINE_XARRAY(mte_pages);
+
+static int save_tags(struct page *page, unsigned long pfn)
+{
+ void *tag_storage, *ret;
+
+ tag_storage = mte_allocate_tag_storage();
+ if (!tag_storage)
+ return -ENOMEM;
+
+ mte_save_page_tags(page_address(page), tag_storage);
+
+ ret = xa_store(&mte_pages, pfn, tag_storage, GFP_KERNEL);
+ if (WARN(xa_is_err(ret), "Failed to store MTE tags")) {
+ mte_free_tag_storage(tag_storage);
+ return xa_err(ret);
+ } else if (WARN(ret, "swsusp: %s: Duplicate entry", __func__)) {
+ mte_free_tag_storage(ret);
+ }
+
+ return 0;
+}
+
+static void swsusp_mte_free_storage(void)
+{
+ XA_STATE(xa_state, &mte_pages, 0);
+ void *tags;
+
+ xa_lock(&mte_pages);
+ xas_for_each(&xa_state, tags, ULONG_MAX) {
+ mte_free_tag_storage(tags);
+ }
+ xa_unlock(&mte_pages);
+
+ xa_destroy(&mte_pages);
+}
+
+static int swsusp_mte_save_tags(void)
+{
+ struct zone *zone;
+ unsigned long pfn, max_zone_pfn;
+ int ret = 0;
+ int n = 0;
+
+ if (!system_supports_mte())
+ return 0;
+
+ for_each_populated_zone(zone) {
+ max_zone_pfn = zone_end_pfn(zone);
+ for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
+ struct page *page = pfn_to_online_page(pfn);
+
+ if (!page)
+ continue;
+
+ if (!test_bit(PG_mte_tagged, &page->flags))
+ continue;
+
+ ret = save_tags(page, pfn);
+ if (ret) {
+ swsusp_mte_free_storage();
+ goto out;
+ }
+
+ n++;
+ }
+ }
+ pr_info("Saved %d MTE pages\n", n);
+
+out:
+ return ret;
+}
+
+static void swsusp_mte_restore_tags(void)
+{
+ XA_STATE(xa_state, &mte_pages, 0);
+ int n = 0;
+ void *tags;
+
+ xa_lock(&mte_pages);
+ xas_for_each(&xa_state, tags, ULONG_MAX) {
+ unsigned long pfn = xa_state.xa_index;
+ struct page *page = pfn_to_online_page(pfn);
+
+ mte_restore_page_tags(page_address(page), tags);
+
+ mte_free_tag_storage(tags);
+ n++;
+ }
+ xa_unlock(&mte_pages);
+
+ pr_info("Restored %d MTE pages\n", n);
+
+ xa_destroy(&mte_pages);
+}
+
+#else /* CONFIG_ARM64_MTE */
+
+static int swsusp_mte_save_tags(void)
+{
+ return 0;
+}
+
+static void swsusp_mte_restore_tags(void)
+{
+}
+
+#endif /* CONFIG_ARM64_MTE */
+
int swsusp_arch_suspend(void)
{
int ret = 0;
@@ -294,6 +406,10 @@ int swsusp_arch_suspend(void)
/* make the crash dump kernel image visible/saveable */
crash_prepare_suspend();
+ ret = swsusp_mte_save_tags();
+ if (ret)
+ return ret;
+
sleep_cpu = smp_processor_id();
ret = swsusp_save();
} else {
@@ -307,6 +423,8 @@ int swsusp_arch_suspend(void)
dcache_clean_range(__hyp_text_start, __hyp_text_end);
}
+ swsusp_mte_restore_tags();
+
/* make the crash dump kernel image protected again */
crash_post_resume();
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH v4 25/26] arm64: mte: Kconfig entry
From: Catalin Marinas @ 2020-05-15 17:16 UTC (permalink / raw)
To: linux-arm-kernel
Cc: linux-arch, Szabolcs Nagy, Andrey Konovalov, Kevin Brodsky,
Peter Collingbourne, linux-mm, Vincenzo Frascino, Will Deacon,
Dave P Martin
In-Reply-To: <20200515171612.1020-1-catalin.marinas@arm.com>
From: Vincenzo Frascino <vincenzo.frascino@arm.com>
Add Memory Tagging Extension support to the arm64 kbuild.
Signed-off-by: Vincenzo Frascino <vincenzo.frascino@arm.com>
Co-developed-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
---
Notes:
v4:
- select ARCH_USES_PG_ARCH_2
- remove ARCH_NO_SWAP
- default y
arch/arm64/Kconfig | 33 +++++++++++++++++++++++++++++++++
1 file changed, 33 insertions(+)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 40fb05d96c60..70deb14d78e0 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -1606,6 +1606,39 @@ config ARCH_RANDOM
endmenu
+menu "ARMv8.5 architectural features"
+
+config ARM64_AS_HAS_MTE
+ def_bool $(as-instr,.arch armv8.5-a+memtag)
+
+config ARM64_MTE
+ bool "Memory Tagging Extension support"
+ default y
+ depends on ARM64_AS_HAS_MTE && ARM64_TAGGED_ADDR_ABI
+ select ARCH_USES_HIGH_VMA_FLAGS
+ select ARCH_USES_PG_ARCH_2
+ help
+ Memory Tagging (part of the ARMv8.5 Extensions) provides
+ architectural support for run-time, always-on detection of
+ various classes of memory error to aid with software debugging
+ to eliminate vulnerabilities arising from memory-unsafe
+ languages.
+
+ This option enables the support for the Memory Tagging
+ Extension at EL0 (i.e. for userspace).
+
+ Selecting this option allows the feature to be detected at
+ runtime. Any secondary CPU not implementing this feature will
+ not be allowed a late bring-up.
+
+ Userspace binaries that want to use this feature must
+ explicitly opt in. The mechanism for the userspace is
+ described in:
+
+ Documentation/arm64/memory-tagging-extension.rst.
+
+endmenu
+
config ARM64_SVE
bool "ARM Scalable Vector Extension support"
default y
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH v4 23/26] arm64: mte: Check the DT memory nodes for MTE support
From: Catalin Marinas @ 2020-05-15 17:16 UTC (permalink / raw)
To: linux-arm-kernel
Cc: linux-arch, Mark Rutland, Szabolcs Nagy, Andrey Konovalov,
Kevin Brodsky, Rob Herring, Peter Collingbourne, linux-mm,
Vincenzo Frascino, Will Deacon, Dave P Martin
In-Reply-To: <20200515171612.1020-1-catalin.marinas@arm.com>
Even if the ID_AA64PFR1_EL1 register advertises the presence of MTE, it
is not guaranteed that the memory system on the SoC supports the
feature. In the absence of system-wide MTE support, the behaviour is
undefined and the kernel should not enable the MTE memory type in
MAIR_EL1.
For FDT, add an 'arm,armv8.5-memtag' property to the /memory nodes and
check for its presence during MTE probing. For example:
memory@80000000 {
device_type = "memory";
arm,armv8.5-memtag;
reg = <0x00000000 0x80000000 0 0x80000000>,
<0x00000008 0x80000000 0 0x80000000>;
};
If the /memory nodes are not present in DT or if at least one node does
not support MTE, the feature will be disabled. On EFI systems, it is
assumed that the memory description matches the EFI memory map (if not,
it is considered a firmware bug).
MTE is not currently supported on ACPI systems.
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Rob Herring <Rob.Herring@arm.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Will Deacon <will@kernel.org>
Reviewed-by: Suzuki K Poulose <suzuki.poulose@arm.com>
---
Notes:
Ongoing (internal) discussions on whether this is the right approach.
The issue may need to be solved similarly for ACPI systems.
v4:
- Reworked the CPUID field handling to ensure that it is not exposed to
user (via MRS emulation) when the feature is not described in the DT.
Previously, only the HWCAP was hidden.
New in v3.
arch/arm64/boot/dts/arm/fvp-base-revc.dts | 1 +
arch/arm64/include/asm/cpufeature.h | 6 ++-
arch/arm64/kernel/cpufeature.c | 59 ++++++++++++++++++++---
3 files changed, 58 insertions(+), 8 deletions(-)
diff --git a/arch/arm64/boot/dts/arm/fvp-base-revc.dts b/arch/arm64/boot/dts/arm/fvp-base-revc.dts
index 66381d89c1ce..c620a289f15e 100644
--- a/arch/arm64/boot/dts/arm/fvp-base-revc.dts
+++ b/arch/arm64/boot/dts/arm/fvp-base-revc.dts
@@ -94,6 +94,7 @@
memory@80000000 {
device_type = "memory";
+ arm,armv8.5-memtag;
reg = <0x00000000 0x80000000 0 0x80000000>,
<0x00000008 0x80000000 0 0x80000000>;
};
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index afc315814563..0a24d36bf231 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -61,6 +61,7 @@ struct arm64_ftr_bits {
u8 shift;
u8 width;
s64 safe_val; /* safe value for FTR_EXACT features */
+ s64 (*filter)(const struct arm64_ftr_bits *, s64);
};
/*
@@ -542,7 +543,10 @@ cpuid_feature_extract_field(u64 features, int field, bool sign)
static inline s64 arm64_ftr_value(const struct arm64_ftr_bits *ftrp, u64 val)
{
- return (s64)cpuid_feature_extract_field_width(val, ftrp->shift, ftrp->width, ftrp->sign);
+ s64 fval = (s64)cpuid_feature_extract_field_width(val, ftrp->shift, ftrp->width, ftrp->sign);
+ if (ftrp->filter)
+ fval = ftrp->filter(ftrp, fval);
+ return fval;
}
static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0)
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index d2fe8ff72324..aaadc1cbc006 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -7,6 +7,7 @@
#define pr_fmt(fmt) "CPU features: " fmt
+#include <linux/acpi.h>
#include <linux/bsearch.h>
#include <linux/cpumask.h>
#include <linux/crash_dump.h>
@@ -14,6 +15,7 @@
#include <linux/stop_machine.h>
#include <linux/types.h>
#include <linux/mm.h>
+#include <linux/of.h>
#include <linux/cpu.h>
#include <asm/cpu.h>
#include <asm/cpufeature.h>
@@ -87,23 +89,28 @@ DEFINE_STATIC_KEY_ARRAY_FALSE(cpu_hwcap_keys, ARM64_NCAPS);
EXPORT_SYMBOL(cpu_hwcap_keys);
#define __ARM64_FTR_BITS(SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
- { \
.sign = SIGNED, \
.visible = VISIBLE, \
.strict = STRICT, \
.type = TYPE, \
.shift = SHIFT, \
.width = WIDTH, \
- .safe_val = SAFE_VAL, \
- }
+ .safe_val = SAFE_VAL
/* Define a feature with unsigned values */
#define ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
- __ARM64_FTR_BITS(FTR_UNSIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
+ { __ARM64_FTR_BITS(FTR_UNSIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL), }
/* Define a feature with a signed value */
#define S_ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
- __ARM64_FTR_BITS(FTR_SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
+ { __ARM64_FTR_BITS(FTR_SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL), }
+
+/* Define a feature with a filter function to process the field value */
+#define FILTERED_ARM64_FTR_BITS(SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL, filter_fn) \
+ { \
+ __ARM64_FTR_BITS(SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL), \
+ .filter = filter_fn, \
+ }
#define ARM64_FTR_END \
{ \
@@ -118,6 +125,42 @@ static void cpu_enable_cnp(struct arm64_cpu_capabilities const *cap);
static bool __system_matches_cap(unsigned int n);
+#ifdef CONFIG_ARM64_MTE
+s64 mte_ftr_filter(const struct arm64_ftr_bits *ftrp, s64 val)
+{
+ struct device_node *np;
+ static bool memory_checked = false;
+ static bool mte_capable = true;
+
+ /* EL0-only MTE is not supported by Linux, don't expose it */
+ if (val < ID_AA64PFR1_MTE)
+ return ID_AA64PFR1_MTE_NI;
+
+ if (memory_checked)
+ return mte_capable ? val : ID_AA64PFR1_MTE_NI;
+
+ if (!acpi_disabled) {
+ pr_warn("MTE not supported on ACPI systems\n");
+ return ID_AA64PFR1_MTE_NI;
+ }
+
+ /* check the DT "memory" nodes for MTE support */
+ for_each_node_by_type(np, "memory") {
+ memory_checked = true;
+ mte_capable &= of_property_read_bool(np, "arm,armv8.5-memtag");
+ }
+
+ if (!memory_checked || !mte_capable) {
+ pr_warn("System memory is not MTE-capable\n");
+ memory_checked = true;
+ mte_capable = false;
+ return ID_AA64PFR1_MTE_NI;
+ }
+
+ return val;
+}
+#endif
+
/*
* NOTE: Any changes to the visibility of features should be kept in
* sync with the documentation of the CPU feature register ABI.
@@ -182,8 +225,10 @@ static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = {
ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_SSBS_SHIFT, 4, ID_AA64PFR1_SSBS_PSTATE_NI),
- ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_MTE),
- FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_MTE_SHIFT, 4, ID_AA64PFR1_MTE_NI),
+#ifdef CONFIG_ARM64_MTE
+ FILTERED_ARM64_FTR_BITS(FTR_UNSIGNED, FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE,
+ ID_AA64PFR1_MTE_SHIFT, 4, ID_AA64PFR1_MTE_NI, mte_ftr_filter),
+#endif
ARM64_FTR_END,
};
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH v4 21/26] arm64: mte: Enable swap of tagged pages
From: Catalin Marinas @ 2020-05-15 17:16 UTC (permalink / raw)
To: linux-arm-kernel
Cc: linux-arch, Szabolcs Nagy, Andrey Konovalov, Kevin Brodsky,
Steven Price, Peter Collingbourne, linux-mm, Andrew Morton,
Vincenzo Frascino, Will Deacon, Dave P Martin
In-Reply-To: <20200515171612.1020-1-catalin.marinas@arm.com>
From: Steven Price <steven.price@arm.com>
When swapping pages out to disk it is necessary to save any tags that
have been set, and restore when swapping back in. Make use of the new
page flag (PG_ARCH_2, locally named PG_mte_tagged) to identify pages
with tags. When swapping out these pages the tags are stored in memory
and later restored when the pages are brought back in. Because shmem can
swap pages back in without restoring the userspace PTE it is also
necessary to add a hook for shmem.
Signed-off-by: Steven Price <steven.price@arm.com>
[catalin.marinas@arm.com: move function prototypes to mte.h]
[catalin.marinas@arm.com: drop '_tags' from arch_swap_restore_tags()]
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/mte.h | 8 ++++
arch/arm64/include/asm/pgtable.h | 32 +++++++++++++
arch/arm64/kernel/mte.c | 10 ++++
arch/arm64/lib/mte.S | 45 ++++++++++++++++++
arch/arm64/mm/Makefile | 1 +
arch/arm64/mm/mteswap.c | 82 ++++++++++++++++++++++++++++++++
include/asm-generic/pgtable.h | 4 +-
mm/shmem.c | 2 +-
8 files changed, 181 insertions(+), 3 deletions(-)
create mode 100644 arch/arm64/mm/mteswap.c
diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h
index 9d4b1390d07d..ad43cfe88d5f 100644
--- a/arch/arm64/include/asm/mte.h
+++ b/arch/arm64/include/asm/mte.h
@@ -21,6 +21,14 @@ unsigned long mte_copy_tags_from_user(void *to, const void __user *from,
unsigned long n);
unsigned long mte_copy_tags_to_user(void __user *to, void *from,
unsigned long n);
+int mte_save_tags(struct page *page);
+void mte_save_page_tags(const void *page_addr, void *tag_storage);
+bool mte_restore_tags(swp_entry_t entry, struct page *page);
+void mte_restore_page_tags(void *page_addr, const void *tag_storage);
+void mte_invalidate_tags(int type, pgoff_t offset);
+void mte_invalidate_tags_area(int type);
+void *mte_allocate_tag_storage(void);
+void mte_free_tag_storage(char *storage);
#ifdef CONFIG_ARM64_MTE
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index f2cd59b01b27..281d0b0f4fe1 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -851,6 +851,38 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
extern int kern_addr_valid(unsigned long addr);
+#ifdef CONFIG_ARM64_MTE
+
+#define __HAVE_ARCH_PREPARE_TO_SWAP
+static inline int arch_prepare_to_swap(struct page *page)
+{
+ if (system_supports_mte())
+ return mte_save_tags(page);
+ return 0;
+}
+
+#define __HAVE_ARCH_SWAP_INVALIDATE
+static inline void arch_swap_invalidate_page(int type, pgoff_t offset)
+{
+ if (system_supports_mte())
+ mte_invalidate_tags(type, offset);
+}
+
+static inline void arch_swap_invalidate_area(int type)
+{
+ if (system_supports_mte())
+ mte_invalidate_tags_area(type);
+}
+
+#define __HAVE_ARCH_SWAP_RESTORE
+static inline void arch_swap_restore(swp_entry_t entry, struct page *page)
+{
+ if (system_supports_mte() && mte_restore_tags(entry, page))
+ set_bit(PG_mte_tagged, &page->flags);
+}
+
+#endif /* CONFIG_ARM64_MTE */
+
#include <asm-generic/pgtable.h>
/*
diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c
index 6abd6a16a145..7b3041b0f909 100644
--- a/arch/arm64/kernel/mte.c
+++ b/arch/arm64/kernel/mte.c
@@ -10,6 +10,8 @@
#include <linux/sched.h>
#include <linux/sched/mm.h>
#include <linux/string.h>
+#include <linux/swap.h>
+#include <linux/swapops.h>
#include <linux/thread_info.h>
#include <linux/uio.h>
@@ -21,11 +23,19 @@
void mte_sync_tags(pte_t *ptep, pte_t pte)
{
struct page *page = pte_page(pte);
+ pte_t old_pte = READ_ONCE(*ptep);
/* if PG_mte_tagged is set, tags have already been initialised */
if (test_and_set_bit(PG_mte_tagged, &page->flags))
return;
+ if (is_swap_pte(old_pte)) {
+ swp_entry_t entry = pte_to_swp_entry(old_pte);
+
+ if (!non_swap_entry(entry) && mte_restore_tags(entry, page))
+ return;
+ }
+
/* tags of the zero page will be cleared on first mapping */
mte_clear_page_tags(page_address(page), page_size(page));
}
diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S
index 862655a36013..dd4593693e63 100644
--- a/arch/arm64/lib/mte.S
+++ b/arch/arm64/lib/mte.S
@@ -93,3 +93,48 @@ SYM_FUNC_START(mte_copy_tags_to_user)
2: sub x0, x0, x3 // update the number of tags copied
ret
SYM_FUNC_END(mte_copy_tags_to_user)
+
+/*
+ * Save the tags in a page
+ * x0 - page address
+ * x1 - tag storage
+ */
+SYM_FUNC_START(mte_save_page_tags)
+ multitag_transfer_size x7, x5
+1:
+ mov x2, #0
+2:
+ ldgm x5, [x0]
+ orr x2, x2, x5
+ add x0, x0, x7
+ tst x0, #0xFF // 16 tag values fit in a register,
+ b.ne 2b // which is 16*16=256 bytes
+
+ str x2, [x1], #8
+
+ tst x0, #(PAGE_SIZE - 1)
+ b.ne 1b
+
+ ret
+SYM_FUNC_END(mte_save_page_tags)
+
+/*
+ * Restore the tags in a page
+ * x0 - page address
+ * x1 - tag storage
+ */
+SYM_FUNC_START(mte_restore_page_tags)
+ multitag_transfer_size x7, x5
+1:
+ ldr x2, [x1], #8
+2:
+ stgm x2, [x0]
+ add x0, x0, x7
+ tst x0, #0xFF
+ b.ne 2b
+
+ tst x0, #(PAGE_SIZE - 1)
+ b.ne 1b
+
+ ret
+SYM_FUNC_END(mte_restore_page_tags)
diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
index d91030f0ffee..5bcc9e0aa259 100644
--- a/arch/arm64/mm/Makefile
+++ b/arch/arm64/mm/Makefile
@@ -8,6 +8,7 @@ obj-$(CONFIG_PTDUMP_CORE) += dump.o
obj-$(CONFIG_PTDUMP_DEBUGFS) += ptdump_debugfs.o
obj-$(CONFIG_NUMA) += numa.o
obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o
+obj-$(CONFIG_ARM64_MTE) += mteswap.o
KASAN_SANITIZE_physaddr.o += n
obj-$(CONFIG_KASAN) += kasan_init.o
diff --git a/arch/arm64/mm/mteswap.c b/arch/arm64/mm/mteswap.c
new file mode 100644
index 000000000000..847d99814d03
--- /dev/null
+++ b/arch/arm64/mm/mteswap.c
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/pagemap.h>
+#include <linux/xarray.h>
+#include <linux/swap.h>
+#include <linux/swapops.h>
+#include <asm/mte.h>
+
+static DEFINE_XARRAY(mte_pages);
+
+void *mte_allocate_tag_storage(void)
+{
+ /* tags granule is 16 bytes, 2 tags stored per byte */
+ return kmalloc(PAGE_SIZE / 16 / 2, GFP_KERNEL);
+}
+
+void mte_free_tag_storage(char *storage)
+{
+ kfree(storage);
+}
+
+int mte_save_tags(struct page *page)
+{
+ void *tag_storage, *ret;
+
+ if (!test_bit(PG_mte_tagged, &page->flags))
+ return 0;
+
+ tag_storage = mte_allocate_tag_storage();
+ if (!tag_storage)
+ return -ENOMEM;
+
+ mte_save_page_tags(page_address(page), tag_storage);
+
+ /* page_private contains the swap entry.val set in do_swap_page */
+ ret = xa_store(&mte_pages, page_private(page), tag_storage, GFP_KERNEL);
+ if (WARN(xa_is_err(ret), "Failed to store MTE tags")) {
+ mte_free_tag_storage(tag_storage);
+ return xa_err(ret);
+ } else if (ret) {
+ /* Entry is being replaced, free the old entry */
+ mte_free_tag_storage(ret);
+ }
+
+ return 0;
+}
+
+bool mte_restore_tags(swp_entry_t entry, struct page *page)
+{
+ void *tags = xa_load(&mte_pages, entry.val);
+
+ if (!tags)
+ return false;
+
+ mte_restore_page_tags(page_address(page), tags);
+
+ return true;
+}
+
+void mte_invalidate_tags(int type, pgoff_t offset)
+{
+ swp_entry_t entry = swp_entry(type, offset);
+ void *tags = xa_erase(&mte_pages, entry.val);
+
+ mte_free_tag_storage(tags);
+}
+
+void mte_invalidate_tags_area(int type)
+{
+ swp_entry_t entry = swp_entry(type, 0);
+ swp_entry_t last_entry = swp_entry(type + 1, 0);
+ void *tags;
+
+ XA_STATE(xa_state, &mte_pages, entry.val);
+
+ xa_lock(&mte_pages);
+ xas_for_each(&xa_state, tags, last_entry.val - 1) {
+ __xa_erase(&mte_pages, xa_state.xa_index);
+ mte_free_tag_storage(tags);
+ }
+ xa_unlock(&mte_pages);
+}
diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h
index 306cee75b9ec..09eb80160920 100644
--- a/include/asm-generic/pgtable.h
+++ b/include/asm-generic/pgtable.h
@@ -492,8 +492,8 @@ static inline void arch_swap_invalidate_area(int type)
}
#endif
-#ifndef __HAVE_ARCH_SWAP_RESTORE_TAGS
-static inline void arch_swap_restore_tags(swp_entry_t entry, struct page *page)
+#ifndef __HAVE_ARCH_SWAP_RESTORE
+static inline void arch_swap_restore(swp_entry_t entry, struct page *page)
{
}
#endif
diff --git a/mm/shmem.c b/mm/shmem.c
index 0e87925a7cb4..003b0e96c746 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -1662,7 +1662,7 @@ static int shmem_swapin_page(struct inode *inode, pgoff_t index,
* Some architectures may have to restore extra metadata to the
* physical page after reading from swap.
*/
- arch_swap_restore_tags(swap, page);
+ arch_swap_restore(swap, page);
if (shmem_should_replace_page(page, gfp)) {
error = shmem_replace_page(&page, gfp, info, index);
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH v4 20/26] mm: Add arch hooks for saving/restoring tags
From: Catalin Marinas @ 2020-05-15 17:16 UTC (permalink / raw)
To: linux-arm-kernel
Cc: linux-arch, Szabolcs Nagy, Andrey Konovalov, Kevin Brodsky,
Steven Price, Peter Collingbourne, linux-mm, Andrew Morton,
Vincenzo Frascino, Will Deacon, Dave P Martin
In-Reply-To: <20200515171612.1020-1-catalin.marinas@arm.com>
From: Steven Price <steven.price@arm.com>
Arm's Memory Tagging Extension (MTE) adds some metadata (tags) to
every physical page, when swapping pages out to disk it is necessary to
save these tags, and later restore them when reading the pages back.
Add some hooks along with dummy implementations to enable the
arch code to handle this.
Three new hooks are added to the swap code:
* arch_prepare_to_swap() and
* arch_swap_invalidate_page() / arch_swap_invalidate_area().
One new hook is added to shmem:
* arch_swap_restore_tags()
Signed-off-by: Steven Price <steven.price@arm.com>
[catalin.marinas@arm.com: add unlock_page() on the error path]
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
---
include/asm-generic/pgtable.h | 23 +++++++++++++++++++++++
mm/page_io.c | 10 ++++++++++
mm/shmem.c | 6 ++++++
mm/swapfile.c | 2 ++
4 files changed, 41 insertions(+)
diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h
index 329b8c8ca703..306cee75b9ec 100644
--- a/include/asm-generic/pgtable.h
+++ b/include/asm-generic/pgtable.h
@@ -475,6 +475,29 @@ static inline int arch_unmap_one(struct mm_struct *mm,
}
#endif
+#ifndef __HAVE_ARCH_PREPARE_TO_SWAP
+static inline int arch_prepare_to_swap(struct page *page)
+{
+ return 0;
+}
+#endif
+
+#ifndef __HAVE_ARCH_SWAP_INVALIDATE
+static inline void arch_swap_invalidate_page(int type, pgoff_t offset)
+{
+}
+
+static inline void arch_swap_invalidate_area(int type)
+{
+}
+#endif
+
+#ifndef __HAVE_ARCH_SWAP_RESTORE_TAGS
+static inline void arch_swap_restore_tags(swp_entry_t entry, struct page *page)
+{
+}
+#endif
+
#ifndef __HAVE_ARCH_PGD_OFFSET_GATE
#define pgd_offset_gate(mm, addr) pgd_offset(mm, addr)
#endif
diff --git a/mm/page_io.c b/mm/page_io.c
index 76965be1d40e..dae85d56e45b 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -253,6 +253,16 @@ int swap_writepage(struct page *page, struct writeback_control *wbc)
unlock_page(page);
goto out;
}
+ /*
+ * Arch code may have to preserve more data than just the page
+ * contents, e.g. memory tags.
+ */
+ ret = arch_prepare_to_swap(page);
+ if (ret) {
+ set_page_dirty(page);
+ unlock_page(page);
+ goto out;
+ }
if (frontswap_store(page) == 0) {
set_page_writeback(page);
unlock_page(page);
diff --git a/mm/shmem.c b/mm/shmem.c
index 73754ed7af69..0e87925a7cb4 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -1658,6 +1658,12 @@ static int shmem_swapin_page(struct inode *inode, pgoff_t index,
}
wait_on_page_writeback(page);
+ /*
+ * Some architectures may have to restore extra metadata to the
+ * physical page after reading from swap.
+ */
+ arch_swap_restore_tags(swap, page);
+
if (shmem_should_replace_page(page, gfp)) {
error = shmem_replace_page(&page, gfp, info, index);
if (error)
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 5871a2aa86a5..b39c6520b0cf 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -722,6 +722,7 @@ static void swap_range_free(struct swap_info_struct *si, unsigned long offset,
else
swap_slot_free_notify = NULL;
while (offset <= end) {
+ arch_swap_invalidate_page(si->type, offset);
frontswap_invalidate_page(si->type, offset);
if (swap_slot_free_notify)
swap_slot_free_notify(si->bdev, offset);
@@ -2645,6 +2646,7 @@ SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
frontswap_map = frontswap_map_get(p);
spin_unlock(&p->lock);
spin_unlock(&swap_lock);
+ arch_swap_invalidate_area(p->type);
frontswap_invalidate_area(p->type);
frontswap_map_set(p, NULL);
mutex_unlock(&swapon_mutex);
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH v4 18/26] arm64: mte: Add PTRACE_{PEEK,POKE}MTETAGS support
From: Catalin Marinas @ 2020-05-15 17:16 UTC (permalink / raw)
To: linux-arm-kernel
Cc: linux-arch, Luis Machado, Omair Javaid, Szabolcs Nagy,
Andrey Konovalov, Kevin Brodsky, Peter Collingbourne, linux-mm,
Alan Hayward, Vincenzo Frascino, Will Deacon, Dave P Martin
In-Reply-To: <20200515171612.1020-1-catalin.marinas@arm.com>
Add support for bulk setting/getting of the MTE tags in a tracee's
address space at 'addr' in the ptrace() syscall prototype. 'data' points
to a struct iovec in the tracer's address space with iov_base
representing the address of a tracer's buffer of length iov_len. The
tags to be copied to/from the tracer's buffer are stored as one tag per
byte.
On successfully copying at least one tag, ptrace() returns 0 and updates
the tracer's iov_len with the number of tags copied. In case of error,
either -EIO or -EFAULT is returned, trying to follow the ptrace() man
page.
Note that the tag copying functions are not performance critical,
therefore they lack optimisations found in typical memory copy routines.
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Cc: Alan Hayward <Alan.Hayward@arm.com>
Cc: Luis Machado <luis.machado@linaro.org>
Cc: Omair Javaid <omair.javaid@linaro.org>
---
Notes:
v4:
- Following the change to only clear the tags in a page if it is mapped
to user with PROT_MTE, ptrace() now will refuse to access tags in
pages not previously mapped with PROT_MTE (PG_mte_tagged set). This is
primarily to avoid leaking uninitialised tags to user via ptrace().
- Fix SYM_FUNC_END argument typo.
- Rename MTE_ALLOC_* to MTE_GRANULE_*.
- Use uao_user_alternative for the user access in case we ever want to
call mte_copy_tags_* with a kernel buffer. It also matches the other
uaccess routines in the kernel.
- Simplify arch_ptrace() slightly.
- Reorder down_write_killable() with access_ok() in
__access_remote_tags().
- Handle copy length 0 in mte_copy_tags_{to,from}_user().
- Use put_user() instead of __put_user().
New in v3.
arch/arm64/include/asm/mte.h | 17 ++++
arch/arm64/include/uapi/asm/ptrace.h | 3 +
arch/arm64/kernel/mte.c | 139 +++++++++++++++++++++++++++
arch/arm64/kernel/ptrace.c | 7 ++
arch/arm64/lib/mte.S | 53 ++++++++++
5 files changed, 219 insertions(+)
diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h
index 7435f6619bf1..9d4b1390d07d 100644
--- a/arch/arm64/include/asm/mte.h
+++ b/arch/arm64/include/asm/mte.h
@@ -5,6 +5,11 @@
#ifndef __ASM_MTE_H
#define __ASM_MTE_H
+#define MTE_GRANULE_SIZE UL(16)
+#define MTE_GRANULE_MASK (~(MTE_GRANULE_SIZE - 1))
+#define MTE_TAG_SHIFT 56
+#define MTE_TAG_SIZE 4
+
#ifndef __ASSEMBLY__
#include <linux/page-flags.h>
@@ -12,6 +17,10 @@
#include <asm/pgtable-types.h>
void mte_clear_page_tags(void *addr, size_t size);
+unsigned long mte_copy_tags_from_user(void *to, const void __user *from,
+ unsigned long n);
+unsigned long mte_copy_tags_to_user(void __user *to, void *from,
+ unsigned long n);
#ifdef CONFIG_ARM64_MTE
@@ -25,6 +34,8 @@ void mte_thread_switch(struct task_struct *next);
void mte_suspend_exit(void);
long set_mte_ctrl(unsigned long arg);
long get_mte_ctrl(void);
+int mte_ptrace_copy_tags(struct task_struct *child, long request,
+ unsigned long addr, unsigned long data);
#else
@@ -54,6 +65,12 @@ static inline long get_mte_ctrl(void)
{
return 0;
}
+static inline int mte_ptrace_copy_tags(struct task_struct *child,
+ long request, unsigned long addr,
+ unsigned long data)
+{
+ return -EIO;
+}
#endif
diff --git a/arch/arm64/include/uapi/asm/ptrace.h b/arch/arm64/include/uapi/asm/ptrace.h
index 1daf6dda8af0..cd2a4a164de3 100644
--- a/arch/arm64/include/uapi/asm/ptrace.h
+++ b/arch/arm64/include/uapi/asm/ptrace.h
@@ -67,6 +67,9 @@
/* syscall emulation path in ptrace */
#define PTRACE_SYSEMU 31
#define PTRACE_SYSEMU_SINGLESTEP 32
+/* MTE allocation tag access */
+#define PTRACE_PEEKMTETAGS 33
+#define PTRACE_POKEMTETAGS 34
#ifndef __ASSEMBLY__
diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c
index cda09bc8caf4..6abd6a16a145 100644
--- a/arch/arm64/kernel/mte.c
+++ b/arch/arm64/kernel/mte.c
@@ -4,14 +4,18 @@
*/
#include <linux/bitops.h>
+#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/prctl.h>
#include <linux/sched.h>
+#include <linux/sched/mm.h>
#include <linux/string.h>
#include <linux/thread_info.h>
+#include <linux/uio.h>
#include <asm/cpufeature.h>
#include <asm/mte.h>
+#include <asm/ptrace.h>
#include <asm/sysreg.h>
void mte_sync_tags(pte_t *ptep, pte_t pte)
@@ -173,3 +177,138 @@ long get_mte_ctrl(void)
return ret;
}
+
+/*
+ * Access MTE tags in another process' address space as given in mm. Update
+ * the number of tags copied. Return 0 if any tags copied, error otherwise.
+ * Inspired by __access_remote_vm().
+ */
+static int __access_remote_tags(struct task_struct *tsk, struct mm_struct *mm,
+ unsigned long addr, struct iovec *kiov,
+ unsigned int gup_flags)
+{
+ struct vm_area_struct *vma;
+ void __user *buf = kiov->iov_base;
+ size_t len = kiov->iov_len;
+ int ret;
+ int write = gup_flags & FOLL_WRITE;
+
+ if (!access_ok(buf, len))
+ return -EFAULT;
+
+ if (down_read_killable(&mm->mmap_sem))
+ return -EIO;
+
+ while (len) {
+ unsigned long tags, offset;
+ void *maddr;
+ struct page *page = NULL;
+
+ ret = get_user_pages_remote(tsk, mm, addr, 1, gup_flags,
+ &page, &vma, NULL);
+ if (ret <= 0)
+ break;
+
+ /*
+ * Only copy tags if the page has been mapped as PROT_MTE
+ * (PG_mte_tagged set). Otherwise the tags are not valid and
+ * not accessible to user. Moreover, an mprotect(PROT_MTE)
+ * would cause the existing tags to be cleared if the page
+ * was never mapped with PROT_MTE.
+ */
+ if (!test_bit(PG_mte_tagged, &page->flags)) {
+ ret = -EOPNOTSUPP;
+ put_page(page);
+ break;
+ }
+
+ /* limit access to the end of the page */
+ offset = offset_in_page(addr);
+ tags = min(len, (PAGE_SIZE - offset) / MTE_GRANULE_SIZE);
+
+ maddr = page_address(page);
+ if (write) {
+ tags = mte_copy_tags_from_user(maddr + offset, buf, tags);
+ set_page_dirty_lock(page);
+ } else {
+ tags = mte_copy_tags_to_user(buf, maddr + offset, tags);
+ }
+ put_page(page);
+
+ /* error accessing the tracer's buffer */
+ if (!tags)
+ break;
+
+ len -= tags;
+ buf += tags;
+ addr += tags * MTE_GRANULE_SIZE;
+ }
+ up_read(&mm->mmap_sem);
+
+ /* return an error if no tags copied */
+ kiov->iov_len = buf - kiov->iov_base;
+ if (!kiov->iov_len) {
+ /* check for error accessing the tracee's address space */
+ if (ret <= 0)
+ return -EIO;
+ else
+ return -EFAULT;
+ }
+
+ return 0;
+}
+
+/*
+ * Copy MTE tags in another process' address space at 'addr' to/from tracer's
+ * iovec buffer. Return 0 on success. Inspired by ptrace_access_vm().
+ */
+static int access_remote_tags(struct task_struct *tsk, unsigned long addr,
+ struct iovec *kiov, unsigned int gup_flags)
+{
+ struct mm_struct *mm;
+ int ret;
+
+ mm = get_task_mm(tsk);
+ if (!mm)
+ return -EPERM;
+
+ if (!tsk->ptrace || (current != tsk->parent) ||
+ ((get_dumpable(mm) != SUID_DUMP_USER) &&
+ !ptracer_capable(tsk, mm->user_ns))) {
+ mmput(mm);
+ return -EPERM;
+ }
+
+ ret = __access_remote_tags(tsk, mm, addr, kiov, gup_flags);
+ mmput(mm);
+
+ return ret;
+}
+
+int mte_ptrace_copy_tags(struct task_struct *child, long request,
+ unsigned long addr, unsigned long data)
+{
+ int ret;
+ struct iovec kiov;
+ struct iovec __user *uiov = (void __user *)data;
+ unsigned int gup_flags = FOLL_FORCE;
+
+ if (!system_supports_mte())
+ return -EIO;
+
+ if (get_user(kiov.iov_base, &uiov->iov_base) ||
+ get_user(kiov.iov_len, &uiov->iov_len))
+ return -EFAULT;
+
+ if (request == PTRACE_POKEMTETAGS)
+ gup_flags |= FOLL_WRITE;
+
+ /* align addr to the MTE tag granule */
+ addr &= MTE_GRANULE_MASK;
+
+ ret = access_remote_tags(child, addr, &kiov, gup_flags);
+ if (!ret)
+ ret = put_user(kiov.iov_len, &uiov->iov_len);
+
+ return ret;
+}
diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c
index 077e352495eb..c8bda5f5b321 100644
--- a/arch/arm64/kernel/ptrace.c
+++ b/arch/arm64/kernel/ptrace.c
@@ -34,6 +34,7 @@
#include <asm/cpufeature.h>
#include <asm/debug-monitors.h>
#include <asm/fpsimd.h>
+#include <asm/mte.h>
#include <asm/pgtable.h>
#include <asm/pointer_auth.h>
#include <asm/stacktrace.h>
@@ -1797,6 +1798,12 @@ const struct user_regset_view *task_user_regset_view(struct task_struct *task)
long arch_ptrace(struct task_struct *child, long request,
unsigned long addr, unsigned long data)
{
+ switch (request) {
+ case PTRACE_PEEKMTETAGS:
+ case PTRACE_POKEMTETAGS:
+ return mte_ptrace_copy_tags(child, request, addr, data);
+ }
+
return ptrace_request(child, request, addr, data);
}
diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S
index a531b52fa5ba..862655a36013 100644
--- a/arch/arm64/lib/mte.S
+++ b/arch/arm64/lib/mte.S
@@ -4,7 +4,9 @@
*/
#include <linux/linkage.h>
+#include <asm/alternative.h>
#include <asm/assembler.h>
+#include <asm/mte.h>
#include <asm/page.h>
.arch armv8.5-a+memtag
@@ -40,3 +42,54 @@ SYM_FUNC_START(mte_copy_page_tags)
b.ne 1b
2:
SYM_FUNC_END(mte_copy_page_tags)
+
+/*
+ * Read tags from a user buffer (one tag per byte) and set the corresponding
+ * tags at the given kernel address. Used by PTRACE_POKEMTETAGS.
+ * x0 - kernel address (to)
+ * x1 - user buffer (from)
+ * x2 - number of tags/bytes (n)
+ * Returns:
+ * x0 - number of tags read/set
+ */
+SYM_FUNC_START(mte_copy_tags_from_user)
+ mov x3, x1
+ cbz x2, 2f
+1:
+ uao_user_alternative 2f, ldrb, ldtrb, w4, x1, 0
+ lsl x4, x4, #MTE_TAG_SHIFT
+ stg x4, [x0], #MTE_GRANULE_SIZE
+ add x1, x1, #1
+ subs x2, x2, #1
+ b.ne 1b
+
+ // exception handling and function return
+2: sub x0, x1, x3 // update the number of tags set
+ ret
+SYM_FUNC_END(mte_copy_tags_from_user)
+
+/*
+ * Get the tags from a kernel address range and write the tag values to the
+ * given user buffer (one tag per byte). Used by PTRACE_PEEKMTETAGS.
+ * x0 - user buffer (to)
+ * x1 - kernel address (from)
+ * x2 - number of tags/bytes (n)
+ * Returns:
+ * x0 - number of tags read/set
+ */
+SYM_FUNC_START(mte_copy_tags_to_user)
+ mov x3, x0
+ cbz x2, 2f
+1:
+ ldg x4, [x1]
+ ubfx x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE
+ uao_user_alternative 2f, strb, sttrb, w4, x0, 0
+ add x0, x0, #1
+ add x1, x1, #MTE_GRANULE_SIZE
+ subs x2, x2, #1
+ b.ne 1b
+
+ // exception handling and function return
+2: sub x0, x0, x3 // update the number of tags copied
+ ret
+SYM_FUNC_END(mte_copy_tags_to_user)
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH v4 19/26] fs: Handle intra-page faults in copy_mount_options()
From: Catalin Marinas @ 2020-05-15 17:16 UTC (permalink / raw)
To: linux-arm-kernel
Cc: linux-arch, Szabolcs Nagy, Andrey Konovalov, Kevin Brodsky,
Peter Collingbourne, linux-mm, Alexander Viro, Vincenzo Frascino,
Will Deacon, Dave P Martin
In-Reply-To: <20200515171612.1020-1-catalin.marinas@arm.com>
The copy_mount_options() function takes a user pointer argument but no
size. It tries to read up to a PAGE_SIZE. However, copy_from_user() is
not guaranteed to return all the accessible bytes if, for example, the
access crosses a page boundary and gets a fault on the second page. To
work around this, the current copy_mount_options() implementation
performs two copy_from_user() passes, first to the end of the current
page and the second to what's left in the subsequent page.
On arm64 with MTE enabled, access to a user page may trigger a fault
after part of the buffer has been copied (when the user pointer tag,
bits 56-59, no longer matches the allocation tag stored in memory).
Allow copy_mount_options() to handle such intra-page faults by returning
-EFAULT only if the first copy_from_user() has not copied any bytes.
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Reviewed-by: Kevin Brodsky <kevin.brodsky@arm.com>
---
Notes:
v4:
- Rewrite to avoid arch_has_exact_copy_from_user()
New in v3.
fs/namespace.c | 24 +++++++++++++++++++++---
1 file changed, 21 insertions(+), 3 deletions(-)
diff --git a/fs/namespace.c b/fs/namespace.c
index a28e4db075ed..70aa60ccce57 100644
--- a/fs/namespace.c
+++ b/fs/namespace.c
@@ -3016,7 +3016,7 @@ static void shrink_submounts(struct mount *mnt)
void *copy_mount_options(const void __user * data)
{
char *copy;
- unsigned size;
+ unsigned size, left;
if (!data)
return NULL;
@@ -3027,12 +3027,30 @@ void *copy_mount_options(const void __user * data)
size = PAGE_SIZE - offset_in_page(data);
- if (copy_from_user(copy, data, size)) {
+ /*
+ * Attempt to copy to the end of the first user page. On success,
+ * left == 0, copy the rest from the second user page (if it is
+ * accessible). copy_from_user() will zero the part of the kernel
+ * buffer not copied into.
+ *
+ * On architectures with intra-page faults (arm64 with MTE), the read
+ * from the first page may fail after copying part of the user data
+ * (left > 0 && left < size). Do not attempt the second copy in this
+ * case as the end of the valid user buffer has already been reached.
+ * Ensure, however, that the second part of the kernel buffer is
+ * zeroed.
+ */
+ left = copy_from_user(copy, data, size);
+ if (left == size) {
kfree(copy);
return ERR_PTR(-EFAULT);
}
if (size != PAGE_SIZE) {
- if (copy_from_user(copy + size, data + size, PAGE_SIZE - size))
+ if (left == 0)
+ /* return not relevant, just silence the compiler */
+ left = copy_from_user(copy + size, data + size,
+ PAGE_SIZE - size);
+ else
memset(copy + size, 0, PAGE_SIZE - size);
}
return copy;
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH v4 17/26] arm64: mte: Restore the GCR_EL1 register after a suspend
From: Catalin Marinas @ 2020-05-15 17:16 UTC (permalink / raw)
To: linux-arm-kernel
Cc: linux-arch, Szabolcs Nagy, Andrey Konovalov, Kevin Brodsky,
Peter Collingbourne, linux-mm, Vincenzo Frascino, Will Deacon,
Dave P Martin
In-Reply-To: <20200515171612.1020-1-catalin.marinas@arm.com>
The CPU resume/suspend routines only take care of the common system
registers. Restore GCR_EL1 in addition via the __cpu_suspend_exit()
function.
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
Reviewed-by: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>
---
Notes:
New in v3.
arch/arm64/include/asm/mte.h | 4 ++++
arch/arm64/kernel/mte.c | 8 ++++++++
arch/arm64/kernel/suspend.c | 4 ++++
3 files changed, 16 insertions(+)
diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h
index c79dbffd30c4..7435f6619bf1 100644
--- a/arch/arm64/include/asm/mte.h
+++ b/arch/arm64/include/asm/mte.h
@@ -22,6 +22,7 @@ void mte_sync_tags(pte_t *ptep, pte_t pte);
void mte_copy_page_tags(void *kto, const void *kfrom);
void flush_mte_state(void);
void mte_thread_switch(struct task_struct *next);
+void mte_suspend_exit(void);
long set_mte_ctrl(unsigned long arg);
long get_mte_ctrl(void);
@@ -42,6 +43,9 @@ static inline void flush_mte_state(void)
static inline void mte_thread_switch(struct task_struct *next)
{
}
+static inline void mte_suspend_exit(void)
+{
+}
static inline long set_mte_ctrl(unsigned long arg)
{
return 0;
diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c
index 999300e2110e..cda09bc8caf4 100644
--- a/arch/arm64/kernel/mte.c
+++ b/arch/arm64/kernel/mte.c
@@ -116,6 +116,14 @@ void mte_thread_switch(struct task_struct *next)
update_gcr_el1_excl(next->thread.gcr_incl);
}
+void mte_suspend_exit(void)
+{
+ if (!system_supports_mte())
+ return;
+
+ update_gcr_el1_excl(current->thread.gcr_incl);
+}
+
long set_mte_ctrl(unsigned long arg)
{
u64 tcf0;
diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c
index 9405d1b7f4b0..1d405b73d009 100644
--- a/arch/arm64/kernel/suspend.c
+++ b/arch/arm64/kernel/suspend.c
@@ -9,6 +9,7 @@
#include <asm/daifflags.h>
#include <asm/debug-monitors.h>
#include <asm/exec.h>
+#include <asm/mte.h>
#include <asm/pgtable.h>
#include <asm/memory.h>
#include <asm/mmu_context.h>
@@ -74,6 +75,9 @@ void notrace __cpu_suspend_exit(void)
*/
if (arm64_get_ssbd_state() == ARM64_SSBD_FORCE_DISABLE)
arm64_set_ssbd_mitigation(false);
+
+ /* Restore additional MTE-specific configuration */
+ mte_suspend_exit();
}
/*
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH v4 16/26] arm64: mte: Allow user control of the generated random tags via prctl()
From: Catalin Marinas @ 2020-05-15 17:16 UTC (permalink / raw)
To: linux-arm-kernel
Cc: linux-arch, Szabolcs Nagy, Andrey Konovalov, Kevin Brodsky,
Peter Collingbourne, linux-mm, Vincenzo Frascino, Will Deacon,
Dave P Martin
In-Reply-To: <20200515171612.1020-1-catalin.marinas@arm.com>
The IRG, ADDG and SUBG instructions insert a random tag in the resulting
address. Certain tags can be excluded via the GCR_EL1.Exclude bitmap
when, for example, the user wants a certain colour for freed buffers.
Since the GCR_EL1 register is not accessible at EL0, extend the
prctl(PR_SET_TAGGED_ADDR_CTRL) interface to include a 16-bit field in
the first argument for controlling which tags can be generated by the
above instruction (an include rather than exclude mask). Note that by
default all non-zero tags are excluded. This setting is per-thread.
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
---
Notes:
v2:
- Switch from an exclude mask to an include one for the prctl()
interface.
- Reset the allowed tags mask during flush_thread().
arch/arm64/include/asm/processor.h | 1 +
arch/arm64/include/asm/sysreg.h | 7 ++++++
arch/arm64/kernel/mte.c | 35 +++++++++++++++++++++++++++---
arch/arm64/kernel/process.c | 2 +-
include/uapi/linux/prctl.h | 3 +++
5 files changed, 44 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 80e7f0573309..996b882a32d9 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -153,6 +153,7 @@ struct thread_struct {
#endif
#ifdef CONFIG_ARM64_MTE
u64 sctlr_tcf0;
+ u64 gcr_incl;
#endif
};
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 86236ae6c4e7..cb247f2f75ca 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -981,6 +981,13 @@
write_sysreg(__scs_new, sysreg); \
} while (0)
+#define sysreg_clear_set_s(sysreg, clear, set) do { \
+ u64 __scs_val = read_sysreg_s(sysreg); \
+ u64 __scs_new = (__scs_val & ~(u64)(clear)) | (set); \
+ if (__scs_new != __scs_val) \
+ write_sysreg_s(__scs_new, sysreg); \
+} while (0)
+
#endif
#endif /* __ASM_SYSREG_H */
diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c
index 3e425f124360..999300e2110e 100644
--- a/arch/arm64/kernel/mte.c
+++ b/arch/arm64/kernel/mte.c
@@ -71,6 +71,25 @@ static void set_sctlr_el1_tcf0(u64 tcf0)
preempt_enable();
}
+static void update_gcr_el1_excl(u64 incl)
+{
+ u64 excl = ~incl & SYS_GCR_EL1_EXCL_MASK;
+
+ /*
+ * Note that 'incl' is an include mask (controlled by the user via
+ * prctl()) while GCR_EL1 accepts an exclude mask.
+ * No need for ISB since this only affects EL0 currently, implicit
+ * with ERET.
+ */
+ sysreg_clear_set_s(SYS_GCR_EL1, SYS_GCR_EL1_EXCL_MASK, excl);
+}
+
+static void set_gcr_el1_excl(u64 incl)
+{
+ current->thread.gcr_incl = incl;
+ update_gcr_el1_excl(incl);
+}
+
void flush_mte_state(void)
{
if (!system_supports_mte())
@@ -82,6 +101,8 @@ void flush_mte_state(void)
clear_thread_flag(TIF_MTE_ASYNC_FAULT);
/* disable tag checking */
set_sctlr_el1_tcf0(SCTLR_EL1_TCF0_NONE);
+ /* reset tag generation mask */
+ set_gcr_el1_excl(0);
}
void mte_thread_switch(struct task_struct *next)
@@ -92,6 +113,7 @@ void mte_thread_switch(struct task_struct *next)
/* avoid expensive SCTLR_EL1 accesses if no change */
if (current->thread.sctlr_tcf0 != next->thread.sctlr_tcf0)
update_sctlr_el1_tcf0(next->thread.sctlr_tcf0);
+ update_gcr_el1_excl(next->thread.gcr_incl);
}
long set_mte_ctrl(unsigned long arg)
@@ -116,23 +138,30 @@ long set_mte_ctrl(unsigned long arg)
}
set_sctlr_el1_tcf0(tcf0);
+ set_gcr_el1_excl((arg & PR_MTE_TAG_MASK) >> PR_MTE_TAG_SHIFT);
return 0;
}
long get_mte_ctrl(void)
{
+ unsigned long ret;
+
if (!system_supports_mte())
return 0;
+ ret = current->thread.gcr_incl << PR_MTE_TAG_SHIFT;
+
switch (current->thread.sctlr_tcf0) {
case SCTLR_EL1_TCF0_NONE:
return PR_MTE_TCF_NONE;
case SCTLR_EL1_TCF0_SYNC:
- return PR_MTE_TCF_SYNC;
+ ret |= PR_MTE_TCF_SYNC;
+ break;
case SCTLR_EL1_TCF0_ASYNC:
- return PR_MTE_TCF_ASYNC;
+ ret |= PR_MTE_TCF_ASYNC;
+ break;
}
- return 0;
+ return ret;
}
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index ff6031a398d0..697571be259b 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -601,7 +601,7 @@ long set_tagged_addr_ctrl(unsigned long arg)
return -EINVAL;
if (system_supports_mte())
- valid_mask |= PR_MTE_TCF_MASK;
+ valid_mask |= PR_MTE_TCF_MASK | PR_MTE_TAG_MASK;
if (arg & ~valid_mask)
return -EINVAL;
diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h
index 2390ab324afa..7f0827705c9a 100644
--- a/include/uapi/linux/prctl.h
+++ b/include/uapi/linux/prctl.h
@@ -239,6 +239,9 @@ struct prctl_mm_map {
# define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT)
# define PR_MTE_TCF_ASYNC (2UL << PR_MTE_TCF_SHIFT)
# define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT)
+/* MTE tag inclusion mask */
+# define PR_MTE_TAG_SHIFT 3
+# define PR_MTE_TAG_MASK (0xffffUL << PR_MTE_TAG_SHIFT)
/* Control reclaim behavior when allocating memory */
#define PR_SET_IO_FLUSHER 57
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH v4 15/26] arm64: mte: Allow user control of the tag check mode via prctl()
From: Catalin Marinas @ 2020-05-15 17:16 UTC (permalink / raw)
To: linux-arm-kernel
Cc: linux-arch, Szabolcs Nagy, Andrey Konovalov, Kevin Brodsky,
Peter Collingbourne, linux-mm, Vincenzo Frascino, Will Deacon,
Dave P Martin
In-Reply-To: <20200515171612.1020-1-catalin.marinas@arm.com>
By default, even if PROT_MTE is set on a memory range, there is no tag
check fault reporting (SIGSEGV). Introduce a set of option to the
exiting prctl(PR_SET_TAGGED_ADDR_CTRL) to allow user control of the tag
check fault mode:
PR_MTE_TCF_NONE - no reporting (default)
PR_MTE_TCF_SYNC - synchronous tag check fault reporting
PR_MTE_TCF_ASYNC - asynchronous tag check fault reporting
These options translate into the corresponding SCTLR_EL1.TCF0 bitfield,
context-switched by the kernel. Note that uaccess done by the kernel is
not checked and cannot be configured by the user.
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
---
Notes:
v3:
- Use SCTLR_EL1_TCF0_NONE instead of 0 for consistency.
- Move mte_thread_switch() in this patch from an earlier one. In
addition, it is called after the dsb() in __switch_to() so that any
asynchronous tag check faults have been registered in the TFSR_EL1
registers (to be added with the in-kernel MTE support.
v2:
- Handle SCTLR_EL1_TCF0_NONE explicitly for consistency with PR_MTE_TCF_NONE.
- Fix SCTLR_EL1 register setting in flush_mte_state() (thanks to Peter
Collingbourne).
- Added ISB to update_sctlr_el1_tcf0() since, with the latest
architecture update/fix, the TCF0 field is used by the uaccess
routines.
arch/arm64/include/asm/mte.h | 14 ++++++
arch/arm64/include/asm/processor.h | 3 ++
arch/arm64/kernel/mte.c | 77 ++++++++++++++++++++++++++++++
arch/arm64/kernel/process.c | 26 ++++++++--
include/uapi/linux/prctl.h | 6 +++
5 files changed, 123 insertions(+), 3 deletions(-)
diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h
index c1a09499c678..c79dbffd30c4 100644
--- a/arch/arm64/include/asm/mte.h
+++ b/arch/arm64/include/asm/mte.h
@@ -21,6 +21,9 @@ void mte_clear_page_tags(void *addr, size_t size);
void mte_sync_tags(pte_t *ptep, pte_t pte);
void mte_copy_page_tags(void *kto, const void *kfrom);
void flush_mte_state(void);
+void mte_thread_switch(struct task_struct *next);
+long set_mte_ctrl(unsigned long arg);
+long get_mte_ctrl(void);
#else
@@ -36,6 +39,17 @@ static inline void mte_copy_page_tags(void *kto, const void *kfrom)
static inline void flush_mte_state(void)
{
}
+static inline void mte_thread_switch(struct task_struct *next)
+{
+}
+static inline long set_mte_ctrl(unsigned long arg)
+{
+ return 0;
+}
+static inline long get_mte_ctrl(void)
+{
+ return 0;
+}
#endif
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 240fe5e5b720..80e7f0573309 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -151,6 +151,9 @@ struct thread_struct {
struct ptrauth_keys_user keys_user;
struct ptrauth_keys_kernel keys_kernel;
#endif
+#ifdef CONFIG_ARM64_MTE
+ u64 sctlr_tcf0;
+#endif
};
static inline void arch_thread_struct_whitelist(unsigned long *offset,
diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c
index da2d70178a4b..3e425f124360 100644
--- a/arch/arm64/kernel/mte.c
+++ b/arch/arm64/kernel/mte.c
@@ -5,6 +5,8 @@
#include <linux/bitops.h>
#include <linux/mm.h>
+#include <linux/prctl.h>
+#include <linux/sched.h>
#include <linux/string.h>
#include <linux/thread_info.h>
@@ -49,6 +51,26 @@ int memcmp_pages(struct page *page1, struct page *page2)
return ret;
}
+static void update_sctlr_el1_tcf0(u64 tcf0)
+{
+ /* ISB required for the kernel uaccess routines */
+ sysreg_clear_set(sctlr_el1, SCTLR_EL1_TCF0_MASK, tcf0);
+ isb();
+}
+
+static void set_sctlr_el1_tcf0(u64 tcf0)
+{
+ /*
+ * mte_thread_switch() checks current->thread.sctlr_tcf0 as an
+ * optimisation. Disable preemption so that it does not see
+ * the variable update before the SCTLR_EL1.TCF0 one.
+ */
+ preempt_disable();
+ current->thread.sctlr_tcf0 = tcf0;
+ update_sctlr_el1_tcf0(tcf0);
+ preempt_enable();
+}
+
void flush_mte_state(void)
{
if (!system_supports_mte())
@@ -58,4 +80,59 @@ void flush_mte_state(void)
dsb(ish);
write_sysreg_s(0, SYS_TFSRE0_EL1);
clear_thread_flag(TIF_MTE_ASYNC_FAULT);
+ /* disable tag checking */
+ set_sctlr_el1_tcf0(SCTLR_EL1_TCF0_NONE);
+}
+
+void mte_thread_switch(struct task_struct *next)
+{
+ if (!system_supports_mte())
+ return;
+
+ /* avoid expensive SCTLR_EL1 accesses if no change */
+ if (current->thread.sctlr_tcf0 != next->thread.sctlr_tcf0)
+ update_sctlr_el1_tcf0(next->thread.sctlr_tcf0);
+}
+
+long set_mte_ctrl(unsigned long arg)
+{
+ u64 tcf0;
+
+ if (!system_supports_mte())
+ return 0;
+
+ switch (arg & PR_MTE_TCF_MASK) {
+ case PR_MTE_TCF_NONE:
+ tcf0 = SCTLR_EL1_TCF0_NONE;
+ break;
+ case PR_MTE_TCF_SYNC:
+ tcf0 = SCTLR_EL1_TCF0_SYNC;
+ break;
+ case PR_MTE_TCF_ASYNC:
+ tcf0 = SCTLR_EL1_TCF0_ASYNC;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ set_sctlr_el1_tcf0(tcf0);
+
+ return 0;
+}
+
+long get_mte_ctrl(void)
+{
+ if (!system_supports_mte())
+ return 0;
+
+ switch (current->thread.sctlr_tcf0) {
+ case SCTLR_EL1_TCF0_NONE:
+ return PR_MTE_TCF_NONE;
+ case SCTLR_EL1_TCF0_SYNC:
+ return PR_MTE_TCF_SYNC;
+ case SCTLR_EL1_TCF0_ASYNC:
+ return PR_MTE_TCF_ASYNC;
+ }
+
+ return 0;
}
diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 740047c9cd13..ff6031a398d0 100644
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -529,6 +529,13 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
*/
dsb(ish);
+ /*
+ * MTE thread switching must happen after the DSB above to ensure that
+ * any asynchronous tag check faults have been logged in the TFSR*_EL1
+ * registers.
+ */
+ mte_thread_switch(next);
+
/* the actual thread switch */
last = cpu_switch_to(prev, next);
@@ -588,9 +595,15 @@ static unsigned int tagged_addr_disabled;
long set_tagged_addr_ctrl(unsigned long arg)
{
+ unsigned long valid_mask = PR_TAGGED_ADDR_ENABLE;
+
if (is_compat_task())
return -EINVAL;
- if (arg & ~PR_TAGGED_ADDR_ENABLE)
+
+ if (system_supports_mte())
+ valid_mask |= PR_MTE_TCF_MASK;
+
+ if (arg & ~valid_mask)
return -EINVAL;
/*
@@ -600,6 +613,9 @@ long set_tagged_addr_ctrl(unsigned long arg)
if (arg & PR_TAGGED_ADDR_ENABLE && tagged_addr_disabled)
return -EINVAL;
+ if (set_mte_ctrl(arg) != 0)
+ return -EINVAL;
+
update_thread_flag(TIF_TAGGED_ADDR, arg & PR_TAGGED_ADDR_ENABLE);
return 0;
@@ -607,13 +623,17 @@ long set_tagged_addr_ctrl(unsigned long arg)
long get_tagged_addr_ctrl(void)
{
+ long ret = 0;
+
if (is_compat_task())
return -EINVAL;
if (test_thread_flag(TIF_TAGGED_ADDR))
- return PR_TAGGED_ADDR_ENABLE;
+ ret = PR_TAGGED_ADDR_ENABLE;
- return 0;
+ ret |= get_mte_ctrl();
+
+ return ret;
}
/*
diff --git a/include/uapi/linux/prctl.h b/include/uapi/linux/prctl.h
index 07b4f8131e36..2390ab324afa 100644
--- a/include/uapi/linux/prctl.h
+++ b/include/uapi/linux/prctl.h
@@ -233,6 +233,12 @@ struct prctl_mm_map {
#define PR_SET_TAGGED_ADDR_CTRL 55
#define PR_GET_TAGGED_ADDR_CTRL 56
# define PR_TAGGED_ADDR_ENABLE (1UL << 0)
+/* MTE tag check fault modes */
+# define PR_MTE_TCF_SHIFT 1
+# define PR_MTE_TCF_NONE (0UL << PR_MTE_TCF_SHIFT)
+# define PR_MTE_TCF_SYNC (1UL << PR_MTE_TCF_SHIFT)
+# define PR_MTE_TCF_ASYNC (2UL << PR_MTE_TCF_SHIFT)
+# define PR_MTE_TCF_MASK (3UL << PR_MTE_TCF_SHIFT)
/* Control reclaim behavior when allocating memory */
#define PR_SET_IO_FLUSHER 57
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH v4 14/26] mm: Allow arm64 mmap(PROT_MTE) on RAM-based files
From: Catalin Marinas @ 2020-05-15 17:16 UTC (permalink / raw)
To: linux-arm-kernel
Cc: linux-arch, Szabolcs Nagy, Andrey Konovalov, Kevin Brodsky,
Peter Collingbourne, linux-mm, Andrew Morton, Vincenzo Frascino,
Will Deacon, Dave P Martin
In-Reply-To: <20200515171612.1020-1-catalin.marinas@arm.com>
Since arm64 memory (allocation) tags can only be stored in RAM, mapping
files with PROT_MTE is not allowed by default. RAM-based files like
those in a tmpfs mount or memfd_create() can support memory tagging, so
update the vm_flags accordingly in shmem_mmap().
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
---
mm/shmem.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/mm/shmem.c b/mm/shmem.c
index d722eb830317..73754ed7af69 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -2221,6 +2221,9 @@ static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
vma->vm_flags &= ~(VM_MAYWRITE);
}
+ /* arm64 - allow memory tagging on RAM-based files */
+ vma->vm_flags |= VM_MTE_ALLOWED;
+
file_accessed(file);
vma->vm_ops = &shmem_vm_ops;
if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH v4 13/26] arm64: mte: Validate the PROT_MTE request via arch_validate_flags()
From: Catalin Marinas @ 2020-05-15 17:15 UTC (permalink / raw)
To: linux-arm-kernel
Cc: linux-arch, Szabolcs Nagy, Andrey Konovalov, Kevin Brodsky,
Peter Collingbourne, linux-mm, Vincenzo Frascino, Will Deacon,
Dave P Martin
In-Reply-To: <20200515171612.1020-1-catalin.marinas@arm.com>
Make use of the newly introduced arch_validate_flags() hook to
sanity-check the PROT_MTE request passed to mmap() and mprotect(). If
the mapping does not support MTE, these syscalls will return -EINVAL.
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
---
arch/arm64/include/asm/mman.h | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/mman.h b/arch/arm64/include/asm/mman.h
index c77a23869223..5c356d1ca266 100644
--- a/arch/arm64/include/asm/mman.h
+++ b/arch/arm64/include/asm/mman.h
@@ -44,7 +44,11 @@ static inline unsigned long arch_calc_vm_flag_bits(unsigned long flags)
static inline pgprot_t arch_vm_get_page_prot(unsigned long vm_flags)
{
- return (vm_flags & VM_MTE) && (vm_flags & VM_MTE_ALLOWED) ?
+ /*
+ * Checking for VM_MTE only is sufficient since arch_validate_flags()
+ * does not permit (VM_MTE & !VM_MTE_ALLOWED).
+ */
+ return (vm_flags & VM_MTE) ?
__pgprot(PTE_ATTRINDX(MT_NORMAL_TAGGED)) :
__pgprot(0);
}
@@ -61,4 +65,14 @@ static inline bool arch_validate_prot(unsigned long prot, unsigned long addr)
}
#define arch_validate_prot arch_validate_prot
+static inline bool arch_validate_flags(unsigned long flags)
+{
+ if (!system_supports_mte())
+ return true;
+
+ /* only allow VM_MTE if VM_MTE_ALLOWED has been set previously */
+ return !(flags & VM_MTE) || (flags & VM_MTE_ALLOWED);
+}
+#define arch_validate_flags arch_validate_flags
+
#endif /* !__ASM_MMAN_H__ */
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH v4 12/26] mm: Introduce arch_validate_flags()
From: Catalin Marinas @ 2020-05-15 17:15 UTC (permalink / raw)
To: linux-arm-kernel
Cc: linux-arch, Szabolcs Nagy, Andrey Konovalov, Kevin Brodsky,
Peter Collingbourne, linux-mm, Andrew Morton, Vincenzo Frascino,
Will Deacon, Dave P Martin
In-Reply-To: <20200515171612.1020-1-catalin.marinas@arm.com>
Similarly to arch_validate_prot() called from do_mprotect_pkey(), an
architecture may need to sanity-check the new vm_flags.
Define a dummy function always returning true. In addition to
do_mprotect_pkey(), also invoke it from mmap_region() prior to updating
vma->vm_page_prot to allow the architecture code to veto potentially
inconsistent vm_flags.
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
---
Notes:
v2:
- Some comments updated.
include/linux/mman.h | 13 +++++++++++++
mm/mmap.c | 9 +++++++++
mm/mprotect.c | 6 ++++++
3 files changed, 28 insertions(+)
diff --git a/include/linux/mman.h b/include/linux/mman.h
index 15c1162b9d65..09dd414b81b6 100644
--- a/include/linux/mman.h
+++ b/include/linux/mman.h
@@ -103,6 +103,19 @@ static inline bool arch_validate_prot(unsigned long prot, unsigned long addr)
#define arch_validate_prot arch_validate_prot
#endif
+#ifndef arch_validate_flags
+/*
+ * This is called from mmap() and mprotect() with the updated vma->vm_flags.
+ *
+ * Returns true if the VM_* flags are valid.
+ */
+static inline bool arch_validate_flags(unsigned long flags)
+{
+ return true;
+}
+#define arch_validate_flags arch_validate_flags
+#endif
+
/*
* Optimisation macro. It is equivalent to:
* (x & bit1) ? bit2 : 0
diff --git a/mm/mmap.c b/mm/mmap.c
index f609e9ec4a25..d5fc93c2072e 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -1792,6 +1792,15 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
vma_set_anonymous(vma);
}
+ /* Allow architectures to sanity-check the vm_flags */
+ if (!arch_validate_flags(vma->vm_flags)) {
+ error = -EINVAL;
+ if (file)
+ goto unmap_and_free_vma;
+ else
+ goto free_vma;
+ }
+
vma_link(mm, vma, prev, rb_link, rb_parent);
/* Once vma denies write, undo our temporary denial count */
if (file) {
diff --git a/mm/mprotect.c b/mm/mprotect.c
index 494192ca954b..04b1d2cf0e74 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -603,6 +603,12 @@ static int do_mprotect_pkey(unsigned long start, size_t len,
goto out;
}
+ /* Allow architectures to sanity-check the new flags */
+ if (!arch_validate_flags(newflags)) {
+ error = -EINVAL;
+ goto out;
+ }
+
error = security_file_mprotect(vma, reqprot, prot);
if (error)
goto out;
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH v4 11/26] arm64: mte: Add PROT_MTE support to mmap() and mprotect()
From: Catalin Marinas @ 2020-05-15 17:15 UTC (permalink / raw)
To: linux-arm-kernel
Cc: linux-arch, Szabolcs Nagy, Andrey Konovalov, Kevin Brodsky,
Peter Collingbourne, linux-mm, Vincenzo Frascino, Will Deacon,
Dave P Martin
In-Reply-To: <20200515171612.1020-1-catalin.marinas@arm.com>
To enable tagging on a memory range, the user must explicitly opt in via
a new PROT_MTE flag passed to mmap() or mprotect(). Since this is a new
memory type in the AttrIndx field of a pte, simplify the or'ing of these
bits over the protection_map[] attributes by making MT_NORMAL index 0.
There are two conditions for arch_vm_get_page_prot() to return the
MT_NORMAL_TAGGED memory type: (1) the user requested it via PROT_MTE,
registered as VM_MTE in the vm_flags, and (2) the vma supports MTE,
decided during the mmap() call (only) and registered as VM_MTE_ALLOWED.
arch_calc_vm_prot_bits() is responsible for registering the user request
as VM_MTE. The newly introduced arch_calc_vm_flag_bits() sets
VM_MTE_ALLOWED if the mapping is MAP_ANONYMOUS. An MTE-capable
filesystem (RAM-based) may be able to set VM_MTE_ALLOWED during its
mmap() file ops call.
In addition, update VM_DATA_DEFAULT_FLAGS to allow mprotect(PROT_MTE) on
stack or brk area.
The Linux mmap() syscall currently ignores unknown PROT_* flags. In the
presence of MTE, an mmap(PROT_MTE) on a file which does not support MTE
will not report an error and the memory will not be mapped as Normal
Tagged. For consistency, mprotect(PROT_MTE) will not report an error
either if the memory range does not support MTE. Two subsequent patches
in the series will propose tightening of this behaviour.
Co-developed-by: Vincenzo Frascino <vincenzo.frascino@arm.com>
Signed-off-by: Vincenzo Frascino <vincenzo.frascino@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will@kernel.org>
---
Notes:
v2:
- Add VM_MTE_ALLOWED to show_smap_vma_flags().
arch/arm64/include/asm/memory.h | 18 +++++----
arch/arm64/include/asm/mman.h | 64 ++++++++++++++++++++++++++++++
arch/arm64/include/asm/page.h | 2 +-
arch/arm64/include/asm/pgtable.h | 7 +++-
arch/arm64/include/uapi/asm/mman.h | 14 +++++++
fs/proc/task_mmu.c | 4 ++
include/linux/mm.h | 8 ++++
7 files changed, 108 insertions(+), 9 deletions(-)
create mode 100644 arch/arm64/include/asm/mman.h
create mode 100644 arch/arm64/include/uapi/asm/mman.h
diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index 472c77a68225..770535b7ca35 100644
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -129,14 +129,18 @@
/*
* Memory types available.
+ *
+ * IMPORTANT: MT_NORMAL must be index 0 since vm_get_page_prot() may 'or' in
+ * the MT_NORMAL_TAGGED memory type for PROT_MTE mappings. Note
+ * that protection_map[] only contains MT_NORMAL attributes.
*/
-#define MT_DEVICE_nGnRnE 0
-#define MT_DEVICE_nGnRE 1
-#define MT_DEVICE_GRE 2
-#define MT_NORMAL_NC 3
-#define MT_NORMAL 4
-#define MT_NORMAL_WT 5
-#define MT_NORMAL_TAGGED 6
+#define MT_NORMAL 0
+#define MT_NORMAL_TAGGED 1
+#define MT_NORMAL_NC 2
+#define MT_NORMAL_WT 3
+#define MT_DEVICE_nGnRnE 4
+#define MT_DEVICE_nGnRE 5
+#define MT_DEVICE_GRE 6
/*
* Memory types for Stage-2 translation
diff --git a/arch/arm64/include/asm/mman.h b/arch/arm64/include/asm/mman.h
new file mode 100644
index 000000000000..c77a23869223
--- /dev/null
+++ b/arch/arm64/include/asm/mman.h
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __ASM_MMAN_H__
+#define __ASM_MMAN_H__
+
+#include <uapi/asm/mman.h>
+
+/*
+ * There are two conditions required for returning a Normal Tagged memory type
+ * in arch_vm_get_page_prot(): (1) the user requested it via PROT_MTE passed
+ * to mmap() or mprotect() and (2) the corresponding vma supports MTE. We
+ * register (1) as VM_MTE in the vma->vm_flags and (2) as VM_MTE_ALLOWED. Note
+ * that the latter can only be set during the mmap() call since mprotect()
+ * does not accept MAP_* flags.
+ */
+static inline unsigned long arch_calc_vm_prot_bits(unsigned long prot,
+ unsigned long pkey)
+{
+ if (!system_supports_mte())
+ return 0;
+
+ if (prot & PROT_MTE)
+ return VM_MTE;
+
+ return 0;
+}
+#define arch_calc_vm_prot_bits arch_calc_vm_prot_bits
+
+static inline unsigned long arch_calc_vm_flag_bits(unsigned long flags)
+{
+ if (!system_supports_mte())
+ return 0;
+
+ /*
+ * Only allow MTE on anonymous mappings as these are guaranteed to be
+ * backed by tags-capable memory. The vm_flags may be overridden by a
+ * filesystem supporting MTE (RAM-based).
+ */
+ if (flags & MAP_ANONYMOUS)
+ return VM_MTE_ALLOWED;
+
+ return 0;
+}
+#define arch_calc_vm_flag_bits arch_calc_vm_flag_bits
+
+static inline pgprot_t arch_vm_get_page_prot(unsigned long vm_flags)
+{
+ return (vm_flags & VM_MTE) && (vm_flags & VM_MTE_ALLOWED) ?
+ __pgprot(PTE_ATTRINDX(MT_NORMAL_TAGGED)) :
+ __pgprot(0);
+}
+#define arch_vm_get_page_prot arch_vm_get_page_prot
+
+static inline bool arch_validate_prot(unsigned long prot, unsigned long addr)
+{
+ unsigned long supported = PROT_READ | PROT_WRITE | PROT_EXEC | PROT_SEM;
+
+ if (system_supports_mte())
+ supported |= PROT_MTE;
+
+ return (prot & ~supported) == 0;
+}
+#define arch_validate_prot arch_validate_prot
+
+#endif /* !__ASM_MMAN_H__ */
diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h
index c01b52add377..673033e0393b 100644
--- a/arch/arm64/include/asm/page.h
+++ b/arch/arm64/include/asm/page.h
@@ -36,7 +36,7 @@ extern int pfn_valid(unsigned long);
#endif /* !__ASSEMBLY__ */
-#define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_TSK_EXEC
+#define VM_DATA_DEFAULT_FLAGS (VM_DATA_FLAGS_TSK_EXEC | VM_MTE_ALLOWED)
#include <asm-generic/getorder.h>
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 647a3f0c7874..f2cd59b01b27 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -665,8 +665,13 @@ static inline phys_addr_t pgd_page_paddr(pgd_t pgd)
static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
{
+ /*
+ * Normal and Normal-Tagged are two different memory types and indices
+ * in MAIR_EL1. The mask below has to include PTE_ATTRINDX_MASK.
+ */
const pteval_t mask = PTE_USER | PTE_PXN | PTE_UXN | PTE_RDONLY |
- PTE_PROT_NONE | PTE_VALID | PTE_WRITE;
+ PTE_PROT_NONE | PTE_VALID | PTE_WRITE |
+ PTE_ATTRINDX_MASK;
/* preserve the hardware dirty information */
if (pte_hw_dirty(pte))
pte = pte_mkdirty(pte);
diff --git a/arch/arm64/include/uapi/asm/mman.h b/arch/arm64/include/uapi/asm/mman.h
new file mode 100644
index 000000000000..d7677ee84878
--- /dev/null
+++ b/arch/arm64/include/uapi/asm/mman.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+#ifndef _UAPI__ASM_MMAN_H
+#define _UAPI__ASM_MMAN_H
+
+#include <asm-generic/mman.h>
+
+/*
+ * The generic mman.h file reserves 0x10 and 0x20 for arch-specific PROT_*
+ * flags.
+ */
+/* 0x10 reserved for PROT_BTI */
+#define PROT_MTE 0x20 /* Normal Tagged mapping */
+
+#endif /* !_UAPI__ASM_MMAN_H */
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 8d382d4ec067..2f26112ebb77 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -647,6 +647,10 @@ static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
[ilog2(VM_MERGEABLE)] = "mg",
[ilog2(VM_UFFD_MISSING)]= "um",
[ilog2(VM_UFFD_WP)] = "uw",
+#ifdef CONFIG_ARM64_MTE
+ [ilog2(VM_MTE)] = "mt",
+ [ilog2(VM_MTE_ALLOWED)] = "",
+#endif
#ifdef CONFIG_ARCH_HAS_PKEYS
/* These come out via ProtectionKey: */
[ilog2(VM_PKEY_BIT0)] = "",
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 5a323422d783..132ca88e407d 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -336,6 +336,14 @@ extern unsigned int kobjsize(const void *objp);
# define VM_MPX VM_NONE
#endif
+#if defined(CONFIG_ARM64_MTE)
+# define VM_MTE VM_HIGH_ARCH_0 /* Use Tagged memory for access control */
+# define VM_MTE_ALLOWED VM_HIGH_ARCH_1 /* Tagged memory permitted */
+#else
+# define VM_MTE VM_NONE
+# define VM_MTE_ALLOWED VM_NONE
+#endif
+
#ifndef VM_GROWSUP
# define VM_GROWSUP VM_NONE
#endif
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
* [PATCH v4 10/26] mm: Introduce arch_calc_vm_flag_bits()
From: Catalin Marinas @ 2020-05-15 17:15 UTC (permalink / raw)
To: linux-arm-kernel
Cc: linux-arch, Szabolcs Nagy, Andrey Konovalov, Kevin Brodsky,
Peter Collingbourne, linux-mm, Andrew Morton, Vincenzo Frascino,
Will Deacon, Dave P Martin
In-Reply-To: <20200515171612.1020-1-catalin.marinas@arm.com>
From: Kevin Brodsky <Kevin.Brodsky@arm.com>
Similarly to arch_calc_vm_prot_bits(), introduce a dummy
arch_calc_vm_flag_bits() invoked from calc_vm_flag_bits(). This macro
can be overridden by architectures to insert specific VM_* flags derived
from the mmap() MAP_* flags.
Signed-off-by: Kevin Brodsky <Kevin.Brodsky@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
---
Notes:
v2:
- Updated the comment above arch_calc_vm_prot_bits().
- Changed author since this patch had already been posted (internally).
include/linux/mman.h | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/include/linux/mman.h b/include/linux/mman.h
index 4b08e9c9c538..15c1162b9d65 100644
--- a/include/linux/mman.h
+++ b/include/linux/mman.h
@@ -74,13 +74,17 @@ static inline void vm_unacct_memory(long pages)
}
/*
- * Allow architectures to handle additional protection bits
+ * Allow architectures to handle additional protection and flag bits
*/
#ifndef arch_calc_vm_prot_bits
#define arch_calc_vm_prot_bits(prot, pkey) 0
#endif
+#ifndef arch_calc_vm_flag_bits
+#define arch_calc_vm_flag_bits(flags) 0
+#endif
+
#ifndef arch_vm_get_page_prot
#define arch_vm_get_page_prot(vm_flags) __pgprot(0)
#endif
@@ -131,7 +135,8 @@ calc_vm_flag_bits(unsigned long flags)
return _calc_vm_trans(flags, MAP_GROWSDOWN, VM_GROWSDOWN ) |
_calc_vm_trans(flags, MAP_DENYWRITE, VM_DENYWRITE ) |
_calc_vm_trans(flags, MAP_LOCKED, VM_LOCKED ) |
- _calc_vm_trans(flags, MAP_SYNC, VM_SYNC );
+ _calc_vm_trans(flags, MAP_SYNC, VM_SYNC ) |
+ arch_calc_vm_flag_bits(flags);
}
unsigned long vm_commit_limit(void);
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox