* [RFC v3 24/27] x86/mm: Make the x86 GOT read-only
From: Thomas Garnier via Virtualization @ 2017-10-04 21:20 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
The GOT is changed during early boot when relocations are applied. Make
it read-only directly. This table exists only for PIE binary.
Position Independent Executable (PIE) support will allow to extended the
KASLR randomization range below the -2G memory limit.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
include/asm-generic/vmlinux.lds.h | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index e549bff87c5b..a2301c292e26 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -279,6 +279,17 @@
VMLINUX_SYMBOL(__end_ro_after_init) = .;
#endif
+#ifdef CONFIG_X86_PIE
+#define RO_GOT_X86 \
+ .got : AT(ADDR(.got) - LOAD_OFFSET) { \
+ VMLINUX_SYMBOL(__start_got) = .; \
+ *(.got); \
+ VMLINUX_SYMBOL(__end_got) = .; \
+ }
+#else
+#define RO_GOT_X86
+#endif
+
/*
* Read only Data
*/
@@ -335,6 +346,7 @@
VMLINUX_SYMBOL(__end_builtin_fw) = .; \
} \
\
+ RO_GOT_X86 \
TRACEDATA \
\
/* Kernel symbol table: Normal symbols */ \
--
2.14.2.920.gcf0c67979c-goog
^ permalink raw reply related
* [RFC v3 24/27] x86/mm: Make the x86 GOT read-only
From: Thomas Garnier @ 2017-10-04 21:20 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter, Bor
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
The GOT is changed during early boot when relocations are applied. Make
it read-only directly. This table exists only for PIE binary.
Position Independent Executable (PIE) support will allow to extended the
KASLR randomization range below the -2G memory limit.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
include/asm-generic/vmlinux.lds.h | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index e549bff87c5b..a2301c292e26 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -279,6 +279,17 @@
VMLINUX_SYMBOL(__end_ro_after_init) = .;
#endif
+#ifdef CONFIG_X86_PIE
+#define RO_GOT_X86 \
+ .got : AT(ADDR(.got) - LOAD_OFFSET) { \
+ VMLINUX_SYMBOL(__start_got) = .; \
+ *(.got); \
+ VMLINUX_SYMBOL(__end_got) = .; \
+ }
+#else
+#define RO_GOT_X86
+#endif
+
/*
* Read only Data
*/
@@ -335,6 +346,7 @@
VMLINUX_SYMBOL(__end_builtin_fw) = .; \
} \
\
+ RO_GOT_X86 \
TRACEDATA \
\
/* Kernel symbol table: Normal symbols */ \
--
2.14.2.920.gcf0c67979c-goog
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related
* [RFC v3 24/27] x86/mm: Make the x86 GOT read-only
From: Thomas Garnier @ 2017-10-04 21:20 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
The GOT is changed during early boot when relocations are applied. Make
it read-only directly. This table exists only for PIE binary.
Position Independent Executable (PIE) support will allow to extended the
KASLR randomization range below the -2G memory limit.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
include/asm-generic/vmlinux.lds.h | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index e549bff87c5b..a2301c292e26 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -279,6 +279,17 @@
VMLINUX_SYMBOL(__end_ro_after_init) = .;
#endif
+#ifdef CONFIG_X86_PIE
+#define RO_GOT_X86 \
+ .got : AT(ADDR(.got) - LOAD_OFFSET) { \
+ VMLINUX_SYMBOL(__start_got) = .; \
+ *(.got); \
+ VMLINUX_SYMBOL(__end_got) = .; \
+ }
+#else
+#define RO_GOT_X86
+#endif
+
/*
* Read only Data
*/
@@ -335,6 +346,7 @@
VMLINUX_SYMBOL(__end_builtin_fw) = .; \
} \
\
+ RO_GOT_X86 \
TRACEDATA \
\
/* Kernel symbol table: Normal symbols */ \
--
2.14.2.920.gcf0c67979c-goog
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related
* [kernel-hardening] [RFC v3 24/27] x86/mm: Make the x86 GOT read-only
From: Thomas Garnier @ 2017-10-04 21:20 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter, Boris Ostrovsky, Alexey Dobriyan,
Andrew Morton, Paul Gortmaker, Chris Metcalf, Paul E . McKenney,
Nicolas Pitre, Borislav Petkov, Luis R . Rodriguez,
Greg Kroah-Hartman, Christopher Li, Steven Rostedt, Jason Baron,
Dou Liyang, Rafael J . Wysocki, Mika Westerberg, Lukas Wunner,
Masahiro Yamada, Alexei Starovoitov, Daniel Borkmann,
Markus Trippelsdorf, Paolo Bonzini, Radim Krčmář,
Joerg Roedel, Rik van Riel, David Howells, Ard Biesheuvel,
Waiman Long, Kyle Huey, Andrey Ryabinin, Jonathan Corbet,
Matthew Wilcox, Michal Hocko, Peter Foley, Paul Bolle,
Jiri Kosina, Rob Landley, H . J . Lu, Baoquan He,
Jan H . Schönherr, Daniel Micay
Cc: x86, linux-crypto, linux-kernel, linux-pm, virtualization,
xen-devel, linux-arch, linux-sparse, kvm, linux-doc,
kernel-hardening
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
The GOT is changed during early boot when relocations are applied. Make
it read-only directly. This table exists only for PIE binary.
Position Independent Executable (PIE) support will allow to extended the
KASLR randomization range below the -2G memory limit.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
include/asm-generic/vmlinux.lds.h | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index e549bff87c5b..a2301c292e26 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -279,6 +279,17 @@
VMLINUX_SYMBOL(__end_ro_after_init) = .;
#endif
+#ifdef CONFIG_X86_PIE
+#define RO_GOT_X86 \
+ .got : AT(ADDR(.got) - LOAD_OFFSET) { \
+ VMLINUX_SYMBOL(__start_got) = .; \
+ *(.got); \
+ VMLINUX_SYMBOL(__end_got) = .; \
+ }
+#else
+#define RO_GOT_X86
+#endif
+
/*
* Read only Data
*/
@@ -335,6 +346,7 @@
VMLINUX_SYMBOL(__end_builtin_fw) = .; \
} \
\
+ RO_GOT_X86 \
TRACEDATA \
\
/* Kernel symbol table: Normal symbols */ \
--
2.14.2.920.gcf0c67979c-goog
^ permalink raw reply related
* [RFC v3 23/27] x86/modules: Adapt module loading for PIE support
From: Thomas Garnier via Virtualization @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
Adapt module loading to support PIE relocations. Generate dynamic GOT if
a symbol requires it but no entry exist in the kernel GOT.
Position Independent Executable (PIE) support will allow to extended the
KASLR randomization range below the -2G memory limit.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
arch/x86/Makefile | 4 +
arch/x86/include/asm/module.h | 14 +++
arch/x86/kernel/module.c | 204 ++++++++++++++++++++++++++++++++++++++++--
3 files changed, 217 insertions(+), 5 deletions(-)
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 4cb4f0495ddc..42774185a58a 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -143,7 +143,11 @@ else
KBUILD_CFLAGS += $(cflags-y)
KBUILD_CFLAGS += -mno-red-zone
+ifdef CONFIG_X86_PIE
+ KBUILD_LDFLAGS_MODULE += -T $(srctree)/arch/x86/kernel/module.lds
+else
KBUILD_CFLAGS += -mcmodel=kernel
+endif
# -funit-at-a-time shrinks the kernel .text considerably
# unfortunately it makes reading oopses harder.
diff --git a/arch/x86/include/asm/module.h b/arch/x86/include/asm/module.h
index 9eb7c718aaf8..8e0bd52bbadf 100644
--- a/arch/x86/include/asm/module.h
+++ b/arch/x86/include/asm/module.h
@@ -4,12 +4,23 @@
#include <asm-generic/module.h>
#include <asm/orc_types.h>
+#ifdef CONFIG_X86_PIE
+struct mod_got_sec {
+ struct elf64_shdr *got;
+ int got_num_entries;
+ int got_max_entries;
+};
+#endif
+
struct mod_arch_specific {
#ifdef CONFIG_ORC_UNWINDER
unsigned int num_orcs;
int *orc_unwind_ip;
struct orc_entry *orc_unwind;
#endif
+#ifdef CONFIG_X86_PIE
+ struct mod_got_sec core;
+#endif
};
#ifdef CONFIG_X86_64
@@ -70,4 +81,7 @@ struct mod_arch_specific {
# define MODULE_ARCH_VERMAGIC MODULE_PROC_FAMILY
#endif
+
+u64 module_find_got_entry(struct module *mod, u64 addr);
+
#endif /* _ASM_X86_MODULE_H */
diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
index 62e7d70aadd5..3b9b43a9d63b 100644
--- a/arch/x86/kernel/module.c
+++ b/arch/x86/kernel/module.c
@@ -30,6 +30,7 @@
#include <linux/gfp.h>
#include <linux/jump_label.h>
#include <linux/random.h>
+#include <linux/sort.h>
#include <asm/text-patching.h>
#include <asm/page.h>
@@ -77,6 +78,195 @@ static unsigned long int get_module_load_offset(void)
}
#endif
+#ifdef CONFIG_X86_PIE
+static u64 find_got_kernel_entry(Elf64_Sym *sym, const Elf64_Rela *rela)
+{
+ u64 *pos;
+
+ for (pos = (u64*)__start_got; pos < (u64*)__end_got; pos++) {
+ if (*pos == sym->st_value)
+ return (u64)pos + rela->r_addend;
+ }
+
+ return 0;
+}
+
+/* Search the GOT entry for a specific address and a module (optional) */
+u64 module_find_got_entry(struct module *mod, u64 addr)
+{
+ int i;
+ u64 *pos;
+
+ for (pos = (u64*)__start_got; pos < (u64*)__end_got; pos++) {
+ if (*pos == addr)
+ return (u64)pos;
+ }
+
+ if (!mod)
+ return 0;
+
+ pos = (u64*)mod->arch.core.got->sh_addr;
+ for (i = 0; i < mod->arch.core.got_num_entries; i++) {
+ if (pos[i] == addr)
+ return (u64)&pos[i];
+ }
+ return 0;
+}
+
+static u64 module_emit_got_entry(struct module *mod, void *loc,
+ const Elf64_Rela *rela, Elf64_Sym *sym)
+{
+ struct mod_got_sec *gotsec = &mod->arch.core;
+ u64 *got = (u64*)gotsec->got->sh_addr;
+ int i = gotsec->got_num_entries;
+ u64 ret;
+
+ /* Check if we can use the kernel GOT */
+ ret = find_got_kernel_entry(sym, rela);
+ if (ret)
+ return ret;
+
+ got[i] = sym->st_value;
+
+ /*
+ * Check if the entry we just created is a duplicate. Given that the
+ * relocations are sorted, this will be the last entry we allocated.
+ * (if one exists).
+ */
+ if (i > 0 && got[i] == got[i - 2]) {
+ ret = (u64)&got[i - 1];
+ } else {
+ gotsec->got_num_entries++;
+ BUG_ON(gotsec->got_num_entries > gotsec->got_max_entries);
+ ret = (u64)&got[i];
+ }
+
+ return ret + rela->r_addend;
+}
+
+#define cmp_3way(a,b) ((a) < (b) ? -1 : (a) > (b))
+
+static int cmp_rela(const void *a, const void *b)
+{
+ const Elf64_Rela *x = a, *y = b;
+ int i;
+
+ /* sort by type, symbol index and addend */
+ i = cmp_3way(ELF64_R_TYPE(x->r_info), ELF64_R_TYPE(y->r_info));
+ if (i == 0)
+ i = cmp_3way(ELF64_R_SYM(x->r_info), ELF64_R_SYM(y->r_info));
+ if (i == 0)
+ i = cmp_3way(x->r_addend, y->r_addend);
+ return i;
+}
+
+static bool duplicate_rel(const Elf64_Rela *rela, int num)
+{
+ /*
+ * Entries are sorted by type, symbol index and addend. That means
+ * that, if a duplicate entry exists, it must be in the preceding
+ * slot.
+ */
+ return num > 0 && cmp_rela(rela + num, rela + num - 1) == 0;
+}
+
+static unsigned int count_gots(Elf64_Sym *syms, Elf64_Rela *rela, int num)
+{
+ unsigned int ret = 0;
+ Elf64_Sym *s;
+ int i;
+
+ for (i = 0; i < num; i++) {
+ switch (ELF64_R_TYPE(rela[i].r_info)) {
+ case R_X86_64_GOTPCREL:
+ s = syms + ELF64_R_SYM(rela[i].r_info);
+
+ /*
+ * Use the kernel GOT when possible, else reserve a
+ * custom one for this module.
+ */
+ if (!duplicate_rel(rela, i) &&
+ !find_got_kernel_entry(s, rela + i))
+ ret++;
+ break;
+ }
+ }
+ return ret;
+}
+
+/*
+ * Generate GOT entries for GOTPCREL relocations that do not exists in the
+ * kernel GOT. Based on arm64 module-plts implementation.
+ */
+int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
+ char *secstrings, struct module *mod)
+{
+ unsigned long gots = 0;
+ Elf_Shdr *symtab = NULL;
+ Elf64_Sym *syms = NULL;
+ char *strings, *name;
+ int i;
+
+ /*
+ * Find the empty .got section so we can expand it to store the PLT
+ * entries. Record the symtab address as well.
+ */
+ for (i = 0; i < ehdr->e_shnum; i++) {
+ if (!strcmp(secstrings + sechdrs[i].sh_name, ".got")) {
+ mod->arch.core.got = sechdrs + i;
+ } else if (sechdrs[i].sh_type == SHT_SYMTAB) {
+ symtab = sechdrs + i;
+ syms = (Elf64_Sym *)symtab->sh_addr;
+ }
+ }
+
+ if (!mod->arch.core.got) {
+ pr_err("%s: module GOT section missing\n", mod->name);
+ return -ENOEXEC;
+ }
+ if (!syms) {
+ pr_err("%s: module symtab section missing\n", mod->name);
+ return -ENOEXEC;
+ }
+
+ for (i = 0; i < ehdr->e_shnum; i++) {
+ Elf64_Rela *rels = (void *)ehdr + sechdrs[i].sh_offset;
+ int numrels = sechdrs[i].sh_size / sizeof(Elf64_Rela);
+
+ if (sechdrs[i].sh_type != SHT_RELA)
+ continue;
+
+ /* sort by type, symbol index and addend */
+ sort(rels, numrels, sizeof(Elf64_Rela), cmp_rela, NULL);
+
+ gots += count_gots(syms, rels, numrels);
+ }
+
+ mod->arch.core.got->sh_type = SHT_NOBITS;
+ mod->arch.core.got->sh_flags = SHF_ALLOC;
+ mod->arch.core.got->sh_addralign = L1_CACHE_BYTES;
+ mod->arch.core.got->sh_size = (gots + 1) * sizeof(u64);
+ mod->arch.core.got_num_entries = 0;
+ mod->arch.core.got_max_entries = gots;
+
+ /*
+ * If a _GLOBAL_OFFSET_TABLE_ symbol exists, make it absolute for
+ * modules to correctly reference it. Similar to s390 implementation.
+ */
+ strings = (void *) ehdr + sechdrs[symtab->sh_link].sh_offset;
+ for (i = 0; i < symtab->sh_size/sizeof(Elf_Sym); i++) {
+ if (syms[i].st_shndx != SHN_UNDEF)
+ continue;
+ name = strings + syms[i].st_name;
+ if (!strcmp(name, "_GLOBAL_OFFSET_TABLE_")) {
+ syms[i].st_shndx = SHN_ABS;
+ break;
+ }
+ }
+ return 0;
+}
+#endif
+
void *module_alloc(unsigned long size)
{
void *p;
@@ -184,13 +374,18 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
if ((s64)val != *(s32 *)loc)
goto overflow;
break;
+#ifdef CONFIG_X86_PIE
+ case R_X86_64_GOTPCREL:
+ val = module_emit_got_entry(me, loc, rel + i, sym);
+ /* fallthrough */
+#endif
+ case R_X86_64_PLT32:
case R_X86_64_PC32:
val -= (u64)loc;
*(u32 *)loc = val;
-#if 0
- if ((s64)val != *(s32 *)loc)
+ if (IS_ENABLED(CONFIG_X86_PIE) &&
+ (s64)val != *(s32 *)loc)
goto overflow;
-#endif
break;
default:
pr_err("%s: Unknown rela relocation: %llu\n",
@@ -203,8 +398,7 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
overflow:
pr_err("overflow in relocation type %d val %Lx\n",
(int)ELF64_R_TYPE(rel[i].r_info), val);
- pr_err("`%s' likely not compiled with -mcmodel=kernel\n",
- me->name);
+ pr_err("`%s' likely too far from the kernel\n", me->name);
return -ENOEXEC;
}
#endif
--
2.14.2.920.gcf0c67979c-goog
^ permalink raw reply related
* [RFC v3 23/27] x86/modules: Adapt module loading for PIE support
From: Thomas Garnier @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter, Bor
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
Adapt module loading to support PIE relocations. Generate dynamic GOT if
a symbol requires it but no entry exist in the kernel GOT.
Position Independent Executable (PIE) support will allow to extended the
KASLR randomization range below the -2G memory limit.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
arch/x86/Makefile | 4 +
arch/x86/include/asm/module.h | 14 +++
arch/x86/kernel/module.c | 204 ++++++++++++++++++++++++++++++++++++++++--
3 files changed, 217 insertions(+), 5 deletions(-)
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 4cb4f0495ddc..42774185a58a 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -143,7 +143,11 @@ else
KBUILD_CFLAGS += $(cflags-y)
KBUILD_CFLAGS += -mno-red-zone
+ifdef CONFIG_X86_PIE
+ KBUILD_LDFLAGS_MODULE += -T $(srctree)/arch/x86/kernel/module.lds
+else
KBUILD_CFLAGS += -mcmodel=kernel
+endif
# -funit-at-a-time shrinks the kernel .text considerably
# unfortunately it makes reading oopses harder.
diff --git a/arch/x86/include/asm/module.h b/arch/x86/include/asm/module.h
index 9eb7c718aaf8..8e0bd52bbadf 100644
--- a/arch/x86/include/asm/module.h
+++ b/arch/x86/include/asm/module.h
@@ -4,12 +4,23 @@
#include <asm-generic/module.h>
#include <asm/orc_types.h>
+#ifdef CONFIG_X86_PIE
+struct mod_got_sec {
+ struct elf64_shdr *got;
+ int got_num_entries;
+ int got_max_entries;
+};
+#endif
+
struct mod_arch_specific {
#ifdef CONFIG_ORC_UNWINDER
unsigned int num_orcs;
int *orc_unwind_ip;
struct orc_entry *orc_unwind;
#endif
+#ifdef CONFIG_X86_PIE
+ struct mod_got_sec core;
+#endif
};
#ifdef CONFIG_X86_64
@@ -70,4 +81,7 @@ struct mod_arch_specific {
# define MODULE_ARCH_VERMAGIC MODULE_PROC_FAMILY
#endif
+
+u64 module_find_got_entry(struct module *mod, u64 addr);
+
#endif /* _ASM_X86_MODULE_H */
diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
index 62e7d70aadd5..3b9b43a9d63b 100644
--- a/arch/x86/kernel/module.c
+++ b/arch/x86/kernel/module.c
@@ -30,6 +30,7 @@
#include <linux/gfp.h>
#include <linux/jump_label.h>
#include <linux/random.h>
+#include <linux/sort.h>
#include <asm/text-patching.h>
#include <asm/page.h>
@@ -77,6 +78,195 @@ static unsigned long int get_module_load_offset(void)
}
#endif
+#ifdef CONFIG_X86_PIE
+static u64 find_got_kernel_entry(Elf64_Sym *sym, const Elf64_Rela *rela)
+{
+ u64 *pos;
+
+ for (pos = (u64*)__start_got; pos < (u64*)__end_got; pos++) {
+ if (*pos == sym->st_value)
+ return (u64)pos + rela->r_addend;
+ }
+
+ return 0;
+}
+
+/* Search the GOT entry for a specific address and a module (optional) */
+u64 module_find_got_entry(struct module *mod, u64 addr)
+{
+ int i;
+ u64 *pos;
+
+ for (pos = (u64*)__start_got; pos < (u64*)__end_got; pos++) {
+ if (*pos == addr)
+ return (u64)pos;
+ }
+
+ if (!mod)
+ return 0;
+
+ pos = (u64*)mod->arch.core.got->sh_addr;
+ for (i = 0; i < mod->arch.core.got_num_entries; i++) {
+ if (pos[i] == addr)
+ return (u64)&pos[i];
+ }
+ return 0;
+}
+
+static u64 module_emit_got_entry(struct module *mod, void *loc,
+ const Elf64_Rela *rela, Elf64_Sym *sym)
+{
+ struct mod_got_sec *gotsec = &mod->arch.core;
+ u64 *got = (u64*)gotsec->got->sh_addr;
+ int i = gotsec->got_num_entries;
+ u64 ret;
+
+ /* Check if we can use the kernel GOT */
+ ret = find_got_kernel_entry(sym, rela);
+ if (ret)
+ return ret;
+
+ got[i] = sym->st_value;
+
+ /*
+ * Check if the entry we just created is a duplicate. Given that the
+ * relocations are sorted, this will be the last entry we allocated.
+ * (if one exists).
+ */
+ if (i > 0 && got[i] == got[i - 2]) {
+ ret = (u64)&got[i - 1];
+ } else {
+ gotsec->got_num_entries++;
+ BUG_ON(gotsec->got_num_entries > gotsec->got_max_entries);
+ ret = (u64)&got[i];
+ }
+
+ return ret + rela->r_addend;
+}
+
+#define cmp_3way(a,b) ((a) < (b) ? -1 : (a) > (b))
+
+static int cmp_rela(const void *a, const void *b)
+{
+ const Elf64_Rela *x = a, *y = b;
+ int i;
+
+ /* sort by type, symbol index and addend */
+ i = cmp_3way(ELF64_R_TYPE(x->r_info), ELF64_R_TYPE(y->r_info));
+ if (i == 0)
+ i = cmp_3way(ELF64_R_SYM(x->r_info), ELF64_R_SYM(y->r_info));
+ if (i == 0)
+ i = cmp_3way(x->r_addend, y->r_addend);
+ return i;
+}
+
+static bool duplicate_rel(const Elf64_Rela *rela, int num)
+{
+ /*
+ * Entries are sorted by type, symbol index and addend. That means
+ * that, if a duplicate entry exists, it must be in the preceding
+ * slot.
+ */
+ return num > 0 && cmp_rela(rela + num, rela + num - 1) == 0;
+}
+
+static unsigned int count_gots(Elf64_Sym *syms, Elf64_Rela *rela, int num)
+{
+ unsigned int ret = 0;
+ Elf64_Sym *s;
+ int i;
+
+ for (i = 0; i < num; i++) {
+ switch (ELF64_R_TYPE(rela[i].r_info)) {
+ case R_X86_64_GOTPCREL:
+ s = syms + ELF64_R_SYM(rela[i].r_info);
+
+ /*
+ * Use the kernel GOT when possible, else reserve a
+ * custom one for this module.
+ */
+ if (!duplicate_rel(rela, i) &&
+ !find_got_kernel_entry(s, rela + i))
+ ret++;
+ break;
+ }
+ }
+ return ret;
+}
+
+/*
+ * Generate GOT entries for GOTPCREL relocations that do not exists in the
+ * kernel GOT. Based on arm64 module-plts implementation.
+ */
+int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
+ char *secstrings, struct module *mod)
+{
+ unsigned long gots = 0;
+ Elf_Shdr *symtab = NULL;
+ Elf64_Sym *syms = NULL;
+ char *strings, *name;
+ int i;
+
+ /*
+ * Find the empty .got section so we can expand it to store the PLT
+ * entries. Record the symtab address as well.
+ */
+ for (i = 0; i < ehdr->e_shnum; i++) {
+ if (!strcmp(secstrings + sechdrs[i].sh_name, ".got")) {
+ mod->arch.core.got = sechdrs + i;
+ } else if (sechdrs[i].sh_type == SHT_SYMTAB) {
+ symtab = sechdrs + i;
+ syms = (Elf64_Sym *)symtab->sh_addr;
+ }
+ }
+
+ if (!mod->arch.core.got) {
+ pr_err("%s: module GOT section missing\n", mod->name);
+ return -ENOEXEC;
+ }
+ if (!syms) {
+ pr_err("%s: module symtab section missing\n", mod->name);
+ return -ENOEXEC;
+ }
+
+ for (i = 0; i < ehdr->e_shnum; i++) {
+ Elf64_Rela *rels = (void *)ehdr + sechdrs[i].sh_offset;
+ int numrels = sechdrs[i].sh_size / sizeof(Elf64_Rela);
+
+ if (sechdrs[i].sh_type != SHT_RELA)
+ continue;
+
+ /* sort by type, symbol index and addend */
+ sort(rels, numrels, sizeof(Elf64_Rela), cmp_rela, NULL);
+
+ gots += count_gots(syms, rels, numrels);
+ }
+
+ mod->arch.core.got->sh_type = SHT_NOBITS;
+ mod->arch.core.got->sh_flags = SHF_ALLOC;
+ mod->arch.core.got->sh_addralign = L1_CACHE_BYTES;
+ mod->arch.core.got->sh_size = (gots + 1) * sizeof(u64);
+ mod->arch.core.got_num_entries = 0;
+ mod->arch.core.got_max_entries = gots;
+
+ /*
+ * If a _GLOBAL_OFFSET_TABLE_ symbol exists, make it absolute for
+ * modules to correctly reference it. Similar to s390 implementation.
+ */
+ strings = (void *) ehdr + sechdrs[symtab->sh_link].sh_offset;
+ for (i = 0; i < symtab->sh_size/sizeof(Elf_Sym); i++) {
+ if (syms[i].st_shndx != SHN_UNDEF)
+ continue;
+ name = strings + syms[i].st_name;
+ if (!strcmp(name, "_GLOBAL_OFFSET_TABLE_")) {
+ syms[i].st_shndx = SHN_ABS;
+ break;
+ }
+ }
+ return 0;
+}
+#endif
+
void *module_alloc(unsigned long size)
{
void *p;
@@ -184,13 +374,18 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
if ((s64)val != *(s32 *)loc)
goto overflow;
break;
+#ifdef CONFIG_X86_PIE
+ case R_X86_64_GOTPCREL:
+ val = module_emit_got_entry(me, loc, rel + i, sym);
+ /* fallthrough */
+#endif
+ case R_X86_64_PLT32:
case R_X86_64_PC32:
val -= (u64)loc;
*(u32 *)loc = val;
-#if 0
- if ((s64)val != *(s32 *)loc)
+ if (IS_ENABLED(CONFIG_X86_PIE) &&
+ (s64)val != *(s32 *)loc)
goto overflow;
-#endif
break;
default:
pr_err("%s: Unknown rela relocation: %llu\n",
@@ -203,8 +398,7 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
overflow:
pr_err("overflow in relocation type %d val %Lx\n",
(int)ELF64_R_TYPE(rel[i].r_info), val);
- pr_err("`%s' likely not compiled with -mcmodel=kernel\n",
- me->name);
+ pr_err("`%s' likely too far from the kernel\n", me->name);
return -ENOEXEC;
}
#endif
--
2.14.2.920.gcf0c67979c-goog
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related
* [RFC v3 23/27] x86/modules: Adapt module loading for PIE support
From: Thomas Garnier @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
Adapt module loading to support PIE relocations. Generate dynamic GOT if
a symbol requires it but no entry exist in the kernel GOT.
Position Independent Executable (PIE) support will allow to extended the
KASLR randomization range below the -2G memory limit.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
arch/x86/Makefile | 4 +
arch/x86/include/asm/module.h | 14 +++
arch/x86/kernel/module.c | 204 ++++++++++++++++++++++++++++++++++++++++--
3 files changed, 217 insertions(+), 5 deletions(-)
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 4cb4f0495ddc..42774185a58a 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -143,7 +143,11 @@ else
KBUILD_CFLAGS += $(cflags-y)
KBUILD_CFLAGS += -mno-red-zone
+ifdef CONFIG_X86_PIE
+ KBUILD_LDFLAGS_MODULE += -T $(srctree)/arch/x86/kernel/module.lds
+else
KBUILD_CFLAGS += -mcmodel=kernel
+endif
# -funit-at-a-time shrinks the kernel .text considerably
# unfortunately it makes reading oopses harder.
diff --git a/arch/x86/include/asm/module.h b/arch/x86/include/asm/module.h
index 9eb7c718aaf8..8e0bd52bbadf 100644
--- a/arch/x86/include/asm/module.h
+++ b/arch/x86/include/asm/module.h
@@ -4,12 +4,23 @@
#include <asm-generic/module.h>
#include <asm/orc_types.h>
+#ifdef CONFIG_X86_PIE
+struct mod_got_sec {
+ struct elf64_shdr *got;
+ int got_num_entries;
+ int got_max_entries;
+};
+#endif
+
struct mod_arch_specific {
#ifdef CONFIG_ORC_UNWINDER
unsigned int num_orcs;
int *orc_unwind_ip;
struct orc_entry *orc_unwind;
#endif
+#ifdef CONFIG_X86_PIE
+ struct mod_got_sec core;
+#endif
};
#ifdef CONFIG_X86_64
@@ -70,4 +81,7 @@ struct mod_arch_specific {
# define MODULE_ARCH_VERMAGIC MODULE_PROC_FAMILY
#endif
+
+u64 module_find_got_entry(struct module *mod, u64 addr);
+
#endif /* _ASM_X86_MODULE_H */
diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
index 62e7d70aadd5..3b9b43a9d63b 100644
--- a/arch/x86/kernel/module.c
+++ b/arch/x86/kernel/module.c
@@ -30,6 +30,7 @@
#include <linux/gfp.h>
#include <linux/jump_label.h>
#include <linux/random.h>
+#include <linux/sort.h>
#include <asm/text-patching.h>
#include <asm/page.h>
@@ -77,6 +78,195 @@ static unsigned long int get_module_load_offset(void)
}
#endif
+#ifdef CONFIG_X86_PIE
+static u64 find_got_kernel_entry(Elf64_Sym *sym, const Elf64_Rela *rela)
+{
+ u64 *pos;
+
+ for (pos = (u64*)__start_got; pos < (u64*)__end_got; pos++) {
+ if (*pos == sym->st_value)
+ return (u64)pos + rela->r_addend;
+ }
+
+ return 0;
+}
+
+/* Search the GOT entry for a specific address and a module (optional) */
+u64 module_find_got_entry(struct module *mod, u64 addr)
+{
+ int i;
+ u64 *pos;
+
+ for (pos = (u64*)__start_got; pos < (u64*)__end_got; pos++) {
+ if (*pos == addr)
+ return (u64)pos;
+ }
+
+ if (!mod)
+ return 0;
+
+ pos = (u64*)mod->arch.core.got->sh_addr;
+ for (i = 0; i < mod->arch.core.got_num_entries; i++) {
+ if (pos[i] == addr)
+ return (u64)&pos[i];
+ }
+ return 0;
+}
+
+static u64 module_emit_got_entry(struct module *mod, void *loc,
+ const Elf64_Rela *rela, Elf64_Sym *sym)
+{
+ struct mod_got_sec *gotsec = &mod->arch.core;
+ u64 *got = (u64*)gotsec->got->sh_addr;
+ int i = gotsec->got_num_entries;
+ u64 ret;
+
+ /* Check if we can use the kernel GOT */
+ ret = find_got_kernel_entry(sym, rela);
+ if (ret)
+ return ret;
+
+ got[i] = sym->st_value;
+
+ /*
+ * Check if the entry we just created is a duplicate. Given that the
+ * relocations are sorted, this will be the last entry we allocated.
+ * (if one exists).
+ */
+ if (i > 0 && got[i] == got[i - 2]) {
+ ret = (u64)&got[i - 1];
+ } else {
+ gotsec->got_num_entries++;
+ BUG_ON(gotsec->got_num_entries > gotsec->got_max_entries);
+ ret = (u64)&got[i];
+ }
+
+ return ret + rela->r_addend;
+}
+
+#define cmp_3way(a,b) ((a) < (b) ? -1 : (a) > (b))
+
+static int cmp_rela(const void *a, const void *b)
+{
+ const Elf64_Rela *x = a, *y = b;
+ int i;
+
+ /* sort by type, symbol index and addend */
+ i = cmp_3way(ELF64_R_TYPE(x->r_info), ELF64_R_TYPE(y->r_info));
+ if (i == 0)
+ i = cmp_3way(ELF64_R_SYM(x->r_info), ELF64_R_SYM(y->r_info));
+ if (i == 0)
+ i = cmp_3way(x->r_addend, y->r_addend);
+ return i;
+}
+
+static bool duplicate_rel(const Elf64_Rela *rela, int num)
+{
+ /*
+ * Entries are sorted by type, symbol index and addend. That means
+ * that, if a duplicate entry exists, it must be in the preceding
+ * slot.
+ */
+ return num > 0 && cmp_rela(rela + num, rela + num - 1) == 0;
+}
+
+static unsigned int count_gots(Elf64_Sym *syms, Elf64_Rela *rela, int num)
+{
+ unsigned int ret = 0;
+ Elf64_Sym *s;
+ int i;
+
+ for (i = 0; i < num; i++) {
+ switch (ELF64_R_TYPE(rela[i].r_info)) {
+ case R_X86_64_GOTPCREL:
+ s = syms + ELF64_R_SYM(rela[i].r_info);
+
+ /*
+ * Use the kernel GOT when possible, else reserve a
+ * custom one for this module.
+ */
+ if (!duplicate_rel(rela, i) &&
+ !find_got_kernel_entry(s, rela + i))
+ ret++;
+ break;
+ }
+ }
+ return ret;
+}
+
+/*
+ * Generate GOT entries for GOTPCREL relocations that do not exists in the
+ * kernel GOT. Based on arm64 module-plts implementation.
+ */
+int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
+ char *secstrings, struct module *mod)
+{
+ unsigned long gots = 0;
+ Elf_Shdr *symtab = NULL;
+ Elf64_Sym *syms = NULL;
+ char *strings, *name;
+ int i;
+
+ /*
+ * Find the empty .got section so we can expand it to store the PLT
+ * entries. Record the symtab address as well.
+ */
+ for (i = 0; i < ehdr->e_shnum; i++) {
+ if (!strcmp(secstrings + sechdrs[i].sh_name, ".got")) {
+ mod->arch.core.got = sechdrs + i;
+ } else if (sechdrs[i].sh_type == SHT_SYMTAB) {
+ symtab = sechdrs + i;
+ syms = (Elf64_Sym *)symtab->sh_addr;
+ }
+ }
+
+ if (!mod->arch.core.got) {
+ pr_err("%s: module GOT section missing\n", mod->name);
+ return -ENOEXEC;
+ }
+ if (!syms) {
+ pr_err("%s: module symtab section missing\n", mod->name);
+ return -ENOEXEC;
+ }
+
+ for (i = 0; i < ehdr->e_shnum; i++) {
+ Elf64_Rela *rels = (void *)ehdr + sechdrs[i].sh_offset;
+ int numrels = sechdrs[i].sh_size / sizeof(Elf64_Rela);
+
+ if (sechdrs[i].sh_type != SHT_RELA)
+ continue;
+
+ /* sort by type, symbol index and addend */
+ sort(rels, numrels, sizeof(Elf64_Rela), cmp_rela, NULL);
+
+ gots += count_gots(syms, rels, numrels);
+ }
+
+ mod->arch.core.got->sh_type = SHT_NOBITS;
+ mod->arch.core.got->sh_flags = SHF_ALLOC;
+ mod->arch.core.got->sh_addralign = L1_CACHE_BYTES;
+ mod->arch.core.got->sh_size = (gots + 1) * sizeof(u64);
+ mod->arch.core.got_num_entries = 0;
+ mod->arch.core.got_max_entries = gots;
+
+ /*
+ * If a _GLOBAL_OFFSET_TABLE_ symbol exists, make it absolute for
+ * modules to correctly reference it. Similar to s390 implementation.
+ */
+ strings = (void *) ehdr + sechdrs[symtab->sh_link].sh_offset;
+ for (i = 0; i < symtab->sh_size/sizeof(Elf_Sym); i++) {
+ if (syms[i].st_shndx != SHN_UNDEF)
+ continue;
+ name = strings + syms[i].st_name;
+ if (!strcmp(name, "_GLOBAL_OFFSET_TABLE_")) {
+ syms[i].st_shndx = SHN_ABS;
+ break;
+ }
+ }
+ return 0;
+}
+#endif
+
void *module_alloc(unsigned long size)
{
void *p;
@@ -184,13 +374,18 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
if ((s64)val != *(s32 *)loc)
goto overflow;
break;
+#ifdef CONFIG_X86_PIE
+ case R_X86_64_GOTPCREL:
+ val = module_emit_got_entry(me, loc, rel + i, sym);
+ /* fallthrough */
+#endif
+ case R_X86_64_PLT32:
case R_X86_64_PC32:
val -= (u64)loc;
*(u32 *)loc = val;
-#if 0
- if ((s64)val != *(s32 *)loc)
+ if (IS_ENABLED(CONFIG_X86_PIE) &&
+ (s64)val != *(s32 *)loc)
goto overflow;
-#endif
break;
default:
pr_err("%s: Unknown rela relocation: %llu\n",
@@ -203,8 +398,7 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
overflow:
pr_err("overflow in relocation type %d val %Lx\n",
(int)ELF64_R_TYPE(rel[i].r_info), val);
- pr_err("`%s' likely not compiled with -mcmodel=kernel\n",
- me->name);
+ pr_err("`%s' likely too far from the kernel\n", me->name);
return -ENOEXEC;
}
#endif
--
2.14.2.920.gcf0c67979c-goog
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related
* [kernel-hardening] [RFC v3 23/27] x86/modules: Adapt module loading for PIE support
From: Thomas Garnier @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter, Boris Ostrovsky, Alexey Dobriyan,
Andrew Morton, Paul Gortmaker, Chris Metcalf, Paul E . McKenney,
Nicolas Pitre, Borislav Petkov, Luis R . Rodriguez,
Greg Kroah-Hartman, Christopher Li, Steven Rostedt, Jason Baron,
Dou Liyang, Rafael J . Wysocki, Mika Westerberg, Lukas Wunner,
Masahiro Yamada, Alexei Starovoitov, Daniel Borkmann,
Markus Trippelsdorf, Paolo Bonzini, Radim Krčmář,
Joerg Roedel, Rik van Riel, David Howells, Ard Biesheuvel,
Waiman Long, Kyle Huey, Andrey Ryabinin, Jonathan Corbet,
Matthew Wilcox, Michal Hocko, Peter Foley, Paul Bolle,
Jiri Kosina, Rob Landley, H . J . Lu, Baoquan He,
Jan H . Schönherr, Daniel Micay
Cc: x86, linux-crypto, linux-kernel, linux-pm, virtualization,
xen-devel, linux-arch, linux-sparse, kvm, linux-doc,
kernel-hardening
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
Adapt module loading to support PIE relocations. Generate dynamic GOT if
a symbol requires it but no entry exist in the kernel GOT.
Position Independent Executable (PIE) support will allow to extended the
KASLR randomization range below the -2G memory limit.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
arch/x86/Makefile | 4 +
arch/x86/include/asm/module.h | 14 +++
arch/x86/kernel/module.c | 204 ++++++++++++++++++++++++++++++++++++++++--
3 files changed, 217 insertions(+), 5 deletions(-)
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 4cb4f0495ddc..42774185a58a 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -143,7 +143,11 @@ else
KBUILD_CFLAGS += $(cflags-y)
KBUILD_CFLAGS += -mno-red-zone
+ifdef CONFIG_X86_PIE
+ KBUILD_LDFLAGS_MODULE += -T $(srctree)/arch/x86/kernel/module.lds
+else
KBUILD_CFLAGS += -mcmodel=kernel
+endif
# -funit-at-a-time shrinks the kernel .text considerably
# unfortunately it makes reading oopses harder.
diff --git a/arch/x86/include/asm/module.h b/arch/x86/include/asm/module.h
index 9eb7c718aaf8..8e0bd52bbadf 100644
--- a/arch/x86/include/asm/module.h
+++ b/arch/x86/include/asm/module.h
@@ -4,12 +4,23 @@
#include <asm-generic/module.h>
#include <asm/orc_types.h>
+#ifdef CONFIG_X86_PIE
+struct mod_got_sec {
+ struct elf64_shdr *got;
+ int got_num_entries;
+ int got_max_entries;
+};
+#endif
+
struct mod_arch_specific {
#ifdef CONFIG_ORC_UNWINDER
unsigned int num_orcs;
int *orc_unwind_ip;
struct orc_entry *orc_unwind;
#endif
+#ifdef CONFIG_X86_PIE
+ struct mod_got_sec core;
+#endif
};
#ifdef CONFIG_X86_64
@@ -70,4 +81,7 @@ struct mod_arch_specific {
# define MODULE_ARCH_VERMAGIC MODULE_PROC_FAMILY
#endif
+
+u64 module_find_got_entry(struct module *mod, u64 addr);
+
#endif /* _ASM_X86_MODULE_H */
diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
index 62e7d70aadd5..3b9b43a9d63b 100644
--- a/arch/x86/kernel/module.c
+++ b/arch/x86/kernel/module.c
@@ -30,6 +30,7 @@
#include <linux/gfp.h>
#include <linux/jump_label.h>
#include <linux/random.h>
+#include <linux/sort.h>
#include <asm/text-patching.h>
#include <asm/page.h>
@@ -77,6 +78,195 @@ static unsigned long int get_module_load_offset(void)
}
#endif
+#ifdef CONFIG_X86_PIE
+static u64 find_got_kernel_entry(Elf64_Sym *sym, const Elf64_Rela *rela)
+{
+ u64 *pos;
+
+ for (pos = (u64*)__start_got; pos < (u64*)__end_got; pos++) {
+ if (*pos == sym->st_value)
+ return (u64)pos + rela->r_addend;
+ }
+
+ return 0;
+}
+
+/* Search the GOT entry for a specific address and a module (optional) */
+u64 module_find_got_entry(struct module *mod, u64 addr)
+{
+ int i;
+ u64 *pos;
+
+ for (pos = (u64*)__start_got; pos < (u64*)__end_got; pos++) {
+ if (*pos == addr)
+ return (u64)pos;
+ }
+
+ if (!mod)
+ return 0;
+
+ pos = (u64*)mod->arch.core.got->sh_addr;
+ for (i = 0; i < mod->arch.core.got_num_entries; i++) {
+ if (pos[i] == addr)
+ return (u64)&pos[i];
+ }
+ return 0;
+}
+
+static u64 module_emit_got_entry(struct module *mod, void *loc,
+ const Elf64_Rela *rela, Elf64_Sym *sym)
+{
+ struct mod_got_sec *gotsec = &mod->arch.core;
+ u64 *got = (u64*)gotsec->got->sh_addr;
+ int i = gotsec->got_num_entries;
+ u64 ret;
+
+ /* Check if we can use the kernel GOT */
+ ret = find_got_kernel_entry(sym, rela);
+ if (ret)
+ return ret;
+
+ got[i] = sym->st_value;
+
+ /*
+ * Check if the entry we just created is a duplicate. Given that the
+ * relocations are sorted, this will be the last entry we allocated.
+ * (if one exists).
+ */
+ if (i > 0 && got[i] == got[i - 2]) {
+ ret = (u64)&got[i - 1];
+ } else {
+ gotsec->got_num_entries++;
+ BUG_ON(gotsec->got_num_entries > gotsec->got_max_entries);
+ ret = (u64)&got[i];
+ }
+
+ return ret + rela->r_addend;
+}
+
+#define cmp_3way(a,b) ((a) < (b) ? -1 : (a) > (b))
+
+static int cmp_rela(const void *a, const void *b)
+{
+ const Elf64_Rela *x = a, *y = b;
+ int i;
+
+ /* sort by type, symbol index and addend */
+ i = cmp_3way(ELF64_R_TYPE(x->r_info), ELF64_R_TYPE(y->r_info));
+ if (i == 0)
+ i = cmp_3way(ELF64_R_SYM(x->r_info), ELF64_R_SYM(y->r_info));
+ if (i == 0)
+ i = cmp_3way(x->r_addend, y->r_addend);
+ return i;
+}
+
+static bool duplicate_rel(const Elf64_Rela *rela, int num)
+{
+ /*
+ * Entries are sorted by type, symbol index and addend. That means
+ * that, if a duplicate entry exists, it must be in the preceding
+ * slot.
+ */
+ return num > 0 && cmp_rela(rela + num, rela + num - 1) == 0;
+}
+
+static unsigned int count_gots(Elf64_Sym *syms, Elf64_Rela *rela, int num)
+{
+ unsigned int ret = 0;
+ Elf64_Sym *s;
+ int i;
+
+ for (i = 0; i < num; i++) {
+ switch (ELF64_R_TYPE(rela[i].r_info)) {
+ case R_X86_64_GOTPCREL:
+ s = syms + ELF64_R_SYM(rela[i].r_info);
+
+ /*
+ * Use the kernel GOT when possible, else reserve a
+ * custom one for this module.
+ */
+ if (!duplicate_rel(rela, i) &&
+ !find_got_kernel_entry(s, rela + i))
+ ret++;
+ break;
+ }
+ }
+ return ret;
+}
+
+/*
+ * Generate GOT entries for GOTPCREL relocations that do not exists in the
+ * kernel GOT. Based on arm64 module-plts implementation.
+ */
+int module_frob_arch_sections(Elf_Ehdr *ehdr, Elf_Shdr *sechdrs,
+ char *secstrings, struct module *mod)
+{
+ unsigned long gots = 0;
+ Elf_Shdr *symtab = NULL;
+ Elf64_Sym *syms = NULL;
+ char *strings, *name;
+ int i;
+
+ /*
+ * Find the empty .got section so we can expand it to store the PLT
+ * entries. Record the symtab address as well.
+ */
+ for (i = 0; i < ehdr->e_shnum; i++) {
+ if (!strcmp(secstrings + sechdrs[i].sh_name, ".got")) {
+ mod->arch.core.got = sechdrs + i;
+ } else if (sechdrs[i].sh_type == SHT_SYMTAB) {
+ symtab = sechdrs + i;
+ syms = (Elf64_Sym *)symtab->sh_addr;
+ }
+ }
+
+ if (!mod->arch.core.got) {
+ pr_err("%s: module GOT section missing\n", mod->name);
+ return -ENOEXEC;
+ }
+ if (!syms) {
+ pr_err("%s: module symtab section missing\n", mod->name);
+ return -ENOEXEC;
+ }
+
+ for (i = 0; i < ehdr->e_shnum; i++) {
+ Elf64_Rela *rels = (void *)ehdr + sechdrs[i].sh_offset;
+ int numrels = sechdrs[i].sh_size / sizeof(Elf64_Rela);
+
+ if (sechdrs[i].sh_type != SHT_RELA)
+ continue;
+
+ /* sort by type, symbol index and addend */
+ sort(rels, numrels, sizeof(Elf64_Rela), cmp_rela, NULL);
+
+ gots += count_gots(syms, rels, numrels);
+ }
+
+ mod->arch.core.got->sh_type = SHT_NOBITS;
+ mod->arch.core.got->sh_flags = SHF_ALLOC;
+ mod->arch.core.got->sh_addralign = L1_CACHE_BYTES;
+ mod->arch.core.got->sh_size = (gots + 1) * sizeof(u64);
+ mod->arch.core.got_num_entries = 0;
+ mod->arch.core.got_max_entries = gots;
+
+ /*
+ * If a _GLOBAL_OFFSET_TABLE_ symbol exists, make it absolute for
+ * modules to correctly reference it. Similar to s390 implementation.
+ */
+ strings = (void *) ehdr + sechdrs[symtab->sh_link].sh_offset;
+ for (i = 0; i < symtab->sh_size/sizeof(Elf_Sym); i++) {
+ if (syms[i].st_shndx != SHN_UNDEF)
+ continue;
+ name = strings + syms[i].st_name;
+ if (!strcmp(name, "_GLOBAL_OFFSET_TABLE_")) {
+ syms[i].st_shndx = SHN_ABS;
+ break;
+ }
+ }
+ return 0;
+}
+#endif
+
void *module_alloc(unsigned long size)
{
void *p;
@@ -184,13 +374,18 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
if ((s64)val != *(s32 *)loc)
goto overflow;
break;
+#ifdef CONFIG_X86_PIE
+ case R_X86_64_GOTPCREL:
+ val = module_emit_got_entry(me, loc, rel + i, sym);
+ /* fallthrough */
+#endif
+ case R_X86_64_PLT32:
case R_X86_64_PC32:
val -= (u64)loc;
*(u32 *)loc = val;
-#if 0
- if ((s64)val != *(s32 *)loc)
+ if (IS_ENABLED(CONFIG_X86_PIE) &&
+ (s64)val != *(s32 *)loc)
goto overflow;
-#endif
break;
default:
pr_err("%s: Unknown rela relocation: %llu\n",
@@ -203,8 +398,7 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
overflow:
pr_err("overflow in relocation type %d val %Lx\n",
(int)ELF64_R_TYPE(rel[i].r_info), val);
- pr_err("`%s' likely not compiled with -mcmodel=kernel\n",
- me->name);
+ pr_err("`%s' likely too far from the kernel\n", me->name);
return -ENOEXEC;
}
#endif
--
2.14.2.920.gcf0c67979c-goog
^ permalink raw reply related
* [RFC v3 22/27] x86/modules: Add option to start module section after kernel
From: Thomas Garnier via Virtualization @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
Add an option so the module section is just after the mapped kernel. It
will ensure position independent modules are always at the right
distance from the kernel and do not require mcmodule=large. It also
optimize the available size for modules by getting rid of the empty
space on kernel randomization range.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
Documentation/x86/x86_64/mm.txt | 3 +++
arch/x86/Kconfig | 4 ++++
arch/x86/include/asm/pgtable_64_types.h | 6 +++++-
arch/x86/kernel/head64.c | 5 ++++-
arch/x86/mm/dump_pagetables.c | 4 ++--
5 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/Documentation/x86/x86_64/mm.txt b/Documentation/x86/x86_64/mm.txt
index b0798e281aa6..b51d66386e32 100644
--- a/Documentation/x86/x86_64/mm.txt
+++ b/Documentation/x86/x86_64/mm.txt
@@ -73,4 +73,7 @@ Note that if CONFIG_RANDOMIZE_MEMORY is enabled, the direct mapping of all
physical memory, vmalloc/ioremap space and virtual memory map are randomized.
Their order is preserved but their base will be offset early at boot time.
+If CONFIG_DYNAMIC_MODULE_BASE is enabled, the module section follows the end of
+the mapped kernel.
+
-Andi Kleen, Jul 2004
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 777197fab6dd..1e4b399c64e5 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -2133,6 +2133,10 @@ config RANDOMIZE_MEMORY_PHYSICAL_PADDING
If unsure, leave at the default value.
+# Module section starts just after the end of the kernel module
+config DYNAMIC_MODULE_BASE
+ bool
+
config X86_GLOBAL_STACKPROTECTOR
bool
depends on CC_STACKPROTECTOR
diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h
index 06470da156ba..e00fc429b898 100644
--- a/arch/x86/include/asm/pgtable_64_types.h
+++ b/arch/x86/include/asm/pgtable_64_types.h
@@ -6,6 +6,7 @@
#ifndef __ASSEMBLY__
#include <linux/types.h>
#include <asm/kaslr.h>
+#include <asm/sections.h>
/*
* These are used to make use of C type-checking..
@@ -18,7 +19,6 @@ typedef unsigned long pgdval_t;
typedef unsigned long pgprotval_t;
typedef struct { pteval_t pte; } pte_t;
-
#endif /* !__ASSEMBLY__ */
#define SHARED_KERNEL_PMD 0
@@ -93,7 +93,11 @@ typedef struct { pteval_t pte; } pte_t;
#define VMEMMAP_START __VMEMMAP_BASE
#endif /* CONFIG_RANDOMIZE_MEMORY */
#define VMALLOC_END (VMALLOC_START + _AC((VMALLOC_SIZE_TB << 40) - 1, UL))
+#ifdef CONFIG_DYNAMIC_MODULE_BASE
+#define MODULES_VADDR ALIGN(((unsigned long)_end + PAGE_SIZE), PMD_SIZE)
+#else
#define MODULES_VADDR (__START_KERNEL_map + KERNEL_IMAGE_SIZE)
+#endif
/* The module sections ends with the start of the fixmap */
#define MODULES_END __fix_to_virt(__end_of_fixed_addresses + 1)
#define MODULES_LEN (MODULES_END - MODULES_VADDR)
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 675f1dba3b21..b6363f0d11a7 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -321,12 +321,15 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
* Build-time sanity checks on the kernel image and module
* area mappings. (these are purely build-time and produce no code)
*/
+#ifndef CONFIG_DYNAMIC_MODULE_BASE
BUILD_BUG_ON(MODULES_VADDR < __START_KERNEL_map);
BUILD_BUG_ON(MODULES_VADDR - __START_KERNEL_map < KERNEL_IMAGE_SIZE);
- BUILD_BUG_ON(MODULES_LEN + KERNEL_IMAGE_SIZE > 2*PUD_SIZE);
+ BUILD_BUG_ON(!IS_ENABLED(CONFIG_RANDOMIZE_BASE_LARGE) &&
+ MODULES_LEN + KERNEL_IMAGE_SIZE > 2*PUD_SIZE);
BUILD_BUG_ON((__START_KERNEL_map & ~PMD_MASK) != 0);
BUILD_BUG_ON((MODULES_VADDR & ~PMD_MASK) != 0);
BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
+#endif
BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
(__START_KERNEL & PGDIR_MASK)));
BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);
diff --git a/arch/x86/mm/dump_pagetables.c b/arch/x86/mm/dump_pagetables.c
index 8691a57da63e..8565b2b45848 100644
--- a/arch/x86/mm/dump_pagetables.c
+++ b/arch/x86/mm/dump_pagetables.c
@@ -95,7 +95,7 @@ static struct addr_marker address_markers[] = {
{ EFI_VA_END, "EFI Runtime Services" },
# endif
{ __START_KERNEL_map, "High Kernel Mapping" },
- { MODULES_VADDR, "Modules" },
+ { 0/* MODULES_VADDR */, "Modules" },
{ MODULES_END, "End Modules" },
#else
{ PAGE_OFFSET, "Kernel Mapping" },
@@ -529,7 +529,7 @@ static int __init pt_dump_init(void)
# endif
address_markers[FIXADDR_START_NR].start_address = FIXADDR_START;
#endif
-
+ address_markers[MODULES_VADDR_NR].start_address = MODULES_VADDR;
return 0;
}
__initcall(pt_dump_init);
--
2.14.2.920.gcf0c67979c-goog
^ permalink raw reply related
* [RFC v3 22/27] x86/modules: Add option to start module section after kernel
From: Thomas Garnier @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter, Bor
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
Add an option so the module section is just after the mapped kernel. It
will ensure position independent modules are always at the right
distance from the kernel and do not require mcmodule=large. It also
optimize the available size for modules by getting rid of the empty
space on kernel randomization range.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
Documentation/x86/x86_64/mm.txt | 3 +++
arch/x86/Kconfig | 4 ++++
arch/x86/include/asm/pgtable_64_types.h | 6 +++++-
arch/x86/kernel/head64.c | 5 ++++-
arch/x86/mm/dump_pagetables.c | 4 ++--
5 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/Documentation/x86/x86_64/mm.txt b/Documentation/x86/x86_64/mm.txt
index b0798e281aa6..b51d66386e32 100644
--- a/Documentation/x86/x86_64/mm.txt
+++ b/Documentation/x86/x86_64/mm.txt
@@ -73,4 +73,7 @@ Note that if CONFIG_RANDOMIZE_MEMORY is enabled, the direct mapping of all
physical memory, vmalloc/ioremap space and virtual memory map are randomized.
Their order is preserved but their base will be offset early at boot time.
+If CONFIG_DYNAMIC_MODULE_BASE is enabled, the module section follows the end of
+the mapped kernel.
+
-Andi Kleen, Jul 2004
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 777197fab6dd..1e4b399c64e5 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -2133,6 +2133,10 @@ config RANDOMIZE_MEMORY_PHYSICAL_PADDING
If unsure, leave at the default value.
+# Module section starts just after the end of the kernel module
+config DYNAMIC_MODULE_BASE
+ bool
+
config X86_GLOBAL_STACKPROTECTOR
bool
depends on CC_STACKPROTECTOR
diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h
index 06470da156ba..e00fc429b898 100644
--- a/arch/x86/include/asm/pgtable_64_types.h
+++ b/arch/x86/include/asm/pgtable_64_types.h
@@ -6,6 +6,7 @@
#ifndef __ASSEMBLY__
#include <linux/types.h>
#include <asm/kaslr.h>
+#include <asm/sections.h>
/*
* These are used to make use of C type-checking..
@@ -18,7 +19,6 @@ typedef unsigned long pgdval_t;
typedef unsigned long pgprotval_t;
typedef struct { pteval_t pte; } pte_t;
-
#endif /* !__ASSEMBLY__ */
#define SHARED_KERNEL_PMD 0
@@ -93,7 +93,11 @@ typedef struct { pteval_t pte; } pte_t;
#define VMEMMAP_START __VMEMMAP_BASE
#endif /* CONFIG_RANDOMIZE_MEMORY */
#define VMALLOC_END (VMALLOC_START + _AC((VMALLOC_SIZE_TB << 40) - 1, UL))
+#ifdef CONFIG_DYNAMIC_MODULE_BASE
+#define MODULES_VADDR ALIGN(((unsigned long)_end + PAGE_SIZE), PMD_SIZE)
+#else
#define MODULES_VADDR (__START_KERNEL_map + KERNEL_IMAGE_SIZE)
+#endif
/* The module sections ends with the start of the fixmap */
#define MODULES_END __fix_to_virt(__end_of_fixed_addresses + 1)
#define MODULES_LEN (MODULES_END - MODULES_VADDR)
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 675f1dba3b21..b6363f0d11a7 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -321,12 +321,15 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
* Build-time sanity checks on the kernel image and module
* area mappings. (these are purely build-time and produce no code)
*/
+#ifndef CONFIG_DYNAMIC_MODULE_BASE
BUILD_BUG_ON(MODULES_VADDR < __START_KERNEL_map);
BUILD_BUG_ON(MODULES_VADDR - __START_KERNEL_map < KERNEL_IMAGE_SIZE);
- BUILD_BUG_ON(MODULES_LEN + KERNEL_IMAGE_SIZE > 2*PUD_SIZE);
+ BUILD_BUG_ON(!IS_ENABLED(CONFIG_RANDOMIZE_BASE_LARGE) &&
+ MODULES_LEN + KERNEL_IMAGE_SIZE > 2*PUD_SIZE);
BUILD_BUG_ON((__START_KERNEL_map & ~PMD_MASK) != 0);
BUILD_BUG_ON((MODULES_VADDR & ~PMD_MASK) != 0);
BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
+#endif
BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
(__START_KERNEL & PGDIR_MASK)));
BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);
diff --git a/arch/x86/mm/dump_pagetables.c b/arch/x86/mm/dump_pagetables.c
index 8691a57da63e..8565b2b45848 100644
--- a/arch/x86/mm/dump_pagetables.c
+++ b/arch/x86/mm/dump_pagetables.c
@@ -95,7 +95,7 @@ static struct addr_marker address_markers[] = {
{ EFI_VA_END, "EFI Runtime Services" },
# endif
{ __START_KERNEL_map, "High Kernel Mapping" },
- { MODULES_VADDR, "Modules" },
+ { 0/* MODULES_VADDR */, "Modules" },
{ MODULES_END, "End Modules" },
#else
{ PAGE_OFFSET, "Kernel Mapping" },
@@ -529,7 +529,7 @@ static int __init pt_dump_init(void)
# endif
address_markers[FIXADDR_START_NR].start_address = FIXADDR_START;
#endif
-
+ address_markers[MODULES_VADDR_NR].start_address = MODULES_VADDR;
return 0;
}
__initcall(pt_dump_init);
--
2.14.2.920.gcf0c67979c-goog
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related
* [RFC v3 22/27] x86/modules: Add option to start module section after kernel
From: Thomas Garnier @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
Add an option so the module section is just after the mapped kernel. It
will ensure position independent modules are always at the right
distance from the kernel and do not require mcmodule=large. It also
optimize the available size for modules by getting rid of the empty
space on kernel randomization range.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
Documentation/x86/x86_64/mm.txt | 3 +++
arch/x86/Kconfig | 4 ++++
arch/x86/include/asm/pgtable_64_types.h | 6 +++++-
arch/x86/kernel/head64.c | 5 ++++-
arch/x86/mm/dump_pagetables.c | 4 ++--
5 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/Documentation/x86/x86_64/mm.txt b/Documentation/x86/x86_64/mm.txt
index b0798e281aa6..b51d66386e32 100644
--- a/Documentation/x86/x86_64/mm.txt
+++ b/Documentation/x86/x86_64/mm.txt
@@ -73,4 +73,7 @@ Note that if CONFIG_RANDOMIZE_MEMORY is enabled, the direct mapping of all
physical memory, vmalloc/ioremap space and virtual memory map are randomized.
Their order is preserved but their base will be offset early at boot time.
+If CONFIG_DYNAMIC_MODULE_BASE is enabled, the module section follows the end of
+the mapped kernel.
+
-Andi Kleen, Jul 2004
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 777197fab6dd..1e4b399c64e5 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -2133,6 +2133,10 @@ config RANDOMIZE_MEMORY_PHYSICAL_PADDING
If unsure, leave at the default value.
+# Module section starts just after the end of the kernel module
+config DYNAMIC_MODULE_BASE
+ bool
+
config X86_GLOBAL_STACKPROTECTOR
bool
depends on CC_STACKPROTECTOR
diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h
index 06470da156ba..e00fc429b898 100644
--- a/arch/x86/include/asm/pgtable_64_types.h
+++ b/arch/x86/include/asm/pgtable_64_types.h
@@ -6,6 +6,7 @@
#ifndef __ASSEMBLY__
#include <linux/types.h>
#include <asm/kaslr.h>
+#include <asm/sections.h>
/*
* These are used to make use of C type-checking..
@@ -18,7 +19,6 @@ typedef unsigned long pgdval_t;
typedef unsigned long pgprotval_t;
typedef struct { pteval_t pte; } pte_t;
-
#endif /* !__ASSEMBLY__ */
#define SHARED_KERNEL_PMD 0
@@ -93,7 +93,11 @@ typedef struct { pteval_t pte; } pte_t;
#define VMEMMAP_START __VMEMMAP_BASE
#endif /* CONFIG_RANDOMIZE_MEMORY */
#define VMALLOC_END (VMALLOC_START + _AC((VMALLOC_SIZE_TB << 40) - 1, UL))
+#ifdef CONFIG_DYNAMIC_MODULE_BASE
+#define MODULES_VADDR ALIGN(((unsigned long)_end + PAGE_SIZE), PMD_SIZE)
+#else
#define MODULES_VADDR (__START_KERNEL_map + KERNEL_IMAGE_SIZE)
+#endif
/* The module sections ends with the start of the fixmap */
#define MODULES_END __fix_to_virt(__end_of_fixed_addresses + 1)
#define MODULES_LEN (MODULES_END - MODULES_VADDR)
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 675f1dba3b21..b6363f0d11a7 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -321,12 +321,15 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
* Build-time sanity checks on the kernel image and module
* area mappings. (these are purely build-time and produce no code)
*/
+#ifndef CONFIG_DYNAMIC_MODULE_BASE
BUILD_BUG_ON(MODULES_VADDR < __START_KERNEL_map);
BUILD_BUG_ON(MODULES_VADDR - __START_KERNEL_map < KERNEL_IMAGE_SIZE);
- BUILD_BUG_ON(MODULES_LEN + KERNEL_IMAGE_SIZE > 2*PUD_SIZE);
+ BUILD_BUG_ON(!IS_ENABLED(CONFIG_RANDOMIZE_BASE_LARGE) &&
+ MODULES_LEN + KERNEL_IMAGE_SIZE > 2*PUD_SIZE);
BUILD_BUG_ON((__START_KERNEL_map & ~PMD_MASK) != 0);
BUILD_BUG_ON((MODULES_VADDR & ~PMD_MASK) != 0);
BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
+#endif
BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
(__START_KERNEL & PGDIR_MASK)));
BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);
diff --git a/arch/x86/mm/dump_pagetables.c b/arch/x86/mm/dump_pagetables.c
index 8691a57da63e..8565b2b45848 100644
--- a/arch/x86/mm/dump_pagetables.c
+++ b/arch/x86/mm/dump_pagetables.c
@@ -95,7 +95,7 @@ static struct addr_marker address_markers[] = {
{ EFI_VA_END, "EFI Runtime Services" },
# endif
{ __START_KERNEL_map, "High Kernel Mapping" },
- { MODULES_VADDR, "Modules" },
+ { 0/* MODULES_VADDR */, "Modules" },
{ MODULES_END, "End Modules" },
#else
{ PAGE_OFFSET, "Kernel Mapping" },
@@ -529,7 +529,7 @@ static int __init pt_dump_init(void)
# endif
address_markers[FIXADDR_START_NR].start_address = FIXADDR_START;
#endif
-
+ address_markers[MODULES_VADDR_NR].start_address = MODULES_VADDR;
return 0;
}
__initcall(pt_dump_init);
--
2.14.2.920.gcf0c67979c-goog
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related
* [kernel-hardening] [RFC v3 22/27] x86/modules: Add option to start module section after kernel
From: Thomas Garnier @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter, Boris Ostrovsky, Alexey Dobriyan,
Andrew Morton, Paul Gortmaker, Chris Metcalf, Paul E . McKenney,
Nicolas Pitre, Borislav Petkov, Luis R . Rodriguez,
Greg Kroah-Hartman, Christopher Li, Steven Rostedt, Jason Baron,
Dou Liyang, Rafael J . Wysocki, Mika Westerberg, Lukas Wunner,
Masahiro Yamada, Alexei Starovoitov, Daniel Borkmann,
Markus Trippelsdorf, Paolo Bonzini, Radim Krčmář,
Joerg Roedel, Rik van Riel, David Howells, Ard Biesheuvel,
Waiman Long, Kyle Huey, Andrey Ryabinin, Jonathan Corbet,
Matthew Wilcox, Michal Hocko, Peter Foley, Paul Bolle,
Jiri Kosina, Rob Landley, H . J . Lu, Baoquan He,
Jan H . Schönherr, Daniel Micay
Cc: x86, linux-crypto, linux-kernel, linux-pm, virtualization,
xen-devel, linux-arch, linux-sparse, kvm, linux-doc,
kernel-hardening
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
Add an option so the module section is just after the mapped kernel. It
will ensure position independent modules are always at the right
distance from the kernel and do not require mcmodule=large. It also
optimize the available size for modules by getting rid of the empty
space on kernel randomization range.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
Documentation/x86/x86_64/mm.txt | 3 +++
arch/x86/Kconfig | 4 ++++
arch/x86/include/asm/pgtable_64_types.h | 6 +++++-
arch/x86/kernel/head64.c | 5 ++++-
arch/x86/mm/dump_pagetables.c | 4 ++--
5 files changed, 18 insertions(+), 4 deletions(-)
diff --git a/Documentation/x86/x86_64/mm.txt b/Documentation/x86/x86_64/mm.txt
index b0798e281aa6..b51d66386e32 100644
--- a/Documentation/x86/x86_64/mm.txt
+++ b/Documentation/x86/x86_64/mm.txt
@@ -73,4 +73,7 @@ Note that if CONFIG_RANDOMIZE_MEMORY is enabled, the direct mapping of all
physical memory, vmalloc/ioremap space and virtual memory map are randomized.
Their order is preserved but their base will be offset early at boot time.
+If CONFIG_DYNAMIC_MODULE_BASE is enabled, the module section follows the end of
+the mapped kernel.
+
-Andi Kleen, Jul 2004
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 777197fab6dd..1e4b399c64e5 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -2133,6 +2133,10 @@ config RANDOMIZE_MEMORY_PHYSICAL_PADDING
If unsure, leave at the default value.
+# Module section starts just after the end of the kernel module
+config DYNAMIC_MODULE_BASE
+ bool
+
config X86_GLOBAL_STACKPROTECTOR
bool
depends on CC_STACKPROTECTOR
diff --git a/arch/x86/include/asm/pgtable_64_types.h b/arch/x86/include/asm/pgtable_64_types.h
index 06470da156ba..e00fc429b898 100644
--- a/arch/x86/include/asm/pgtable_64_types.h
+++ b/arch/x86/include/asm/pgtable_64_types.h
@@ -6,6 +6,7 @@
#ifndef __ASSEMBLY__
#include <linux/types.h>
#include <asm/kaslr.h>
+#include <asm/sections.h>
/*
* These are used to make use of C type-checking..
@@ -18,7 +19,6 @@ typedef unsigned long pgdval_t;
typedef unsigned long pgprotval_t;
typedef struct { pteval_t pte; } pte_t;
-
#endif /* !__ASSEMBLY__ */
#define SHARED_KERNEL_PMD 0
@@ -93,7 +93,11 @@ typedef struct { pteval_t pte; } pte_t;
#define VMEMMAP_START __VMEMMAP_BASE
#endif /* CONFIG_RANDOMIZE_MEMORY */
#define VMALLOC_END (VMALLOC_START + _AC((VMALLOC_SIZE_TB << 40) - 1, UL))
+#ifdef CONFIG_DYNAMIC_MODULE_BASE
+#define MODULES_VADDR ALIGN(((unsigned long)_end + PAGE_SIZE), PMD_SIZE)
+#else
#define MODULES_VADDR (__START_KERNEL_map + KERNEL_IMAGE_SIZE)
+#endif
/* The module sections ends with the start of the fixmap */
#define MODULES_END __fix_to_virt(__end_of_fixed_addresses + 1)
#define MODULES_LEN (MODULES_END - MODULES_VADDR)
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 675f1dba3b21..b6363f0d11a7 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -321,12 +321,15 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
* Build-time sanity checks on the kernel image and module
* area mappings. (these are purely build-time and produce no code)
*/
+#ifndef CONFIG_DYNAMIC_MODULE_BASE
BUILD_BUG_ON(MODULES_VADDR < __START_KERNEL_map);
BUILD_BUG_ON(MODULES_VADDR - __START_KERNEL_map < KERNEL_IMAGE_SIZE);
- BUILD_BUG_ON(MODULES_LEN + KERNEL_IMAGE_SIZE > 2*PUD_SIZE);
+ BUILD_BUG_ON(!IS_ENABLED(CONFIG_RANDOMIZE_BASE_LARGE) &&
+ MODULES_LEN + KERNEL_IMAGE_SIZE > 2*PUD_SIZE);
BUILD_BUG_ON((__START_KERNEL_map & ~PMD_MASK) != 0);
BUILD_BUG_ON((MODULES_VADDR & ~PMD_MASK) != 0);
BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
+#endif
BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
(__START_KERNEL & PGDIR_MASK)));
BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);
diff --git a/arch/x86/mm/dump_pagetables.c b/arch/x86/mm/dump_pagetables.c
index 8691a57da63e..8565b2b45848 100644
--- a/arch/x86/mm/dump_pagetables.c
+++ b/arch/x86/mm/dump_pagetables.c
@@ -95,7 +95,7 @@ static struct addr_marker address_markers[] = {
{ EFI_VA_END, "EFI Runtime Services" },
# endif
{ __START_KERNEL_map, "High Kernel Mapping" },
- { MODULES_VADDR, "Modules" },
+ { 0/* MODULES_VADDR */, "Modules" },
{ MODULES_END, "End Modules" },
#else
{ PAGE_OFFSET, "Kernel Mapping" },
@@ -529,7 +529,7 @@ static int __init pt_dump_init(void)
# endif
address_markers[FIXADDR_START_NR].start_address = FIXADDR_START;
#endif
-
+ address_markers[MODULES_VADDR_NR].start_address = MODULES_VADDR;
return 0;
}
__initcall(pt_dump_init);
--
2.14.2.920.gcf0c67979c-goog
^ permalink raw reply related
* [RFC v3 21/27] x86/mm/dump_pagetables: Fix address markers index on x86_64
From: Thomas Garnier via Virtualization @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
The address_markers_idx enum is not aligned with the table when EFI is
enabled. Add an EFI_VA_END_NR entry in this case.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
arch/x86/mm/dump_pagetables.c | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/arch/x86/mm/dump_pagetables.c b/arch/x86/mm/dump_pagetables.c
index 5e3ac6fe6c9e..8691a57da63e 100644
--- a/arch/x86/mm/dump_pagetables.c
+++ b/arch/x86/mm/dump_pagetables.c
@@ -52,12 +52,15 @@ enum address_markers_idx {
LOW_KERNEL_NR,
VMALLOC_START_NR,
VMEMMAP_START_NR,
-#ifdef CONFIG_KASAN
+# ifdef CONFIG_KASAN
KASAN_SHADOW_START_NR,
KASAN_SHADOW_END_NR,
-#endif
+# endif
# ifdef CONFIG_X86_ESPFIX64
ESPFIX_START_NR,
+# endif
+# ifdef CONFIG_EFI
+ EFI_VA_END_NR,
# endif
HIGH_KERNEL_NR,
MODULES_VADDR_NR,
--
2.14.2.920.gcf0c67979c-goog
^ permalink raw reply related
* [RFC v3 21/27] x86/mm/dump_pagetables: Fix address markers index on x86_64
From: Thomas Garnier @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter, Bor
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
The address_markers_idx enum is not aligned with the table when EFI is
enabled. Add an EFI_VA_END_NR entry in this case.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
arch/x86/mm/dump_pagetables.c | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/arch/x86/mm/dump_pagetables.c b/arch/x86/mm/dump_pagetables.c
index 5e3ac6fe6c9e..8691a57da63e 100644
--- a/arch/x86/mm/dump_pagetables.c
+++ b/arch/x86/mm/dump_pagetables.c
@@ -52,12 +52,15 @@ enum address_markers_idx {
LOW_KERNEL_NR,
VMALLOC_START_NR,
VMEMMAP_START_NR,
-#ifdef CONFIG_KASAN
+# ifdef CONFIG_KASAN
KASAN_SHADOW_START_NR,
KASAN_SHADOW_END_NR,
-#endif
+# endif
# ifdef CONFIG_X86_ESPFIX64
ESPFIX_START_NR,
+# endif
+# ifdef CONFIG_EFI
+ EFI_VA_END_NR,
# endif
HIGH_KERNEL_NR,
MODULES_VADDR_NR,
--
2.14.2.920.gcf0c67979c-goog
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related
* [RFC v3 21/27] x86/mm/dump_pagetables: Fix address markers index on x86_64
From: Thomas Garnier @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
The address_markers_idx enum is not aligned with the table when EFI is
enabled. Add an EFI_VA_END_NR entry in this case.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
arch/x86/mm/dump_pagetables.c | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/arch/x86/mm/dump_pagetables.c b/arch/x86/mm/dump_pagetables.c
index 5e3ac6fe6c9e..8691a57da63e 100644
--- a/arch/x86/mm/dump_pagetables.c
+++ b/arch/x86/mm/dump_pagetables.c
@@ -52,12 +52,15 @@ enum address_markers_idx {
LOW_KERNEL_NR,
VMALLOC_START_NR,
VMEMMAP_START_NR,
-#ifdef CONFIG_KASAN
+# ifdef CONFIG_KASAN
KASAN_SHADOW_START_NR,
KASAN_SHADOW_END_NR,
-#endif
+# endif
# ifdef CONFIG_X86_ESPFIX64
ESPFIX_START_NR,
+# endif
+# ifdef CONFIG_EFI
+ EFI_VA_END_NR,
# endif
HIGH_KERNEL_NR,
MODULES_VADDR_NR,
--
2.14.2.920.gcf0c67979c-goog
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related
* [kernel-hardening] [RFC v3 21/27] x86/mm/dump_pagetables: Fix address markers index on x86_64
From: Thomas Garnier @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter, Boris Ostrovsky, Alexey Dobriyan,
Andrew Morton, Paul Gortmaker, Chris Metcalf, Paul E . McKenney,
Nicolas Pitre, Borislav Petkov, Luis R . Rodriguez,
Greg Kroah-Hartman, Christopher Li, Steven Rostedt, Jason Baron,
Dou Liyang, Rafael J . Wysocki, Mika Westerberg, Lukas Wunner,
Masahiro Yamada, Alexei Starovoitov, Daniel Borkmann,
Markus Trippelsdorf, Paolo Bonzini, Radim Krčmář,
Joerg Roedel, Rik van Riel, David Howells, Ard Biesheuvel,
Waiman Long, Kyle Huey, Andrey Ryabinin, Jonathan Corbet,
Matthew Wilcox, Michal Hocko, Peter Foley, Paul Bolle,
Jiri Kosina, Rob Landley, H . J . Lu, Baoquan He,
Jan H . Schönherr, Daniel Micay
Cc: x86, linux-crypto, linux-kernel, linux-pm, virtualization,
xen-devel, linux-arch, linux-sparse, kvm, linux-doc,
kernel-hardening
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
The address_markers_idx enum is not aligned with the table when EFI is
enabled. Add an EFI_VA_END_NR entry in this case.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
arch/x86/mm/dump_pagetables.c | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/arch/x86/mm/dump_pagetables.c b/arch/x86/mm/dump_pagetables.c
index 5e3ac6fe6c9e..8691a57da63e 100644
--- a/arch/x86/mm/dump_pagetables.c
+++ b/arch/x86/mm/dump_pagetables.c
@@ -52,12 +52,15 @@ enum address_markers_idx {
LOW_KERNEL_NR,
VMALLOC_START_NR,
VMEMMAP_START_NR,
-#ifdef CONFIG_KASAN
+# ifdef CONFIG_KASAN
KASAN_SHADOW_START_NR,
KASAN_SHADOW_END_NR,
-#endif
+# endif
# ifdef CONFIG_X86_ESPFIX64
ESPFIX_START_NR,
+# endif
+# ifdef CONFIG_EFI
+ EFI_VA_END_NR,
# endif
HIGH_KERNEL_NR,
MODULES_VADDR_NR,
--
2.14.2.920.gcf0c67979c-goog
^ permalink raw reply related
* [RFC v3 20/27] x86/ftrace: Adapt function tracing for PIE support
From: Thomas Garnier via Virtualization @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
When using -fPIE/PIC with function tracing, the compiler generates a
call through the GOT (call *__fentry__@GOTPCREL). This instruction
takes 6 bytes instead of 5 on the usual relative call.
With this change, function tracing supports 6 bytes on traceable
function and can still replace relative calls on the ftrace assembly
functions.
Position Independent Executable (PIE) support will allow to extended the
KASLR randomization range below the -2G memory limit.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
arch/x86/include/asm/ftrace.h | 23 +++++-
arch/x86/include/asm/sections.h | 4 +
arch/x86/kernel/ftrace.c | 168 ++++++++++++++++++++++++++--------------
arch/x86/kernel/module.lds | 3 +
4 files changed, 139 insertions(+), 59 deletions(-)
create mode 100644 arch/x86/kernel/module.lds
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index eccd0ac6bc38..b8bbcc7fad7f 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -1,6 +1,7 @@
#ifndef _ASM_X86_FTRACE_H
#define _ASM_X86_FTRACE_H
+
#ifdef CONFIG_FUNCTION_TRACER
#ifdef CC_USING_FENTRY
# define MCOUNT_ADDR ((unsigned long)(__fentry__))
@@ -8,7 +9,19 @@
# define MCOUNT_ADDR ((unsigned long)(mcount))
# define HAVE_FUNCTION_GRAPH_FP_TEST
#endif
-#define MCOUNT_INSN_SIZE 5 /* sizeof mcount call */
+
+#define MCOUNT_RELINSN_SIZE 5 /* sizeof relative (call or jump) */
+#define MCOUNT_GOTCALL_SIZE 6 /* sizeof call *got */
+
+/*
+ * MCOUNT_INSN_SIZE is the highest size of instructions based on the
+ * configuration.
+ */
+#ifdef CONFIG_X86_PIE
+#define MCOUNT_INSN_SIZE MCOUNT_GOTCALL_SIZE
+#else
+#define MCOUNT_INSN_SIZE MCOUNT_RELINSN_SIZE
+#endif
#ifdef CONFIG_DYNAMIC_FTRACE
#define ARCH_SUPPORTS_FTRACE_OPS 1
@@ -17,6 +30,8 @@
#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
#ifndef __ASSEMBLY__
+#include <asm/sections.h>
+
extern void mcount(void);
extern atomic_t modifying_ftrace_code;
extern void __fentry__(void);
@@ -24,9 +39,11 @@ extern void __fentry__(void);
static inline unsigned long ftrace_call_adjust(unsigned long addr)
{
/*
- * addr is the address of the mcount call instruction.
- * recordmcount does the necessary offset calculation.
+ * addr is the address of the mcount call instruction. PIE has always a
+ * byte added to the start of the function.
*/
+ if (IS_ENABLED(CONFIG_X86_PIE))
+ addr -= 1;
return addr;
}
diff --git a/arch/x86/include/asm/sections.h b/arch/x86/include/asm/sections.h
index 2f75f30cb2f6..6b2d496cf1aa 100644
--- a/arch/x86/include/asm/sections.h
+++ b/arch/x86/include/asm/sections.h
@@ -11,4 +11,8 @@ extern struct exception_table_entry __stop___ex_table[];
extern char __end_rodata_hpage_align[];
#endif
+#if defined(CONFIG_X86_PIE)
+extern char __start_got[], __end_got[];
+#endif
+
#endif /* _ASM_X86_SECTIONS_H */
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index 9bef1bbeba63..41d8c4c4306d 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -58,12 +58,17 @@ static int ftrace_calc_offset(long ip, long addr)
return (int)(addr - ip);
}
-static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
+static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr,
+ unsigned int size)
{
static union ftrace_code_union calc;
+ /* On PIE, fill the rest of the buffer with nops */
+ if (IS_ENABLED(CONFIG_X86_PIE))
+ memset(calc.code, ideal_nops[1][0], sizeof(calc.code));
+
calc.e8 = 0xe8;
- calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
+ calc.offset = ftrace_calc_offset(ip + MCOUNT_RELINSN_SIZE, addr);
/*
* No locking needed, this must be called via kstop_machine
@@ -72,6 +77,44 @@ static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
return calc.code;
}
+#ifdef CONFIG_X86_PIE
+union ftrace_code_got_union {
+ char code[MCOUNT_INSN_SIZE];
+ struct {
+ unsigned short ff15;
+ int offset;
+ } __attribute__((packed));
+};
+
+/* Used to identify a mcount GOT call on PIE */
+static unsigned char *ftrace_original_call(struct module* mod, unsigned long ip,
+ unsigned long addr,
+ unsigned int size)
+{
+ static union ftrace_code_got_union calc;
+ unsigned long gotaddr;
+
+ calc.ff15 = 0x15ff;
+
+ gotaddr = module_find_got_entry(mod, addr);
+ if (!gotaddr) {
+ pr_err("Failed to find GOT entry for 0x%lx\n", addr);
+ return NULL;
+ }
+
+ calc.offset = ftrace_calc_offset(ip + MCOUNT_GOTCALL_SIZE, gotaddr);
+ return calc.code;
+}
+#else
+static unsigned char *ftrace_original_call(struct module* mod, unsigned long ip,
+ unsigned long addr,
+ unsigned int size)
+{
+ return ftrace_call_replace(ip, addr, size);
+}
+
+#endif
+
static inline int
within(unsigned long addr, unsigned long start, unsigned long end)
{
@@ -94,16 +137,18 @@ static unsigned long text_ip_addr(unsigned long ip)
return ip;
}
-static const unsigned char *ftrace_nop_replace(void)
+static const unsigned char *ftrace_nop_replace(unsigned int size)
{
- return ideal_nops[NOP_ATOMIC5];
+ return ideal_nops[size == 5 ? NOP_ATOMIC5 : size];
}
static int
-ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
- unsigned const char *new_code)
+ftrace_modify_code_direct(struct dyn_ftrace *rec, unsigned const char *old_code,
+ unsigned const char *new_code)
{
unsigned char replaced[MCOUNT_INSN_SIZE];
+ unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
ftrace_expected = old_code;
@@ -116,17 +161,17 @@ ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
*/
/* read the text we want to modify */
- if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
+ if (probe_kernel_read(replaced, (void *)ip, size))
return -EFAULT;
/* Make sure it is what we expect it to be */
- if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
+ if (memcmp(replaced, old_code, size) != 0)
return -EINVAL;
ip = text_ip_addr(ip);
/* replace the text with the new text */
- if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))
+ if (probe_kernel_write((void *)ip, new_code, size))
return -EPERM;
sync_core();
@@ -139,9 +184,7 @@ int ftrace_make_nop(struct module *mod,
{
unsigned const char *new, *old;
unsigned long ip = rec->ip;
-
- old = ftrace_call_replace(ip, addr);
- new = ftrace_nop_replace();
+ unsigned int size = MCOUNT_INSN_SIZE;
/*
* On boot up, and when modules are loaded, the MCOUNT_ADDR
@@ -151,14 +194,20 @@ int ftrace_make_nop(struct module *mod,
* We do not want to use the breakpoint version in this case,
* just modify the code directly.
*/
- if (addr == MCOUNT_ADDR)
- return ftrace_modify_code_direct(rec->ip, old, new);
+ if (addr != MCOUNT_ADDR) {
+ ftrace_expected = NULL;
- ftrace_expected = NULL;
+ /* Normal cases use add_brk_on_nop */
+ WARN_ONCE(1, "invalid use of ftrace_make_nop");
+ return -EINVAL;
+ }
- /* Normal cases use add_brk_on_nop */
- WARN_ONCE(1, "invalid use of ftrace_make_nop");
- return -EINVAL;
+ old = ftrace_original_call(mod, ip, addr, size);
+ if (!old)
+ return -EINVAL;
+ new = ftrace_nop_replace(size);
+
+ return ftrace_modify_code_direct(rec, old, new);
}
int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
@@ -166,11 +215,11 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
unsigned const char *new, *old;
unsigned long ip = rec->ip;
- old = ftrace_nop_replace();
- new = ftrace_call_replace(ip, addr);
+ old = ftrace_nop_replace(MCOUNT_INSN_SIZE);
+ new = ftrace_call_replace(ip, addr, MCOUNT_INSN_SIZE);
/* Should only be called when module is loaded */
- return ftrace_modify_code_direct(rec->ip, old, new);
+ return ftrace_modify_code_direct(rec, old, new);
}
/*
@@ -233,7 +282,7 @@ static int update_ftrace_func(unsigned long ip, void *new)
unsigned char old[MCOUNT_INSN_SIZE];
int ret;
- memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);
+ memcpy(old, (void *)ip, MCOUNT_RELINSN_SIZE);
ftrace_update_func = ip;
/* Make sure the breakpoints see the ftrace_update_func update */
@@ -255,13 +304,14 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
unsigned char *new;
int ret;
- new = ftrace_call_replace(ip, (unsigned long)func);
+ new = ftrace_call_replace(ip, (unsigned long)func, MCOUNT_RELINSN_SIZE);
ret = update_ftrace_func(ip, new);
/* Also update the regs callback function */
if (!ret) {
ip = (unsigned long)(&ftrace_regs_call);
- new = ftrace_call_replace(ip, (unsigned long)func);
+ new = ftrace_call_replace(ip, (unsigned long)func,
+ MCOUNT_RELINSN_SIZE);
ret = update_ftrace_func(ip, new);
}
@@ -309,18 +359,18 @@ static int ftrace_write(unsigned long ip, const char *val, int size)
return 0;
}
-static int add_break(unsigned long ip, const char *old)
+static int add_break(unsigned long ip, const char *old, unsigned int size)
{
unsigned char replaced[MCOUNT_INSN_SIZE];
unsigned char brk = BREAKPOINT_INSTRUCTION;
- if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
+ if (probe_kernel_read(replaced, (void *)ip, size))
return -EFAULT;
ftrace_expected = old;
/* Make sure it is what we expect it to be */
- if (memcmp(replaced, old, MCOUNT_INSN_SIZE) != 0)
+ if (memcmp(replaced, old, size) != 0)
return -EINVAL;
return ftrace_write(ip, &brk, 1);
@@ -330,20 +380,22 @@ static int add_brk_on_call(struct dyn_ftrace *rec, unsigned long addr)
{
unsigned const char *old;
unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
- old = ftrace_call_replace(ip, addr);
+ old = ftrace_call_replace(ip, addr, size);
- return add_break(rec->ip, old);
+ return add_break(rec->ip, old, size);
}
static int add_brk_on_nop(struct dyn_ftrace *rec)
{
unsigned const char *old;
+ unsigned int size = MCOUNT_INSN_SIZE;
- old = ftrace_nop_replace();
+ old = ftrace_nop_replace(size);
- return add_break(rec->ip, old);
+ return add_break(rec->ip, old, size);
}
static int add_breakpoints(struct dyn_ftrace *rec, int enable)
@@ -386,22 +438,23 @@ static int remove_breakpoint(struct dyn_ftrace *rec)
const unsigned char *nop;
unsigned long ftrace_addr;
unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
/* If we fail the read, just give up */
- if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))
+ if (probe_kernel_read(ins, (void *)ip, size))
return -EFAULT;
/* If this does not have a breakpoint, we are done */
if (ins[0] != brk)
return 0;
- nop = ftrace_nop_replace();
+ nop = ftrace_nop_replace(size);
/*
* If the last 4 bytes of the instruction do not match
* a nop, then we assume that this is a call to ftrace_addr.
*/
- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) {
+ if (memcmp(&ins[1], &nop[1], size - 1) != 0) {
/*
* For extra paranoidism, we check if the breakpoint is on
* a call that would actually jump to the ftrace_addr.
@@ -409,18 +462,18 @@ static int remove_breakpoint(struct dyn_ftrace *rec)
* a disaster.
*/
ftrace_addr = ftrace_get_addr_new(rec);
- nop = ftrace_call_replace(ip, ftrace_addr);
+ nop = ftrace_call_replace(ip, ftrace_addr, size);
- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) == 0)
+ if (memcmp(&ins[1], &nop[1], size - 1) == 0)
goto update;
/* Check both ftrace_addr and ftrace_old_addr */
ftrace_addr = ftrace_get_addr_curr(rec);
- nop = ftrace_call_replace(ip, ftrace_addr);
+ nop = ftrace_call_replace(ip, ftrace_addr, size);
ftrace_expected = nop;
- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
+ if (memcmp(&ins[1], &nop[1], size - 1) != 0)
return -EINVAL;
}
@@ -428,30 +481,33 @@ static int remove_breakpoint(struct dyn_ftrace *rec)
return ftrace_write(ip, nop, 1);
}
-static int add_update_code(unsigned long ip, unsigned const char *new)
+static int add_update_code(unsigned long ip, unsigned const char *new,
+ unsigned int size)
{
/* skip breakpoint */
ip++;
new++;
- return ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1);
+ return ftrace_write(ip, new, size - 1);
}
static int add_update_call(struct dyn_ftrace *rec, unsigned long addr)
{
unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
unsigned const char *new;
- new = ftrace_call_replace(ip, addr);
- return add_update_code(ip, new);
+ new = ftrace_call_replace(ip, addr, size);
+ return add_update_code(ip, new, size);
}
static int add_update_nop(struct dyn_ftrace *rec)
{
unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
unsigned const char *new;
- new = ftrace_nop_replace();
- return add_update_code(ip, new);
+ new = ftrace_nop_replace(size);
+ return add_update_code(ip, new, size);
}
static int add_update(struct dyn_ftrace *rec, int enable)
@@ -485,7 +541,7 @@ static int finish_update_call(struct dyn_ftrace *rec, unsigned long addr)
unsigned long ip = rec->ip;
unsigned const char *new;
- new = ftrace_call_replace(ip, addr);
+ new = ftrace_call_replace(ip, addr, MCOUNT_INSN_SIZE);
return ftrace_write(ip, new, 1);
}
@@ -495,7 +551,7 @@ static int finish_update_nop(struct dyn_ftrace *rec)
unsigned long ip = rec->ip;
unsigned const char *new;
- new = ftrace_nop_replace();
+ new = ftrace_nop_replace(MCOUNT_INSN_SIZE);
return ftrace_write(ip, new, 1);
}
@@ -619,13 +675,13 @@ ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
{
int ret;
- ret = add_break(ip, old_code);
+ ret = add_break(ip, old_code, MCOUNT_RELINSN_SIZE);
if (ret)
goto out;
run_sync();
- ret = add_update_code(ip, new_code);
+ ret = add_update_code(ip, new_code, MCOUNT_RELINSN_SIZE);
if (ret)
goto fail_update;
@@ -670,7 +726,7 @@ static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
/* Jmp not a call (ignore the .e8) */
calc.e8 = 0xe9;
- calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
+ calc.offset = ftrace_calc_offset(ip + MCOUNT_RELINSN_SIZE, addr);
/*
* ftrace external locks synchronize the access to the static variable.
@@ -766,11 +822,11 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
* the jmp to ftrace_epilogue, as well as the address of
* the ftrace_ops this trampoline is used for.
*/
- trampoline = alloc_tramp(size + MCOUNT_INSN_SIZE + sizeof(void *));
+ trampoline = alloc_tramp(size + MCOUNT_RELINSN_SIZE + sizeof(void *));
if (!trampoline)
return 0;
- *tramp_size = size + MCOUNT_INSN_SIZE + sizeof(void *);
+ *tramp_size = size + MCOUNT_RELINSN_SIZE + sizeof(void *);
/* Copy ftrace_caller onto the trampoline memory */
ret = probe_kernel_read(trampoline, (void *)start_offset, size);
@@ -783,7 +839,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
/* The trampoline ends with a jmp to ftrace_epilogue */
jmp = ftrace_jmp_replace(ip, (unsigned long)ftrace_epilogue);
- memcpy(trampoline + size, jmp, MCOUNT_INSN_SIZE);
+ memcpy(trampoline + size, jmp, MCOUNT_RELINSN_SIZE);
/*
* The address of the ftrace_ops that is used for this trampoline
@@ -793,7 +849,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
* the global function_trace_op variable.
*/
- ptr = (unsigned long *)(trampoline + size + MCOUNT_INSN_SIZE);
+ ptr = (unsigned long *)(trampoline + size + MCOUNT_RELINSN_SIZE);
*ptr = (unsigned long)ops;
op_offset -= start_offset;
@@ -868,7 +924,7 @@ void arch_ftrace_update_trampoline(struct ftrace_ops *ops)
func = ftrace_ops_get_func(ops);
/* Do a safe modify in case the trampoline is executing */
- new = ftrace_call_replace(ip, (unsigned long)func);
+ new = ftrace_call_replace(ip, (unsigned long)func, MCOUNT_RELINSN_SIZE);
ret = update_ftrace_func(ip, new);
set_memory_ro(ops->trampoline, npages);
@@ -882,7 +938,7 @@ static void *addr_from_call(void *ptr)
union ftrace_code_union calc;
int ret;
- ret = probe_kernel_read(&calc, ptr, MCOUNT_INSN_SIZE);
+ ret = probe_kernel_read(&calc, ptr, MCOUNT_RELINSN_SIZE);
if (WARN_ON_ONCE(ret < 0))
return NULL;
@@ -892,7 +948,7 @@ static void *addr_from_call(void *ptr)
return NULL;
}
- return ptr + MCOUNT_INSN_SIZE + calc.offset;
+ return ptr + MCOUNT_RELINSN_SIZE + calc.offset;
}
void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
diff --git a/arch/x86/kernel/module.lds b/arch/x86/kernel/module.lds
new file mode 100644
index 000000000000..fd6e95a4b454
--- /dev/null
+++ b/arch/x86/kernel/module.lds
@@ -0,0 +1,3 @@
+SECTIONS {
+ .got (NOLOAD) : { BYTE(0) }
+}
--
2.14.2.920.gcf0c67979c-goog
^ permalink raw reply related
* [RFC v3 20/27] x86/ftrace: Adapt function tracing for PIE support
From: Thomas Garnier @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter, Bor
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
When using -fPIE/PIC with function tracing, the compiler generates a
call through the GOT (call *__fentry__@GOTPCREL). This instruction
takes 6 bytes instead of 5 on the usual relative call.
With this change, function tracing supports 6 bytes on traceable
function and can still replace relative calls on the ftrace assembly
functions.
Position Independent Executable (PIE) support will allow to extended the
KASLR randomization range below the -2G memory limit.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
arch/x86/include/asm/ftrace.h | 23 +++++-
arch/x86/include/asm/sections.h | 4 +
arch/x86/kernel/ftrace.c | 168 ++++++++++++++++++++++++++--------------
arch/x86/kernel/module.lds | 3 +
4 files changed, 139 insertions(+), 59 deletions(-)
create mode 100644 arch/x86/kernel/module.lds
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index eccd0ac6bc38..b8bbcc7fad7f 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -1,6 +1,7 @@
#ifndef _ASM_X86_FTRACE_H
#define _ASM_X86_FTRACE_H
+
#ifdef CONFIG_FUNCTION_TRACER
#ifdef CC_USING_FENTRY
# define MCOUNT_ADDR ((unsigned long)(__fentry__))
@@ -8,7 +9,19 @@
# define MCOUNT_ADDR ((unsigned long)(mcount))
# define HAVE_FUNCTION_GRAPH_FP_TEST
#endif
-#define MCOUNT_INSN_SIZE 5 /* sizeof mcount call */
+
+#define MCOUNT_RELINSN_SIZE 5 /* sizeof relative (call or jump) */
+#define MCOUNT_GOTCALL_SIZE 6 /* sizeof call *got */
+
+/*
+ * MCOUNT_INSN_SIZE is the highest size of instructions based on the
+ * configuration.
+ */
+#ifdef CONFIG_X86_PIE
+#define MCOUNT_INSN_SIZE MCOUNT_GOTCALL_SIZE
+#else
+#define MCOUNT_INSN_SIZE MCOUNT_RELINSN_SIZE
+#endif
#ifdef CONFIG_DYNAMIC_FTRACE
#define ARCH_SUPPORTS_FTRACE_OPS 1
@@ -17,6 +30,8 @@
#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
#ifndef __ASSEMBLY__
+#include <asm/sections.h>
+
extern void mcount(void);
extern atomic_t modifying_ftrace_code;
extern void __fentry__(void);
@@ -24,9 +39,11 @@ extern void __fentry__(void);
static inline unsigned long ftrace_call_adjust(unsigned long addr)
{
/*
- * addr is the address of the mcount call instruction.
- * recordmcount does the necessary offset calculation.
+ * addr is the address of the mcount call instruction. PIE has always a
+ * byte added to the start of the function.
*/
+ if (IS_ENABLED(CONFIG_X86_PIE))
+ addr -= 1;
return addr;
}
diff --git a/arch/x86/include/asm/sections.h b/arch/x86/include/asm/sections.h
index 2f75f30cb2f6..6b2d496cf1aa 100644
--- a/arch/x86/include/asm/sections.h
+++ b/arch/x86/include/asm/sections.h
@@ -11,4 +11,8 @@ extern struct exception_table_entry __stop___ex_table[];
extern char __end_rodata_hpage_align[];
#endif
+#if defined(CONFIG_X86_PIE)
+extern char __start_got[], __end_got[];
+#endif
+
#endif /* _ASM_X86_SECTIONS_H */
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index 9bef1bbeba63..41d8c4c4306d 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -58,12 +58,17 @@ static int ftrace_calc_offset(long ip, long addr)
return (int)(addr - ip);
}
-static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
+static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr,
+ unsigned int size)
{
static union ftrace_code_union calc;
+ /* On PIE, fill the rest of the buffer with nops */
+ if (IS_ENABLED(CONFIG_X86_PIE))
+ memset(calc.code, ideal_nops[1][0], sizeof(calc.code));
+
calc.e8 = 0xe8;
- calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
+ calc.offset = ftrace_calc_offset(ip + MCOUNT_RELINSN_SIZE, addr);
/*
* No locking needed, this must be called via kstop_machine
@@ -72,6 +77,44 @@ static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
return calc.code;
}
+#ifdef CONFIG_X86_PIE
+union ftrace_code_got_union {
+ char code[MCOUNT_INSN_SIZE];
+ struct {
+ unsigned short ff15;
+ int offset;
+ } __attribute__((packed));
+};
+
+/* Used to identify a mcount GOT call on PIE */
+static unsigned char *ftrace_original_call(struct module* mod, unsigned long ip,
+ unsigned long addr,
+ unsigned int size)
+{
+ static union ftrace_code_got_union calc;
+ unsigned long gotaddr;
+
+ calc.ff15 = 0x15ff;
+
+ gotaddr = module_find_got_entry(mod, addr);
+ if (!gotaddr) {
+ pr_err("Failed to find GOT entry for 0x%lx\n", addr);
+ return NULL;
+ }
+
+ calc.offset = ftrace_calc_offset(ip + MCOUNT_GOTCALL_SIZE, gotaddr);
+ return calc.code;
+}
+#else
+static unsigned char *ftrace_original_call(struct module* mod, unsigned long ip,
+ unsigned long addr,
+ unsigned int size)
+{
+ return ftrace_call_replace(ip, addr, size);
+}
+
+#endif
+
static inline int
within(unsigned long addr, unsigned long start, unsigned long end)
{
@@ -94,16 +137,18 @@ static unsigned long text_ip_addr(unsigned long ip)
return ip;
}
-static const unsigned char *ftrace_nop_replace(void)
+static const unsigned char *ftrace_nop_replace(unsigned int size)
{
- return ideal_nops[NOP_ATOMIC5];
+ return ideal_nops[size == 5 ? NOP_ATOMIC5 : size];
}
static int
-ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
- unsigned const char *new_code)
+ftrace_modify_code_direct(struct dyn_ftrace *rec, unsigned const char *old_code,
+ unsigned const char *new_code)
{
unsigned char replaced[MCOUNT_INSN_SIZE];
+ unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
ftrace_expected = old_code;
@@ -116,17 +161,17 @@ ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
*/
/* read the text we want to modify */
- if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
+ if (probe_kernel_read(replaced, (void *)ip, size))
return -EFAULT;
/* Make sure it is what we expect it to be */
- if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
+ if (memcmp(replaced, old_code, size) != 0)
return -EINVAL;
ip = text_ip_addr(ip);
/* replace the text with the new text */
- if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))
+ if (probe_kernel_write((void *)ip, new_code, size))
return -EPERM;
sync_core();
@@ -139,9 +184,7 @@ int ftrace_make_nop(struct module *mod,
{
unsigned const char *new, *old;
unsigned long ip = rec->ip;
-
- old = ftrace_call_replace(ip, addr);
- new = ftrace_nop_replace();
+ unsigned int size = MCOUNT_INSN_SIZE;
/*
* On boot up, and when modules are loaded, the MCOUNT_ADDR
@@ -151,14 +194,20 @@ int ftrace_make_nop(struct module *mod,
* We do not want to use the breakpoint version in this case,
* just modify the code directly.
*/
- if (addr == MCOUNT_ADDR)
- return ftrace_modify_code_direct(rec->ip, old, new);
+ if (addr != MCOUNT_ADDR) {
+ ftrace_expected = NULL;
- ftrace_expected = NULL;
+ /* Normal cases use add_brk_on_nop */
+ WARN_ONCE(1, "invalid use of ftrace_make_nop");
+ return -EINVAL;
+ }
- /* Normal cases use add_brk_on_nop */
- WARN_ONCE(1, "invalid use of ftrace_make_nop");
- return -EINVAL;
+ old = ftrace_original_call(mod, ip, addr, size);
+ if (!old)
+ return -EINVAL;
+ new = ftrace_nop_replace(size);
+
+ return ftrace_modify_code_direct(rec, old, new);
}
int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
@@ -166,11 +215,11 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
unsigned const char *new, *old;
unsigned long ip = rec->ip;
- old = ftrace_nop_replace();
- new = ftrace_call_replace(ip, addr);
+ old = ftrace_nop_replace(MCOUNT_INSN_SIZE);
+ new = ftrace_call_replace(ip, addr, MCOUNT_INSN_SIZE);
/* Should only be called when module is loaded */
- return ftrace_modify_code_direct(rec->ip, old, new);
+ return ftrace_modify_code_direct(rec, old, new);
}
/*
@@ -233,7 +282,7 @@ static int update_ftrace_func(unsigned long ip, void *new)
unsigned char old[MCOUNT_INSN_SIZE];
int ret;
- memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);
+ memcpy(old, (void *)ip, MCOUNT_RELINSN_SIZE);
ftrace_update_func = ip;
/* Make sure the breakpoints see the ftrace_update_func update */
@@ -255,13 +304,14 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
unsigned char *new;
int ret;
- new = ftrace_call_replace(ip, (unsigned long)func);
+ new = ftrace_call_replace(ip, (unsigned long)func, MCOUNT_RELINSN_SIZE);
ret = update_ftrace_func(ip, new);
/* Also update the regs callback function */
if (!ret) {
ip = (unsigned long)(&ftrace_regs_call);
- new = ftrace_call_replace(ip, (unsigned long)func);
+ new = ftrace_call_replace(ip, (unsigned long)func,
+ MCOUNT_RELINSN_SIZE);
ret = update_ftrace_func(ip, new);
}
@@ -309,18 +359,18 @@ static int ftrace_write(unsigned long ip, const char *val, int size)
return 0;
}
-static int add_break(unsigned long ip, const char *old)
+static int add_break(unsigned long ip, const char *old, unsigned int size)
{
unsigned char replaced[MCOUNT_INSN_SIZE];
unsigned char brk = BREAKPOINT_INSTRUCTION;
- if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
+ if (probe_kernel_read(replaced, (void *)ip, size))
return -EFAULT;
ftrace_expected = old;
/* Make sure it is what we expect it to be */
- if (memcmp(replaced, old, MCOUNT_INSN_SIZE) != 0)
+ if (memcmp(replaced, old, size) != 0)
return -EINVAL;
return ftrace_write(ip, &brk, 1);
@@ -330,20 +380,22 @@ static int add_brk_on_call(struct dyn_ftrace *rec, unsigned long addr)
{
unsigned const char *old;
unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
- old = ftrace_call_replace(ip, addr);
+ old = ftrace_call_replace(ip, addr, size);
- return add_break(rec->ip, old);
+ return add_break(rec->ip, old, size);
}
static int add_brk_on_nop(struct dyn_ftrace *rec)
{
unsigned const char *old;
+ unsigned int size = MCOUNT_INSN_SIZE;
- old = ftrace_nop_replace();
+ old = ftrace_nop_replace(size);
- return add_break(rec->ip, old);
+ return add_break(rec->ip, old, size);
}
static int add_breakpoints(struct dyn_ftrace *rec, int enable)
@@ -386,22 +438,23 @@ static int remove_breakpoint(struct dyn_ftrace *rec)
const unsigned char *nop;
unsigned long ftrace_addr;
unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
/* If we fail the read, just give up */
- if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))
+ if (probe_kernel_read(ins, (void *)ip, size))
return -EFAULT;
/* If this does not have a breakpoint, we are done */
if (ins[0] != brk)
return 0;
- nop = ftrace_nop_replace();
+ nop = ftrace_nop_replace(size);
/*
* If the last 4 bytes of the instruction do not match
* a nop, then we assume that this is a call to ftrace_addr.
*/
- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) {
+ if (memcmp(&ins[1], &nop[1], size - 1) != 0) {
/*
* For extra paranoidism, we check if the breakpoint is on
* a call that would actually jump to the ftrace_addr.
@@ -409,18 +462,18 @@ static int remove_breakpoint(struct dyn_ftrace *rec)
* a disaster.
*/
ftrace_addr = ftrace_get_addr_new(rec);
- nop = ftrace_call_replace(ip, ftrace_addr);
+ nop = ftrace_call_replace(ip, ftrace_addr, size);
- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) == 0)
+ if (memcmp(&ins[1], &nop[1], size - 1) == 0)
goto update;
/* Check both ftrace_addr and ftrace_old_addr */
ftrace_addr = ftrace_get_addr_curr(rec);
- nop = ftrace_call_replace(ip, ftrace_addr);
+ nop = ftrace_call_replace(ip, ftrace_addr, size);
ftrace_expected = nop;
- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
+ if (memcmp(&ins[1], &nop[1], size - 1) != 0)
return -EINVAL;
}
@@ -428,30 +481,33 @@ static int remove_breakpoint(struct dyn_ftrace *rec)
return ftrace_write(ip, nop, 1);
}
-static int add_update_code(unsigned long ip, unsigned const char *new)
+static int add_update_code(unsigned long ip, unsigned const char *new,
+ unsigned int size)
{
/* skip breakpoint */
ip++;
new++;
- return ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1);
+ return ftrace_write(ip, new, size - 1);
}
static int add_update_call(struct dyn_ftrace *rec, unsigned long addr)
{
unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
unsigned const char *new;
- new = ftrace_call_replace(ip, addr);
- return add_update_code(ip, new);
+ new = ftrace_call_replace(ip, addr, size);
+ return add_update_code(ip, new, size);
}
static int add_update_nop(struct dyn_ftrace *rec)
{
unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
unsigned const char *new;
- new = ftrace_nop_replace();
- return add_update_code(ip, new);
+ new = ftrace_nop_replace(size);
+ return add_update_code(ip, new, size);
}
static int add_update(struct dyn_ftrace *rec, int enable)
@@ -485,7 +541,7 @@ static int finish_update_call(struct dyn_ftrace *rec, unsigned long addr)
unsigned long ip = rec->ip;
unsigned const char *new;
- new = ftrace_call_replace(ip, addr);
+ new = ftrace_call_replace(ip, addr, MCOUNT_INSN_SIZE);
return ftrace_write(ip, new, 1);
}
@@ -495,7 +551,7 @@ static int finish_update_nop(struct dyn_ftrace *rec)
unsigned long ip = rec->ip;
unsigned const char *new;
- new = ftrace_nop_replace();
+ new = ftrace_nop_replace(MCOUNT_INSN_SIZE);
return ftrace_write(ip, new, 1);
}
@@ -619,13 +675,13 @@ ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
{
int ret;
- ret = add_break(ip, old_code);
+ ret = add_break(ip, old_code, MCOUNT_RELINSN_SIZE);
if (ret)
goto out;
run_sync();
- ret = add_update_code(ip, new_code);
+ ret = add_update_code(ip, new_code, MCOUNT_RELINSN_SIZE);
if (ret)
goto fail_update;
@@ -670,7 +726,7 @@ static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
/* Jmp not a call (ignore the .e8) */
calc.e8 = 0xe9;
- calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
+ calc.offset = ftrace_calc_offset(ip + MCOUNT_RELINSN_SIZE, addr);
/*
* ftrace external locks synchronize the access to the static variable.
@@ -766,11 +822,11 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
* the jmp to ftrace_epilogue, as well as the address of
* the ftrace_ops this trampoline is used for.
*/
- trampoline = alloc_tramp(size + MCOUNT_INSN_SIZE + sizeof(void *));
+ trampoline = alloc_tramp(size + MCOUNT_RELINSN_SIZE + sizeof(void *));
if (!trampoline)
return 0;
- *tramp_size = size + MCOUNT_INSN_SIZE + sizeof(void *);
+ *tramp_size = size + MCOUNT_RELINSN_SIZE + sizeof(void *);
/* Copy ftrace_caller onto the trampoline memory */
ret = probe_kernel_read(trampoline, (void *)start_offset, size);
@@ -783,7 +839,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
/* The trampoline ends with a jmp to ftrace_epilogue */
jmp = ftrace_jmp_replace(ip, (unsigned long)ftrace_epilogue);
- memcpy(trampoline + size, jmp, MCOUNT_INSN_SIZE);
+ memcpy(trampoline + size, jmp, MCOUNT_RELINSN_SIZE);
/*
* The address of the ftrace_ops that is used for this trampoline
@@ -793,7 +849,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
* the global function_trace_op variable.
*/
- ptr = (unsigned long *)(trampoline + size + MCOUNT_INSN_SIZE);
+ ptr = (unsigned long *)(trampoline + size + MCOUNT_RELINSN_SIZE);
*ptr = (unsigned long)ops;
op_offset -= start_offset;
@@ -868,7 +924,7 @@ void arch_ftrace_update_trampoline(struct ftrace_ops *ops)
func = ftrace_ops_get_func(ops);
/* Do a safe modify in case the trampoline is executing */
- new = ftrace_call_replace(ip, (unsigned long)func);
+ new = ftrace_call_replace(ip, (unsigned long)func, MCOUNT_RELINSN_SIZE);
ret = update_ftrace_func(ip, new);
set_memory_ro(ops->trampoline, npages);
@@ -882,7 +938,7 @@ static void *addr_from_call(void *ptr)
union ftrace_code_union calc;
int ret;
- ret = probe_kernel_read(&calc, ptr, MCOUNT_INSN_SIZE);
+ ret = probe_kernel_read(&calc, ptr, MCOUNT_RELINSN_SIZE);
if (WARN_ON_ONCE(ret < 0))
return NULL;
@@ -892,7 +948,7 @@ static void *addr_from_call(void *ptr)
return NULL;
}
- return ptr + MCOUNT_INSN_SIZE + calc.offset;
+ return ptr + MCOUNT_RELINSN_SIZE + calc.offset;
}
void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
diff --git a/arch/x86/kernel/module.lds b/arch/x86/kernel/module.lds
new file mode 100644
index 000000000000..fd6e95a4b454
--- /dev/null
+++ b/arch/x86/kernel/module.lds
@@ -0,0 +1,3 @@
+SECTIONS {
+ .got (NOLOAD) : { BYTE(0) }
+}
--
2.14.2.920.gcf0c67979c-goog
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related
* [RFC v3 20/27] x86/ftrace: Adapt function tracing for PIE support
From: Thomas Garnier @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
When using -fPIE/PIC with function tracing, the compiler generates a
call through the GOT (call *__fentry__@GOTPCREL). This instruction
takes 6 bytes instead of 5 on the usual relative call.
With this change, function tracing supports 6 bytes on traceable
function and can still replace relative calls on the ftrace assembly
functions.
Position Independent Executable (PIE) support will allow to extended the
KASLR randomization range below the -2G memory limit.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
arch/x86/include/asm/ftrace.h | 23 +++++-
arch/x86/include/asm/sections.h | 4 +
arch/x86/kernel/ftrace.c | 168 ++++++++++++++++++++++++++--------------
arch/x86/kernel/module.lds | 3 +
4 files changed, 139 insertions(+), 59 deletions(-)
create mode 100644 arch/x86/kernel/module.lds
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index eccd0ac6bc38..b8bbcc7fad7f 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -1,6 +1,7 @@
#ifndef _ASM_X86_FTRACE_H
#define _ASM_X86_FTRACE_H
+
#ifdef CONFIG_FUNCTION_TRACER
#ifdef CC_USING_FENTRY
# define MCOUNT_ADDR ((unsigned long)(__fentry__))
@@ -8,7 +9,19 @@
# define MCOUNT_ADDR ((unsigned long)(mcount))
# define HAVE_FUNCTION_GRAPH_FP_TEST
#endif
-#define MCOUNT_INSN_SIZE 5 /* sizeof mcount call */
+
+#define MCOUNT_RELINSN_SIZE 5 /* sizeof relative (call or jump) */
+#define MCOUNT_GOTCALL_SIZE 6 /* sizeof call *got */
+
+/*
+ * MCOUNT_INSN_SIZE is the highest size of instructions based on the
+ * configuration.
+ */
+#ifdef CONFIG_X86_PIE
+#define MCOUNT_INSN_SIZE MCOUNT_GOTCALL_SIZE
+#else
+#define MCOUNT_INSN_SIZE MCOUNT_RELINSN_SIZE
+#endif
#ifdef CONFIG_DYNAMIC_FTRACE
#define ARCH_SUPPORTS_FTRACE_OPS 1
@@ -17,6 +30,8 @@
#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
#ifndef __ASSEMBLY__
+#include <asm/sections.h>
+
extern void mcount(void);
extern atomic_t modifying_ftrace_code;
extern void __fentry__(void);
@@ -24,9 +39,11 @@ extern void __fentry__(void);
static inline unsigned long ftrace_call_adjust(unsigned long addr)
{
/*
- * addr is the address of the mcount call instruction.
- * recordmcount does the necessary offset calculation.
+ * addr is the address of the mcount call instruction. PIE has always a
+ * byte added to the start of the function.
*/
+ if (IS_ENABLED(CONFIG_X86_PIE))
+ addr -= 1;
return addr;
}
diff --git a/arch/x86/include/asm/sections.h b/arch/x86/include/asm/sections.h
index 2f75f30cb2f6..6b2d496cf1aa 100644
--- a/arch/x86/include/asm/sections.h
+++ b/arch/x86/include/asm/sections.h
@@ -11,4 +11,8 @@ extern struct exception_table_entry __stop___ex_table[];
extern char __end_rodata_hpage_align[];
#endif
+#if defined(CONFIG_X86_PIE)
+extern char __start_got[], __end_got[];
+#endif
+
#endif /* _ASM_X86_SECTIONS_H */
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index 9bef1bbeba63..41d8c4c4306d 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -58,12 +58,17 @@ static int ftrace_calc_offset(long ip, long addr)
return (int)(addr - ip);
}
-static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
+static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr,
+ unsigned int size)
{
static union ftrace_code_union calc;
+ /* On PIE, fill the rest of the buffer with nops */
+ if (IS_ENABLED(CONFIG_X86_PIE))
+ memset(calc.code, ideal_nops[1][0], sizeof(calc.code));
+
calc.e8 = 0xe8;
- calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
+ calc.offset = ftrace_calc_offset(ip + MCOUNT_RELINSN_SIZE, addr);
/*
* No locking needed, this must be called via kstop_machine
@@ -72,6 +77,44 @@ static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
return calc.code;
}
+#ifdef CONFIG_X86_PIE
+union ftrace_code_got_union {
+ char code[MCOUNT_INSN_SIZE];
+ struct {
+ unsigned short ff15;
+ int offset;
+ } __attribute__((packed));
+};
+
+/* Used to identify a mcount GOT call on PIE */
+static unsigned char *ftrace_original_call(struct module* mod, unsigned long ip,
+ unsigned long addr,
+ unsigned int size)
+{
+ static union ftrace_code_got_union calc;
+ unsigned long gotaddr;
+
+ calc.ff15 = 0x15ff;
+
+ gotaddr = module_find_got_entry(mod, addr);
+ if (!gotaddr) {
+ pr_err("Failed to find GOT entry for 0x%lx\n", addr);
+ return NULL;
+ }
+
+ calc.offset = ftrace_calc_offset(ip + MCOUNT_GOTCALL_SIZE, gotaddr);
+ return calc.code;
+}
+#else
+static unsigned char *ftrace_original_call(struct module* mod, unsigned long ip,
+ unsigned long addr,
+ unsigned int size)
+{
+ return ftrace_call_replace(ip, addr, size);
+}
+
+#endif
+
static inline int
within(unsigned long addr, unsigned long start, unsigned long end)
{
@@ -94,16 +137,18 @@ static unsigned long text_ip_addr(unsigned long ip)
return ip;
}
-static const unsigned char *ftrace_nop_replace(void)
+static const unsigned char *ftrace_nop_replace(unsigned int size)
{
- return ideal_nops[NOP_ATOMIC5];
+ return ideal_nops[size == 5 ? NOP_ATOMIC5 : size];
}
static int
-ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
- unsigned const char *new_code)
+ftrace_modify_code_direct(struct dyn_ftrace *rec, unsigned const char *old_code,
+ unsigned const char *new_code)
{
unsigned char replaced[MCOUNT_INSN_SIZE];
+ unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
ftrace_expected = old_code;
@@ -116,17 +161,17 @@ ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
*/
/* read the text we want to modify */
- if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
+ if (probe_kernel_read(replaced, (void *)ip, size))
return -EFAULT;
/* Make sure it is what we expect it to be */
- if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
+ if (memcmp(replaced, old_code, size) != 0)
return -EINVAL;
ip = text_ip_addr(ip);
/* replace the text with the new text */
- if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))
+ if (probe_kernel_write((void *)ip, new_code, size))
return -EPERM;
sync_core();
@@ -139,9 +184,7 @@ int ftrace_make_nop(struct module *mod,
{
unsigned const char *new, *old;
unsigned long ip = rec->ip;
-
- old = ftrace_call_replace(ip, addr);
- new = ftrace_nop_replace();
+ unsigned int size = MCOUNT_INSN_SIZE;
/*
* On boot up, and when modules are loaded, the MCOUNT_ADDR
@@ -151,14 +194,20 @@ int ftrace_make_nop(struct module *mod,
* We do not want to use the breakpoint version in this case,
* just modify the code directly.
*/
- if (addr == MCOUNT_ADDR)
- return ftrace_modify_code_direct(rec->ip, old, new);
+ if (addr != MCOUNT_ADDR) {
+ ftrace_expected = NULL;
- ftrace_expected = NULL;
+ /* Normal cases use add_brk_on_nop */
+ WARN_ONCE(1, "invalid use of ftrace_make_nop");
+ return -EINVAL;
+ }
- /* Normal cases use add_brk_on_nop */
- WARN_ONCE(1, "invalid use of ftrace_make_nop");
- return -EINVAL;
+ old = ftrace_original_call(mod, ip, addr, size);
+ if (!old)
+ return -EINVAL;
+ new = ftrace_nop_replace(size);
+
+ return ftrace_modify_code_direct(rec, old, new);
}
int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
@@ -166,11 +215,11 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
unsigned const char *new, *old;
unsigned long ip = rec->ip;
- old = ftrace_nop_replace();
- new = ftrace_call_replace(ip, addr);
+ old = ftrace_nop_replace(MCOUNT_INSN_SIZE);
+ new = ftrace_call_replace(ip, addr, MCOUNT_INSN_SIZE);
/* Should only be called when module is loaded */
- return ftrace_modify_code_direct(rec->ip, old, new);
+ return ftrace_modify_code_direct(rec, old, new);
}
/*
@@ -233,7 +282,7 @@ static int update_ftrace_func(unsigned long ip, void *new)
unsigned char old[MCOUNT_INSN_SIZE];
int ret;
- memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);
+ memcpy(old, (void *)ip, MCOUNT_RELINSN_SIZE);
ftrace_update_func = ip;
/* Make sure the breakpoints see the ftrace_update_func update */
@@ -255,13 +304,14 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
unsigned char *new;
int ret;
- new = ftrace_call_replace(ip, (unsigned long)func);
+ new = ftrace_call_replace(ip, (unsigned long)func, MCOUNT_RELINSN_SIZE);
ret = update_ftrace_func(ip, new);
/* Also update the regs callback function */
if (!ret) {
ip = (unsigned long)(&ftrace_regs_call);
- new = ftrace_call_replace(ip, (unsigned long)func);
+ new = ftrace_call_replace(ip, (unsigned long)func,
+ MCOUNT_RELINSN_SIZE);
ret = update_ftrace_func(ip, new);
}
@@ -309,18 +359,18 @@ static int ftrace_write(unsigned long ip, const char *val, int size)
return 0;
}
-static int add_break(unsigned long ip, const char *old)
+static int add_break(unsigned long ip, const char *old, unsigned int size)
{
unsigned char replaced[MCOUNT_INSN_SIZE];
unsigned char brk = BREAKPOINT_INSTRUCTION;
- if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
+ if (probe_kernel_read(replaced, (void *)ip, size))
return -EFAULT;
ftrace_expected = old;
/* Make sure it is what we expect it to be */
- if (memcmp(replaced, old, MCOUNT_INSN_SIZE) != 0)
+ if (memcmp(replaced, old, size) != 0)
return -EINVAL;
return ftrace_write(ip, &brk, 1);
@@ -330,20 +380,22 @@ static int add_brk_on_call(struct dyn_ftrace *rec, unsigned long addr)
{
unsigned const char *old;
unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
- old = ftrace_call_replace(ip, addr);
+ old = ftrace_call_replace(ip, addr, size);
- return add_break(rec->ip, old);
+ return add_break(rec->ip, old, size);
}
static int add_brk_on_nop(struct dyn_ftrace *rec)
{
unsigned const char *old;
+ unsigned int size = MCOUNT_INSN_SIZE;
- old = ftrace_nop_replace();
+ old = ftrace_nop_replace(size);
- return add_break(rec->ip, old);
+ return add_break(rec->ip, old, size);
}
static int add_breakpoints(struct dyn_ftrace *rec, int enable)
@@ -386,22 +438,23 @@ static int remove_breakpoint(struct dyn_ftrace *rec)
const unsigned char *nop;
unsigned long ftrace_addr;
unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
/* If we fail the read, just give up */
- if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))
+ if (probe_kernel_read(ins, (void *)ip, size))
return -EFAULT;
/* If this does not have a breakpoint, we are done */
if (ins[0] != brk)
return 0;
- nop = ftrace_nop_replace();
+ nop = ftrace_nop_replace(size);
/*
* If the last 4 bytes of the instruction do not match
* a nop, then we assume that this is a call to ftrace_addr.
*/
- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) {
+ if (memcmp(&ins[1], &nop[1], size - 1) != 0) {
/*
* For extra paranoidism, we check if the breakpoint is on
* a call that would actually jump to the ftrace_addr.
@@ -409,18 +462,18 @@ static int remove_breakpoint(struct dyn_ftrace *rec)
* a disaster.
*/
ftrace_addr = ftrace_get_addr_new(rec);
- nop = ftrace_call_replace(ip, ftrace_addr);
+ nop = ftrace_call_replace(ip, ftrace_addr, size);
- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) == 0)
+ if (memcmp(&ins[1], &nop[1], size - 1) == 0)
goto update;
/* Check both ftrace_addr and ftrace_old_addr */
ftrace_addr = ftrace_get_addr_curr(rec);
- nop = ftrace_call_replace(ip, ftrace_addr);
+ nop = ftrace_call_replace(ip, ftrace_addr, size);
ftrace_expected = nop;
- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
+ if (memcmp(&ins[1], &nop[1], size - 1) != 0)
return -EINVAL;
}
@@ -428,30 +481,33 @@ static int remove_breakpoint(struct dyn_ftrace *rec)
return ftrace_write(ip, nop, 1);
}
-static int add_update_code(unsigned long ip, unsigned const char *new)
+static int add_update_code(unsigned long ip, unsigned const char *new,
+ unsigned int size)
{
/* skip breakpoint */
ip++;
new++;
- return ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1);
+ return ftrace_write(ip, new, size - 1);
}
static int add_update_call(struct dyn_ftrace *rec, unsigned long addr)
{
unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
unsigned const char *new;
- new = ftrace_call_replace(ip, addr);
- return add_update_code(ip, new);
+ new = ftrace_call_replace(ip, addr, size);
+ return add_update_code(ip, new, size);
}
static int add_update_nop(struct dyn_ftrace *rec)
{
unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
unsigned const char *new;
- new = ftrace_nop_replace();
- return add_update_code(ip, new);
+ new = ftrace_nop_replace(size);
+ return add_update_code(ip, new, size);
}
static int add_update(struct dyn_ftrace *rec, int enable)
@@ -485,7 +541,7 @@ static int finish_update_call(struct dyn_ftrace *rec, unsigned long addr)
unsigned long ip = rec->ip;
unsigned const char *new;
- new = ftrace_call_replace(ip, addr);
+ new = ftrace_call_replace(ip, addr, MCOUNT_INSN_SIZE);
return ftrace_write(ip, new, 1);
}
@@ -495,7 +551,7 @@ static int finish_update_nop(struct dyn_ftrace *rec)
unsigned long ip = rec->ip;
unsigned const char *new;
- new = ftrace_nop_replace();
+ new = ftrace_nop_replace(MCOUNT_INSN_SIZE);
return ftrace_write(ip, new, 1);
}
@@ -619,13 +675,13 @@ ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
{
int ret;
- ret = add_break(ip, old_code);
+ ret = add_break(ip, old_code, MCOUNT_RELINSN_SIZE);
if (ret)
goto out;
run_sync();
- ret = add_update_code(ip, new_code);
+ ret = add_update_code(ip, new_code, MCOUNT_RELINSN_SIZE);
if (ret)
goto fail_update;
@@ -670,7 +726,7 @@ static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
/* Jmp not a call (ignore the .e8) */
calc.e8 = 0xe9;
- calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
+ calc.offset = ftrace_calc_offset(ip + MCOUNT_RELINSN_SIZE, addr);
/*
* ftrace external locks synchronize the access to the static variable.
@@ -766,11 +822,11 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
* the jmp to ftrace_epilogue, as well as the address of
* the ftrace_ops this trampoline is used for.
*/
- trampoline = alloc_tramp(size + MCOUNT_INSN_SIZE + sizeof(void *));
+ trampoline = alloc_tramp(size + MCOUNT_RELINSN_SIZE + sizeof(void *));
if (!trampoline)
return 0;
- *tramp_size = size + MCOUNT_INSN_SIZE + sizeof(void *);
+ *tramp_size = size + MCOUNT_RELINSN_SIZE + sizeof(void *);
/* Copy ftrace_caller onto the trampoline memory */
ret = probe_kernel_read(trampoline, (void *)start_offset, size);
@@ -783,7 +839,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
/* The trampoline ends with a jmp to ftrace_epilogue */
jmp = ftrace_jmp_replace(ip, (unsigned long)ftrace_epilogue);
- memcpy(trampoline + size, jmp, MCOUNT_INSN_SIZE);
+ memcpy(trampoline + size, jmp, MCOUNT_RELINSN_SIZE);
/*
* The address of the ftrace_ops that is used for this trampoline
@@ -793,7 +849,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
* the global function_trace_op variable.
*/
- ptr = (unsigned long *)(trampoline + size + MCOUNT_INSN_SIZE);
+ ptr = (unsigned long *)(trampoline + size + MCOUNT_RELINSN_SIZE);
*ptr = (unsigned long)ops;
op_offset -= start_offset;
@@ -868,7 +924,7 @@ void arch_ftrace_update_trampoline(struct ftrace_ops *ops)
func = ftrace_ops_get_func(ops);
/* Do a safe modify in case the trampoline is executing */
- new = ftrace_call_replace(ip, (unsigned long)func);
+ new = ftrace_call_replace(ip, (unsigned long)func, MCOUNT_RELINSN_SIZE);
ret = update_ftrace_func(ip, new);
set_memory_ro(ops->trampoline, npages);
@@ -882,7 +938,7 @@ static void *addr_from_call(void *ptr)
union ftrace_code_union calc;
int ret;
- ret = probe_kernel_read(&calc, ptr, MCOUNT_INSN_SIZE);
+ ret = probe_kernel_read(&calc, ptr, MCOUNT_RELINSN_SIZE);
if (WARN_ON_ONCE(ret < 0))
return NULL;
@@ -892,7 +948,7 @@ static void *addr_from_call(void *ptr)
return NULL;
}
- return ptr + MCOUNT_INSN_SIZE + calc.offset;
+ return ptr + MCOUNT_RELINSN_SIZE + calc.offset;
}
void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
diff --git a/arch/x86/kernel/module.lds b/arch/x86/kernel/module.lds
new file mode 100644
index 000000000000..fd6e95a4b454
--- /dev/null
+++ b/arch/x86/kernel/module.lds
@@ -0,0 +1,3 @@
+SECTIONS {
+ .got (NOLOAD) : { BYTE(0) }
+}
--
2.14.2.920.gcf0c67979c-goog
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related
* [kernel-hardening] [RFC v3 20/27] x86/ftrace: Adapt function tracing for PIE support
From: Thomas Garnier @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter, Boris Ostrovsky, Alexey Dobriyan,
Andrew Morton, Paul Gortmaker, Chris Metcalf, Paul E . McKenney,
Nicolas Pitre, Borislav Petkov, Luis R . Rodriguez,
Greg Kroah-Hartman, Christopher Li, Steven Rostedt, Jason Baron,
Dou Liyang, Rafael J . Wysocki, Mika Westerberg, Lukas Wunner,
Masahiro Yamada, Alexei Starovoitov, Daniel Borkmann,
Markus Trippelsdorf, Paolo Bonzini, Radim Krčmář,
Joerg Roedel, Rik van Riel, David Howells, Ard Biesheuvel,
Waiman Long, Kyle Huey, Andrey Ryabinin, Jonathan Corbet,
Matthew Wilcox, Michal Hocko, Peter Foley, Paul Bolle,
Jiri Kosina, Rob Landley, H . J . Lu, Baoquan He,
Jan H . Schönherr, Daniel Micay
Cc: x86, linux-crypto, linux-kernel, linux-pm, virtualization,
xen-devel, linux-arch, linux-sparse, kvm, linux-doc,
kernel-hardening
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
When using -fPIE/PIC with function tracing, the compiler generates a
call through the GOT (call *__fentry__@GOTPCREL). This instruction
takes 6 bytes instead of 5 on the usual relative call.
With this change, function tracing supports 6 bytes on traceable
function and can still replace relative calls on the ftrace assembly
functions.
Position Independent Executable (PIE) support will allow to extended the
KASLR randomization range below the -2G memory limit.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
arch/x86/include/asm/ftrace.h | 23 +++++-
arch/x86/include/asm/sections.h | 4 +
arch/x86/kernel/ftrace.c | 168 ++++++++++++++++++++++++++--------------
arch/x86/kernel/module.lds | 3 +
4 files changed, 139 insertions(+), 59 deletions(-)
create mode 100644 arch/x86/kernel/module.lds
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index eccd0ac6bc38..b8bbcc7fad7f 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -1,6 +1,7 @@
#ifndef _ASM_X86_FTRACE_H
#define _ASM_X86_FTRACE_H
+
#ifdef CONFIG_FUNCTION_TRACER
#ifdef CC_USING_FENTRY
# define MCOUNT_ADDR ((unsigned long)(__fentry__))
@@ -8,7 +9,19 @@
# define MCOUNT_ADDR ((unsigned long)(mcount))
# define HAVE_FUNCTION_GRAPH_FP_TEST
#endif
-#define MCOUNT_INSN_SIZE 5 /* sizeof mcount call */
+
+#define MCOUNT_RELINSN_SIZE 5 /* sizeof relative (call or jump) */
+#define MCOUNT_GOTCALL_SIZE 6 /* sizeof call *got */
+
+/*
+ * MCOUNT_INSN_SIZE is the highest size of instructions based on the
+ * configuration.
+ */
+#ifdef CONFIG_X86_PIE
+#define MCOUNT_INSN_SIZE MCOUNT_GOTCALL_SIZE
+#else
+#define MCOUNT_INSN_SIZE MCOUNT_RELINSN_SIZE
+#endif
#ifdef CONFIG_DYNAMIC_FTRACE
#define ARCH_SUPPORTS_FTRACE_OPS 1
@@ -17,6 +30,8 @@
#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
#ifndef __ASSEMBLY__
+#include <asm/sections.h>
+
extern void mcount(void);
extern atomic_t modifying_ftrace_code;
extern void __fentry__(void);
@@ -24,9 +39,11 @@ extern void __fentry__(void);
static inline unsigned long ftrace_call_adjust(unsigned long addr)
{
/*
- * addr is the address of the mcount call instruction.
- * recordmcount does the necessary offset calculation.
+ * addr is the address of the mcount call instruction. PIE has always a
+ * byte added to the start of the function.
*/
+ if (IS_ENABLED(CONFIG_X86_PIE))
+ addr -= 1;
return addr;
}
diff --git a/arch/x86/include/asm/sections.h b/arch/x86/include/asm/sections.h
index 2f75f30cb2f6..6b2d496cf1aa 100644
--- a/arch/x86/include/asm/sections.h
+++ b/arch/x86/include/asm/sections.h
@@ -11,4 +11,8 @@ extern struct exception_table_entry __stop___ex_table[];
extern char __end_rodata_hpage_align[];
#endif
+#if defined(CONFIG_X86_PIE)
+extern char __start_got[], __end_got[];
+#endif
+
#endif /* _ASM_X86_SECTIONS_H */
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index 9bef1bbeba63..41d8c4c4306d 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -58,12 +58,17 @@ static int ftrace_calc_offset(long ip, long addr)
return (int)(addr - ip);
}
-static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
+static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr,
+ unsigned int size)
{
static union ftrace_code_union calc;
+ /* On PIE, fill the rest of the buffer with nops */
+ if (IS_ENABLED(CONFIG_X86_PIE))
+ memset(calc.code, ideal_nops[1][0], sizeof(calc.code));
+
calc.e8 = 0xe8;
- calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
+ calc.offset = ftrace_calc_offset(ip + MCOUNT_RELINSN_SIZE, addr);
/*
* No locking needed, this must be called via kstop_machine
@@ -72,6 +77,44 @@ static unsigned char *ftrace_call_replace(unsigned long ip, unsigned long addr)
return calc.code;
}
+#ifdef CONFIG_X86_PIE
+union ftrace_code_got_union {
+ char code[MCOUNT_INSN_SIZE];
+ struct {
+ unsigned short ff15;
+ int offset;
+ } __attribute__((packed));
+};
+
+/* Used to identify a mcount GOT call on PIE */
+static unsigned char *ftrace_original_call(struct module* mod, unsigned long ip,
+ unsigned long addr,
+ unsigned int size)
+{
+ static union ftrace_code_got_union calc;
+ unsigned long gotaddr;
+
+ calc.ff15 = 0x15ff;
+
+ gotaddr = module_find_got_entry(mod, addr);
+ if (!gotaddr) {
+ pr_err("Failed to find GOT entry for 0x%lx\n", addr);
+ return NULL;
+ }
+
+ calc.offset = ftrace_calc_offset(ip + MCOUNT_GOTCALL_SIZE, gotaddr);
+ return calc.code;
+}
+#else
+static unsigned char *ftrace_original_call(struct module* mod, unsigned long ip,
+ unsigned long addr,
+ unsigned int size)
+{
+ return ftrace_call_replace(ip, addr, size);
+}
+
+#endif
+
static inline int
within(unsigned long addr, unsigned long start, unsigned long end)
{
@@ -94,16 +137,18 @@ static unsigned long text_ip_addr(unsigned long ip)
return ip;
}
-static const unsigned char *ftrace_nop_replace(void)
+static const unsigned char *ftrace_nop_replace(unsigned int size)
{
- return ideal_nops[NOP_ATOMIC5];
+ return ideal_nops[size == 5 ? NOP_ATOMIC5 : size];
}
static int
-ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
- unsigned const char *new_code)
+ftrace_modify_code_direct(struct dyn_ftrace *rec, unsigned const char *old_code,
+ unsigned const char *new_code)
{
unsigned char replaced[MCOUNT_INSN_SIZE];
+ unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
ftrace_expected = old_code;
@@ -116,17 +161,17 @@ ftrace_modify_code_direct(unsigned long ip, unsigned const char *old_code,
*/
/* read the text we want to modify */
- if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
+ if (probe_kernel_read(replaced, (void *)ip, size))
return -EFAULT;
/* Make sure it is what we expect it to be */
- if (memcmp(replaced, old_code, MCOUNT_INSN_SIZE) != 0)
+ if (memcmp(replaced, old_code, size) != 0)
return -EINVAL;
ip = text_ip_addr(ip);
/* replace the text with the new text */
- if (probe_kernel_write((void *)ip, new_code, MCOUNT_INSN_SIZE))
+ if (probe_kernel_write((void *)ip, new_code, size))
return -EPERM;
sync_core();
@@ -139,9 +184,7 @@ int ftrace_make_nop(struct module *mod,
{
unsigned const char *new, *old;
unsigned long ip = rec->ip;
-
- old = ftrace_call_replace(ip, addr);
- new = ftrace_nop_replace();
+ unsigned int size = MCOUNT_INSN_SIZE;
/*
* On boot up, and when modules are loaded, the MCOUNT_ADDR
@@ -151,14 +194,20 @@ int ftrace_make_nop(struct module *mod,
* We do not want to use the breakpoint version in this case,
* just modify the code directly.
*/
- if (addr == MCOUNT_ADDR)
- return ftrace_modify_code_direct(rec->ip, old, new);
+ if (addr != MCOUNT_ADDR) {
+ ftrace_expected = NULL;
- ftrace_expected = NULL;
+ /* Normal cases use add_brk_on_nop */
+ WARN_ONCE(1, "invalid use of ftrace_make_nop");
+ return -EINVAL;
+ }
- /* Normal cases use add_brk_on_nop */
- WARN_ONCE(1, "invalid use of ftrace_make_nop");
- return -EINVAL;
+ old = ftrace_original_call(mod, ip, addr, size);
+ if (!old)
+ return -EINVAL;
+ new = ftrace_nop_replace(size);
+
+ return ftrace_modify_code_direct(rec, old, new);
}
int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
@@ -166,11 +215,11 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
unsigned const char *new, *old;
unsigned long ip = rec->ip;
- old = ftrace_nop_replace();
- new = ftrace_call_replace(ip, addr);
+ old = ftrace_nop_replace(MCOUNT_INSN_SIZE);
+ new = ftrace_call_replace(ip, addr, MCOUNT_INSN_SIZE);
/* Should only be called when module is loaded */
- return ftrace_modify_code_direct(rec->ip, old, new);
+ return ftrace_modify_code_direct(rec, old, new);
}
/*
@@ -233,7 +282,7 @@ static int update_ftrace_func(unsigned long ip, void *new)
unsigned char old[MCOUNT_INSN_SIZE];
int ret;
- memcpy(old, (void *)ip, MCOUNT_INSN_SIZE);
+ memcpy(old, (void *)ip, MCOUNT_RELINSN_SIZE);
ftrace_update_func = ip;
/* Make sure the breakpoints see the ftrace_update_func update */
@@ -255,13 +304,14 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
unsigned char *new;
int ret;
- new = ftrace_call_replace(ip, (unsigned long)func);
+ new = ftrace_call_replace(ip, (unsigned long)func, MCOUNT_RELINSN_SIZE);
ret = update_ftrace_func(ip, new);
/* Also update the regs callback function */
if (!ret) {
ip = (unsigned long)(&ftrace_regs_call);
- new = ftrace_call_replace(ip, (unsigned long)func);
+ new = ftrace_call_replace(ip, (unsigned long)func,
+ MCOUNT_RELINSN_SIZE);
ret = update_ftrace_func(ip, new);
}
@@ -309,18 +359,18 @@ static int ftrace_write(unsigned long ip, const char *val, int size)
return 0;
}
-static int add_break(unsigned long ip, const char *old)
+static int add_break(unsigned long ip, const char *old, unsigned int size)
{
unsigned char replaced[MCOUNT_INSN_SIZE];
unsigned char brk = BREAKPOINT_INSTRUCTION;
- if (probe_kernel_read(replaced, (void *)ip, MCOUNT_INSN_SIZE))
+ if (probe_kernel_read(replaced, (void *)ip, size))
return -EFAULT;
ftrace_expected = old;
/* Make sure it is what we expect it to be */
- if (memcmp(replaced, old, MCOUNT_INSN_SIZE) != 0)
+ if (memcmp(replaced, old, size) != 0)
return -EINVAL;
return ftrace_write(ip, &brk, 1);
@@ -330,20 +380,22 @@ static int add_brk_on_call(struct dyn_ftrace *rec, unsigned long addr)
{
unsigned const char *old;
unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
- old = ftrace_call_replace(ip, addr);
+ old = ftrace_call_replace(ip, addr, size);
- return add_break(rec->ip, old);
+ return add_break(rec->ip, old, size);
}
static int add_brk_on_nop(struct dyn_ftrace *rec)
{
unsigned const char *old;
+ unsigned int size = MCOUNT_INSN_SIZE;
- old = ftrace_nop_replace();
+ old = ftrace_nop_replace(size);
- return add_break(rec->ip, old);
+ return add_break(rec->ip, old, size);
}
static int add_breakpoints(struct dyn_ftrace *rec, int enable)
@@ -386,22 +438,23 @@ static int remove_breakpoint(struct dyn_ftrace *rec)
const unsigned char *nop;
unsigned long ftrace_addr;
unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
/* If we fail the read, just give up */
- if (probe_kernel_read(ins, (void *)ip, MCOUNT_INSN_SIZE))
+ if (probe_kernel_read(ins, (void *)ip, size))
return -EFAULT;
/* If this does not have a breakpoint, we are done */
if (ins[0] != brk)
return 0;
- nop = ftrace_nop_replace();
+ nop = ftrace_nop_replace(size);
/*
* If the last 4 bytes of the instruction do not match
* a nop, then we assume that this is a call to ftrace_addr.
*/
- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) {
+ if (memcmp(&ins[1], &nop[1], size - 1) != 0) {
/*
* For extra paranoidism, we check if the breakpoint is on
* a call that would actually jump to the ftrace_addr.
@@ -409,18 +462,18 @@ static int remove_breakpoint(struct dyn_ftrace *rec)
* a disaster.
*/
ftrace_addr = ftrace_get_addr_new(rec);
- nop = ftrace_call_replace(ip, ftrace_addr);
+ nop = ftrace_call_replace(ip, ftrace_addr, size);
- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) == 0)
+ if (memcmp(&ins[1], &nop[1], size - 1) == 0)
goto update;
/* Check both ftrace_addr and ftrace_old_addr */
ftrace_addr = ftrace_get_addr_curr(rec);
- nop = ftrace_call_replace(ip, ftrace_addr);
+ nop = ftrace_call_replace(ip, ftrace_addr, size);
ftrace_expected = nop;
- if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
+ if (memcmp(&ins[1], &nop[1], size - 1) != 0)
return -EINVAL;
}
@@ -428,30 +481,33 @@ static int remove_breakpoint(struct dyn_ftrace *rec)
return ftrace_write(ip, nop, 1);
}
-static int add_update_code(unsigned long ip, unsigned const char *new)
+static int add_update_code(unsigned long ip, unsigned const char *new,
+ unsigned int size)
{
/* skip breakpoint */
ip++;
new++;
- return ftrace_write(ip, new, MCOUNT_INSN_SIZE - 1);
+ return ftrace_write(ip, new, size - 1);
}
static int add_update_call(struct dyn_ftrace *rec, unsigned long addr)
{
unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
unsigned const char *new;
- new = ftrace_call_replace(ip, addr);
- return add_update_code(ip, new);
+ new = ftrace_call_replace(ip, addr, size);
+ return add_update_code(ip, new, size);
}
static int add_update_nop(struct dyn_ftrace *rec)
{
unsigned long ip = rec->ip;
+ unsigned int size = MCOUNT_INSN_SIZE;
unsigned const char *new;
- new = ftrace_nop_replace();
- return add_update_code(ip, new);
+ new = ftrace_nop_replace(size);
+ return add_update_code(ip, new, size);
}
static int add_update(struct dyn_ftrace *rec, int enable)
@@ -485,7 +541,7 @@ static int finish_update_call(struct dyn_ftrace *rec, unsigned long addr)
unsigned long ip = rec->ip;
unsigned const char *new;
- new = ftrace_call_replace(ip, addr);
+ new = ftrace_call_replace(ip, addr, MCOUNT_INSN_SIZE);
return ftrace_write(ip, new, 1);
}
@@ -495,7 +551,7 @@ static int finish_update_nop(struct dyn_ftrace *rec)
unsigned long ip = rec->ip;
unsigned const char *new;
- new = ftrace_nop_replace();
+ new = ftrace_nop_replace(MCOUNT_INSN_SIZE);
return ftrace_write(ip, new, 1);
}
@@ -619,13 +675,13 @@ ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
{
int ret;
- ret = add_break(ip, old_code);
+ ret = add_break(ip, old_code, MCOUNT_RELINSN_SIZE);
if (ret)
goto out;
run_sync();
- ret = add_update_code(ip, new_code);
+ ret = add_update_code(ip, new_code, MCOUNT_RELINSN_SIZE);
if (ret)
goto fail_update;
@@ -670,7 +726,7 @@ static unsigned char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
/* Jmp not a call (ignore the .e8) */
calc.e8 = 0xe9;
- calc.offset = ftrace_calc_offset(ip + MCOUNT_INSN_SIZE, addr);
+ calc.offset = ftrace_calc_offset(ip + MCOUNT_RELINSN_SIZE, addr);
/*
* ftrace external locks synchronize the access to the static variable.
@@ -766,11 +822,11 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
* the jmp to ftrace_epilogue, as well as the address of
* the ftrace_ops this trampoline is used for.
*/
- trampoline = alloc_tramp(size + MCOUNT_INSN_SIZE + sizeof(void *));
+ trampoline = alloc_tramp(size + MCOUNT_RELINSN_SIZE + sizeof(void *));
if (!trampoline)
return 0;
- *tramp_size = size + MCOUNT_INSN_SIZE + sizeof(void *);
+ *tramp_size = size + MCOUNT_RELINSN_SIZE + sizeof(void *);
/* Copy ftrace_caller onto the trampoline memory */
ret = probe_kernel_read(trampoline, (void *)start_offset, size);
@@ -783,7 +839,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
/* The trampoline ends with a jmp to ftrace_epilogue */
jmp = ftrace_jmp_replace(ip, (unsigned long)ftrace_epilogue);
- memcpy(trampoline + size, jmp, MCOUNT_INSN_SIZE);
+ memcpy(trampoline + size, jmp, MCOUNT_RELINSN_SIZE);
/*
* The address of the ftrace_ops that is used for this trampoline
@@ -793,7 +849,7 @@ create_trampoline(struct ftrace_ops *ops, unsigned int *tramp_size)
* the global function_trace_op variable.
*/
- ptr = (unsigned long *)(trampoline + size + MCOUNT_INSN_SIZE);
+ ptr = (unsigned long *)(trampoline + size + MCOUNT_RELINSN_SIZE);
*ptr = (unsigned long)ops;
op_offset -= start_offset;
@@ -868,7 +924,7 @@ void arch_ftrace_update_trampoline(struct ftrace_ops *ops)
func = ftrace_ops_get_func(ops);
/* Do a safe modify in case the trampoline is executing */
- new = ftrace_call_replace(ip, (unsigned long)func);
+ new = ftrace_call_replace(ip, (unsigned long)func, MCOUNT_RELINSN_SIZE);
ret = update_ftrace_func(ip, new);
set_memory_ro(ops->trampoline, npages);
@@ -882,7 +938,7 @@ static void *addr_from_call(void *ptr)
union ftrace_code_union calc;
int ret;
- ret = probe_kernel_read(&calc, ptr, MCOUNT_INSN_SIZE);
+ ret = probe_kernel_read(&calc, ptr, MCOUNT_RELINSN_SIZE);
if (WARN_ON_ONCE(ret < 0))
return NULL;
@@ -892,7 +948,7 @@ static void *addr_from_call(void *ptr)
return NULL;
}
- return ptr + MCOUNT_INSN_SIZE + calc.offset;
+ return ptr + MCOUNT_RELINSN_SIZE + calc.offset;
}
void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
diff --git a/arch/x86/kernel/module.lds b/arch/x86/kernel/module.lds
new file mode 100644
index 000000000000..fd6e95a4b454
--- /dev/null
+++ b/arch/x86/kernel/module.lds
@@ -0,0 +1,3 @@
+SECTIONS {
+ .got (NOLOAD) : { BYTE(0) }
+}
--
2.14.2.920.gcf0c67979c-goog
^ permalink raw reply related
* [RFC v3 19/27] x86: Support global stack cookie
From: Thomas Garnier via Virtualization @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
Add an off-by-default configuration option to use a global stack cookie
instead of the default TLS. This configuration option will only be used
with PIE binaries.
For kernel stack cookie, the compiler uses the mcmodel=kernel to switch
between the fs segment to gs segment. A PIE binary does not use
mcmodel=kernel because it can be relocated anywhere, therefore the
compiler will default to the fs segment register. This is going to be
fixed with a compiler change allowing to pick the segment register as
done on PowerPC. In the meantime, this configuration can be used to
support older compilers.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
arch/x86/Kconfig | 4 ++++
arch/x86/Makefile | 9 +++++++++
arch/x86/entry/entry_32.S | 3 ++-
arch/x86/entry/entry_64.S | 3 ++-
arch/x86/include/asm/processor.h | 3 ++-
arch/x86/include/asm/stackprotector.h | 19 ++++++++++++++-----
arch/x86/kernel/asm-offsets.c | 3 ++-
arch/x86/kernel/asm-offsets_32.c | 3 ++-
arch/x86/kernel/asm-offsets_64.c | 3 ++-
arch/x86/kernel/cpu/common.c | 3 ++-
arch/x86/kernel/head_32.S | 3 ++-
arch/x86/kernel/process.c | 5 +++++
12 files changed, 48 insertions(+), 13 deletions(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 063f1e0d51aa..777197fab6dd 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -2133,6 +2133,10 @@ config RANDOMIZE_MEMORY_PHYSICAL_PADDING
If unsure, leave at the default value.
+config X86_GLOBAL_STACKPROTECTOR
+ bool
+ depends on CC_STACKPROTECTOR
+
config HOTPLUG_CPU
bool "Support for hot-pluggable CPUs"
depends on SMP
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 6276572259c8..4cb4f0495ddc 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -60,6 +60,15 @@ endif
KBUILD_CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-3dnow
KBUILD_CFLAGS += $(call cc-option,-mno-avx,)
+ifdef CONFIG_X86_GLOBAL_STACKPROTECTOR
+ ifeq ($(call cc-option, -mstack-protector-guard=global),)
+ $(error Cannot use CONFIG_X86_GLOBAL_STACKPROTECTOR: \
+ -mstack-protector-guard=global not supported \
+ by compiler)
+ endif
+ KBUILD_CFLAGS += -mstack-protector-guard=global
+endif
+
ifeq ($(CONFIG_X86_32),y)
BITS := 32
UTS_MACHINE := i386
diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index 8a13d468635a..ab3e5056722f 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -237,7 +237,8 @@ ENTRY(__switch_to_asm)
movl %esp, TASK_threadsp(%eax)
movl TASK_threadsp(%edx), %esp
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
movl TASK_stack_canary(%edx), %ebx
movl %ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset
#endif
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index d3a52d2342af..01be62c1b436 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -390,7 +390,8 @@ ENTRY(__switch_to_asm)
movq %rsp, TASK_threadsp(%rdi)
movq TASK_threadsp(%rsi), %rsp
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
movq TASK_stack_canary(%rsi), %rbx
movq %rbx, PER_CPU_VAR(irq_stack_union + stack_canary_offset)
#endif
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index b09bd50b06c7..e3a7ef8d5fb8 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -394,7 +394,8 @@ DECLARE_PER_CPU(char *, irq_stack_ptr);
DECLARE_PER_CPU(unsigned int, irq_count);
extern asmlinkage void ignore_sysret(void);
#else /* X86_64 */
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
/*
* Make sure stack canary segment base is cached-aligned:
* "For Intel Atom processors, avoid non zero segment base address
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
index 8abedf1d650e..66462d778dc5 100644
--- a/arch/x86/include/asm/stackprotector.h
+++ b/arch/x86/include/asm/stackprotector.h
@@ -51,6 +51,10 @@
#define GDT_STACK_CANARY_INIT \
[GDT_ENTRY_STACK_CANARY] = GDT_ENTRY_INIT(0x4090, 0, 0x18),
+#ifdef CONFIG_X86_GLOBAL_STACKPROTECTOR
+extern unsigned long __stack_chk_guard;
+#endif
+
/*
* Initialize the stackprotector canary value.
*
@@ -62,7 +66,7 @@ static __always_inline void boot_init_stack_canary(void)
u64 canary;
u64 tsc;
-#ifdef CONFIG_X86_64
+#if defined(CONFIG_X86_64) && !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
BUILD_BUG_ON(offsetof(union irq_stack_union, stack_canary) != 40);
#endif
/*
@@ -76,17 +80,22 @@ static __always_inline void boot_init_stack_canary(void)
canary += tsc + (tsc << 32UL);
canary &= CANARY_MASK;
+#ifdef CONFIG_X86_GLOBAL_STACKPROTECTOR
+ if (__stack_chk_guard == 0)
+ __stack_chk_guard = canary ?: 1;
+#else /* !CONFIG_X86_GLOBAL_STACKPROTECTOR */
current->stack_canary = canary;
#ifdef CONFIG_X86_64
this_cpu_write(irq_stack_union.stack_canary, canary);
-#else
+#else /* CONFIG_X86_32 */
this_cpu_write(stack_canary.canary, canary);
#endif
+#endif
}
static inline void setup_stack_canary_segment(int cpu)
{
-#ifdef CONFIG_X86_32
+#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
unsigned long canary = (unsigned long)&per_cpu(stack_canary, cpu);
struct desc_struct *gdt_table = get_cpu_gdt_rw(cpu);
struct desc_struct desc;
@@ -99,7 +108,7 @@ static inline void setup_stack_canary_segment(int cpu)
static inline void load_stack_canary_segment(void)
{
-#ifdef CONFIG_X86_32
+#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
asm("mov %0, %%gs" : : "r" (__KERNEL_STACK_CANARY) : "memory");
#endif
}
@@ -115,7 +124,7 @@ static inline void setup_stack_canary_segment(int cpu)
static inline void load_stack_canary_segment(void)
{
-#ifdef CONFIG_X86_32
+#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
asm volatile ("mov %0, %%gs" : : "r" (0));
#endif
}
diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index de827d6ac8c2..b30a12cd021e 100644
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -30,7 +30,8 @@
void common(void) {
BLANK();
OFFSET(TASK_threadsp, task_struct, thread.sp);
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
OFFSET(TASK_stack_canary, task_struct, stack_canary);
#endif
diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c
index 710edab9e644..33584e7e486b 100644
--- a/arch/x86/kernel/asm-offsets_32.c
+++ b/arch/x86/kernel/asm-offsets_32.c
@@ -54,7 +54,8 @@ void foo(void)
/* Size of SYSENTER_stack */
DEFINE(SIZEOF_SYSENTER_stack, sizeof(((struct tss_struct *)0)->SYSENTER_stack));
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
BLANK();
OFFSET(stack_canary_offset, stack_canary, canary);
#endif
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index cf42206926af..06feb31a09f5 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -64,7 +64,8 @@ int main(void)
OFFSET(TSS_sp0, tss_struct, x86_tss.sp0);
BLANK();
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
DEFINE(stack_canary_offset, offsetof(union irq_stack_union, stack_canary));
BLANK();
#endif
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 8732c9e719d5..839f4da2ed76 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1431,7 +1431,8 @@ DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) =
(unsigned long)&init_thread_union + THREAD_SIZE;
EXPORT_PER_CPU_SYMBOL(cpu_current_top_of_stack);
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
#endif
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index 9ed3074d0d27..a55a67b33934 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -377,7 +377,8 @@ ENDPROC(startup_32_smp)
*/
__INIT
setup_once:
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
/*
* Configure the stack canary. The linker can't handle this by
* relocation. Manually set base address in stack canary
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index bd6b85fac666..66ea1a35413e 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -73,6 +73,11 @@ EXPORT_PER_CPU_SYMBOL(cpu_tss);
DEFINE_PER_CPU(bool, __tss_limit_invalid);
EXPORT_PER_CPU_SYMBOL_GPL(__tss_limit_invalid);
+#ifdef CONFIG_X86_GLOBAL_STACKPROTECTOR
+unsigned long __stack_chk_guard __read_mostly;
+EXPORT_SYMBOL(__stack_chk_guard);
+#endif
+
/*
* this gets called so that we can store lazy state into memory and copy the
* current task into the new thread.
--
2.14.2.920.gcf0c67979c-goog
^ permalink raw reply related
* [RFC v3 19/27] x86: Support global stack cookie
From: Thomas Garnier @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter, Bor
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
Add an off-by-default configuration option to use a global stack cookie
instead of the default TLS. This configuration option will only be used
with PIE binaries.
For kernel stack cookie, the compiler uses the mcmodel=kernel to switch
between the fs segment to gs segment. A PIE binary does not use
mcmodel=kernel because it can be relocated anywhere, therefore the
compiler will default to the fs segment register. This is going to be
fixed with a compiler change allowing to pick the segment register as
done on PowerPC. In the meantime, this configuration can be used to
support older compilers.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
arch/x86/Kconfig | 4 ++++
arch/x86/Makefile | 9 +++++++++
arch/x86/entry/entry_32.S | 3 ++-
arch/x86/entry/entry_64.S | 3 ++-
arch/x86/include/asm/processor.h | 3 ++-
arch/x86/include/asm/stackprotector.h | 19 ++++++++++++++-----
arch/x86/kernel/asm-offsets.c | 3 ++-
arch/x86/kernel/asm-offsets_32.c | 3 ++-
arch/x86/kernel/asm-offsets_64.c | 3 ++-
arch/x86/kernel/cpu/common.c | 3 ++-
arch/x86/kernel/head_32.S | 3 ++-
arch/x86/kernel/process.c | 5 +++++
12 files changed, 48 insertions(+), 13 deletions(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 063f1e0d51aa..777197fab6dd 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -2133,6 +2133,10 @@ config RANDOMIZE_MEMORY_PHYSICAL_PADDING
If unsure, leave at the default value.
+config X86_GLOBAL_STACKPROTECTOR
+ bool
+ depends on CC_STACKPROTECTOR
+
config HOTPLUG_CPU
bool "Support for hot-pluggable CPUs"
depends on SMP
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 6276572259c8..4cb4f0495ddc 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -60,6 +60,15 @@ endif
KBUILD_CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-3dnow
KBUILD_CFLAGS += $(call cc-option,-mno-avx,)
+ifdef CONFIG_X86_GLOBAL_STACKPROTECTOR
+ ifeq ($(call cc-option, -mstack-protector-guard=global),)
+ $(error Cannot use CONFIG_X86_GLOBAL_STACKPROTECTOR: \
+ -mstack-protector-guard=global not supported \
+ by compiler)
+ endif
+ KBUILD_CFLAGS += -mstack-protector-guard=global
+endif
+
ifeq ($(CONFIG_X86_32),y)
BITS := 32
UTS_MACHINE := i386
diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index 8a13d468635a..ab3e5056722f 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -237,7 +237,8 @@ ENTRY(__switch_to_asm)
movl %esp, TASK_threadsp(%eax)
movl TASK_threadsp(%edx), %esp
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
movl TASK_stack_canary(%edx), %ebx
movl %ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset
#endif
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index d3a52d2342af..01be62c1b436 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -390,7 +390,8 @@ ENTRY(__switch_to_asm)
movq %rsp, TASK_threadsp(%rdi)
movq TASK_threadsp(%rsi), %rsp
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
movq TASK_stack_canary(%rsi), %rbx
movq %rbx, PER_CPU_VAR(irq_stack_union + stack_canary_offset)
#endif
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index b09bd50b06c7..e3a7ef8d5fb8 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -394,7 +394,8 @@ DECLARE_PER_CPU(char *, irq_stack_ptr);
DECLARE_PER_CPU(unsigned int, irq_count);
extern asmlinkage void ignore_sysret(void);
#else /* X86_64 */
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
/*
* Make sure stack canary segment base is cached-aligned:
* "For Intel Atom processors, avoid non zero segment base address
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
index 8abedf1d650e..66462d778dc5 100644
--- a/arch/x86/include/asm/stackprotector.h
+++ b/arch/x86/include/asm/stackprotector.h
@@ -51,6 +51,10 @@
#define GDT_STACK_CANARY_INIT \
[GDT_ENTRY_STACK_CANARY] = GDT_ENTRY_INIT(0x4090, 0, 0x18),
+#ifdef CONFIG_X86_GLOBAL_STACKPROTECTOR
+extern unsigned long __stack_chk_guard;
+#endif
+
/*
* Initialize the stackprotector canary value.
*
@@ -62,7 +66,7 @@ static __always_inline void boot_init_stack_canary(void)
u64 canary;
u64 tsc;
-#ifdef CONFIG_X86_64
+#if defined(CONFIG_X86_64) && !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
BUILD_BUG_ON(offsetof(union irq_stack_union, stack_canary) != 40);
#endif
/*
@@ -76,17 +80,22 @@ static __always_inline void boot_init_stack_canary(void)
canary += tsc + (tsc << 32UL);
canary &= CANARY_MASK;
+#ifdef CONFIG_X86_GLOBAL_STACKPROTECTOR
+ if (__stack_chk_guard == 0)
+ __stack_chk_guard = canary ?: 1;
+#else /* !CONFIG_X86_GLOBAL_STACKPROTECTOR */
current->stack_canary = canary;
#ifdef CONFIG_X86_64
this_cpu_write(irq_stack_union.stack_canary, canary);
-#else
+#else /* CONFIG_X86_32 */
this_cpu_write(stack_canary.canary, canary);
#endif
+#endif
}
static inline void setup_stack_canary_segment(int cpu)
{
-#ifdef CONFIG_X86_32
+#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
unsigned long canary = (unsigned long)&per_cpu(stack_canary, cpu);
struct desc_struct *gdt_table = get_cpu_gdt_rw(cpu);
struct desc_struct desc;
@@ -99,7 +108,7 @@ static inline void setup_stack_canary_segment(int cpu)
static inline void load_stack_canary_segment(void)
{
-#ifdef CONFIG_X86_32
+#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
asm("mov %0, %%gs" : : "r" (__KERNEL_STACK_CANARY) : "memory");
#endif
}
@@ -115,7 +124,7 @@ static inline void setup_stack_canary_segment(int cpu)
static inline void load_stack_canary_segment(void)
{
-#ifdef CONFIG_X86_32
+#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
asm volatile ("mov %0, %%gs" : : "r" (0));
#endif
}
diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index de827d6ac8c2..b30a12cd021e 100644
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -30,7 +30,8 @@
void common(void) {
BLANK();
OFFSET(TASK_threadsp, task_struct, thread.sp);
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
OFFSET(TASK_stack_canary, task_struct, stack_canary);
#endif
diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c
index 710edab9e644..33584e7e486b 100644
--- a/arch/x86/kernel/asm-offsets_32.c
+++ b/arch/x86/kernel/asm-offsets_32.c
@@ -54,7 +54,8 @@ void foo(void)
/* Size of SYSENTER_stack */
DEFINE(SIZEOF_SYSENTER_stack, sizeof(((struct tss_struct *)0)->SYSENTER_stack));
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
BLANK();
OFFSET(stack_canary_offset, stack_canary, canary);
#endif
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index cf42206926af..06feb31a09f5 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -64,7 +64,8 @@ int main(void)
OFFSET(TSS_sp0, tss_struct, x86_tss.sp0);
BLANK();
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
DEFINE(stack_canary_offset, offsetof(union irq_stack_union, stack_canary));
BLANK();
#endif
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 8732c9e719d5..839f4da2ed76 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1431,7 +1431,8 @@ DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) =
(unsigned long)&init_thread_union + THREAD_SIZE;
EXPORT_PER_CPU_SYMBOL(cpu_current_top_of_stack);
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
#endif
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index 9ed3074d0d27..a55a67b33934 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -377,7 +377,8 @@ ENDPROC(startup_32_smp)
*/
__INIT
setup_once:
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
/*
* Configure the stack canary. The linker can't handle this by
* relocation. Manually set base address in stack canary
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index bd6b85fac666..66ea1a35413e 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -73,6 +73,11 @@ EXPORT_PER_CPU_SYMBOL(cpu_tss);
DEFINE_PER_CPU(bool, __tss_limit_invalid);
EXPORT_PER_CPU_SYMBOL_GPL(__tss_limit_invalid);
+#ifdef CONFIG_X86_GLOBAL_STACKPROTECTOR
+unsigned long __stack_chk_guard __read_mostly;
+EXPORT_SYMBOL(__stack_chk_guard);
+#endif
+
/*
* this gets called so that we can store lazy state into memory and copy the
* current task into the new thread.
--
2.14.2.920.gcf0c67979c-goog
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related
* [RFC v3 19/27] x86: Support global stack cookie
From: Thomas Garnier @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
Add an off-by-default configuration option to use a global stack cookie
instead of the default TLS. This configuration option will only be used
with PIE binaries.
For kernel stack cookie, the compiler uses the mcmodel=kernel to switch
between the fs segment to gs segment. A PIE binary does not use
mcmodel=kernel because it can be relocated anywhere, therefore the
compiler will default to the fs segment register. This is going to be
fixed with a compiler change allowing to pick the segment register as
done on PowerPC. In the meantime, this configuration can be used to
support older compilers.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
arch/x86/Kconfig | 4 ++++
arch/x86/Makefile | 9 +++++++++
arch/x86/entry/entry_32.S | 3 ++-
arch/x86/entry/entry_64.S | 3 ++-
arch/x86/include/asm/processor.h | 3 ++-
arch/x86/include/asm/stackprotector.h | 19 ++++++++++++++-----
arch/x86/kernel/asm-offsets.c | 3 ++-
arch/x86/kernel/asm-offsets_32.c | 3 ++-
arch/x86/kernel/asm-offsets_64.c | 3 ++-
arch/x86/kernel/cpu/common.c | 3 ++-
arch/x86/kernel/head_32.S | 3 ++-
arch/x86/kernel/process.c | 5 +++++
12 files changed, 48 insertions(+), 13 deletions(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 063f1e0d51aa..777197fab6dd 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -2133,6 +2133,10 @@ config RANDOMIZE_MEMORY_PHYSICAL_PADDING
If unsure, leave at the default value.
+config X86_GLOBAL_STACKPROTECTOR
+ bool
+ depends on CC_STACKPROTECTOR
+
config HOTPLUG_CPU
bool "Support for hot-pluggable CPUs"
depends on SMP
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 6276572259c8..4cb4f0495ddc 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -60,6 +60,15 @@ endif
KBUILD_CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-3dnow
KBUILD_CFLAGS += $(call cc-option,-mno-avx,)
+ifdef CONFIG_X86_GLOBAL_STACKPROTECTOR
+ ifeq ($(call cc-option, -mstack-protector-guard=global),)
+ $(error Cannot use CONFIG_X86_GLOBAL_STACKPROTECTOR: \
+ -mstack-protector-guard=global not supported \
+ by compiler)
+ endif
+ KBUILD_CFLAGS += -mstack-protector-guard=global
+endif
+
ifeq ($(CONFIG_X86_32),y)
BITS := 32
UTS_MACHINE := i386
diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index 8a13d468635a..ab3e5056722f 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -237,7 +237,8 @@ ENTRY(__switch_to_asm)
movl %esp, TASK_threadsp(%eax)
movl TASK_threadsp(%edx), %esp
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
movl TASK_stack_canary(%edx), %ebx
movl %ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset
#endif
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index d3a52d2342af..01be62c1b436 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -390,7 +390,8 @@ ENTRY(__switch_to_asm)
movq %rsp, TASK_threadsp(%rdi)
movq TASK_threadsp(%rsi), %rsp
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
movq TASK_stack_canary(%rsi), %rbx
movq %rbx, PER_CPU_VAR(irq_stack_union + stack_canary_offset)
#endif
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index b09bd50b06c7..e3a7ef8d5fb8 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -394,7 +394,8 @@ DECLARE_PER_CPU(char *, irq_stack_ptr);
DECLARE_PER_CPU(unsigned int, irq_count);
extern asmlinkage void ignore_sysret(void);
#else /* X86_64 */
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
/*
* Make sure stack canary segment base is cached-aligned:
* "For Intel Atom processors, avoid non zero segment base address
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
index 8abedf1d650e..66462d778dc5 100644
--- a/arch/x86/include/asm/stackprotector.h
+++ b/arch/x86/include/asm/stackprotector.h
@@ -51,6 +51,10 @@
#define GDT_STACK_CANARY_INIT \
[GDT_ENTRY_STACK_CANARY] = GDT_ENTRY_INIT(0x4090, 0, 0x18),
+#ifdef CONFIG_X86_GLOBAL_STACKPROTECTOR
+extern unsigned long __stack_chk_guard;
+#endif
+
/*
* Initialize the stackprotector canary value.
*
@@ -62,7 +66,7 @@ static __always_inline void boot_init_stack_canary(void)
u64 canary;
u64 tsc;
-#ifdef CONFIG_X86_64
+#if defined(CONFIG_X86_64) && !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
BUILD_BUG_ON(offsetof(union irq_stack_union, stack_canary) != 40);
#endif
/*
@@ -76,17 +80,22 @@ static __always_inline void boot_init_stack_canary(void)
canary += tsc + (tsc << 32UL);
canary &= CANARY_MASK;
+#ifdef CONFIG_X86_GLOBAL_STACKPROTECTOR
+ if (__stack_chk_guard == 0)
+ __stack_chk_guard = canary ?: 1;
+#else /* !CONFIG_X86_GLOBAL_STACKPROTECTOR */
current->stack_canary = canary;
#ifdef CONFIG_X86_64
this_cpu_write(irq_stack_union.stack_canary, canary);
-#else
+#else /* CONFIG_X86_32 */
this_cpu_write(stack_canary.canary, canary);
#endif
+#endif
}
static inline void setup_stack_canary_segment(int cpu)
{
-#ifdef CONFIG_X86_32
+#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
unsigned long canary = (unsigned long)&per_cpu(stack_canary, cpu);
struct desc_struct *gdt_table = get_cpu_gdt_rw(cpu);
struct desc_struct desc;
@@ -99,7 +108,7 @@ static inline void setup_stack_canary_segment(int cpu)
static inline void load_stack_canary_segment(void)
{
-#ifdef CONFIG_X86_32
+#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
asm("mov %0, %%gs" : : "r" (__KERNEL_STACK_CANARY) : "memory");
#endif
}
@@ -115,7 +124,7 @@ static inline void setup_stack_canary_segment(int cpu)
static inline void load_stack_canary_segment(void)
{
-#ifdef CONFIG_X86_32
+#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
asm volatile ("mov %0, %%gs" : : "r" (0));
#endif
}
diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index de827d6ac8c2..b30a12cd021e 100644
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -30,7 +30,8 @@
void common(void) {
BLANK();
OFFSET(TASK_threadsp, task_struct, thread.sp);
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
OFFSET(TASK_stack_canary, task_struct, stack_canary);
#endif
diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c
index 710edab9e644..33584e7e486b 100644
--- a/arch/x86/kernel/asm-offsets_32.c
+++ b/arch/x86/kernel/asm-offsets_32.c
@@ -54,7 +54,8 @@ void foo(void)
/* Size of SYSENTER_stack */
DEFINE(SIZEOF_SYSENTER_stack, sizeof(((struct tss_struct *)0)->SYSENTER_stack));
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
BLANK();
OFFSET(stack_canary_offset, stack_canary, canary);
#endif
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index cf42206926af..06feb31a09f5 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -64,7 +64,8 @@ int main(void)
OFFSET(TSS_sp0, tss_struct, x86_tss.sp0);
BLANK();
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
DEFINE(stack_canary_offset, offsetof(union irq_stack_union, stack_canary));
BLANK();
#endif
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 8732c9e719d5..839f4da2ed76 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1431,7 +1431,8 @@ DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) =
(unsigned long)&init_thread_union + THREAD_SIZE;
EXPORT_PER_CPU_SYMBOL(cpu_current_top_of_stack);
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
#endif
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index 9ed3074d0d27..a55a67b33934 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -377,7 +377,8 @@ ENDPROC(startup_32_smp)
*/
__INIT
setup_once:
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
/*
* Configure the stack canary. The linker can't handle this by
* relocation. Manually set base address in stack canary
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index bd6b85fac666..66ea1a35413e 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -73,6 +73,11 @@ EXPORT_PER_CPU_SYMBOL(cpu_tss);
DEFINE_PER_CPU(bool, __tss_limit_invalid);
EXPORT_PER_CPU_SYMBOL_GPL(__tss_limit_invalid);
+#ifdef CONFIG_X86_GLOBAL_STACKPROTECTOR
+unsigned long __stack_chk_guard __read_mostly;
+EXPORT_SYMBOL(__stack_chk_guard);
+#endif
+
/*
* this gets called so that we can store lazy state into memory and copy the
* current task into the new thread.
--
2.14.2.920.gcf0c67979c-goog
_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
^ permalink raw reply related
* [kernel-hardening] [RFC v3 19/27] x86: Support global stack cookie
From: Thomas Garnier @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter, Boris Ostrovsky, Alexey Dobriyan,
Andrew Morton, Paul Gortmaker, Chris Metcalf, Paul E . McKenney,
Nicolas Pitre, Borislav Petkov, Luis R . Rodriguez,
Greg Kroah-Hartman, Christopher Li, Steven Rostedt, Jason Baron,
Dou Liyang, Rafael J . Wysocki, Mika Westerberg, Lukas Wunner,
Masahiro Yamada, Alexei Starovoitov, Daniel Borkmann,
Markus Trippelsdorf, Paolo Bonzini, Radim Krčmář,
Joerg Roedel, Rik van Riel, David Howells, Ard Biesheuvel,
Waiman Long, Kyle Huey, Andrey Ryabinin, Jonathan Corbet,
Matthew Wilcox, Michal Hocko, Peter Foley, Paul Bolle,
Jiri Kosina, Rob Landley, H . J . Lu, Baoquan He,
Jan H . Schönherr, Daniel Micay
Cc: x86, linux-crypto, linux-kernel, linux-pm, virtualization,
xen-devel, linux-arch, linux-sparse, kvm, linux-doc,
kernel-hardening
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
Add an off-by-default configuration option to use a global stack cookie
instead of the default TLS. This configuration option will only be used
with PIE binaries.
For kernel stack cookie, the compiler uses the mcmodel=kernel to switch
between the fs segment to gs segment. A PIE binary does not use
mcmodel=kernel because it can be relocated anywhere, therefore the
compiler will default to the fs segment register. This is going to be
fixed with a compiler change allowing to pick the segment register as
done on PowerPC. In the meantime, this configuration can be used to
support older compilers.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
arch/x86/Kconfig | 4 ++++
arch/x86/Makefile | 9 +++++++++
arch/x86/entry/entry_32.S | 3 ++-
arch/x86/entry/entry_64.S | 3 ++-
arch/x86/include/asm/processor.h | 3 ++-
arch/x86/include/asm/stackprotector.h | 19 ++++++++++++++-----
arch/x86/kernel/asm-offsets.c | 3 ++-
arch/x86/kernel/asm-offsets_32.c | 3 ++-
arch/x86/kernel/asm-offsets_64.c | 3 ++-
arch/x86/kernel/cpu/common.c | 3 ++-
arch/x86/kernel/head_32.S | 3 ++-
arch/x86/kernel/process.c | 5 +++++
12 files changed, 48 insertions(+), 13 deletions(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 063f1e0d51aa..777197fab6dd 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -2133,6 +2133,10 @@ config RANDOMIZE_MEMORY_PHYSICAL_PADDING
If unsure, leave at the default value.
+config X86_GLOBAL_STACKPROTECTOR
+ bool
+ depends on CC_STACKPROTECTOR
+
config HOTPLUG_CPU
bool "Support for hot-pluggable CPUs"
depends on SMP
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 6276572259c8..4cb4f0495ddc 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -60,6 +60,15 @@ endif
KBUILD_CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-3dnow
KBUILD_CFLAGS += $(call cc-option,-mno-avx,)
+ifdef CONFIG_X86_GLOBAL_STACKPROTECTOR
+ ifeq ($(call cc-option, -mstack-protector-guard=global),)
+ $(error Cannot use CONFIG_X86_GLOBAL_STACKPROTECTOR: \
+ -mstack-protector-guard=global not supported \
+ by compiler)
+ endif
+ KBUILD_CFLAGS += -mstack-protector-guard=global
+endif
+
ifeq ($(CONFIG_X86_32),y)
BITS := 32
UTS_MACHINE := i386
diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S
index 8a13d468635a..ab3e5056722f 100644
--- a/arch/x86/entry/entry_32.S
+++ b/arch/x86/entry/entry_32.S
@@ -237,7 +237,8 @@ ENTRY(__switch_to_asm)
movl %esp, TASK_threadsp(%eax)
movl TASK_threadsp(%edx), %esp
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
movl TASK_stack_canary(%edx), %ebx
movl %ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset
#endif
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index d3a52d2342af..01be62c1b436 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -390,7 +390,8 @@ ENTRY(__switch_to_asm)
movq %rsp, TASK_threadsp(%rdi)
movq TASK_threadsp(%rsi), %rsp
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
movq TASK_stack_canary(%rsi), %rbx
movq %rbx, PER_CPU_VAR(irq_stack_union + stack_canary_offset)
#endif
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index b09bd50b06c7..e3a7ef8d5fb8 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -394,7 +394,8 @@ DECLARE_PER_CPU(char *, irq_stack_ptr);
DECLARE_PER_CPU(unsigned int, irq_count);
extern asmlinkage void ignore_sysret(void);
#else /* X86_64 */
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
/*
* Make sure stack canary segment base is cached-aligned:
* "For Intel Atom processors, avoid non zero segment base address
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
index 8abedf1d650e..66462d778dc5 100644
--- a/arch/x86/include/asm/stackprotector.h
+++ b/arch/x86/include/asm/stackprotector.h
@@ -51,6 +51,10 @@
#define GDT_STACK_CANARY_INIT \
[GDT_ENTRY_STACK_CANARY] = GDT_ENTRY_INIT(0x4090, 0, 0x18),
+#ifdef CONFIG_X86_GLOBAL_STACKPROTECTOR
+extern unsigned long __stack_chk_guard;
+#endif
+
/*
* Initialize the stackprotector canary value.
*
@@ -62,7 +66,7 @@ static __always_inline void boot_init_stack_canary(void)
u64 canary;
u64 tsc;
-#ifdef CONFIG_X86_64
+#if defined(CONFIG_X86_64) && !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
BUILD_BUG_ON(offsetof(union irq_stack_union, stack_canary) != 40);
#endif
/*
@@ -76,17 +80,22 @@ static __always_inline void boot_init_stack_canary(void)
canary += tsc + (tsc << 32UL);
canary &= CANARY_MASK;
+#ifdef CONFIG_X86_GLOBAL_STACKPROTECTOR
+ if (__stack_chk_guard == 0)
+ __stack_chk_guard = canary ?: 1;
+#else /* !CONFIG_X86_GLOBAL_STACKPROTECTOR */
current->stack_canary = canary;
#ifdef CONFIG_X86_64
this_cpu_write(irq_stack_union.stack_canary, canary);
-#else
+#else /* CONFIG_X86_32 */
this_cpu_write(stack_canary.canary, canary);
#endif
+#endif
}
static inline void setup_stack_canary_segment(int cpu)
{
-#ifdef CONFIG_X86_32
+#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
unsigned long canary = (unsigned long)&per_cpu(stack_canary, cpu);
struct desc_struct *gdt_table = get_cpu_gdt_rw(cpu);
struct desc_struct desc;
@@ -99,7 +108,7 @@ static inline void setup_stack_canary_segment(int cpu)
static inline void load_stack_canary_segment(void)
{
-#ifdef CONFIG_X86_32
+#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
asm("mov %0, %%gs" : : "r" (__KERNEL_STACK_CANARY) : "memory");
#endif
}
@@ -115,7 +124,7 @@ static inline void setup_stack_canary_segment(int cpu)
static inline void load_stack_canary_segment(void)
{
-#ifdef CONFIG_X86_32
+#if defined(CONFIG_X86_32) && !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
asm volatile ("mov %0, %%gs" : : "r" (0));
#endif
}
diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index de827d6ac8c2..b30a12cd021e 100644
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -30,7 +30,8 @@
void common(void) {
BLANK();
OFFSET(TASK_threadsp, task_struct, thread.sp);
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
OFFSET(TASK_stack_canary, task_struct, stack_canary);
#endif
diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c
index 710edab9e644..33584e7e486b 100644
--- a/arch/x86/kernel/asm-offsets_32.c
+++ b/arch/x86/kernel/asm-offsets_32.c
@@ -54,7 +54,8 @@ void foo(void)
/* Size of SYSENTER_stack */
DEFINE(SIZEOF_SYSENTER_stack, sizeof(((struct tss_struct *)0)->SYSENTER_stack));
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
BLANK();
OFFSET(stack_canary_offset, stack_canary, canary);
#endif
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index cf42206926af..06feb31a09f5 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -64,7 +64,8 @@ int main(void)
OFFSET(TSS_sp0, tss_struct, x86_tss.sp0);
BLANK();
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
DEFINE(stack_canary_offset, offsetof(union irq_stack_union, stack_canary));
BLANK();
#endif
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 8732c9e719d5..839f4da2ed76 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1431,7 +1431,8 @@ DEFINE_PER_CPU(unsigned long, cpu_current_top_of_stack) =
(unsigned long)&init_thread_union + THREAD_SIZE;
EXPORT_PER_CPU_SYMBOL(cpu_current_top_of_stack);
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
#endif
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index 9ed3074d0d27..a55a67b33934 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -377,7 +377,8 @@ ENDPROC(startup_32_smp)
*/
__INIT
setup_once:
-#ifdef CONFIG_CC_STACKPROTECTOR
+#if defined(CONFIG_CC_STACKPROTECTOR) && \
+ !defined(CONFIG_X86_GLOBAL_STACKPROTECTOR)
/*
* Configure the stack canary. The linker can't handle this by
* relocation. Manually set base address in stack canary
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index bd6b85fac666..66ea1a35413e 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -73,6 +73,11 @@ EXPORT_PER_CPU_SYMBOL(cpu_tss);
DEFINE_PER_CPU(bool, __tss_limit_invalid);
EXPORT_PER_CPU_SYMBOL_GPL(__tss_limit_invalid);
+#ifdef CONFIG_X86_GLOBAL_STACKPROTECTOR
+unsigned long __stack_chk_guard __read_mostly;
+EXPORT_SYMBOL(__stack_chk_guard);
+#endif
+
/*
* this gets called so that we can store lazy state into memory and copy the
* current task into the new thread.
--
2.14.2.920.gcf0c67979c-goog
^ permalink raw reply related
* [RFC v3 18/27] kvm: Adapt assembly for PIE support
From: Thomas Garnier via Virtualization @ 2017-10-04 21:19 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Thomas Gleixner, Ingo Molnar,
H . Peter Anvin, Peter Zijlstra, Josh Poimboeuf, Thomas Garnier,
Arnd Bergmann, Kees Cook, Matthias Kaehlcke, Tom Lendacky,
Andy Lutomirski, Kirill A . Shutemov, Borislav Petkov,
Rafael J . Wysocki, Len Brown, Pavel Machek, Juergen Gross,
Chris Wright, Alok Kataria, Rusty Russell, Tejun Heo,
Christoph Lameter
Cc: linux-arch, kvm, linux-pm, x86, linux-doc, linux-kernel,
virtualization, linux-sparse, linux-crypto, kernel-hardening,
xen-devel
In-Reply-To: <20171004212003.28296-1-thgarnie@google.com>
Change the assembly code to use only relative references of symbols for the
kernel to be PIE compatible. The new __ASM_GET_PTR_PRE macro is used to
get the address of a symbol on both 32 and 64-bit with PIE support.
Position Independent Executable (PIE) support will allow to extended the
KASLR randomization range below the -2G memory limit.
Signed-off-by: Thomas Garnier <thgarnie@google.com>
---
arch/x86/include/asm/kvm_host.h | 6 ++++--
arch/x86/kernel/kvm.c | 6 ++++--
arch/x86/kvm/svm.c | 4 ++--
3 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 9d7d856b2d89..14073fda75fb 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1342,9 +1342,11 @@ asmlinkage void kvm_spurious_fault(void);
".pushsection .fixup, \"ax\" \n" \
"667: \n\t" \
cleanup_insn "\n\t" \
- "cmpb $0, kvm_rebooting \n\t" \
+ "cmpb $0, kvm_rebooting" __ASM_SEL(,(%%rip)) " \n\t" \
"jne 668b \n\t" \
- __ASM_SIZE(push) " $666b \n\t" \
+ __ASM_SIZE(push) "%%" _ASM_AX " \n\t" \
+ __ASM_GET_PTR_PRE(666b) "%%" _ASM_AX "\n\t" \
+ "xchg %%" _ASM_AX ", (%%" _ASM_SP ") \n\t" \
"call kvm_spurious_fault \n\t" \
".popsection \n\t" \
_ASM_EXTABLE(666b, 667b)
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index aa60a08b65b1..07176bfc188b 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -620,8 +620,10 @@ asm(
".global __raw_callee_save___kvm_vcpu_is_preempted;"
".type __raw_callee_save___kvm_vcpu_is_preempted, @function;"
"__raw_callee_save___kvm_vcpu_is_preempted:"
-"movq __per_cpu_offset(,%rdi,8), %rax;"
-"cmpb $0, " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rax);"
+"leaq __per_cpu_offset(%rip), %rax;"
+"movq (%rax,%rdi,8), %rax;"
+"addq " __stringify(KVM_STEAL_TIME_preempted) "+steal_time(%rip), %rax;"
+"cmpb $0, (%rax);"
"setne %al;"
"ret;"
".popsection");
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index 0e68f0b3cbf7..364536080438 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -568,12 +568,12 @@ static u32 svm_msrpm_offset(u32 msr)
static inline void clgi(void)
{
- asm volatile (__ex(SVM_CLGI));
+ asm volatile (__ex(SVM_CLGI) : :);
}
static inline void stgi(void)
{
- asm volatile (__ex(SVM_STGI));
+ asm volatile (__ex(SVM_STGI) : :);
}
static inline void invlpga(unsigned long addr, u32 asid)
--
2.14.2.920.gcf0c67979c-goog
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.