* [PATCH 01/12] jump label v10: base patch
2010-07-27 20:53 [PATCH 00/12] jump label v10 Jason Baron
@ 2010-07-27 20:54 ` Jason Baron
2010-07-27 20:54 ` [PATCH 02/12] jump label v10: add module support Jason Baron
` (11 subsequent siblings)
12 siblings, 0 replies; 21+ messages in thread
From: Jason Baron @ 2010-07-27 20:54 UTC (permalink / raw)
To: linux-kernel
Cc: mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi, roland, rth,
mhiramat, fweisbec, avi, davem, vgoyal, sam, tony
base patch to implement 'jump labeling'. Based on a new 'asm goto' inline
assembly gcc mechanism, we can now branch to labels from an 'asm goto'
statment. This allows us to create a 'no-op' fastpath, which can subsequently
be patched with a jump to the slowpath code. This is useful for code which
might be rarely used, but which we'd like to be able to call, if needed.
Tracepoints are the current usecase that these are being implemented for.
Signed-off-by: Jason Baron <jbaron@redhat.com>
---
Makefile | 5 +
arch/Kconfig | 3 +
include/asm-generic/vmlinux.lds.h | 10 ++-
include/linux/jump_label.h | 56 ++++++++++++
kernel/Makefile | 2 +-
kernel/jump_label.c | 179 +++++++++++++++++++++++++++++++++++++
scripts/gcc-goto.sh | 5 +
7 files changed, 258 insertions(+), 2 deletions(-)
create mode 100644 include/linux/jump_label.h
create mode 100644 kernel/jump_label.c
create mode 100644 scripts/gcc-goto.sh
diff --git a/Makefile b/Makefile
index f5787e2..0aafa4d 100644
--- a/Makefile
+++ b/Makefile
@@ -582,6 +582,11 @@ KBUILD_CFLAGS += $(call cc-option,-fno-strict-overflow)
# conserve stack if available
KBUILD_CFLAGS += $(call cc-option,-fconserve-stack)
+# check for 'asm goto'
+ifeq ($(shell $(CONFIG_SHELL) $(srctree)/scripts/gcc-goto.sh $(CC)), y)
+ KBUILD_CFLAGS += -DCC_HAVE_ASM_GOTO
+endif
+
# Add user supplied CPPFLAGS, AFLAGS and CFLAGS as the last assignments
# But warn user when we do so
warn-assign = \
diff --git a/arch/Kconfig b/arch/Kconfig
index 4877a8c..1462d84 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -158,4 +158,7 @@ config HAVE_PERF_EVENTS_NMI
subsystem. Also has support for calculating CPU cycle events
to determine how many clock cycles in a given period.
+config HAVE_ARCH_JUMP_LABEL
+ bool
+
source "kernel/gcov/Kconfig"
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 415b1a9..3c7e9ab 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -163,7 +163,8 @@
BRANCH_PROFILE() \
TRACE_PRINTKS() \
FTRACE_EVENTS() \
- TRACE_SYSCALLS()
+ TRACE_SYSCALLS() \
+ JUMP_TABLE() \
/*
* Data section helpers
@@ -202,6 +203,7 @@
*(__vermagic) /* Kernel version magic */ \
*(__markers_strings) /* Markers: strings */ \
*(__tracepoints_strings)/* Tracepoints: strings */ \
+ *(__jump_strings)/* Jump: strings */ \
} \
\
.rodata1 : AT(ADDR(.rodata1) - LOAD_OFFSET) { \
@@ -553,6 +555,12 @@
#define BUG_TABLE
#endif
+#define JUMP_TABLE() \
+ . = ALIGN(8); \
+ VMLINUX_SYMBOL(__start___jump_table) = .; \
+ *(__jump_table) \
+ VMLINUX_SYMBOL(__stop___jump_table) = .; \
+
#ifdef CONFIG_PM_TRACE
#define TRACEDATA \
. = ALIGN(4); \
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
new file mode 100644
index 0000000..846b72a
--- /dev/null
+++ b/include/linux/jump_label.h
@@ -0,0 +1,56 @@
+#ifndef _LINUX_JUMP_LABEL_H
+#define _LINUX_JUMP_LABEL_H
+
+#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_HAVE_ARCH_JUMP_LABEL)
+# include <asm/jump_label.h>
+# define HAVE_JUMP_LABEL
+#endif
+
+enum jump_label_type {
+ JUMP_LABEL_ENABLE,
+ JUMP_LABEL_DISABLE
+};
+
+#ifdef HAVE_JUMP_LABEL
+
+extern struct jump_entry __start___jump_table[];
+extern struct jump_entry __stop___jump_table[];
+
+#define DEFINE_JUMP_LABEL(name) \
+ const char __jlstrtab_##name[] \
+ __used __attribute__((section("__jump_strings"))) = #name;
+
+extern void arch_jump_label_transform(struct jump_entry *entry,
+ enum jump_label_type type);
+extern const u8 *arch_jump_label_get_nop(void);
+extern void jump_label_update(const char *name, enum jump_label_type type);
+
+#define enable_jump_label(name) \
+ jump_label_update(name, JUMP_LABEL_ENABLE);
+
+#define disable_jump_label(name) \
+ jump_label_update(name, JUMP_LABEL_DISABLE);
+
+#else
+
+#define DEFINE_JUMP_LABEL(name)
+
+#define JUMP_LABEL(tag, label, cond) \
+do { \
+ if (unlikely(cond)) \
+ goto label; \
+} while (0)
+
+static inline int enable_jump_label(const char *name)
+{
+ return 0;
+}
+
+static inline int disable_jump_label(const char *name)
+{
+ return 0;
+}
+
+#endif
+
+#endif
diff --git a/kernel/Makefile b/kernel/Makefile
index ce53fb2..662c96d 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -10,7 +10,7 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o \
kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \
hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \
notifier.o ksysfs.o pm_qos_params.o sched_clock.o cred.o \
- async.o range.o
+ async.o range.o jump_label.o
obj-$(CONFIG_HAVE_EARLY_RES) += early_res.o
obj-y += groups.o
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
new file mode 100644
index 0000000..78d18f0
--- /dev/null
+++ b/kernel/jump_label.c
@@ -0,0 +1,179 @@
+/*
+ * jump label support
+ *
+ * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
+ *
+ */
+#include <linux/jump_label.h>
+#include <linux/memory.h>
+#include <linux/uaccess.h>
+#include <linux/module.h>
+#include <linux/list.h>
+#include <linux/jhash.h>
+#include <linux/slab.h>
+
+#ifdef HAVE_JUMP_LABEL
+
+#define JUMP_LABEL_HASH_BITS 6
+#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS)
+static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE];
+
+/* mutex to protect coming/going of the the jump_label table */
+static DEFINE_MUTEX(jump_label_mutex);
+
+struct jump_label_entry {
+ struct hlist_node hlist;
+ struct jump_entry *table;
+ int nr_entries;
+ /* hang modules off here */
+ struct hlist_head modules;
+ const char *name;
+};
+
+static void swap_jump_label_entries(struct jump_entry *previous, struct jump_entry *next)
+{
+ struct jump_entry tmp;
+
+ tmp = *next;
+ *next = *previous;
+ *previous = tmp;
+}
+
+static void sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop)
+{
+ int swapped = 0;
+ struct jump_entry *iter;
+ struct jump_entry *iter_next;
+
+ do {
+ swapped = 0;
+ iter = start;
+ iter_next = start;
+ iter_next++;
+ for (; iter_next < stop; iter++, iter_next++) {
+ if (strcmp((char *)iter->name,
+ (char *)iter_next->name) > 0) {
+ swap_jump_label_entries(iter, iter_next);
+ swapped = 1;
+ }
+ }
+ } while (swapped == 1);
+}
+
+static struct jump_label_entry *get_jump_label_entry(const char *name)
+{
+ struct hlist_head *head;
+ struct hlist_node *node;
+ struct jump_label_entry *e;
+ u32 hash = jhash(name, strlen(name), 0);
+
+ head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
+ hlist_for_each_entry(e, node, head, hlist) {
+ if (!strcmp((char *)name, (char *)e->name))
+ return e;
+ }
+ return NULL;
+}
+
+static struct jump_label_entry *add_jump_label_entry(const char *name, int nr_entries, struct jump_entry *table)
+{
+ struct hlist_head *head;
+ struct jump_label_entry *e;
+ size_t name_len;
+ u32 hash;
+
+ e = get_jump_label_entry(name);
+ if (e)
+ return ERR_PTR(-EEXIST);
+
+ e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL);
+ if (!e)
+ return ERR_PTR(-ENOMEM);
+
+ name_len = strlen(name) + 1;
+ hash = jhash(name, name_len-1, 0);
+ head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
+ e->name = name;
+ e->table = table;
+ e->nr_entries = nr_entries;
+ INIT_HLIST_HEAD(&(e->modules));
+ hlist_add_head(&e->hlist, head);
+ return e;
+}
+
+static int build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop)
+{
+ struct jump_entry *iter, *iter_begin;
+ struct jump_label_entry *entry;
+ int count;
+
+ sort_jump_label_entries(start, stop);
+ iter = start;
+ while (iter < stop) {
+ entry = get_jump_label_entry((char *)iter->name);
+ if (!entry) {
+ iter_begin = iter;
+ count = 0;
+ while ((iter < stop) &&
+ (strcmp((char *)iter->name,
+ (char *)iter_begin->name) == 0)) {
+ iter++;
+ count++;
+ }
+ entry = add_jump_label_entry((char *)iter_begin->name,
+ count, iter_begin);
+ if (IS_ERR(entry))
+ return PTR_ERR(entry);
+ } else {
+ WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n");
+ return -1;
+ }
+ }
+ return 0;
+}
+
+/***
+ * jump_label_update - update jump label text
+ * @name - name of the jump label
+ * @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE
+ *
+ * Will enable/disable the jump for jump label @name, depending on the
+ * value of @type.
+ *
+ */
+
+void jump_label_update(const char *name, enum jump_label_type type)
+{
+ struct jump_entry *iter;
+ struct jump_label_entry *entry;
+ struct hlist_node *module_node;
+ struct jump_label_module_entry *e_module;
+ int count;
+
+ mutex_lock(&jump_label_mutex);
+ entry = get_jump_label_entry(name);
+ if (entry) {
+ count = entry->nr_entries;
+ iter = entry->table;
+ while (count--) {
+ if (kernel_text_address(iter->code))
+ arch_jump_label_transform(iter, type);
+ iter++;
+ }
+ }
+ mutex_unlock(&jump_label_mutex);
+}
+
+static __init int init_jump_label(void)
+{
+ int ret;
+
+ mutex_lock(&jump_label_mutex);
+ ret = build_jump_label_hashtable(__start___jump_table,
+ __stop___jump_table);
+ mutex_unlock(&jump_label_mutex);
+ return ret;
+}
+early_initcall(init_jump_label);
+
+#endif
diff --git a/scripts/gcc-goto.sh b/scripts/gcc-goto.sh
new file mode 100644
index 0000000..8e82424
--- /dev/null
+++ b/scripts/gcc-goto.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+# Test for gcc 'asm goto' suport
+# Copyright (C) 2010, Jason Baron <jbaron@redhat.com>
+
+echo "int main(void) { entry: asm goto (\"\"::::entry); return 0; }" | $1 -x c - -c -o /dev/null >/dev/null 2>&1 && echo "y"
--
1.7.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 02/12] jump label v10: add module support
2010-07-27 20:53 [PATCH 00/12] jump label v10 Jason Baron
2010-07-27 20:54 ` [PATCH 01/12] jump label v10: base patch Jason Baron
@ 2010-07-27 20:54 ` Jason Baron
2010-07-27 20:54 ` [PATCH 03/12] jump label v10: move ftrace_dyn_arch_init to common code Jason Baron
` (10 subsequent siblings)
12 siblings, 0 replies; 21+ messages in thread
From: Jason Baron @ 2010-07-27 20:54 UTC (permalink / raw)
To: linux-kernel
Cc: mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi, roland, rth,
mhiramat, fweisbec, avi, davem, vgoyal, sam, tony
Add support for 'jump label' for modules.
Signed-off-by: Jason Baron <jbaron@redhat.com>
---
include/linux/jump_label.h | 3 +-
include/linux/module.h | 5 +-
kernel/jump_label.c | 136 ++++++++++++++++++++++++++++++++++++++++++++
kernel/module.c | 7 ++
4 files changed, 149 insertions(+), 2 deletions(-)
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 846b72a..9ac29c0 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -18,7 +18,8 @@ extern struct jump_entry __stop___jump_table[];
#define DEFINE_JUMP_LABEL(name) \
const char __jlstrtab_##name[] \
- __used __attribute__((section("__jump_strings"))) = #name;
+ __used __attribute__((section("__jump_strings"))) = #name; \
+ EXPORT_SYMBOL_GPL(__jlstrtab_##name);
extern void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type);
diff --git a/include/linux/module.h b/include/linux/module.h
index 8a6b9fd..403ac26 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -350,7 +350,10 @@ struct module
struct tracepoint *tracepoints;
unsigned int num_tracepoints;
#endif
-
+#ifdef HAVE_JUMP_LABEL
+ struct jump_entry *jump_entries;
+ unsigned int num_jump_entries;
+#endif
#ifdef CONFIG_TRACING
const char **trace_bprintk_fmt_start;
unsigned int num_trace_bprintk_fmt;
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 78d18f0..ecca4f4 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -30,6 +30,13 @@ struct jump_label_entry {
const char *name;
};
+struct jump_label_module_entry {
+ struct hlist_node hlist;
+ struct jump_entry *table;
+ int nr_entries;
+ struct module *mod;
+};
+
static void swap_jump_label_entries(struct jump_entry *previous, struct jump_entry *next)
{
struct jump_entry tmp;
@@ -160,6 +167,17 @@ void jump_label_update(const char *name, enum jump_label_type type)
arch_jump_label_transform(iter, type);
iter++;
}
+ /* eanble/disable jump labels in modules */
+ hlist_for_each_entry(e_module, module_node, &(entry->modules),
+ hlist) {
+ count = e_module->nr_entries;
+ iter = e_module->table;
+ while (count--) {
+ if (kernel_text_address(iter->code))
+ arch_jump_label_transform(iter, type);
+ iter++;
+ }
+ }
}
mutex_unlock(&jump_label_mutex);
}
@@ -176,4 +194,122 @@ static __init int init_jump_label(void)
}
early_initcall(init_jump_label);
+#ifdef CONFIG_MODULES
+
+static struct jump_label_module_entry *add_jump_label_module_entry(struct jump_label_entry *entry, struct jump_entry *iter_begin, int count, struct module *mod)
+{
+ struct jump_label_module_entry *e;
+
+ e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL);
+ if (!e)
+ return ERR_PTR(-ENOMEM);
+ e->mod = mod;
+ e->nr_entries = count;
+ e->table = iter_begin;
+ hlist_add_head(&e->hlist, &entry->modules);
+ return e;
+}
+
+static int add_jump_label_module(struct module *mod)
+{
+ struct jump_entry *iter, *iter_begin;
+ struct jump_label_entry *entry;
+ struct jump_label_module_entry *module_entry;
+ int count;
+
+ /* if the module doesn't have jump label entries, just return */
+ if (!mod->num_jump_entries)
+ return 0;
+
+ sort_jump_label_entries(mod->jump_entries,
+ mod->jump_entries + mod->num_jump_entries);
+ iter = mod->jump_entries;
+ while (iter < mod->jump_entries + mod->num_jump_entries) {
+ entry = get_jump_label_entry((char *)iter->name);
+ iter_begin = iter;
+ count = 0;
+ while ((iter < mod->jump_entries + mod->num_jump_entries) &&
+ (strcmp((char *)iter->name, (char *)iter_begin->name) == 0)) {
+ iter++;
+ count++;
+ }
+ if (!entry) {
+ entry = add_jump_label_entry((char *)iter_begin->name, 0, NULL);
+ if (IS_ERR(entry))
+ return PTR_ERR(entry);
+ }
+ module_entry = add_jump_label_module_entry(entry, iter_begin,
+ count, mod);
+ if (IS_ERR(module_entry))
+ return PTR_ERR(module_entry);
+ }
+ return 0;
+}
+
+static void remove_jump_label_module(struct module *mod)
+{
+ struct hlist_head *head;
+ struct hlist_node *node, *node_next, *module_node, *module_node_next;
+ struct jump_label_entry *e;
+ struct jump_label_module_entry *e_module;
+ int i;
+
+ /* if the module doesn't have jump label entries, just return */
+ if (!mod->num_jump_entries)
+ return;
+
+ for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
+ head = &jump_label_table[i];
+ hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
+ hlist_for_each_entry_safe(e_module, module_node,
+ module_node_next,
+ &(e->modules), hlist) {
+ if (e_module->mod == mod) {
+ hlist_del(&e_module->hlist);
+ kfree(e_module);
+ }
+ }
+ if (hlist_empty(&e->modules) && (e->nr_entries == 0)) {
+ hlist_del(&e->hlist);
+ kfree(e);
+ }
+ }
+ }
+}
+
+static int jump_label_module_notify(struct notifier_block *self, unsigned long val, void *data)
+{
+ struct module *mod = data;
+ int ret = 0;
+
+ switch (val) {
+ case MODULE_STATE_COMING:
+ mutex_lock(&jump_label_mutex);
+ ret = add_jump_label_module(mod);
+ if (ret)
+ remove_jump_label_module(mod);
+ mutex_unlock(&jump_label_mutex);
+ break;
+ case MODULE_STATE_GOING:
+ mutex_lock(&jump_label_mutex);
+ remove_jump_label_module(mod);
+ mutex_unlock(&jump_label_mutex);
+ break;
+ }
+ return ret;
+}
+
+struct notifier_block jump_label_module_nb = {
+ .notifier_call = jump_label_module_notify,
+ .priority = 0,
+};
+
+static __init int init_jump_label_module(void)
+{
+ return register_module_notifier(&jump_label_module_nb);
+}
+early_initcall(init_jump_label_module);
+
+#endif /* CONFIG_MODULES */
+
#endif
diff --git a/kernel/module.c b/kernel/module.c
index 5d2d281..695a131 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -55,6 +55,7 @@
#include <linux/async.h>
#include <linux/percpu.h>
#include <linux/kmemleak.h>
+#include <linux/jump_label.h>
#define CREATE_TRACE_POINTS
#include <trace/events/module.h>
@@ -2413,6 +2414,12 @@ static noinline struct module *load_module(void __user *umod,
sizeof(*mod->tracepoints),
&mod->num_tracepoints);
#endif
+#ifdef HAVE_JUMP_LABEL
+ mod->jump_entries = section_objs(hdr, sechdrs, secstrings,
+ "__jump_table",
+ sizeof(*mod->jump_entries),
+ &mod->num_jump_entries);
+#endif
#ifdef CONFIG_EVENT_TRACING
mod->trace_events = section_objs(hdr, sechdrs, secstrings,
"_ftrace_events",
--
1.7.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 03/12] jump label v10: move ftrace_dyn_arch_init to common code
2010-07-27 20:53 [PATCH 00/12] jump label v10 Jason Baron
2010-07-27 20:54 ` [PATCH 01/12] jump label v10: base patch Jason Baron
2010-07-27 20:54 ` [PATCH 02/12] jump label v10: add module support Jason Baron
@ 2010-07-27 20:54 ` Jason Baron
2010-07-27 20:54 ` [PATCH 04/12] jump label v10: initialize workqueue tracepoints *before* they are registered Jason Baron
` (9 subsequent siblings)
12 siblings, 0 replies; 21+ messages in thread
From: Jason Baron @ 2010-07-27 20:54 UTC (permalink / raw)
To: linux-kernel
Cc: mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi, roland, rth,
mhiramat, fweisbec, avi, davem, vgoyal, sam, tony
Move Steve's code for finding the best 5-byte no-op from ftrace.c to alternative.c.
The idea is that other consumers (in this case jump label) want to make use of
that code. I've created a global: 'char ideal_nop[5]', that is setup during
setup_arch that can be used.
Signed-off-by: Jason Baron <jbaron@redhat.com>
---
arch/x86/include/asm/alternative.h | 14 +++++++
arch/x86/kernel/alternative.c | 72 +++++++++++++++++++++++++++++++++++-
arch/x86/kernel/ftrace.c | 70 +----------------------------------
arch/x86/kernel/setup.c | 3 +
include/linux/jump_label.h | 10 ++++-
kernel/jump_label.c | 32 ++++++++++++++++
kernel/trace/ftrace.c | 13 +------
7 files changed, 130 insertions(+), 84 deletions(-)
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index bc6abb7..963f5f9 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -4,6 +4,7 @@
#include <linux/types.h>
#include <linux/stddef.h>
#include <linux/stringify.h>
+#include <linux/jump_label.h>
#include <asm/asm.h>
/*
@@ -160,6 +161,8 @@ static inline void apply_paravirt(struct paravirt_patch_site *start,
#define __parainstructions_end NULL
#endif
+extern void *text_poke_early(void *addr, const void *opcode, size_t len);
+
/*
* Clear and restore the kernel write-protection flag on the local CPU.
* Allows the kernel to edit read-only pages.
@@ -180,4 +183,15 @@ static inline void apply_paravirt(struct paravirt_patch_site *start,
extern void *text_poke(void *addr, const void *opcode, size_t len);
extern void *text_poke_smp(void *addr, const void *opcode, size_t len);
+#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL)
+#define IDEAL_NOP_SIZE_5 5
+extern unsigned char ideal_nop5[IDEAL_NOP_SIZE_5];
+extern int arch_init_ideal_nop5(void);
+#else
+static inline int arch_init_ideal_nop5(void)
+{
+ return 0;
+}
+#endif
+
#endif /* _ASM_X86_ALTERNATIVE_H */
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index f65ab8b..df5a4f2 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -195,7 +195,7 @@ static void __init_or_module add_nops(void *insns, unsigned int len)
extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
extern s32 __smp_locks[], __smp_locks_end[];
-static void *text_poke_early(void *addr, const void *opcode, size_t len);
+void *text_poke_early(void *addr, const void *opcode, size_t len);
/* Replace instructions with better alternatives for this CPU type.
This runs before SMP is initialized to avoid SMP problems with
@@ -522,7 +522,7 @@ void __init alternative_instructions(void)
* instructions. And on the local CPU you need to be protected again NMI or MCE
* handlers seeing an inconsistent instruction while you patch.
*/
-static void *__init_or_module text_poke_early(void *addr, const void *opcode,
+void *__init_or_module text_poke_early(void *addr, const void *opcode,
size_t len)
{
unsigned long flags;
@@ -641,3 +641,71 @@ void *__kprobes text_poke_smp(void *addr, const void *opcode, size_t len)
return addr;
}
+#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL)
+
+unsigned char ideal_nop5[IDEAL_NOP_SIZE_5];
+
+int __init arch_init_ideal_nop5(void)
+{
+ extern const unsigned char ftrace_test_p6nop[];
+ extern const unsigned char ftrace_test_nop5[];
+ extern const unsigned char ftrace_test_jmp[];
+ int faulted = 0;
+ unsigned long flags;
+
+ local_irq_save(flags);
+ /*
+ * There is no good nop for all x86 archs.
+ * We will default to using the P6_NOP5, but first we
+ * will test to make sure that the nop will actually
+ * work on this CPU. If it faults, we will then
+ * go to a lesser efficient 5 byte nop. If that fails
+ * we then just use a jmp as our nop. This isn't the most
+ * efficient nop, but we can not use a multi part nop
+ * since we would then risk being preempted in the middle
+ * of that nop, and if we enabled tracing then, it might
+ * cause a system crash.
+ *
+ * TODO: check the cpuid to determine the best nop.
+ */
+ asm volatile (
+ "ftrace_test_jmp:"
+ "jmp ftrace_test_p6nop\n"
+ "nop\n"
+ "nop\n"
+ "nop\n" /* 2 byte jmp + 3 bytes */
+ "ftrace_test_p6nop:"
+ P6_NOP5
+ "jmp 1f\n"
+ "ftrace_test_nop5:"
+ ".byte 0x66,0x66,0x66,0x66,0x90\n"
+ "1:"
+ ".section .fixup, \"ax\"\n"
+ "2: movl $1, %0\n"
+ " jmp ftrace_test_nop5\n"
+ "3: movl $2, %0\n"
+ " jmp 1b\n"
+ ".previous\n"
+ _ASM_EXTABLE(ftrace_test_p6nop, 2b)
+ _ASM_EXTABLE(ftrace_test_nop5, 3b)
+ : "=r"(faulted) : "0" (faulted));
+
+ switch (faulted) {
+ case 0:
+ pr_info("converting mcount calls to 0f 1f 44 00 00\n");
+ memcpy(ideal_nop5, ftrace_test_p6nop, IDEAL_NOP_SIZE_5);
+ break;
+ case 1:
+ pr_info("converting mcount calls to 66 66 66 66 90\n");
+ memcpy(ideal_nop5, ftrace_test_nop5, IDEAL_NOP_SIZE_5);
+ break;
+ case 2:
+ pr_info("converting mcount calls to jmp . + 5\n");
+ memcpy(ideal_nop5, ftrace_test_jmp, IDEAL_NOP_SIZE_5);
+ break;
+ }
+
+ local_irq_restore(flags);
+ return 0;
+}
+#endif
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index cd37469..ba2e0d9 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -257,14 +257,9 @@ do_ftrace_mod_code(unsigned long ip, void *new_code)
return mod_code_status;
}
-
-
-
-static unsigned char ftrace_nop[MCOUNT_INSN_SIZE];
-
static unsigned char *ftrace_nop_replace(void)
{
- return ftrace_nop;
+ return ideal_nop5;
}
static int
@@ -336,69 +331,6 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
return ret;
}
-int __init ftrace_dyn_arch_init(void *data)
-{
- extern const unsigned char ftrace_test_p6nop[];
- extern const unsigned char ftrace_test_nop5[];
- extern const unsigned char ftrace_test_jmp[];
- int faulted = 0;
-
- /*
- * There is no good nop for all x86 archs.
- * We will default to using the P6_NOP5, but first we
- * will test to make sure that the nop will actually
- * work on this CPU. If it faults, we will then
- * go to a lesser efficient 5 byte nop. If that fails
- * we then just use a jmp as our nop. This isn't the most
- * efficient nop, but we can not use a multi part nop
- * since we would then risk being preempted in the middle
- * of that nop, and if we enabled tracing then, it might
- * cause a system crash.
- *
- * TODO: check the cpuid to determine the best nop.
- */
- asm volatile (
- "ftrace_test_jmp:"
- "jmp ftrace_test_p6nop\n"
- "nop\n"
- "nop\n"
- "nop\n" /* 2 byte jmp + 3 bytes */
- "ftrace_test_p6nop:"
- P6_NOP5
- "jmp 1f\n"
- "ftrace_test_nop5:"
- ".byte 0x66,0x66,0x66,0x66,0x90\n"
- "1:"
- ".section .fixup, \"ax\"\n"
- "2: movl $1, %0\n"
- " jmp ftrace_test_nop5\n"
- "3: movl $2, %0\n"
- " jmp 1b\n"
- ".previous\n"
- _ASM_EXTABLE(ftrace_test_p6nop, 2b)
- _ASM_EXTABLE(ftrace_test_nop5, 3b)
- : "=r"(faulted) : "0" (faulted));
-
- switch (faulted) {
- case 0:
- pr_info("converting mcount calls to 0f 1f 44 00 00\n");
- memcpy(ftrace_nop, ftrace_test_p6nop, MCOUNT_INSN_SIZE);
- break;
- case 1:
- pr_info("converting mcount calls to 66 66 66 66 90\n");
- memcpy(ftrace_nop, ftrace_test_nop5, MCOUNT_INSN_SIZE);
- break;
- case 2:
- pr_info("converting mcount calls to jmp . + 5\n");
- memcpy(ftrace_nop, ftrace_test_jmp, MCOUNT_INSN_SIZE);
- break;
- }
-
- /* The return code is retured via data */
- *(unsigned long *)data = 0;
-
- return 0;
-}
#endif
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index b008e78..cb28d82 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -49,6 +49,7 @@
#include <asm/pci-direct.h>
#include <linux/init_ohci1394_dma.h>
#include <linux/kvm_para.h>
+#include <linux/jump_label.h>
#include <linux/errno.h>
#include <linux/kernel.h>
@@ -1069,6 +1070,8 @@ void __init setup_arch(char **cmdline_p)
x86_init.oem.banner();
mcheck_init();
+
+ arch_init_ideal_nop5();
}
#ifdef CONFIG_X86_32
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 9ac29c0..fc2b181 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -11,6 +11,8 @@ enum jump_label_type {
JUMP_LABEL_DISABLE
};
+struct module;
+
#ifdef HAVE_JUMP_LABEL
extern struct jump_entry __start___jump_table[];
@@ -23,8 +25,9 @@ extern struct jump_entry __stop___jump_table[];
extern void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type);
-extern const u8 *arch_jump_label_get_nop(void);
extern void jump_label_update(const char *name, enum jump_label_type type);
+extern void jump_label_apply_nops(struct module *mod);
+extern void arch_jump_label_text_poke_early(jump_label_t addr);
#define enable_jump_label(name) \
jump_label_update(name, JUMP_LABEL_ENABLE);
@@ -52,6 +55,11 @@ static inline int disable_jump_label(const char *name)
return 0;
}
+static inline int jump_label_apply_nops(struct module *mod)
+{
+ return 0;
+}
+
#endif
#endif
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index ecca4f4..ebe9145 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -11,6 +11,7 @@
#include <linux/list.h>
#include <linux/jhash.h>
#include <linux/slab.h>
+#include <linux/err.h>
#ifdef HAVE_JUMP_LABEL
@@ -185,10 +186,18 @@ void jump_label_update(const char *name, enum jump_label_type type)
static __init int init_jump_label(void)
{
int ret;
+ struct jump_entry *iter_start = __start___jump_table;
+ struct jump_entry *iter_stop = __stop___jump_table;
+ struct jump_entry *iter;
mutex_lock(&jump_label_mutex);
ret = build_jump_label_hashtable(__start___jump_table,
__stop___jump_table);
+ iter = iter_start;
+ while (iter < iter_stop) {
+ arch_jump_label_text_poke_early(iter->code);
+ iter++;
+ }
mutex_unlock(&jump_label_mutex);
return ret;
}
@@ -299,6 +308,29 @@ static int jump_label_module_notify(struct notifier_block *self, unsigned long v
return ret;
}
+/***
+ * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
+ * @mod: module to patch
+ *
+ * Allow for run-time selection of the optimal nops. Before the module
+ * loads patch these with arch_get_jump_label_nop(), which is specified by
+ * the arch specific jump label code.
+ */
+void jump_label_apply_nops(struct module *mod)
+{
+ struct jump_entry *iter;
+
+ /* if the module doesn't have jump label entries, just return */
+ if (!mod->num_jump_entries)
+ return;
+
+ iter = mod->jump_entries;
+ while (iter < mod->jump_entries + mod->num_jump_entries) {
+ arch_jump_label_text_poke_early(iter->code);
+ iter++;
+ }
+}
+
struct notifier_block jump_label_module_nb = {
.notifier_call = jump_label_module_notify,
.priority = 0,
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 0d88ce9..1a1cc47 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -2749,20 +2749,9 @@ extern unsigned long __stop_mcount_loc[];
void __init ftrace_init(void)
{
- unsigned long count, addr, flags;
+ unsigned long count;
int ret;
- /* Keep the ftrace pointer to the stub */
- addr = (unsigned long)ftrace_stub;
-
- local_irq_save(flags);
- ftrace_dyn_arch_init(&addr);
- local_irq_restore(flags);
-
- /* ftrace_dyn_arch_init places the return code in addr */
- if (addr)
- goto failed;
-
count = __stop_mcount_loc - __start_mcount_loc;
ret = ftrace_dyn_table_alloc(count);
--
1.7.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 04/12] jump label v10: initialize workqueue tracepoints *before* they are registered
2010-07-27 20:53 [PATCH 00/12] jump label v10 Jason Baron
` (2 preceding siblings ...)
2010-07-27 20:54 ` [PATCH 03/12] jump label v10: move ftrace_dyn_arch_init to common code Jason Baron
@ 2010-07-27 20:54 ` Jason Baron
2010-07-27 20:54 ` [PATCH 05/12] jump label v10: jump_label_text_reserved() to reserve our jump points Jason Baron
` (8 subsequent siblings)
12 siblings, 0 replies; 21+ messages in thread
From: Jason Baron @ 2010-07-27 20:54 UTC (permalink / raw)
To: linux-kernel
Cc: mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi, roland, rth,
mhiramat, fweisbec, avi, davem, vgoyal, sam, tony
Initialize the workqueue data structures *before* they are registered
so that they are ready for callbacks.
Signed-off-by: Jason Baron <jbaron@redhat.com>
---
kernel/trace/trace_workqueue.c | 10 +++++-----
1 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/kernel/trace/trace_workqueue.c b/kernel/trace/trace_workqueue.c
index a7cc379..209b379 100644
--- a/kernel/trace/trace_workqueue.c
+++ b/kernel/trace/trace_workqueue.c
@@ -263,6 +263,11 @@ int __init trace_workqueue_early_init(void)
{
int ret, cpu;
+ for_each_possible_cpu(cpu) {
+ spin_lock_init(&workqueue_cpu_stat(cpu)->lock);
+ INIT_LIST_HEAD(&workqueue_cpu_stat(cpu)->list);
+ }
+
ret = register_trace_workqueue_insertion(probe_workqueue_insertion, NULL);
if (ret)
goto out;
@@ -279,11 +284,6 @@ int __init trace_workqueue_early_init(void)
if (ret)
goto no_creation;
- for_each_possible_cpu(cpu) {
- spin_lock_init(&workqueue_cpu_stat(cpu)->lock);
- INIT_LIST_HEAD(&workqueue_cpu_stat(cpu)->list);
- }
-
return 0;
no_creation:
--
1.7.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 05/12] jump label v10: jump_label_text_reserved() to reserve our jump points
2010-07-27 20:53 [PATCH 00/12] jump label v10 Jason Baron
` (3 preceding siblings ...)
2010-07-27 20:54 ` [PATCH 04/12] jump label v10: initialize workqueue tracepoints *before* they are registered Jason Baron
@ 2010-07-27 20:54 ` Jason Baron
2010-07-27 20:54 ` [PATCH 06/12] jump label v10: convert jump label to use a key Jason Baron
` (7 subsequent siblings)
12 siblings, 0 replies; 21+ messages in thread
From: Jason Baron @ 2010-07-27 20:54 UTC (permalink / raw)
To: linux-kernel
Cc: mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi, roland, rth,
mhiramat, fweisbec, avi, davem, vgoyal, sam, tony
Add a jump_label_text_reserved(void *start, void *end), so that other
pieces of code that want to modify kernel text, can first verify that
jump label has not reserved the instruction.
Signed-off-by: Jason Baron <jbaron@redhat.com>
Acked-by: Masami Hiramatsu <mhiramat@redhat.com>
---
arch/x86/kernel/kprobes.c | 3 +-
include/linux/jump_label.h | 8 ++++-
kernel/jump_label.c | 83 ++++++++++++++++++++++++++++++++++++++++++++
kernel/kprobes.c | 3 +-
4 files changed, 94 insertions(+), 3 deletions(-)
diff --git a/arch/x86/kernel/kprobes.c b/arch/x86/kernel/kprobes.c
index 1bfb6cf..b1a4208 100644
--- a/arch/x86/kernel/kprobes.c
+++ b/arch/x86/kernel/kprobes.c
@@ -1202,7 +1202,8 @@ static int __kprobes copy_optimized_instructions(u8 *dest, u8 *src)
}
/* Check whether the address range is reserved */
if (ftrace_text_reserved(src, src + len - 1) ||
- alternatives_text_reserved(src, src + len - 1))
+ alternatives_text_reserved(src, src + len - 1) ||
+ jump_label_text_reserved(src, src + len - 1))
return -EBUSY;
return len;
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index fc2b181..d3d00b6 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -25,9 +25,10 @@ extern struct jump_entry __stop___jump_table[];
extern void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type);
+extern void arch_jump_label_text_poke_early(jump_label_t addr);
extern void jump_label_update(const char *name, enum jump_label_type type);
+extern int jump_label_text_reserved(void *start, void *end);
extern void jump_label_apply_nops(struct module *mod);
-extern void arch_jump_label_text_poke_early(jump_label_t addr);
#define enable_jump_label(name) \
jump_label_update(name, JUMP_LABEL_ENABLE);
@@ -60,6 +61,11 @@ static inline int jump_label_apply_nops(struct module *mod)
return 0;
}
+static inline int jump_label_text_reserved(void *start, void *end)
+{
+ return 0;
+}
+
#endif
#endif
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index ebe9145..c1e59bb 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -183,6 +183,89 @@ void jump_label_update(const char *name, enum jump_label_type type)
mutex_unlock(&jump_label_mutex);
}
+static int addr_conflict(struct jump_entry *entry, void *start, void *end)
+{
+ if (entry->code <= (unsigned long)end &&
+ entry->code + JUMP_LABEL_NOP_SIZE > (unsigned long)start)
+ return 1;
+
+ return 0;
+}
+
+#ifdef CONFIG_MODULES
+
+static int module_conflict(void *start, void *end)
+{
+ struct hlist_head *head;
+ struct hlist_node *node, *node_next, *module_node, *module_node_next;
+ struct jump_label_entry *e;
+ struct jump_label_module_entry *e_module;
+ struct jump_entry *iter;
+ int i, count;
+ int conflict = 0;
+
+ for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
+ head = &jump_label_table[i];
+ hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
+ hlist_for_each_entry_safe(e_module, module_node,
+ module_node_next,
+ &(e->modules), hlist) {
+ count = e_module->nr_entries;
+ iter = e_module->table;
+ while (count--) {
+ if (addr_conflict(iter, start, end)) {
+ conflict = 1;
+ goto out;
+ }
+ iter++;
+ }
+ }
+ }
+ }
+out:
+ return conflict;
+}
+
+#endif
+
+/***
+ * jump_label_text_reserved - check if addr range is reserved
+ * @start: start text addr
+ * @end: end text addr
+ *
+ * checks if the text addr located between @start and @end
+ * overlaps with any of the jump label patch addresses. Code
+ * that wants to modify kernel text should first verify that
+ * it does not overlap with any of the jump label addresses.
+ *
+ * returns 1 if there is an overlap, 0 otherwise
+ */
+int jump_label_text_reserved(void *start, void *end)
+{
+ struct jump_entry *iter;
+ struct jump_entry *iter_start = __start___jump_table;
+ struct jump_entry *iter_stop = __start___jump_table;
+ int conflict = 0;
+
+ mutex_lock(&jump_label_mutex);
+ iter = iter_start;
+ while (iter < iter_stop) {
+ if (addr_conflict(iter, start, end)) {
+ conflict = 1;
+ goto out;
+ }
+ iter++;
+ }
+
+ /* now check modules */
+#ifdef CONFIG_MODULES
+ conflict = module_conflict(start, end);
+#endif
+out:
+ mutex_unlock(&jump_label_mutex);
+ return conflict;
+}
+
static __init int init_jump_label(void)
{
int ret;
diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index 282035f..2f9bea6 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -1141,7 +1141,8 @@ int __kprobes register_kprobe(struct kprobe *p)
preempt_disable();
if (!kernel_text_address((unsigned long) p->addr) ||
in_kprobes_functions((unsigned long) p->addr) ||
- ftrace_text_reserved(p->addr, p->addr)) {
+ ftrace_text_reserved(p->addr, p->addr) ||
+ jump_label_text_reserved(p->addr, p->addr)) {
preempt_enable();
return -EINVAL;
}
--
1.7.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 06/12] jump label v10: convert jump label to use a key
2010-07-27 20:53 [PATCH 00/12] jump label v10 Jason Baron
` (4 preceding siblings ...)
2010-07-27 20:54 ` [PATCH 05/12] jump label v10: jump_label_text_reserved() to reserve our jump points Jason Baron
@ 2010-07-27 20:54 ` Jason Baron
2010-07-28 20:37 ` David Miller
2010-07-27 20:54 ` [PATCH 07/12] jump label v10: use lib/sort.c Jason Baron
` (6 subsequent siblings)
12 siblings, 1 reply; 21+ messages in thread
From: Jason Baron @ 2010-07-27 20:54 UTC (permalink / raw)
To: linux-kernel
Cc: mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi, roland, rth,
mhiramat, fweisbec, avi, davem, vgoyal, sam, tony
Convert the jump label code to use a key to identify each related set of
jump labels instead of a string. This saves space since we no longer need
to store strings and their pointers, just the keys. In addition we can
simplify the JUMP_LABEL macro to 2 arguments from 3. We use the address
of the conditional variable as the key value. If jump labels are not enabled
then we can use the conditional variable as usual.
Signed-off-by: Jason Baron <jbaron@redhat.com>
---
include/asm-generic/vmlinux.lds.h | 10 +++++---
include/linux/jump_label.h | 37 ++++++++++++++---------------------
kernel/jump_label.c | 38 ++++++++++++++++--------------------
3 files changed, 38 insertions(+), 47 deletions(-)
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 3c7e9ab..8679ec7 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -163,8 +163,7 @@
BRANCH_PROFILE() \
TRACE_PRINTKS() \
FTRACE_EVENTS() \
- TRACE_SYSCALLS() \
- JUMP_TABLE() \
+ TRACE_SYSCALLS()
/*
* Data section helpers
@@ -203,7 +202,6 @@
*(__vermagic) /* Kernel version magic */ \
*(__markers_strings) /* Markers: strings */ \
*(__tracepoints_strings)/* Tracepoints: strings */ \
- *(__jump_strings)/* Jump: strings */ \
} \
\
.rodata1 : AT(ADDR(.rodata1) - LOAD_OFFSET) { \
@@ -212,6 +210,8 @@
\
BUG_TABLE \
\
+ JUMP_TABLE \
+ \
/* PCI quirks */ \
.pci_fixup : AT(ADDR(.pci_fixup) - LOAD_OFFSET) { \
VMLINUX_SYMBOL(__start_pci_fixups_early) = .; \
@@ -555,11 +555,13 @@
#define BUG_TABLE
#endif
-#define JUMP_TABLE() \
+#define JUMP_TABLE \
. = ALIGN(8); \
+ __jump_table : AT(ADDR(__jump_table) - LOAD_OFFSET) { \
VMLINUX_SYMBOL(__start___jump_table) = .; \
*(__jump_table) \
VMLINUX_SYMBOL(__stop___jump_table) = .; \
+ }
#ifdef CONFIG_PM_TRACE
#define TRACEDATA \
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index d3d00b6..c9e17b0 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -18,43 +18,36 @@ struct module;
extern struct jump_entry __start___jump_table[];
extern struct jump_entry __stop___jump_table[];
-#define DEFINE_JUMP_LABEL(name) \
- const char __jlstrtab_##name[] \
- __used __attribute__((section("__jump_strings"))) = #name; \
- EXPORT_SYMBOL_GPL(__jlstrtab_##name);
-
extern void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type);
extern void arch_jump_label_text_poke_early(jump_label_t addr);
-extern void jump_label_update(const char *name, enum jump_label_type type);
+extern void jump_label_update(unsigned long key, enum jump_label_type type);
extern int jump_label_text_reserved(void *start, void *end);
extern void jump_label_apply_nops(struct module *mod);
-#define enable_jump_label(name) \
- jump_label_update(name, JUMP_LABEL_ENABLE);
+#define enable_jump_label(key) \
+ jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE);
-#define disable_jump_label(name) \
- jump_label_update(name, JUMP_LABEL_DISABLE);
+#define disable_jump_label(key) \
+ jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE);
#else
-#define DEFINE_JUMP_LABEL(name)
-
-#define JUMP_LABEL(tag, label, cond) \
+#define JUMP_LABEL(key, label) \
do { \
- if (unlikely(cond)) \
+ if (unlikely(*key)) \
goto label; \
} while (0)
-static inline int enable_jump_label(const char *name)
-{
- return 0;
-}
+#define enable_jump_label(cond_var) \
+do { \
+ *(cond_var) = 1; \
+} while (0)
-static inline int disable_jump_label(const char *name)
-{
- return 0;
-}
+#define disable_jump_label(cond_var) \
+do { \
+ *(cond_var) = 0; \
+} while (0)
static inline int jump_label_apply_nops(struct module *mod)
{
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index c1e59bb..10fb17f 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -28,7 +28,7 @@ struct jump_label_entry {
int nr_entries;
/* hang modules off here */
struct hlist_head modules;
- const char *name;
+ unsigned long key;
};
struct jump_label_module_entry {
@@ -59,8 +59,7 @@ static void sort_jump_label_entries(struct jump_entry *start, struct jump_entry
iter_next = start;
iter_next++;
for (; iter_next < stop; iter++, iter_next++) {
- if (strcmp((char *)iter->name,
- (char *)iter_next->name) > 0) {
+ if (iter->key > iter_next->key) {
swap_jump_label_entries(iter, iter_next);
swapped = 1;
}
@@ -68,29 +67,28 @@ static void sort_jump_label_entries(struct jump_entry *start, struct jump_entry
} while (swapped == 1);
}
-static struct jump_label_entry *get_jump_label_entry(const char *name)
+static struct jump_label_entry *get_jump_label_entry(jump_label_t key)
{
struct hlist_head *head;
struct hlist_node *node;
struct jump_label_entry *e;
- u32 hash = jhash(name, strlen(name), 0);
+ u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0);
head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
hlist_for_each_entry(e, node, head, hlist) {
- if (!strcmp((char *)name, (char *)e->name))
+ if (key == e->key)
return e;
}
return NULL;
}
-static struct jump_label_entry *add_jump_label_entry(const char *name, int nr_entries, struct jump_entry *table)
+static struct jump_label_entry *add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table)
{
struct hlist_head *head;
struct jump_label_entry *e;
- size_t name_len;
u32 hash;
- e = get_jump_label_entry(name);
+ e = get_jump_label_entry(key);
if (e)
return ERR_PTR(-EEXIST);
@@ -98,10 +96,9 @@ static struct jump_label_entry *add_jump_label_entry(const char *name, int nr_en
if (!e)
return ERR_PTR(-ENOMEM);
- name_len = strlen(name) + 1;
- hash = jhash(name, name_len-1, 0);
+ hash = jhash((void *)&key, sizeof(jump_label_t), 0);
head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
- e->name = name;
+ e->key = key;
e->table = table;
e->nr_entries = nr_entries;
INIT_HLIST_HEAD(&(e->modules));
@@ -118,17 +115,16 @@ static int build_jump_label_hashtable(struct jump_entry *start, struct jump_entr
sort_jump_label_entries(start, stop);
iter = start;
while (iter < stop) {
- entry = get_jump_label_entry((char *)iter->name);
+ entry = get_jump_label_entry(iter->key);
if (!entry) {
iter_begin = iter;
count = 0;
while ((iter < stop) &&
- (strcmp((char *)iter->name,
- (char *)iter_begin->name) == 0)) {
+ (iter->key == iter_begin->key)) {
iter++;
count++;
}
- entry = add_jump_label_entry((char *)iter_begin->name,
+ entry = add_jump_label_entry(iter_begin->key,
count, iter_begin);
if (IS_ERR(entry))
return PTR_ERR(entry);
@@ -150,7 +146,7 @@ static int build_jump_label_hashtable(struct jump_entry *start, struct jump_entr
*
*/
-void jump_label_update(const char *name, enum jump_label_type type)
+void jump_label_update(unsigned long key, enum jump_label_type type)
{
struct jump_entry *iter;
struct jump_label_entry *entry;
@@ -159,7 +155,7 @@ void jump_label_update(const char *name, enum jump_label_type type)
int count;
mutex_lock(&jump_label_mutex);
- entry = get_jump_label_entry(name);
+ entry = get_jump_label_entry((jump_label_t)key);
if (entry) {
count = entry->nr_entries;
iter = entry->table;
@@ -317,16 +313,16 @@ static int add_jump_label_module(struct module *mod)
mod->jump_entries + mod->num_jump_entries);
iter = mod->jump_entries;
while (iter < mod->jump_entries + mod->num_jump_entries) {
- entry = get_jump_label_entry((char *)iter->name);
+ entry = get_jump_label_entry(iter->key);
iter_begin = iter;
count = 0;
while ((iter < mod->jump_entries + mod->num_jump_entries) &&
- (strcmp((char *)iter->name, (char *)iter_begin->name) == 0)) {
+ (iter->key == iter_begin->key)) {
iter++;
count++;
}
if (!entry) {
- entry = add_jump_label_entry((char *)iter_begin->name, 0, NULL);
+ entry = add_jump_label_entry(iter_begin->key, 0, NULL);
if (IS_ERR(entry))
return PTR_ERR(entry);
}
--
1.7.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH 06/12] jump label v10: convert jump label to use a key
2010-07-27 20:54 ` [PATCH 06/12] jump label v10: convert jump label to use a key Jason Baron
@ 2010-07-28 20:37 ` David Miller
0 siblings, 0 replies; 21+ messages in thread
From: David Miller @ 2010-07-28 20:37 UTC (permalink / raw)
To: jbaron
Cc: linux-kernel, mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi,
roland, rth, mhiramat, fweisbec, avi, vgoyal, sam, tony
From: Jason Baron <jbaron@redhat.com>
Date: Tue, 27 Jul 2010 16:54:32 -0400
> Convert the jump label code to use a key to identify each related set of
> jump labels instead of a string. This saves space since we no longer need
> to store strings and their pointers, just the keys. In addition we can
> simplify the JUMP_LABEL macro to 2 arguments from 3. We use the address
> of the conditional variable as the key value. If jump labels are not enabled
> then we can use the conditional variable as usual.
>
> Signed-off-by: Jason Baron <jbaron@redhat.com>
It doesn't make any sense to present these changes like this.
If the final implementation of the jump label entries uses this new
key based scheme, well that's what should be in the first patch of
this series which adds the basic jump label infrastructure.
As it is, it's just commit noise.
^ permalink raw reply [flat|nested] 21+ messages in thread
* [PATCH 07/12] jump label v10: use lib/sort.c
2010-07-27 20:53 [PATCH 00/12] jump label v10 Jason Baron
` (5 preceding siblings ...)
2010-07-27 20:54 ` [PATCH 06/12] jump label v10: convert jump label to use a key Jason Baron
@ 2010-07-27 20:54 ` Jason Baron
2010-07-28 20:40 ` David Miller
2010-07-27 20:54 ` [PATCH 08/12] jump label v10: tracepoint support Jason Baron
` (5 subsequent siblings)
12 siblings, 1 reply; 21+ messages in thread
From: Jason Baron @ 2010-07-27 20:54 UTC (permalink / raw)
To: linux-kernel
Cc: mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi, roland, rth,
mhiramat, fweisbec, avi, davem, vgoyal, sam, tony
Make use of the the kernel's generic sort routines.
Signed-off-by: Jason Baron <jbaron@redhat.com>
---
kernel/jump_label.c | 37 ++++++++++++++++---------------------
1 files changed, 16 insertions(+), 21 deletions(-)
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 10fb17f..faef97e 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -12,6 +12,7 @@
#include <linux/jhash.h>
#include <linux/slab.h>
#include <linux/err.h>
+#include <linux/sort.h>
#ifdef HAVE_JUMP_LABEL
@@ -38,33 +39,27 @@ struct jump_label_module_entry {
struct module *mod;
};
-static void swap_jump_label_entries(struct jump_entry *previous, struct jump_entry *next)
+static int jump_label_cmp(const void *a, const void *b)
{
- struct jump_entry tmp;
+ const struct jump_entry *jea = a;
+ const struct jump_entry *jeb = b;
- tmp = *next;
- *next = *previous;
- *previous = tmp;
+ if (jea->key < jeb->key)
+ return -1;
+
+ if (jea->key > jeb->key)
+ return 1;
+
+ return 0;
}
static void sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop)
{
- int swapped = 0;
- struct jump_entry *iter;
- struct jump_entry *iter_next;
-
- do {
- swapped = 0;
- iter = start;
- iter_next = start;
- iter_next++;
- for (; iter_next < stop; iter++, iter_next++) {
- if (iter->key > iter_next->key) {
- swap_jump_label_entries(iter, iter_next);
- swapped = 1;
- }
- }
- } while (swapped == 1);
+ unsigned long size;
+
+ size = (((unsigned long)stop - (unsigned long)start)
+ / sizeof(struct jump_entry));
+ sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
}
static struct jump_label_entry *get_jump_label_entry(jump_label_t key)
--
1.7.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH 07/12] jump label v10: use lib/sort.c
2010-07-27 20:54 ` [PATCH 07/12] jump label v10: use lib/sort.c Jason Baron
@ 2010-07-28 20:40 ` David Miller
0 siblings, 0 replies; 21+ messages in thread
From: David Miller @ 2010-07-28 20:40 UTC (permalink / raw)
To: jbaron
Cc: linux-kernel, mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi,
roland, rth, mhiramat, fweisbec, avi, vgoyal, sam, tony
From: Jason Baron <jbaron@redhat.com>
Date: Tue, 27 Jul 2010 16:54:36 -0400
> Make use of the the kernel's generic sort routines.
>
> Signed-off-by: Jason Baron <jbaron@redhat.com>
Again, this should be incorporated into the first patch.
It's a new feature, so whatever refinements occur over time should be
incorporated into the patch that adds the new feature.
^ permalink raw reply [flat|nested] 21+ messages in thread
* [PATCH 08/12] jump label v10: tracepoint support
2010-07-27 20:53 [PATCH 00/12] jump label v10 Jason Baron
` (6 preceding siblings ...)
2010-07-27 20:54 ` [PATCH 07/12] jump label v10: use lib/sort.c Jason Baron
@ 2010-07-27 20:54 ` Jason Baron
2010-07-27 20:54 ` [PATCH 09/12] jump label v10: convert dynamic debug to use jump labels Jason Baron
` (4 subsequent siblings)
12 siblings, 0 replies; 21+ messages in thread
From: Jason Baron @ 2010-07-27 20:54 UTC (permalink / raw)
To: linux-kernel
Cc: mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi, roland, rth,
mhiramat, fweisbec, avi, davem, vgoyal, sam, tony
Make use of the jump label infrastructure for tracepoints.
Signed-off-by: Jason Baron <jbaron@redhat.com>
---
include/linux/tracepoint.h | 5 ++++-
kernel/tracepoint.c | 14 ++++++++++++--
2 files changed, 16 insertions(+), 3 deletions(-)
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index 103d1b6..a4a90b6 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -17,6 +17,7 @@
#include <linux/errno.h>
#include <linux/types.h>
#include <linux/rcupdate.h>
+#include <linux/jump_label.h>
struct module;
struct tracepoint;
@@ -145,7 +146,9 @@ static inline void tracepoint_update_probe_range(struct tracepoint *begin,
extern struct tracepoint __tracepoint_##name; \
static inline void trace_##name(proto) \
{ \
- if (unlikely(__tracepoint_##name.state)) \
+ JUMP_LABEL(&__tracepoint_##name.state, do_trace); \
+ return; \
+do_trace: \
__DO_TRACE(&__tracepoint_##name, \
TP_PROTO(data_proto), \
TP_ARGS(data_args)); \
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index c77f3ec..d6073a5 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -25,6 +25,7 @@
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/sched.h>
+#include <linux/jump_label.h>
extern struct tracepoint __start___tracepoints[];
extern struct tracepoint __stop___tracepoints[];
@@ -263,7 +264,13 @@ static void set_tracepoint(struct tracepoint_entry **entry,
* is used.
*/
rcu_assign_pointer(elem->funcs, (*entry)->funcs);
- elem->state = active;
+ if (!elem->state && active) {
+ enable_jump_label(&elem->state);
+ elem->state = active;
+ } else if (elem->state && !active) {
+ disable_jump_label(&elem->state);
+ elem->state = active;
+ }
}
/*
@@ -277,7 +284,10 @@ static void disable_tracepoint(struct tracepoint *elem)
if (elem->unregfunc && elem->state)
elem->unregfunc();
- elem->state = 0;
+ if (elem->state) {
+ disable_jump_label(&elem->state);
+ elem->state = 0;
+ }
rcu_assign_pointer(elem->funcs, NULL);
}
--
1.7.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 09/12] jump label v10: convert dynamic debug to use jump labels.
2010-07-27 20:53 [PATCH 00/12] jump label v10 Jason Baron
` (7 preceding siblings ...)
2010-07-27 20:54 ` [PATCH 08/12] jump label v10: tracepoint support Jason Baron
@ 2010-07-27 20:54 ` Jason Baron
2010-07-27 20:54 ` [PATCH 10/12] jump label v10: x86 support Jason Baron
` (3 subsequent siblings)
12 siblings, 0 replies; 21+ messages in thread
From: Jason Baron @ 2010-07-27 20:54 UTC (permalink / raw)
To: linux-kernel
Cc: mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi, roland, rth,
mhiramat, fweisbec, avi, davem, vgoyal, sam, tony
Convert the 'dynamic debug' infrastructure to use jump labels.
Signed-off-by: Jason Baron <jbaron@redhat.com>
---
include/linux/dynamic_debug.h | 39 +++++++++++++-----------
lib/dynamic_debug.c | 42 ++-------------------------
scripts/Makefile.lib | 11 +------
scripts/basic/Makefile | 2 +-
scripts/basic/hash.c | 64 -----------------------------------------
5 files changed, 26 insertions(+), 132 deletions(-)
delete mode 100644 scripts/basic/hash.c
diff --git a/include/linux/dynamic_debug.h b/include/linux/dynamic_debug.h
index 52c0da4..bef3cda 100644
--- a/include/linux/dynamic_debug.h
+++ b/include/linux/dynamic_debug.h
@@ -1,6 +1,8 @@
#ifndef _DYNAMIC_DEBUG_H
#define _DYNAMIC_DEBUG_H
+#include <linux/jump_label.h>
+
/* dynamic_printk_enabled, and dynamic_printk_enabled2 are bitmasks in which
* bit n is set to 1 if any modname hashes into the bucket n, 0 otherwise. They
* use independent hash functions, to reduce the chance of false positives.
@@ -22,8 +24,6 @@ struct _ddebug {
const char *function;
const char *filename;
const char *format;
- char primary_hash;
- char secondary_hash;
unsigned int lineno:24;
/*
* The flags field controls the behaviour at the callsite.
@@ -33,6 +33,7 @@ struct _ddebug {
#define _DPRINTK_FLAGS_PRINT (1<<0) /* printk() a message using the format */
#define _DPRINTK_FLAGS_DEFAULT 0
unsigned int flags:8;
+ char enabled;
} __attribute__((aligned(8)));
@@ -42,33 +43,35 @@ int ddebug_add_module(struct _ddebug *tab, unsigned int n,
#if defined(CONFIG_DYNAMIC_DEBUG)
extern int ddebug_remove_module(const char *mod_name);
-#define __dynamic_dbg_enabled(dd) ({ \
- int __ret = 0; \
- if (unlikely((dynamic_debug_enabled & (1LL << DEBUG_HASH)) && \
- (dynamic_debug_enabled2 & (1LL << DEBUG_HASH2)))) \
- if (unlikely(dd.flags)) \
- __ret = 1; \
- __ret; })
-
#define dynamic_pr_debug(fmt, ...) do { \
+ __label__ do_printk; \
+ __label__ out; \
static struct _ddebug descriptor \
__used \
__attribute__((section("__verbose"), aligned(8))) = \
- { KBUILD_MODNAME, __func__, __FILE__, fmt, DEBUG_HASH, \
- DEBUG_HASH2, __LINE__, _DPRINTK_FLAGS_DEFAULT }; \
- if (__dynamic_dbg_enabled(descriptor)) \
- printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); \
+ { KBUILD_MODNAME, __func__, __FILE__, fmt, __LINE__, \
+ _DPRINTK_FLAGS_DEFAULT }; \
+ JUMP_LABEL(&descriptor.enabled, do_printk); \
+ goto out; \
+do_printk: \
+ printk(KERN_DEBUG pr_fmt(fmt), ##__VA_ARGS__); \
+out: ; \
} while (0)
#define dynamic_dev_dbg(dev, fmt, ...) do { \
+ __label__ do_printk; \
+ __label__ out; \
static struct _ddebug descriptor \
__used \
__attribute__((section("__verbose"), aligned(8))) = \
- { KBUILD_MODNAME, __func__, __FILE__, fmt, DEBUG_HASH, \
- DEBUG_HASH2, __LINE__, _DPRINTK_FLAGS_DEFAULT }; \
- if (__dynamic_dbg_enabled(descriptor)) \
- dev_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__); \
+ { KBUILD_MODNAME, __func__, __FILE__, fmt, __LINE__, \
+ _DPRINTK_FLAGS_DEFAULT }; \
+ JUMP_LABEL(&descriptor.enabled, do_printk); \
+ goto out; \
+do_printk: \
+ dev_printk(KERN_DEBUG, dev, fmt, ##__VA_ARGS__); \
+out: ; \
} while (0)
#else
diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c
index 02afc25..e925c7b 100644
--- a/lib/dynamic_debug.c
+++ b/lib/dynamic_debug.c
@@ -26,19 +26,11 @@
#include <linux/dynamic_debug.h>
#include <linux/debugfs.h>
#include <linux/slab.h>
+#include <linux/jump_label.h>
extern struct _ddebug __start___verbose[];
extern struct _ddebug __stop___verbose[];
-/* dynamic_debug_enabled, and dynamic_debug_enabled2 are bitmasks in which
- * bit n is set to 1 if any modname hashes into the bucket n, 0 otherwise. They
- * use independent hash functions, to reduce the chance of false positives.
- */
-long long dynamic_debug_enabled;
-EXPORT_SYMBOL_GPL(dynamic_debug_enabled);
-long long dynamic_debug_enabled2;
-EXPORT_SYMBOL_GPL(dynamic_debug_enabled2);
-
struct ddebug_table {
struct list_head link;
char *mod_name;
@@ -88,26 +80,6 @@ static char *ddebug_describe_flags(struct _ddebug *dp, char *buf,
}
/*
- * must be called with ddebug_lock held
- */
-
-static int disabled_hash(char hash, bool first_table)
-{
- struct ddebug_table *dt;
- char table_hash_value;
-
- list_for_each_entry(dt, &ddebug_tables, link) {
- if (first_table)
- table_hash_value = dt->ddebugs->primary_hash;
- else
- table_hash_value = dt->ddebugs->secondary_hash;
- if (dt->num_enabled && (hash == table_hash_value))
- return 0;
- }
- return 1;
-}
-
-/*
* Search the tables for _ddebug's which match the given
* `query' and apply the `flags' and `mask' to them. Tells
* the user which ddebug's were changed, or whether none
@@ -170,17 +142,9 @@ static void ddebug_change(const struct ddebug_query *query,
dt->num_enabled++;
dp->flags = newflags;
if (newflags) {
- dynamic_debug_enabled |=
- (1LL << dp->primary_hash);
- dynamic_debug_enabled2 |=
- (1LL << dp->secondary_hash);
+ enable_jump_label(&dp->enabled);
} else {
- if (disabled_hash(dp->primary_hash, true))
- dynamic_debug_enabled &=
- ~(1LL << dp->primary_hash);
- if (disabled_hash(dp->secondary_hash, false))
- dynamic_debug_enabled2 &=
- ~(1LL << dp->secondary_hash);
+ disable_jump_label(&dp->enabled);
}
if (verbose)
printk(KERN_INFO
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 54fd1b7..7bfcf1a 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -101,14 +101,6 @@ basename_flags = -D"KBUILD_BASENAME=KBUILD_STR($(call name-fix,$(basetarget)))"
modname_flags = $(if $(filter 1,$(words $(modname))),\
-D"KBUILD_MODNAME=KBUILD_STR($(call name-fix,$(modname)))")
-#hash values
-ifdef CONFIG_DYNAMIC_DEBUG
-debug_flags = -D"DEBUG_HASH=$(shell ./scripts/basic/hash djb2 $(@D)$(modname))"\
- -D"DEBUG_HASH2=$(shell ./scripts/basic/hash r5 $(@D)$(modname))"
-else
-debug_flags =
-endif
-
orig_c_flags = $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS) $(KBUILD_SUBDIR_CCFLAGS) \
$(ccflags-y) $(CFLAGS_$(basetarget).o)
_c_flags = $(filter-out $(CFLAGS_REMOVE_$(basetarget).o), $(orig_c_flags))
@@ -152,8 +144,7 @@ endif
c_flags = -Wp,-MD,$(depfile) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) \
$(__c_flags) $(modkern_cflags) \
- -D"KBUILD_STR(s)=\#s" $(basename_flags) $(modname_flags) \
- $(debug_flags)
+ -D"KBUILD_STR(s)=\#s" $(basename_flags) $(modname_flags)
a_flags = -Wp,-MD,$(depfile) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) \
$(__a_flags) $(modkern_aflags)
diff --git a/scripts/basic/Makefile b/scripts/basic/Makefile
index 0955995..4c324a1 100644
--- a/scripts/basic/Makefile
+++ b/scripts/basic/Makefile
@@ -9,7 +9,7 @@
# fixdep: Used to generate dependency information during build process
# docproc: Used in Documentation/DocBook
-hostprogs-y := fixdep docproc hash
+hostprogs-y := fixdep docproc
always := $(hostprogs-y)
# fixdep is needed to compile other host programs
diff --git a/scripts/basic/hash.c b/scripts/basic/hash.c
deleted file mode 100644
index 2ef5d3f..0000000
--- a/scripts/basic/hash.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright (C) 2008 Red Hat, Inc., Jason Baron <jbaron@redhat.com>
- *
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#define DYNAMIC_DEBUG_HASH_BITS 6
-
-static const char *program;
-
-static void usage(void)
-{
- printf("Usage: %s <djb2|r5> <modname>\n", program);
- exit(1);
-}
-
-/* djb2 hashing algorithm by Dan Bernstein. From:
- * http://www.cse.yorku.ca/~oz/hash.html
- */
-
-static unsigned int djb2_hash(char *str)
-{
- unsigned long hash = 5381;
- int c;
-
- c = *str;
- while (c) {
- hash = ((hash << 5) + hash) + c;
- c = *++str;
- }
- return (unsigned int)(hash & ((1 << DYNAMIC_DEBUG_HASH_BITS) - 1));
-}
-
-static unsigned int r5_hash(char *str)
-{
- unsigned long hash = 0;
- int c;
-
- c = *str;
- while (c) {
- hash = (hash + (c << 4) + (c >> 4)) * 11;
- c = *++str;
- }
- return (unsigned int)(hash & ((1 << DYNAMIC_DEBUG_HASH_BITS) - 1));
-}
-
-int main(int argc, char *argv[])
-{
- program = argv[0];
-
- if (argc != 3)
- usage();
- if (!strcmp(argv[1], "djb2"))
- printf("%d\n", djb2_hash(argv[2]));
- else if (!strcmp(argv[1], "r5"))
- printf("%d\n", r5_hash(argv[2]));
- else
- usage();
- exit(0);
-}
-
--
1.7.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 10/12] jump label v10: x86 support
2010-07-27 20:53 [PATCH 00/12] jump label v10 Jason Baron
` (8 preceding siblings ...)
2010-07-27 20:54 ` [PATCH 09/12] jump label v10: convert dynamic debug to use jump labels Jason Baron
@ 2010-07-27 20:54 ` Jason Baron
2010-07-27 20:54 ` [PATCH 11/12] jump label v10: add sparc64 support Jason Baron
` (2 subsequent siblings)
12 siblings, 0 replies; 21+ messages in thread
From: Jason Baron @ 2010-07-27 20:54 UTC (permalink / raw)
To: linux-kernel
Cc: mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi, roland, rth,
mhiramat, fweisbec, avi, davem, vgoyal, sam, tony
add x86 support for jump label. I'm keeping this patch separate so its clear to
arch maintainers what was required for x86 support this new feature. hopefully,
it wouldn't be too painful for other arches.
Signed-off-by: Jason Baron <jbaron@redhat.com>
---
arch/x86/Kconfig | 1 +
arch/x86/include/asm/jump_label.h | 47 +++++++++++++++++++++++++++++++++++
arch/x86/kernel/Makefile | 2 +-
arch/x86/kernel/jump_label.c | 49 +++++++++++++++++++++++++++++++++++++
arch/x86/kernel/module.c | 3 ++
5 files changed, 101 insertions(+), 1 deletions(-)
create mode 100644 arch/x86/include/asm/jump_label.h
create mode 100644 arch/x86/kernel/jump_label.c
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 3069a6d..089ef9f 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -59,6 +59,7 @@ config X86
select ANON_INODES
select HAVE_ARCH_KMEMCHECK
select HAVE_USER_RETURN_NOTIFIER
+ select HAVE_ARCH_JUMP_LABEL if !CC_OPTIMIZE_FOR_SIZE
config INSTRUCTION_DECODER
def_bool (KPROBES || PERF_EVENTS)
diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h
new file mode 100644
index 0000000..b4a2cb4
--- /dev/null
+++ b/arch/x86/include/asm/jump_label.h
@@ -0,0 +1,47 @@
+#ifndef _ASM_X86_JUMP_LABEL_H
+#define _ASM_X86_JUMP_LABEL_H
+
+#ifdef __KERNEL__
+
+#include <linux/types.h>
+#include <asm/nops.h>
+
+#define JUMP_LABEL_NOP_SIZE 5
+
+# define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
+
+# define JUMP_LABEL(key, label) \
+ do { \
+ asm goto("1:" \
+ JUMP_LABEL_INITIAL_NOP \
+ ".pushsection __jump_table, \"a\" \n\t"\
+ _ASM_PTR "1b, %l[" #label "], %c0 \n\t" \
+ ".popsection \n\t" \
+ : : "i" (key) : : label); \
+ } while (0)
+
+#endif /* __KERNEL__ */
+
+#ifdef CONFIG_X86_64
+
+typedef u64 jump_label_t;
+
+struct jump_entry {
+ jump_label_t code;
+ jump_label_t target;
+ jump_label_t key;
+};
+
+#else
+
+typedef u32 jump_label_t;
+
+struct jump_entry {
+ jump_label_t code;
+ jump_label_t target;
+ jump_label_t key;
+};
+
+#endif
+
+#endif
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 0925676..24fa171 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -32,7 +32,7 @@ GCOV_PROFILE_paravirt.o := n
obj-y := process_$(BITS).o signal.o entry_$(BITS).o
obj-y += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o
obj-y += time.o ioport.o ldt.o dumpstack.o
-obj-y += setup.o x86_init.o i8259.o irqinit.o
+obj-y += setup.o x86_init.o i8259.o irqinit.o jump_label.o
obj-$(CONFIG_X86_VISWS) += visws_quirks.o
obj-$(CONFIG_X86_32) += probe_roms_32.o
obj-$(CONFIG_X86_32) += sys_i386_32.o i386_ksyms_32.o
diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c
new file mode 100644
index 0000000..b3c0f37
--- /dev/null
+++ b/arch/x86/kernel/jump_label.c
@@ -0,0 +1,49 @@
+/*
+ * jump label x86 support
+ *
+ * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
+ *
+ */
+#include <linux/jump_label.h>
+#include <linux/memory.h>
+#include <linux/uaccess.h>
+#include <linux/module.h>
+#include <linux/list.h>
+#include <linux/jhash.h>
+#include <linux/cpu.h>
+#include <asm/kprobes.h>
+#include <asm/alternative.h>
+
+#ifdef HAVE_JUMP_LABEL
+
+union jump_code_union {
+ char code[JUMP_LABEL_NOP_SIZE];
+ struct {
+ char jump;
+ int offset;
+ } __attribute__((packed));
+};
+
+void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type)
+{
+ union jump_code_union code;
+
+ if (type == JUMP_LABEL_ENABLE) {
+ code.jump = 0xe9;
+ code.offset = entry->target -
+ (entry->code + JUMP_LABEL_NOP_SIZE);
+ } else
+ memcpy(&code, ideal_nop5, JUMP_LABEL_NOP_SIZE);
+ get_online_cpus();
+ mutex_lock(&text_mutex);
+ text_poke_smp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
+ mutex_unlock(&text_mutex);
+ put_online_cpus();
+}
+
+void arch_jump_label_text_poke_early(jump_label_t addr)
+{
+ text_poke_early((void *)addr, ideal_nop5, JUMP_LABEL_NOP_SIZE);
+}
+
+#endif
diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
index e0bc186..5399f58 100644
--- a/arch/x86/kernel/module.c
+++ b/arch/x86/kernel/module.c
@@ -239,6 +239,9 @@ int module_finalize(const Elf_Ehdr *hdr,
apply_paravirt(pseg, pseg + para->sh_size);
}
+ /* make jump label nops */
+ jump_label_apply_nops(me);
+
return module_bug_finalize(hdr, sechdrs, me);
}
--
1.7.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 11/12] jump label v10: add sparc64 support
2010-07-27 20:53 [PATCH 00/12] jump label v10 Jason Baron
` (9 preceding siblings ...)
2010-07-27 20:54 ` [PATCH 10/12] jump label v10: x86 support Jason Baron
@ 2010-07-27 20:54 ` Jason Baron
2010-07-27 20:54 ` [PATCH 12/12] jump label v10: add docs Jason Baron
2010-07-28 19:34 ` [PATCH 00/12] jump label v10 David Miller
12 siblings, 0 replies; 21+ messages in thread
From: Jason Baron @ 2010-07-27 20:54 UTC (permalink / raw)
To: linux-kernel
Cc: mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi, roland, rth,
mhiramat, fweisbec, avi, davem, vgoyal, sam, tony
Signed-off-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Jason Baron <jbaron@redhat.com>
---
arch/sparc/Kconfig | 1 +
arch/sparc/include/asm/jump_label.h | 32 ++++++++++++++++++++++++++++
arch/sparc/kernel/Makefile | 2 +
arch/sparc/kernel/jump_label.c | 39 +++++++++++++++++++++++++++++++++++
arch/sparc/kernel/module.c | 6 +++++
5 files changed, 80 insertions(+), 0 deletions(-)
create mode 100644 arch/sparc/include/asm/jump_label.h
create mode 100644 arch/sparc/kernel/jump_label.c
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index c0015db..e38d976 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -29,6 +29,7 @@ config SPARC
select PERF_USE_VMALLOC
select HAVE_DMA_ATTRS
select HAVE_DMA_API_DEBUG
+ select HAVE_ARCH_JUMP_LABEL if !CC_OPTIMIZE_FOR_SIZE
config SPARC32
def_bool !64BIT
diff --git a/arch/sparc/include/asm/jump_label.h b/arch/sparc/include/asm/jump_label.h
new file mode 100644
index 0000000..62e66d7
--- /dev/null
+++ b/arch/sparc/include/asm/jump_label.h
@@ -0,0 +1,32 @@
+#ifndef _ASM_SPARC_JUMP_LABEL_H
+#define _ASM_SPARC_JUMP_LABEL_H
+
+#ifdef __KERNEL__
+
+#include <linux/types.h>
+#include <asm/system.h>
+
+#define JUMP_LABEL_NOP_SIZE 4
+
+#define JUMP_LABEL(key, label) \
+ do { \
+ asm goto("1:\n\t" \
+ "nop\n\t" \
+ "nop\n\t" \
+ ".pushsection __jump_table, \"a\"\n\t"\
+ ".word 1b, %l[" #label "], %c0\n\t" \
+ ".popsection \n\t" \
+ : : "i" (key) : : label);\
+ } while (0)
+
+#endif /* __KERNEL__ */
+
+typedef u32 jump_label_t;
+
+struct jump_entry {
+ jump_label_t code;
+ jump_label_t target;
+ jump_label_t key;
+};
+
+#endif
diff --git a/arch/sparc/kernel/Makefile b/arch/sparc/kernel/Makefile
index 0c2dc1f..599398f 100644
--- a/arch/sparc/kernel/Makefile
+++ b/arch/sparc/kernel/Makefile
@@ -119,3 +119,5 @@ obj-$(CONFIG_COMPAT) += $(audit--y)
pc--$(CONFIG_PERF_EVENTS) := perf_event.o
obj-$(CONFIG_SPARC64) += $(pc--y)
+
+obj-$(CONFIG_SPARC64) += jump_label.o
diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c
new file mode 100644
index 0000000..10cfbef
--- /dev/null
+++ b/arch/sparc/kernel/jump_label.c
@@ -0,0 +1,39 @@
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/mutex.h>
+#include <linux/cpu.h>
+
+#include <linux/jump_label.h>
+#include <linux/memory.h>
+
+#ifdef HAVE_JUMP_LABEL
+
+void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type)
+{
+ u32 val;
+ u32 *insn = (u32 *) (unsigned long) entry->code;
+
+ if (type == JUMP_LABEL_ENABLE) {
+ s32 off = (s32)entry->target - (s32)entry->code;
+ val = 0x40000000 | ((u32) off >> 2);
+ } else {
+ val = 0x01000000;
+ }
+
+ get_online_cpus();
+ mutex_lock(&text_mutex);
+ *insn = val;
+ flushi(insn);
+ mutex_unlock(&text_mutex);
+ put_online_cpus();
+}
+
+void arch_jump_label_text_poke_early(jump_label_t addr)
+{
+ u32 *insn_p = (u32 *) (unsigned long) addr;
+
+ *insn_p = 0x01000000;
+ flushi(insn_p);
+}
+
+#endif
diff --git a/arch/sparc/kernel/module.c b/arch/sparc/kernel/module.c
index f848aad..ee3c7dd 100644
--- a/arch/sparc/kernel/module.c
+++ b/arch/sparc/kernel/module.c
@@ -18,6 +18,9 @@
#include <asm/spitfire.h>
#ifdef CONFIG_SPARC64
+
+#include <linux/jump_label.h>
+
static void *module_map(unsigned long size)
{
struct vm_struct *area;
@@ -227,6 +230,9 @@ int module_finalize(const Elf_Ehdr *hdr,
const Elf_Shdr *sechdrs,
struct module *me)
{
+ /* make jump label nops */
+ jump_label_apply_nops(me);
+
/* Cheetah's I-cache is fully coherent. */
if (tlb_type == spitfire) {
unsigned long va;
--
1.7.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* [PATCH 12/12] jump label v10: add docs
2010-07-27 20:53 [PATCH 00/12] jump label v10 Jason Baron
` (10 preceding siblings ...)
2010-07-27 20:54 ` [PATCH 11/12] jump label v10: add sparc64 support Jason Baron
@ 2010-07-27 20:54 ` Jason Baron
2010-07-28 19:34 ` [PATCH 00/12] jump label v10 David Miller
12 siblings, 0 replies; 21+ messages in thread
From: Jason Baron @ 2010-07-27 20:54 UTC (permalink / raw)
To: linux-kernel
Cc: mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi, roland, rth,
mhiramat, fweisbec, avi, davem, vgoyal, sam, tony
Add jump label docs as: Documentation/jump-label.txt
Signed-off-by: Jason Baron <jbaron@redhat.com>
---
Documentation/jump-label.txt | 151 ++++++++++++++++++++++++++++++++++++++++++
1 files changed, 151 insertions(+), 0 deletions(-)
create mode 100644 Documentation/jump-label.txt
diff --git a/Documentation/jump-label.txt b/Documentation/jump-label.txt
new file mode 100644
index 0000000..232ab79
--- /dev/null
+++ b/Documentation/jump-label.txt
@@ -0,0 +1,151 @@
+ Jump Label
+ ----------
+
+By: Jason Baron <jbaron@redhat.com>
+
+
+1) motivation
+
+
+Currently, tracepoints are implemented using a conditional. The conditional
+check requires checking a global variable for each tracepoint. Although,
+the overhead of this check is small, it increases under memory pressure. As we
+increase the number of tracepoints in the kernel this may become more of an
+issue. In addition, tracepoints are often dormant (disabled), and provide no
+direct kernel functionality. Thus, it is highly desirable to reduce their
+impact as much as possible. Although tracepoints are the original motivation
+for this work, other kernel code paths should be able to make use of the jump
+label optimization.
+
+
+2) jump label description/usage
+
+
+gcc (v4.5) adds a new 'asm goto' statement that allows branching to a label.
+http://gcc.gnu.org/ml/gcc-patches/2009-07/msg01556.html
+
+Thus, this patch set introduces an architecture specific 'JUMP_LABEL()' macro as
+follows (x86):
+
+# define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
+
+# define JUMP_LABEL(key, label) \
+ do { \
+ asm goto("1:" \
+ JUMP_LABEL_INITIAL_NOP \
+ ".pushsection __jump_table, \"a\" \n\t"\
+ _ASM_PTR "1b, %l[" #label "], %c0 \n\t" \
+ ".popsection \n\t" \
+ : : "i" (key) : : label); \
+ } while (0)
+
+
+For architectures that have not yet introduced jump label support its simply:
+
+#define JUMP_LABEL(key, label) \
+ if (unlikely(*key)) \
+ goto label;
+
+which then can be used as:
+
+ ....
+ JUMP_LABEL(trace_name, trace_label, jump_enabled);
+ printk("not doing tracing\n");
+ return;
+trace_label:
+ printk("doing tracing: %d\n", file);
+ ....
+
+The 'key' argument is thus a pointer to a conditional argument that can be used
+if the optimization is not enabled. Otherwise, this address serves as a unique
+key to identify the particular instance of the jump label.
+
+Thus, when tracing is disabled, we simply have a no-op followed by a jump around
+the dormant (disabled) tracing code. The 'JUMP_LABEL()' macro, produces a
+'jump_table' which has the following format:
+
+[instruction address] [jump target] [tracepoint key]
+
+Thus, to enable a tracepoint, we simply patch the 'instruction address' with
+a jump to the 'jump target'.
+
+The call to enable a jump label is: enable_jump_label(key); to disable:
+disable_jump_label(key);
+
+
+3) architecture interface
+
+
+There are a few functions and macros which arches must implement in order to
+take advantage of this optimization. As previously mentioned, if there is no
+architecture support we simply fall back to a traditional, load, test, and
+jump sequence.
+
+* add "HAVE_ARCH_JUMP_LABEL" to arch/<arch>/Kconfig to indicate support
+
+* #define JUMP_LABEL_NOP_SIZE, arch/x86/include/asm/jump_label.h
+
+* #define "JUMP_LABEL(key, label)", arch/x86/include/asm/jump_label.h
+
+* void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type)
+ see: arch/x86/kernel/jump_label.c
+
+* void arch_jump_label_text_poke_early(jump_label_t addr)
+ see: arch/x86/kernel/jump_label.c
+
+* finally add a definition for "struct jump_entry".
+ see: arch/x86/include/asm/jump_label.h
+
+
+4) Jump label analysis (x86)
+
+
+I've tested the performance of using 'get_cycles()' calls around the
+tracepoint call sites. For an Intel Core 2 Quad cpu (in cycles, averages):
+
+ idle after tbench run
+ ---- ----------------
+old code 32 88
+new code 2 4
+
+
+The performance improvement can be reproduced reliably on both Intel and AMD
+hardware.
+
+In terms of code analysis the current code for the disabled case is a 'cmpl'
+followed by a 'je' around the tracepoint code. so:
+
+cmpl - 83 3d 0e 77 87 00 00 - 7 bytes
+je - 74 3e - 2 bytes
+
+total of 9 instruction bytes.
+
+The new code is a 'nopl' followed by a 'jmp'. Thus:
+
+nopl - 0f 1f 44 00 00 - 5 bytes
+jmp - eb 3e - 2 bytes
+
+total of 7 instruction bytes.
+
+So, the new code also accounts for 2 less bytes in the instruction cache per tracepoint.
+
+The optimization depends on !CC_OPTIMIZE_FOR_SIZE. When CC_OPTIMIZE_FOR_SIZE is
+set, gcc does not always out of line the not taken label path in the same way
+that the "if unlikely()" paths are made out of line. Thus, with
+CC_OPTIMIZE_FOR_SIZE set, this optimization is not always optimal. This may be
+solved in subsequent gcc versions, that allow us to move labels out of line,
+while still optimizing for size. In the case of !CC_OPTIMIZE_FOR_SIZE this
+optimization is seen on high level benchmarks such as tbench where I can get
+~1-2% higher throughput. In addition we are within .5% of the performance of no
+tracepoints compiled in at all.
+
+
+5) Acknowledgments
+
+
+Thanks to Roland McGrath and Richard Henderson for helping come up with the
+initial 'asm goto' and jump label design.
+
+Thanks to Mathieu Desnoyers and H. Peter Anvin for calling attention to this
+issue, and outlining the requirements of a solution. Mathieu also implemened a
+solution in the form of the "Immediate Values" work.
--
1.7.1
^ permalink raw reply related [flat|nested] 21+ messages in thread* Re: [PATCH 00/12] jump label v10
2010-07-27 20:53 [PATCH 00/12] jump label v10 Jason Baron
` (11 preceding siblings ...)
2010-07-27 20:54 ` [PATCH 12/12] jump label v10: add docs Jason Baron
@ 2010-07-28 19:34 ` David Miller
2010-07-28 20:14 ` Jason Baron
12 siblings, 1 reply; 21+ messages in thread
From: David Miller @ 2010-07-28 19:34 UTC (permalink / raw)
To: jbaron
Cc: linux-kernel, mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi,
roland, rth, mhiramat, fweisbec, avi, vgoyal, sam, tony
Can you please indicate which tree this is against when you post these
patches?
They don't apply cleanly to Linus's current tree which is where I tend
to try first when the tree is unspecified, as I imagine others do as
well.
^ permalink raw reply [flat|nested] 21+ messages in thread* Re: [PATCH 00/12] jump label v10
2010-07-28 19:34 ` [PATCH 00/12] jump label v10 David Miller
@ 2010-07-28 20:14 ` Jason Baron
2010-07-28 20:24 ` David Miller
2010-07-29 5:25 ` David Miller
0 siblings, 2 replies; 21+ messages in thread
From: Jason Baron @ 2010-07-28 20:14 UTC (permalink / raw)
To: David Miller
Cc: linux-kernel, mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi,
roland, rth, mhiramat, fweisbec, avi, vgoyal, sam, tony
On Wed, Jul 28, 2010 at 12:34:27PM -0700, David Miller wrote:
> Can you please indicate which tree this is against when you post these
> patches?
>
> They don't apply cleanly to Linus's current tree which is where I tend
> to try first when the tree is unspecified, as I imagine others do as
> well.
>
Hi,
Patches are against Ingo's latest -tip tree.
thanks,
-Jason
^ permalink raw reply [flat|nested] 21+ messages in thread
* Re: [PATCH 00/12] jump label v10
2010-07-28 20:14 ` Jason Baron
@ 2010-07-28 20:24 ` David Miller
2010-07-28 20:42 ` Jason Baron
2010-07-29 5:25 ` David Miller
1 sibling, 1 reply; 21+ messages in thread
From: David Miller @ 2010-07-28 20:24 UTC (permalink / raw)
To: jbaron
Cc: linux-kernel, mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi,
roland, rth, mhiramat, fweisbec, avi, vgoyal, sam, tony
From: Jason Baron <jbaron@redhat.com>
Date: Wed, 28 Jul 2010 16:14:59 -0400
> Patches are against Ingo's latest -tip tree.
After applying patch #5:
kernel/kprobes.c: In function 'register_kprobe':
kernel/kprobes.c:1145:6: error: implicit declaration of function 'jump_label_text_reserved'
make[1]: *** [kernel/kprobes.o] Error 1
make[1]: *** Waiting for unfinished jobs....
make: *** [kernel] Error 2
^ permalink raw reply [flat|nested] 21+ messages in thread
* Re: [PATCH 00/12] jump label v10
2010-07-28 20:24 ` David Miller
@ 2010-07-28 20:42 ` Jason Baron
2010-07-28 21:09 ` David Miller
0 siblings, 1 reply; 21+ messages in thread
From: Jason Baron @ 2010-07-28 20:42 UTC (permalink / raw)
To: David Miller
Cc: linux-kernel, mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi,
roland, rth, mhiramat, fweisbec, avi, vgoyal, sam, tony
On Wed, Jul 28, 2010 at 01:24:27PM -0700, David Miller wrote:
> > Patches are against Ingo's latest -tip tree.
>
> After applying patch #5:
>
> kernel/kprobes.c: In function 'register_kprobe':
> kernel/kprobes.c:1145:6: error: implicit declaration of function 'jump_label_text_reserved'
> make[1]: *** [kernel/kprobes.o] Error 1
> make[1]: *** Waiting for unfinished jobs....
> make: *** [kernel] Error 2
ok,
Not sure exactly what your .config is, but pulling in jump_label.h
explicitly into kprobes.c should fix that. if it doesn't please post
your .config.
diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index 2f9bea6..a53bff3 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -47,6 +47,7 @@
#include <linux/memory.h>
#include <linux/ftrace.h>
#include <linux/cpu.h>
+#include <linux/jump_label.h>
#include <asm-generic/sections.h>
#include <asm/cacheflush.h>
thanks,
-Jason
^ permalink raw reply related [flat|nested] 21+ messages in thread
* Re: [PATCH 00/12] jump label v10
2010-07-28 20:42 ` Jason Baron
@ 2010-07-28 21:09 ` David Miller
0 siblings, 0 replies; 21+ messages in thread
From: David Miller @ 2010-07-28 21:09 UTC (permalink / raw)
To: jbaron
Cc: linux-kernel, mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi,
roland, rth, mhiramat, fweisbec, avi, vgoyal, sam, tony
From: Jason Baron <jbaron@redhat.com>
Date: Wed, 28 Jul 2010 16:42:26 -0400
> Not sure exactly what your .config is,
arch/sparc/configs/sparc64_defconfig with tracepoints and the function
graph tracer enabled
> but pulling in jump_label.h explicitly into kprobes.c should fix
> that.
Yes, it does.
^ permalink raw reply [flat|nested] 21+ messages in thread
* Re: [PATCH 00/12] jump label v10
2010-07-28 20:14 ` Jason Baron
2010-07-28 20:24 ` David Miller
@ 2010-07-29 5:25 ` David Miller
1 sibling, 0 replies; 21+ messages in thread
From: David Miller @ 2010-07-29 5:25 UTC (permalink / raw)
To: jbaron
Cc: linux-kernel, mingo, mathieu.desnoyers, hpa, tglx, rostedt, andi,
roland, rth, mhiramat, fweisbec, avi, vgoyal, sam, tony
Jason, here is an updated version of patch #11 which makes sparc
properly use a branch instead of a call instruction.
Thanks.
--------------------
jump label: add sparc64 support
Signed-off-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Jason Baron <jbaron@redhat.com>
---
arch/sparc/Kconfig | 1 +
arch/sparc/include/asm/jump_label.h | 32 ++++++++++++++++++++++++
arch/sparc/kernel/Makefile | 2 +
arch/sparc/kernel/jump_label.c | 46 +++++++++++++++++++++++++++++++++++
arch/sparc/kernel/module.c | 6 ++++
5 files changed, 87 insertions(+), 0 deletions(-)
create mode 100644 arch/sparc/include/asm/jump_label.h
create mode 100644 arch/sparc/kernel/jump_label.c
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index c0015db..e38d976 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -29,6 +29,7 @@ config SPARC
select PERF_USE_VMALLOC
select HAVE_DMA_ATTRS
select HAVE_DMA_API_DEBUG
+ select HAVE_ARCH_JUMP_LABEL if !CC_OPTIMIZE_FOR_SIZE
config SPARC32
def_bool !64BIT
diff --git a/arch/sparc/include/asm/jump_label.h b/arch/sparc/include/asm/jump_label.h
new file mode 100644
index 0000000..62e66d7
--- /dev/null
+++ b/arch/sparc/include/asm/jump_label.h
@@ -0,0 +1,32 @@
+#ifndef _ASM_SPARC_JUMP_LABEL_H
+#define _ASM_SPARC_JUMP_LABEL_H
+
+#ifdef __KERNEL__
+
+#include <linux/types.h>
+#include <asm/system.h>
+
+#define JUMP_LABEL_NOP_SIZE 4
+
+#define JUMP_LABEL(key, label) \
+ do { \
+ asm goto("1:\n\t" \
+ "nop\n\t" \
+ "nop\n\t" \
+ ".pushsection __jump_table, \"a\"\n\t"\
+ ".word 1b, %l[" #label "], %c0\n\t" \
+ ".popsection \n\t" \
+ : : "i" (key) : : label);\
+ } while (0)
+
+#endif /* __KERNEL__ */
+
+typedef u32 jump_label_t;
+
+struct jump_entry {
+ jump_label_t code;
+ jump_label_t target;
+ jump_label_t key;
+};
+
+#endif
diff --git a/arch/sparc/kernel/Makefile b/arch/sparc/kernel/Makefile
index 0c2dc1f..599398f 100644
--- a/arch/sparc/kernel/Makefile
+++ b/arch/sparc/kernel/Makefile
@@ -119,3 +119,5 @@ obj-$(CONFIG_COMPAT) += $(audit--y)
pc--$(CONFIG_PERF_EVENTS) := perf_event.o
obj-$(CONFIG_SPARC64) += $(pc--y)
+
+obj-$(CONFIG_SPARC64) += jump_label.o
diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c
new file mode 100644
index 0000000..00b3fcb
--- /dev/null
+++ b/arch/sparc/kernel/jump_label.c
@@ -0,0 +1,46 @@
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/mutex.h>
+#include <linux/cpu.h>
+
+#include <linux/jump_label.h>
+#include <linux/memory.h>
+
+#ifdef HAVE_JUMP_LABEL
+
+void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type)
+{
+ u32 val;
+ u32 *insn = (u32 *) (unsigned long) entry->code;
+
+ if (type == JUMP_LABEL_ENABLE) {
+ s32 off = (s32)entry->target - (s32)entry->code;
+
+#ifdef CONFIG_SPARC64
+ /* ba,pt %xcc, . + (off << 2) */
+ val = 0x10680000 | ((u32) off >> 2);
+#else
+ /* ba . + (off << 2) */
+ val = 0x10800000 | ((u32) off >> 2);
+#endif
+ } else {
+ val = 0x01000000;
+ }
+
+ get_online_cpus();
+ mutex_lock(&text_mutex);
+ *insn = val;
+ flushi(insn);
+ mutex_unlock(&text_mutex);
+ put_online_cpus();
+}
+
+void arch_jump_label_text_poke_early(jump_label_t addr)
+{
+ u32 *insn_p = (u32 *) (unsigned long) addr;
+
+ *insn_p = 0x01000000;
+ flushi(insn_p);
+}
+
+#endif
diff --git a/arch/sparc/kernel/module.c b/arch/sparc/kernel/module.c
index f848aad..ee3c7dd 100644
--- a/arch/sparc/kernel/module.c
+++ b/arch/sparc/kernel/module.c
@@ -18,6 +18,9 @@
#include <asm/spitfire.h>
#ifdef CONFIG_SPARC64
+
+#include <linux/jump_label.h>
+
static void *module_map(unsigned long size)
{
struct vm_struct *area;
@@ -227,6 +230,9 @@ int module_finalize(const Elf_Ehdr *hdr,
const Elf_Shdr *sechdrs,
struct module *me)
{
+ /* make jump label nops */
+ jump_label_apply_nops(me);
+
/* Cheetah's I-cache is fully coherent. */
if (tlb_type == spitfire) {
unsigned long va;
--
1.6.6.1
^ permalink raw reply related [flat|nested] 21+ messages in thread