[PATCH 0/5] jump label v5

[PATCH 0/5] jump label v5

[Jason Baron]

Hi,

Refresh of jump labeling patches aginst latest -tip tree. For bacground see:
 see previous posting: http://marc.info/?l=linux-kernel&m=125858436505941&w=2

The biggest change is dropping Masami's 'text_poke_smp()' since its now
in the -tip tree.

thanks,

-Jason

Jason Baron (4):
  jump label: base patch
  jump label: x86 support
  jump label: tracepoint support
  jump label: add module support

Mathieu Desnoyers (1):
  jump label: notifier atomic call chain notrace

 arch/x86/include/asm/jump_label.h |   31 ++++
 arch/x86/kernel/Makefile          |    2 +-
 arch/x86/kernel/jump_label.c      |   53 +++++++
 arch/x86/kernel/ptrace.c          |    1 +
 include/asm-generic/vmlinux.lds.h |   10 +-
 include/linux/jump_label.h        |   58 +++++++
 include/linux/module.h            |    5 +-
 include/linux/tracepoint.h        |   34 +++--
 kernel/Makefile                   |    2 +-
 kernel/jump_label.c               |  301 +++++++++++++++++++++++++++++++++++++
 kernel/module.c                   |    7 +
 kernel/notifier.c                 |    6 +-
 kernel/tracepoint.c               |    8 +
 13 files changed, 496 insertions(+), 22 deletions(-)
 create mode 100644 arch/x86/include/asm/jump_label.h
 create mode 100644 arch/x86/kernel/jump_label.c
 create mode 100644 include/linux/jump_label.h
 create mode 100644 kernel/jump_label.c

[PATCH 1/5] jump label: notifier atomic call chain notrace

[Jason Baron]

From: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>

In LTTng, being able to use the atomic notifier from cpu idle entry to
ensure the tracer flush the last events in the current subbuffer
requires the rcu read-side to be marked "notrace", otherwise it can end
up calling back into lockdep and the tracer.

Also apply to the the die notifier.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
Signed-off-by: Jason Baron <jbaron@redhat.com>
---
 kernel/notifier.c |    6 +++---
 1 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/kernel/notifier.c b/kernel/notifier.c
index 2488ba7..88453a7 100644
--- a/kernel/notifier.c
+++ b/kernel/notifier.c
@@ -148,7 +148,7 @@ int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
 	spin_lock_irqsave(&nh->lock, flags);
 	ret = notifier_chain_unregister(&nh->head, n);
 	spin_unlock_irqrestore(&nh->lock, flags);
-	synchronize_rcu();
+	synchronize_sched();
 	return ret;
 }
 EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
@@ -178,9 +178,9 @@ int __kprobes __atomic_notifier_call_chain(struct atomic_notifier_head *nh,
 {
 	int ret;
 
-	rcu_read_lock();
+	rcu_read_lock_sched_notrace();
 	ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
-	rcu_read_unlock();
+	rcu_read_unlock_sched_notrace();
 	return ret;
 }
 EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain);
-- 
1.6.5.1

[PATCH 2/5] jump label: base patch

[Jason Baron]

base patch to implement 'jump labeling'. Based on a new 'asm goto' inline
assembly gcc mechanism, we can now branch to labels from an 'asm goto'
statment. This allows us to create a 'no-op' fastpath, which can subsequently
be patched with a jump to the slowpath code. This is useful for code which
might be rarely used, but which we'd like to be able to call, if needed.
Tracepoints are the current usecase that these are being implemented for.

Signed-off-by: Jason Baron <jbaron@redhat.com>
---
 include/asm-generic/vmlinux.lds.h |   10 ++-
 include/linux/jump_label.h        |   57 +++++++++++++
 kernel/Makefile                   |    2 +-
 kernel/jump_label.c               |  165 +++++++++++++++++++++++++++++++++++++
 4 files changed, 232 insertions(+), 2 deletions(-)
 create mode 100644 include/linux/jump_label.h
 create mode 100644 kernel/jump_label.c

diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 67e6520..83a469d 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -167,7 +167,8 @@
 	BRANCH_PROFILE()						\
 	TRACE_PRINTKS()							\
 	FTRACE_EVENTS()							\
-	TRACE_SYSCALLS()
+	TRACE_SYSCALLS()						\
+	JUMP_TABLE()							\
 
 /*
  * Data section helpers
@@ -206,6 +207,7 @@
 		*(__vermagic)		/* Kernel version magic */	\
 		*(__markers_strings)	/* Markers: strings */		\
 		*(__tracepoints_strings)/* Tracepoints: strings */	\
+		*(__jump_strings)/* Jump: strings */	\
 	}								\
 									\
 	.rodata1          : AT(ADDR(.rodata1) - LOAD_OFFSET) {		\
@@ -557,6 +559,12 @@
 #define BUG_TABLE
 #endif
 
+#define JUMP_TABLE()							\
+	. = ALIGN(64);							\
+		VMLINUX_SYMBOL(__start___jump_table) = .;		\
+		*(__jump_table)						\
+		VMLINUX_SYMBOL(__stop___jump_table) = .;		\
+
 #ifdef CONFIG_PM_TRACE
 #define TRACEDATA							\
 	. = ALIGN(4);							\
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
new file mode 100644
index 0000000..3d42e8c
--- /dev/null
+++ b/include/linux/jump_label.h
@@ -0,0 +1,57 @@
+#ifndef _LINUX_JUMP_LABEL_H
+#define _LINUX_JUMP_LABEL_H
+
+#include <asm/jump_label.h>
+
+struct jump_entry {
+	unsigned long code;
+	unsigned long target;
+	char *name;
+};
+
+enum jump_label_type {
+	JUMP_LABEL_ENABLE,
+	JUMP_LABEL_DISABLE
+};
+
+#ifdef __HAVE_ARCH_JUMP_LABEL
+
+extern struct jump_entry __start___jump_table[];
+extern struct jump_entry __stop___jump_table[];
+
+#define DEFINE_JUMP_LABEL(name)						\
+	const char __jlstrtab_##name[]					\
+	__used __attribute__((section("__jump_strings")))  = #name;
+
+extern void arch_jump_label_transform(struct jump_entry *entry,
+				 enum jump_label_type type);
+
+extern int jump_label_update(const char *name, enum jump_label_type type);
+
+#define enable_jump_label(name) \
+	jump_label_update(name, JUMP_LABEL_ENABLE);
+
+#define disable_jump_label(name) \
+	jump_label_update(name, JUMP_LABEL_DISABLE);
+
+#else
+
+#define DEFINE_JUMP_LABEL(name)
+
+#define JUMP_LABEL(tag, label, cond)		\
+	if (unlikely(cond))			\
+		goto label;
+
+static inline int enable_jump_label(const char *name)
+{
+	return 0;
+}
+
+static inline int disable_jump_label(const char *name)
+{
+	return 0;
+}
+
+#endif
+
+#endif
diff --git a/kernel/Makefile b/kernel/Makefile
index d5c3006..59ff12e 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -10,7 +10,7 @@ obj-y     = sched.o fork.o exec_domain.o panic.o printk.o \
 	    kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \
 	    hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \
 	    notifier.o ksysfs.o pm_qos_params.o sched_clock.o cred.o \
-	    async.o range.o
+	    async.o range.o jump_label.o
 obj-$(CONFIG_HAVE_EARLY_RES) += early_res.o
 obj-y += groups.o
 
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
new file mode 100644
index 0000000..671fcbb
--- /dev/null
+++ b/kernel/jump_label.c
@@ -0,0 +1,165 @@
+/*
+ * jump label support
+ *
+ * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
+ *
+ */
+#include <linux/jump_label.h>
+#include <linux/memory.h>
+#include <linux/uaccess.h>
+#include <linux/module.h>
+#include <asm/alternative.h>
+#include <linux/list.h>
+#include <linux/jhash.h>
+
+#ifdef __HAVE_ARCH_JUMP_LABEL
+
+#define JUMP_LABEL_HASH_BITS 6
+#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS)
+static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE];
+
+/* mutex to protect coming/going of the the jump_label table */
+static DEFINE_MUTEX(jump_label_mutex);
+
+struct jump_label_entry {
+	struct hlist_node hlist;
+	struct jump_entry *table;
+	int nr_entries;
+	/* hang modules off here */
+	struct hlist_head modules;
+	char name[0];
+};
+
+static void swap_jump_label_entries(struct jump_entry *previous, struct jump_entry *next)
+{
+	struct jump_entry tmp;
+
+	tmp = *next;
+	*next = *previous;
+	*previous = tmp;
+}
+
+static void sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop)
+{
+	int swapped = 0;
+	struct jump_entry *iter;
+	struct jump_entry *iter_next;
+
+	do {
+		swapped = 0;
+		iter = start;
+		iter_next = start;
+		iter_next++;
+		for (; iter_next < stop; iter++, iter_next++) {
+			if (strcmp(iter->name, iter_next->name) > 0) {
+				swap_jump_label_entries(iter, iter_next);
+				swapped = 1;
+			}
+		}
+	} while (swapped == 1);
+}
+
+static struct jump_label_entry *get_jump_label_entry(const char *name)
+{
+	struct hlist_head *head;
+	struct hlist_node *node;
+	struct jump_label_entry *e;
+	u32 hash = jhash(name, strlen(name), 0);
+
+	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
+	hlist_for_each_entry(e, node, head, hlist) {
+		if (!strcmp(name, e->name))
+			return e;
+	}
+	return NULL;
+}
+
+static struct jump_label_entry *add_jump_label_entry(const char *name, int nr_entries, struct jump_entry *table)
+{
+	struct hlist_head *head;
+	struct hlist_node *node;
+	struct jump_label_entry *e;
+	size_t name_len = strlen(name) + 1;
+	u32 hash = jhash(name, name_len-1, 0);
+
+	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
+	hlist_for_each_entry(e, node, head, hlist) {
+		if (!strcmp(name, e->name))
+			return ERR_PTR(-EEXIST);
+	}
+	e = kmalloc(sizeof(struct jump_label_entry) + name_len, GFP_KERNEL);
+	if (!e)
+		return ERR_PTR(-ENOMEM);
+	memcpy(&e->name[0], name, name_len);
+	e->table = table;
+	e->nr_entries = nr_entries;
+	INIT_HLIST_HEAD(&(e->modules));
+	hlist_add_head(&e->hlist, head);
+	return e;
+}
+
+static int build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop)
+{
+	struct jump_entry *iter, *iter_begin;
+	struct jump_label_entry *entry;
+	int count;
+
+	sort_jump_label_entries(start, stop);
+	iter = start;
+	while (iter < stop) {
+		entry = get_jump_label_entry(iter->name);
+		if (!entry) {
+			iter_begin = iter;
+			count = 0;
+			while ((iter < stop) &&
+				(strcmp(iter->name, iter_begin->name) == 0)) {
+				iter++;
+				count++;
+			}
+			entry = add_jump_label_entry(iter_begin->name, count,
+						     iter_begin);
+			if (IS_ERR(entry))
+				return PTR_ERR(entry);
+			continue;
+		}
+		WARN(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n");
+	}
+	return 0;
+}
+
+int jump_label_update(const char *name, enum jump_label_type type)
+{
+	struct jump_entry *iter;
+	struct jump_label_entry *entry;
+	struct hlist_node *module_node;
+	struct jump_label_module_entry *e_module;
+	int count;
+
+	mutex_lock(&jump_label_mutex);
+	entry = get_jump_label_entry(name);
+	if (entry) {
+		count = entry->nr_entries;
+		iter = entry->table;
+		while (count--) {
+			if (kernel_text_address(iter->code))
+				arch_jump_label_transform(iter, type);
+			iter++;
+		}
+	}
+	mutex_unlock(&jump_label_mutex);
+	return 0;
+}
+
+static int init_jump_label(void)
+{
+	int ret;
+
+	mutex_lock(&jump_label_mutex);
+	ret = build_jump_label_hashtable(__start___jump_table,
+					 __stop___jump_table);
+	mutex_unlock(&jump_label_mutex);
+	return ret;
+}
+early_initcall(init_jump_label);
+
+#endif
-- 
1.6.5.1

[PATCH 3/5] jump label: x86 support

[Jason Baron]

add x86 support for jump label. I'm keeping this patch separate so its clear to
arch maintainers what was required for x86 support this new feature. hopefully,
it wouldn't be too painful for other arches.

Signed-off-by: Jason Baron <jbaron@redhat.com>
---
 arch/x86/include/asm/jump_label.h |   31 +++++++++++++++++++++
 arch/x86/kernel/Makefile          |    2 +-
 arch/x86/kernel/jump_label.c      |   53 +++++++++++++++++++++++++++++++++++++
 3 files changed, 85 insertions(+), 1 deletions(-)
 create mode 100644 arch/x86/include/asm/jump_label.h
 create mode 100644 arch/x86/kernel/jump_label.c

diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h
new file mode 100644
index 0000000..b8ebdc8
--- /dev/null
+++ b/arch/x86/include/asm/jump_label.h
@@ -0,0 +1,31 @@
+#ifndef _ASM_X86_JUMP_LABEL_H
+#define _ASM_X86_JUMP_LABEL_H
+
+#include <asm/nops.h>
+
+#if (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))
+# define __HAVE_ARCH_JUMP_LABEL
+#endif
+
+#ifdef __HAVE_ARCH_JUMP_LABEL
+
+# ifdef CONFIG_X86_64
+#  define JUMP_LABEL_NOP P6_NOP5
+# else
+#  define JUMP_LABEL_NOP ".byte 0xe9 \n\t .long 0\n\t"
+# endif
+
+# define JUMP_LABEL(tag, label, cond)				\
+	do {							\
+		extern const char __jlstrtab_##tag[];		\
+		asm goto("1:"					\
+			JUMP_LABEL_NOP				\
+			".pushsection __jump_table,  \"a\" \n\t"\
+			_ASM_PTR "1b, %l[" #label "], %c0 \n\t" \
+			".popsection \n\t"			\
+			: :  "i" (__jlstrtab_##tag) :  : label);\
+	} while (0)
+
+# endif
+
+#endif
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 4c58352..7cd3bf4 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -32,7 +32,7 @@ GCOV_PROFILE_paravirt.o		:= n
 obj-y			:= process_$(BITS).o signal.o entry_$(BITS).o
 obj-y			+= traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o
 obj-y			+= time.o ioport.o ldt.o dumpstack.o
-obj-y			+= setup.o x86_init.o i8259.o irqinit.o
+obj-y			+= setup.o x86_init.o i8259.o irqinit.o jump_label.o
 obj-$(CONFIG_X86_VISWS)	+= visws_quirks.o
 obj-$(CONFIG_X86_32)	+= probe_roms_32.o
 obj-$(CONFIG_X86_32)	+= sys_i386_32.o i386_ksyms_32.o
diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c
new file mode 100644
index 0000000..7fc4f84
--- /dev/null
+++ b/arch/x86/kernel/jump_label.c
@@ -0,0 +1,53 @@
+/*
+ * jump label x86 support
+ *
+ * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
+ *
+ */
+#include <linux/jump_label.h>
+#include <linux/memory.h>
+#include <linux/uaccess.h>
+#include <linux/module.h>
+#include <linux/list.h>
+#include <linux/jhash.h>
+#include <linux/cpu.h>
+#include <asm/kprobes.h>
+
+#ifdef __HAVE_ARCH_JUMP_LABEL
+
+union jump_code_union {
+	char code[RELATIVEJUMP_SIZE];
+	struct {
+		char jump;
+		int offset;
+	} __attribute__((packed));
+};
+
+void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type)
+{
+	union jump_code_union code;
+
+	if (type == JUMP_LABEL_ENABLE) {
+		code.jump = 0xe9;
+		code.offset = entry->target - (entry->code + RELATIVEJUMP_SIZE);
+	} else {
+#ifdef CONFIG_X86_64
+		/* opcode for P6_NOP5 */
+		code.code[0] = 0x0f;
+		code.code[1] = 0x1f;
+		code.code[2] = 0x44;
+		code.code[3] = 0x00;
+		code.code[4] = 0x00;
+#else
+		code.jump = 0xe9;
+		code.offset = 0;
+#endif
+	}
+	get_online_cpus();
+	mutex_lock(&text_mutex);
+	text_poke_smp((void *)entry->code, &code, RELATIVEJUMP_SIZE);
+	mutex_unlock(&text_mutex);
+	put_online_cpus();
+}
+
+#endif
-- 
1.6.5.1

[PATCH 4/5] jump label: tracepoint support

[Jason Baron]

Make use of the jump label infrastructure for tracepoints.

Signed-off-by: Jason Baron <jbaron@redhat.com>
---
 include/linux/tracepoint.h |   34 +++++++++++++++++++---------------
 kernel/tracepoint.c        |    8 ++++++++
 2 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index f59604e..c18b9c0 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -16,6 +16,7 @@
 
 #include <linux/types.h>
 #include <linux/rcupdate.h>
+#include <linux/jump_label.h>
 
 struct module;
 struct tracepoint;
@@ -63,20 +64,22 @@ struct tracepoint {
  * not add unwanted padding between the beginning of the section and the
  * structure. Force alignment to the same alignment as the section start.
  */
-#define DECLARE_TRACE(name, proto, args)				\
-	extern struct tracepoint __tracepoint_##name;			\
-	static inline void trace_##name(proto)				\
-	{								\
-		if (unlikely(__tracepoint_##name.state))		\
-			__DO_TRACE(&__tracepoint_##name,		\
-				TP_PROTO(proto), TP_ARGS(args));	\
-	}								\
-	static inline int register_trace_##name(void (*probe)(proto))	\
-	{								\
-		return tracepoint_probe_register(#name, (void *)probe);	\
-	}								\
-	static inline int unregister_trace_##name(void (*probe)(proto))	\
-	{								\
+#define DECLARE_TRACE(name, proto, args)				 \
+	extern struct tracepoint __tracepoint_##name;			 \
+	static inline void trace_##name(proto)				 \
+	{								 \
+		JUMP_LABEL(name, do_trace, __tracepoint_##name.state);   \
+		return;							 \
+do_trace:								 \
+		__DO_TRACE(&__tracepoint_##name,			 \
+			   TP_PROTO(proto), TP_ARGS(args));		 \
+	}								 \
+	static inline int register_trace_##name(void (*probe)(proto))	 \
+	{								 \
+		return tracepoint_probe_register(#name, (void *)probe);	 \
+	}								 \
+	static inline int unregister_trace_##name(void (*probe)(proto))	 \
+	{								 \
 		return tracepoint_probe_unregister(#name, (void *)probe);\
 	}
 
@@ -86,7 +89,8 @@ struct tracepoint {
 	__attribute__((section("__tracepoints_strings"))) = #name;	\
 	struct tracepoint __tracepoint_##name				\
 	__attribute__((section("__tracepoints"), aligned(32))) =	\
-		{ __tpstrtab_##name, 0, reg, unreg, NULL }
+		{ __tpstrtab_##name, 0, reg, unreg, NULL };		\
+	DEFINE_JUMP_LABEL(name);
 
 #define DEFINE_TRACE(name)						\
 	DEFINE_TRACE_FN(name, NULL, NULL);
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index cc89be5..8acced8 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -25,6 +25,7 @@
 #include <linux/err.h>
 #include <linux/slab.h>
 #include <linux/sched.h>
+#include <linux/jump_label.h>
 
 extern struct tracepoint __start___tracepoints[];
 extern struct tracepoint __stop___tracepoints[];
@@ -256,6 +257,10 @@ static void set_tracepoint(struct tracepoint_entry **entry,
 	 * is used.
 	 */
 	rcu_assign_pointer(elem->funcs, (*entry)->funcs);
+	if (!elem->state && active)
+		enable_jump_label(elem->name);
+	if (elem->state && !active)
+		disable_jump_label(elem->name);
 	elem->state = active;
 }
 
@@ -270,6 +275,9 @@ static void disable_tracepoint(struct tracepoint *elem)
 	if (elem->unregfunc && elem->state)
 		elem->unregfunc();
 
+	if (elem->state)
+		disable_jump_label(elem->name);
+
 	elem->state = 0;
 	rcu_assign_pointer(elem->funcs, NULL);
 }
-- 
1.6.5.1

[PATCH 5/5] jump label: add module support

[Jason Baron]

Add support for 'jump label' for modules.

Signed-off-by: Jason Baron <jbaron@redhat.com>
---
 arch/x86/kernel/ptrace.c   |    1 +
 include/linux/jump_label.h |    3 +-
 include/linux/module.h     |    5 +-
 kernel/jump_label.c        |  136 ++++++++++++++++++++++++++++++++++++++++++++
 kernel/module.c            |    7 ++
 5 files changed, 150 insertions(+), 2 deletions(-)

diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c
index 2d96aab..21854d2 100644
--- a/arch/x86/kernel/ptrace.c
+++ b/arch/x86/kernel/ptrace.c
@@ -24,6 +24,7 @@
 #include <linux/workqueue.h>
 #include <linux/perf_event.h>
 #include <linux/hw_breakpoint.h>
+#include <linux/module.h>
 
 #include <asm/uaccess.h>
 #include <asm/pgtable.h>
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 3d42e8c..fa81c0a 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -21,7 +21,8 @@ extern struct jump_entry __stop___jump_table[];
 
 #define DEFINE_JUMP_LABEL(name)						\
 	const char __jlstrtab_##name[]					\
-	__used __attribute__((section("__jump_strings")))  = #name;
+	__used __attribute__((section("__jump_strings")))  = #name;	\
+	EXPORT_SYMBOL_GPL(__jlstrtab_##name);
 
 extern void arch_jump_label_transform(struct jump_entry *entry,
 				 enum jump_label_type type);
diff --git a/include/linux/module.h b/include/linux/module.h
index dd618eb..6d7225e 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -339,7 +339,10 @@ struct module
 	struct tracepoint *tracepoints;
 	unsigned int num_tracepoints;
 #endif
-
+#ifdef __HAVE_ARCH_JUMP_LABEL
+	struct jump_entry *jump_entries;
+	unsigned int num_jump_entries;
+#endif
 #ifdef CONFIG_TRACING
 	const char **trace_bprintk_fmt_start;
 	unsigned int num_trace_bprintk_fmt;
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 671fcbb..d7a085d 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -30,6 +30,13 @@ struct jump_label_entry {
 	char name[0];
 };
 
+struct jump_label_module_entry {
+	struct hlist_node hlist;
+	struct jump_entry *table;
+	int nr_entries;
+	struct module *mod;
+};
+
 static void swap_jump_label_entries(struct jump_entry *previous, struct jump_entry *next)
 {
 	struct jump_entry tmp;
@@ -145,6 +152,17 @@ int jump_label_update(const char *name, enum jump_label_type type)
 				arch_jump_label_transform(iter, type);
 			iter++;
 		}
+		/* eanble/disable jump labels in modules */
+		hlist_for_each_entry(e_module, module_node, &(entry->modules),
+							hlist) {
+			count = e_module->nr_entries;
+			iter = e_module->table;
+			while (count--) {
+				if (kernel_text_address(iter->code))
+					arch_jump_label_transform(iter, type);
+				iter++;
+			}
+		}
 	}
 	mutex_unlock(&jump_label_mutex);
 	return 0;
@@ -162,4 +180,122 @@ static int init_jump_label(void)
 }
 early_initcall(init_jump_label);
 
+#ifdef CONFIG_MODULES
+
+static struct jump_label_module_entry *add_jump_label_module_entry(struct jump_label_entry *entry, struct jump_entry *iter_begin, int count, struct module *mod)
+{
+	struct jump_label_module_entry *e;
+
+	e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL);
+	if (!e)
+		return ERR_PTR(-ENOMEM);
+	e->mod = mod;
+	e->nr_entries = count;
+	e->table = iter_begin;
+	hlist_add_head(&e->hlist, &entry->modules);
+	return e;
+}
+
+static int add_jump_label_module(struct module *mod)
+{
+	struct jump_entry *iter, *iter_begin;
+	struct jump_label_entry *entry;
+	struct jump_label_module_entry *module_entry;
+	int count;
+
+	/* if the module doesn't have jump label entries, just return */
+	if (!mod->num_jump_entries)
+		return 0;
+
+	sort_jump_label_entries(mod->jump_entries,
+				mod->jump_entries + mod->num_jump_entries);
+	iter = mod->jump_entries;
+	while (iter < mod->jump_entries + mod->num_jump_entries) {
+		entry = get_jump_label_entry(iter->name);
+		iter_begin = iter;
+		count = 0;
+		while ((iter < mod->jump_entries + mod->num_jump_entries) &&
+			(strcmp(iter->name, iter_begin->name) == 0)) {
+				iter++;
+				count++;
+		}
+		if (!entry) {
+			entry = add_jump_label_entry(iter_begin->name, 0, NULL);
+			if (IS_ERR(entry))
+				return PTR_ERR(entry);
+		}
+		module_entry = add_jump_label_module_entry(entry, iter_begin,
+							   count, mod);
+		if (IS_ERR(module_entry))
+			return PTR_ERR(module_entry);
+	}
+	return 0;
+}
+
+static void remove_jump_label_module(struct module *mod)
+{
+	struct hlist_head *head;
+	struct hlist_node *node, *node_next, *module_node, *module_node_next;
+	struct jump_label_entry *e;
+	struct jump_label_module_entry *e_module;
+	int i;
+
+	/* if the module doesn't have jump label entries, just return */
+	if (!mod->num_jump_entries)
+		return;
+
+	for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
+		head = &jump_label_table[i];
+		hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
+			hlist_for_each_entry_safe(e_module, module_node,
+						  module_node_next,
+						  &(e->modules), hlist) {
+				if (e_module->mod == mod) {
+					hlist_del(&e_module->hlist);
+					kfree(e_module);
+				}
+			}
+			if (hlist_empty(&e->modules) && (e->nr_entries == 0)) {
+				hlist_del(&e->hlist);
+				kfree(e);
+			}
+		}
+	}
+}
+
+static int jump_label_module_notify(struct notifier_block *self, unsigned long val, void *data)
+{
+	struct module *mod = data;
+	int ret = 0;
+
+	switch (val) {
+	case MODULE_STATE_COMING:
+		mutex_lock(&jump_label_mutex);
+		ret = add_jump_label_module(mod);
+		if (ret)
+			remove_jump_label_module(mod);
+		mutex_unlock(&jump_label_mutex);
+		break;
+	case MODULE_STATE_GOING:
+		mutex_lock(&jump_label_mutex);
+		remove_jump_label_module(mod);
+		mutex_unlock(&jump_label_mutex);
+		break;
+	}
+	return ret;
+}
+
+struct notifier_block jump_label_module_nb = {
+	.notifier_call = jump_label_module_notify,
+	.priority = 0,
+};
+
+static int init_jump_label_module(void)
+{
+	return register_module_notifier(&jump_label_module_nb);
+}
+early_initcall(init_jump_label_module);
+
+#endif /* CONFIG_MODULES */
+
 #endif
diff --git a/kernel/module.c b/kernel/module.c
index c968d36..a8c34a2 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -55,6 +55,7 @@
 #include <linux/async.h>
 #include <linux/percpu.h>
 #include <linux/kmemleak.h>
+#include <linux/jump_label.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/module.h>
@@ -2249,6 +2250,12 @@ static noinline struct module *load_module(void __user *umod,
 					sizeof(*mod->tracepoints),
 					&mod->num_tracepoints);
 #endif
+#ifdef __HAVE_ARCH_JUMP_LABEL
+	mod->jump_entries = section_objs(hdr, sechdrs, secstrings,
+					"__jump_table",
+					sizeof(*mod->jump_entries),
+					&mod->num_jump_entries);
+#endif
 #ifdef CONFIG_EVENT_TRACING
 	mod->trace_events = section_objs(hdr, sechdrs, secstrings,
 					 "_ftrace_events",
-- 
1.6.5.1

Re: [PATCH 3/5] jump label: x86 support

[Steven Rostedt]

On Mon, 2010-03-22 at 12:07 -0400, Jason Baron wrote:
> add x86 support for jump label. I'm keeping this patch separate so its clear to
> arch maintainers what was required for x86 support this new feature. hopefully,
> it wouldn't be too painful for other arches.
> 
> Signed-off-by: Jason Baron <jbaron@redhat.com>


> +#ifdef __HAVE_ARCH_JUMP_LABEL
> +
> +# ifdef CONFIG_X86_64
> +#  define JUMP_LABEL_NOP P6_NOP5
> +# else
> +#  define JUMP_LABEL_NOP ".byte 0xe9 \n\t .long 0\n\t"
> +# endif

Are you sure P6_NOP5 can't happen on non 64bit? Just because it is not
configured does not mean that the CPU can not handle it. Look at the
code I did in arch/x86/kernel/ftrace.c to determine what nop to use.
Maybe we can make that generic and have at boot up, the kernel determine
a proper 5byte nop.

> +
> +# define JUMP_LABEL(tag, label, cond)				\
> +	do {							\
> +		extern const char __jlstrtab_##tag[];		\
> +		asm goto("1:"					\
> +			JUMP_LABEL_NOP				\
> +			".pushsection __jump_table,  \"a\" \n\t"\
> +			_ASM_PTR "1b, %l[" #label "], %c0 \n\t" \
> +			".popsection \n\t"			\
> +			: :  "i" (__jlstrtab_##tag) :  : label);\
> +	} while (0)
> +
> +# endif
> +
> +#endif
> diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
> index 4c58352..7cd3bf4 100644
> --- a/arch/x86/kernel/Makefile
> +++ b/arch/x86/kernel/Makefile
> @@ -32,7 +32,7 @@ GCOV_PROFILE_paravirt.o		:= n
>  obj-y			:= process_$(BITS).o signal.o entry_$(BITS).o
>  obj-y			+= traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o
>  obj-y			+= time.o ioport.o ldt.o dumpstack.o
> -obj-y			+= setup.o x86_init.o i8259.o irqinit.o
> +obj-y			+= setup.o x86_init.o i8259.o irqinit.o jump_label.o
>  obj-$(CONFIG_X86_VISWS)	+= visws_quirks.o
>  obj-$(CONFIG_X86_32)	+= probe_roms_32.o
>  obj-$(CONFIG_X86_32)	+= sys_i386_32.o i386_ksyms_32.o
> diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c
> new file mode 100644
> index 0000000..7fc4f84
> --- /dev/null
> +++ b/arch/x86/kernel/jump_label.c
> @@ -0,0 +1,53 @@
> +/*
> + * jump label x86 support
> + *
> + * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
> + *
> + */
> +#include <linux/jump_label.h>
> +#include <linux/memory.h>
> +#include <linux/uaccess.h>
> +#include <linux/module.h>
> +#include <linux/list.h>
> +#include <linux/jhash.h>
> +#include <linux/cpu.h>
> +#include <asm/kprobes.h>
> +
> +#ifdef __HAVE_ARCH_JUMP_LABEL
> +
> +union jump_code_union {
> +	char code[RELATIVEJUMP_SIZE];
> +	struct {
> +		char jump;
> +		int offset;
> +	} __attribute__((packed));
> +};
> +
> +void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type)
> +{
> +	union jump_code_union code;
> +
> +	if (type == JUMP_LABEL_ENABLE) {
> +		code.jump = 0xe9;
> +		code.offset = entry->target - (entry->code + RELATIVEJUMP_SIZE);
> +	} else {
> +#ifdef CONFIG_X86_64
> +		/* opcode for P6_NOP5 */
> +		code.code[0] = 0x0f;
> +		code.code[1] = 0x1f;
> +		code.code[2] = 0x44;
> +		code.code[3] = 0x00;
> +		code.code[4] = 0x00;
> +#else
> +		code.jump = 0xe9;
> +		code.offset = 0;
> +#endif

Same here, and this should also just use the JUMP_LABEL_NOP insead of
spelling it out as it does here. memcpy should work.

-- Steve


> +	}
> +	get_online_cpus();
> +	mutex_lock(&text_mutex);
> +	text_poke_smp((void *)entry->code, &code, RELATIVEJUMP_SIZE);
> +	mutex_unlock(&text_mutex);
> +	put_online_cpus();
> +}
> +
> +#endif

Re: [PATCH 4/5] jump label: tracepoint support

[Steven Rostedt]

On Mon, 2010-03-22 at 12:07 -0400, Jason Baron wrote:
> Make use of the jump label infrastructure for tracepoints.
> 
> Signed-off-by: Jason Baron <jbaron@redhat.com>
> ---
>  include/linux/tracepoint.h |   34 +++++++++++++++++++---------------
>  kernel/tracepoint.c        |    8 ++++++++
>  2 files changed, 27 insertions(+), 15 deletions(-)
> 
> diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
> index f59604e..c18b9c0 100644
> --- a/include/linux/tracepoint.h
> +++ b/include/linux/tracepoint.h
> @@ -16,6 +16,7 @@
>  
>  #include <linux/types.h>
>  #include <linux/rcupdate.h>
> +#include <linux/jump_label.h>
>  
>  struct module;
>  struct tracepoint;
> @@ -63,20 +64,22 @@ struct tracepoint {
>   * not add unwanted padding between the beginning of the section and the
>   * structure. Force alignment to the same alignment as the section start.
>   */
> -#define DECLARE_TRACE(name, proto, args)				\
> -	extern struct tracepoint __tracepoint_##name;			\
> -	static inline void trace_##name(proto)				\
> -	{								\
> -		if (unlikely(__tracepoint_##name.state))		\
> -			__DO_TRACE(&__tracepoint_##name,		\
> -				TP_PROTO(proto), TP_ARGS(args));	\
> -	}								\
> -	static inline int register_trace_##name(void (*probe)(proto))	\
> -	{								\
> -		return tracepoint_probe_register(#name, (void *)probe);	\
> -	}								\
> -	static inline int unregister_trace_##name(void (*probe)(proto))	\
> -	{								\
> +#define DECLARE_TRACE(name, proto, args)				 \
> +	extern struct tracepoint __tracepoint_##name;			 \
> +	static inline void trace_##name(proto)				 \
> +	{								 \
> +		JUMP_LABEL(name, do_trace, __tracepoint_##name.state);   \
> +		return;							 \
> +do_trace:								 \
> +		__DO_TRACE(&__tracepoint_##name,			 \
> +			   TP_PROTO(proto), TP_ARGS(args));		 \

Does this still work on all archs, and when jump labels are not
supported?

I may have missed a patch (I have not received patch 2 yet).

-- Steve

> +	}								 \
> +	static inline int register_trace_##name(void (*probe)(proto))	 \
> +	{								 \
> +		return tracepoint_probe_register(#name, (void *)probe);	 \
> +	}								 \
> +	static inline int unregister_trace_##name(void (*probe)(proto))	 \
> +	{								 \
>  		return tracepoint_probe_unregister(#name, (void *)probe);\
>  	}
>  
> @@ -86,7 +89,8 @@ struct tracepoint {
>  	__attribute__((section("__tracepoints_strings"))) = #name;	\
>  	struct tracepoint __tracepoint_##name				\
>  	__attribute__((section("__tracepoints"), aligned(32))) =	\
> -		{ __tpstrtab_##name, 0, reg, unreg, NULL }
> +		{ __tpstrtab_##name, 0, reg, unreg, NULL };		\
> +	DEFINE_JUMP_LABEL(name);
>  
>  #define DEFINE_TRACE(name)						\
>  	DEFINE_TRACE_FN(name, NULL, NULL);
> diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
> index cc89be5..8acced8 100644
> --- a/kernel/tracepoint.c
> +++ b/kernel/tracepoint.c
> @@ -25,6 +25,7 @@
>  #include <linux/err.h>
>  #include <linux/slab.h>
>  #include <linux/sched.h>
> +#include <linux/jump_label.h>
>  
>  extern struct tracepoint __start___tracepoints[];
>  extern struct tracepoint __stop___tracepoints[];
> @@ -256,6 +257,10 @@ static void set_tracepoint(struct tracepoint_entry **entry,
>  	 * is used.
>  	 */
>  	rcu_assign_pointer(elem->funcs, (*entry)->funcs);
> +	if (!elem->state && active)
> +		enable_jump_label(elem->name);
> +	if (elem->state && !active)
> +		disable_jump_label(elem->name);
>  	elem->state = active;
>  }
>  
> @@ -270,6 +275,9 @@ static void disable_tracepoint(struct tracepoint *elem)
>  	if (elem->unregfunc && elem->state)
>  		elem->unregfunc();
>  
> +	if (elem->state)
> +		disable_jump_label(elem->name);
> +
>  	elem->state = 0;
>  	rcu_assign_pointer(elem->funcs, NULL);
>  }

Re: [PATCH 1/5] jump label: notifier atomic call chain notrace

[Masami Hiramatsu]

Jason Baron wrote:
> From: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
> 
> In LTTng, being able to use the atomic notifier from cpu idle entry to
> ensure the tracer flush the last events in the current subbuffer
> requires the rcu read-side to be marked "notrace", otherwise it can end
> up calling back into lockdep and the tracer.
> 
> Also apply to the the die notifier.

Looks good for me and it'll be good for kprobe-tracer too:)


> 
> Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
> Signed-off-by: Jason Baron <jbaron@redhat.com>

Reviewed-by: Masami Hiramatsu <mhiramat@redhat.com>

> ---
>  kernel/notifier.c |    6 +++---
>  1 files changed, 3 insertions(+), 3 deletions(-)
> 
> diff --git a/kernel/notifier.c b/kernel/notifier.c
> index 2488ba7..88453a7 100644
> --- a/kernel/notifier.c
> +++ b/kernel/notifier.c
> @@ -148,7 +148,7 @@ int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
>  	spin_lock_irqsave(&nh->lock, flags);
>  	ret = notifier_chain_unregister(&nh->head, n);
>  	spin_unlock_irqrestore(&nh->lock, flags);
> -	synchronize_rcu();
> +	synchronize_sched();
>  	return ret;
>  }
>  EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
> @@ -178,9 +178,9 @@ int __kprobes __atomic_notifier_call_chain(struct atomic_notifier_head *nh,
>  {
>  	int ret;
>  
> -	rcu_read_lock();
> +	rcu_read_lock_sched_notrace();
>  	ret = notifier_call_chain(&nh->head, val, v, nr_to_call, nr_calls);
> -	rcu_read_unlock();
> +	rcu_read_unlock_sched_notrace();
>  	return ret;
>  }
>  EXPORT_SYMBOL_GPL(__atomic_notifier_call_chain);

-- 
Masami Hiramatsu
e-mail: mhiramat@redhat.com

Re: [PATCH 2/5] jump label: base patch

[Steven Rostedt]

On Mon, 2010-03-22 at 12:07 -0400, Jason Baron wrote:
> base patch to implement 'jump labeling'. Based on a new 'asm goto' inline
> assembly gcc mechanism, we can now branch to labels from an 'asm goto'
> statment. This allows us to create a 'no-op' fastpath, which can subsequently
> be patched with a jump to the slowpath code. This is useful for code which
> might be rarely used, but which we'd like to be able to call, if needed.
> Tracepoints are the current usecase that these are being implemented for.
> 
> Signed-off-by: Jason Baron <jbaron@redhat.com>
> ---
>  include/asm-generic/vmlinux.lds.h |   10 ++-
>  include/linux/jump_label.h        |   57 +++++++++++++
>  kernel/Makefile                   |    2 +-
>  kernel/jump_label.c               |  165 +++++++++++++++++++++++++++++++++++++
>  4 files changed, 232 insertions(+), 2 deletions(-)
>  create mode 100644 include/linux/jump_label.h
>  create mode 100644 kernel/jump_label.c
> 
> diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
> index 67e6520..83a469d 100644
> --- a/include/asm-generic/vmlinux.lds.h
> +++ b/include/asm-generic/vmlinux.lds.h
> @@ -167,7 +167,8 @@
>  	BRANCH_PROFILE()						\
>  	TRACE_PRINTKS()							\
>  	FTRACE_EVENTS()							\
> -	TRACE_SYSCALLS()
> +	TRACE_SYSCALLS()						\
> +	JUMP_TABLE()							\
>  
>  /*
>   * Data section helpers
> @@ -206,6 +207,7 @@
>  		*(__vermagic)		/* Kernel version magic */	\
>  		*(__markers_strings)	/* Markers: strings */		\
>  		*(__tracepoints_strings)/* Tracepoints: strings */	\
> +		*(__jump_strings)/* Jump: strings */	\
>  	}								\
>  									\
>  	.rodata1          : AT(ADDR(.rodata1) - LOAD_OFFSET) {		\
> @@ -557,6 +559,12 @@
>  #define BUG_TABLE
>  #endif
>  
> +#define JUMP_TABLE()							\
> +	. = ALIGN(64);							\
> +		VMLINUX_SYMBOL(__start___jump_table) = .;		\
> +		*(__jump_table)						\
> +		VMLINUX_SYMBOL(__stop___jump_table) = .;		\
> +
>  #ifdef CONFIG_PM_TRACE
>  #define TRACEDATA							\
>  	. = ALIGN(4);							\
> diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
> new file mode 100644
> index 0000000..3d42e8c
> --- /dev/null
> +++ b/include/linux/jump_label.h
> @@ -0,0 +1,57 @@
> +#ifndef _LINUX_JUMP_LABEL_H
> +#define _LINUX_JUMP_LABEL_H
> +
> +#include <asm/jump_label.h>
> +
> +struct jump_entry {
> +	unsigned long code;
> +	unsigned long target;
> +	char *name;
> +};
> +
> +enum jump_label_type {
> +	JUMP_LABEL_ENABLE,
> +	JUMP_LABEL_DISABLE
> +};
> +
> +#ifdef __HAVE_ARCH_JUMP_LABEL
> +
> +extern struct jump_entry __start___jump_table[];
> +extern struct jump_entry __stop___jump_table[];
> +
> +#define DEFINE_JUMP_LABEL(name)						\
> +	const char __jlstrtab_##name[]					\
> +	__used __attribute__((section("__jump_strings")))  = #name;
> +
> +extern void arch_jump_label_transform(struct jump_entry *entry,
> +				 enum jump_label_type type);
> +
> +extern int jump_label_update(const char *name, enum jump_label_type type);
> +
> +#define enable_jump_label(name) \
> +	jump_label_update(name, JUMP_LABEL_ENABLE);
> +
> +#define disable_jump_label(name) \
> +	jump_label_update(name, JUMP_LABEL_DISABLE);
> +
> +#else
> +
> +#define DEFINE_JUMP_LABEL(name)
> +
> +#define JUMP_LABEL(tag, label, cond)		\
> +	if (unlikely(cond))			\
> +		goto label;
> +
> +static inline int enable_jump_label(const char *name)
> +{
> +	return 0;
> +}
> +
> +static inline int disable_jump_label(const char *name)
> +{
> +	return 0;
> +}
> +
> +#endif
> +
> +#endif
> diff --git a/kernel/Makefile b/kernel/Makefile
> index d5c3006..59ff12e 100644
> --- a/kernel/Makefile
> +++ b/kernel/Makefile
> @@ -10,7 +10,7 @@ obj-y     = sched.o fork.o exec_domain.o panic.o printk.o \
>  	    kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \
>  	    hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \
>  	    notifier.o ksysfs.o pm_qos_params.o sched_clock.o cred.o \
> -	    async.o range.o
> +	    async.o range.o jump_label.o
>  obj-$(CONFIG_HAVE_EARLY_RES) += early_res.o
>  obj-y += groups.o
>  
> diff --git a/kernel/jump_label.c b/kernel/jump_label.c
> new file mode 100644
> index 0000000..671fcbb
> --- /dev/null
> +++ b/kernel/jump_label.c
> @@ -0,0 +1,165 @@
> +/*
> + * jump label support
> + *
> + * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
> + *
> + */
> +#include <linux/jump_label.h>
> +#include <linux/memory.h>
> +#include <linux/uaccess.h>
> +#include <linux/module.h>
> +#include <asm/alternative.h>
> +#include <linux/list.h>
> +#include <linux/jhash.h>
> +
> +#ifdef __HAVE_ARCH_JUMP_LABEL
> +
> +#define JUMP_LABEL_HASH_BITS 6
> +#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS)
> +static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE];
> +
> +/* mutex to protect coming/going of the the jump_label table */
> +static DEFINE_MUTEX(jump_label_mutex);
> +
> +struct jump_label_entry {
> +	struct hlist_node hlist;
> +	struct jump_entry *table;
> +	int nr_entries;
> +	/* hang modules off here */
> +	struct hlist_head modules;
> +	char name[0];
> +};
> +
> +static void swap_jump_label_entries(struct jump_entry *previous, struct jump_entry *next)
> +{
> +	struct jump_entry tmp;
> +
> +	tmp = *next;
> +	*next = *previous;
> +	*previous = tmp;
> +}
> +
> +static void sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop)
> +{
> +	int swapped = 0;
> +	struct jump_entry *iter;
> +	struct jump_entry *iter_next;
> +
> +	do {
> +		swapped = 0;
> +		iter = start;
> +		iter_next = start;
> +		iter_next++;
> +		for (; iter_next < stop; iter++, iter_next++) {
> +			if (strcmp(iter->name, iter_next->name) > 0) {
> +				swap_jump_label_entries(iter, iter_next);
> +				swapped = 1;
> +			}
> +		}
> +	} while (swapped == 1);

I'm curious to how long this takes with a few hundred trace points
(which we currently have). Perhaps this could be sorted at compile time?

> +}
> +
> +static struct jump_label_entry *get_jump_label_entry(const char *name)
> +{
> +	struct hlist_head *head;
> +	struct hlist_node *node;
> +	struct jump_label_entry *e;
> +	u32 hash = jhash(name, strlen(name), 0);
> +
> +	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
> +	hlist_for_each_entry(e, node, head, hlist) {
> +		if (!strcmp(name, e->name))
> +			return e;
> +	}
> +	return NULL;
> +}
> +
> +static struct jump_label_entry *add_jump_label_entry(const char *name, int nr_entries, struct jump_entry *table)
> +{
> +	struct hlist_head *head;
> +	struct hlist_node *node;
> +	struct jump_label_entry *e;
> +	size_t name_len = strlen(name) + 1;
> +	u32 hash = jhash(name, name_len-1, 0);
> +
> +	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
> +	hlist_for_each_entry(e, node, head, hlist) {
> +		if (!strcmp(name, e->name))
> +			return ERR_PTR(-EEXIST);
> +	}

I see duplicate code here for hashing.

Why not:

	e = get_jump_label_entry(name);
	if (e)
		return ERR_PTR(-EEXIST);

?

> +	e = kmalloc(sizeof(struct jump_label_entry) + name_len, GFP_KERNEL);
> +	if (!e)
> +		return ERR_PTR(-ENOMEM);
> +	memcpy(&e->name[0], name, name_len);
> +	e->table = table;
> +	e->nr_entries = nr_entries;
> +	INIT_HLIST_HEAD(&(e->modules));
> +	hlist_add_head(&e->hlist, head);
> +	return e;
> +}
> +
> +static int build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop)
> +{
> +	struct jump_entry *iter, *iter_begin;
> +	struct jump_label_entry *entry;
> +	int count;
> +
> +	sort_jump_label_entries(start, stop);
> +	iter = start;
> +	while (iter < stop) {
> +		entry = get_jump_label_entry(iter->name);
> +		if (!entry) {
> +			iter_begin = iter;
> +			count = 0;
> +			while ((iter < stop) &&
> +				(strcmp(iter->name, iter_begin->name) == 0)) {
> +				iter++;
> +				count++;
> +			}
> +			entry = add_jump_label_entry(iter_begin->name, count,
> +						     iter_begin);
> +			if (IS_ERR(entry))
> +				return PTR_ERR(entry);
> +			continue;
> +		}
> +		WARN(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n");
> +	}
> +	return 0;
> +}
> +
> +int jump_label_update(const char *name, enum jump_label_type type)

Non static function without kernel documentation. 

-- Steve


> +{
> +	struct jump_entry *iter;
> +	struct jump_label_entry *entry;
> +	struct hlist_node *module_node;
> +	struct jump_label_module_entry *e_module;
> +	int count;
> +
> +	mutex_lock(&jump_label_mutex);
> +	entry = get_jump_label_entry(name);
> +	if (entry) {
> +		count = entry->nr_entries;
> +		iter = entry->table;
> +		while (count--) {
> +			if (kernel_text_address(iter->code))
> +				arch_jump_label_transform(iter, type);
> +			iter++;
> +		}
> +	}
> +	mutex_unlock(&jump_label_mutex);
> +	return 0;
> +}
> +
> +static int init_jump_label(void)
> +{
> +	int ret;
> +
> +	mutex_lock(&jump_label_mutex);
> +	ret = build_jump_label_hashtable(__start___jump_table,
> +					 __stop___jump_table);
> +	mutex_unlock(&jump_label_mutex);
> +	return ret;
> +}
> +early_initcall(init_jump_label);
> +
> +#endif

Re: [PATCH 3/5] jump label: x86 support

[Jason Baron]

On Mon, Mar 22, 2010 at 12:40:25PM -0400, Steven Rostedt wrote:
> > add x86 support for jump label. I'm keeping this patch separate so its clear to
> > arch maintainers what was required for x86 support this new feature. hopefully,
> > it wouldn't be too painful for other arches.
> > 
> > Signed-off-by: Jason Baron <jbaron@redhat.com>
> 
> 
> > +#ifdef __HAVE_ARCH_JUMP_LABEL
> > +
> > +# ifdef CONFIG_X86_64
> > +#  define JUMP_LABEL_NOP P6_NOP5
> > +# else
> > +#  define JUMP_LABEL_NOP ".byte 0xe9 \n\t .long 0\n\t"
> > +# endif
> 
> Are you sure P6_NOP5 can't happen on non 64bit? Just because it is not
> configured does not mean that the CPU can not handle it. Look at the
> code I did in arch/x86/kernel/ftrace.c to determine what nop to use.
> Maybe we can make that generic and have at boot up, the kernel determine
> a proper 5byte nop.
> 

indeed, i've looked at the ftrace nop code...I think was concerned that
I would need a baseline nop that would work for all boxes. But I guess a
jmp + 5 would be it?

-Jason

Re: [PATCH 4/5] jump label: tracepoint support

[Jason Baron]

On Mon, Mar 22, 2010 at 12:43:34PM -0400, Steven Rostedt wrote:
> On Mon, 2010-03-22 at 12:07 -0400, Jason Baron wrote:
> > Make use of the jump label infrastructure for tracepoints.
> > 
> > Signed-off-by: Jason Baron <jbaron@redhat.com>
> > ---
> >  include/linux/tracepoint.h |   34 +++++++++++++++++++---------------
> >  kernel/tracepoint.c        |    8 ++++++++
> >  2 files changed, 27 insertions(+), 15 deletions(-)
> > 
> > diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
> > index f59604e..c18b9c0 100644
> > --- a/include/linux/tracepoint.h
> > +++ b/include/linux/tracepoint.h
> > @@ -16,6 +16,7 @@
> >  
> >  #include <linux/types.h>
> >  #include <linux/rcupdate.h>
> > +#include <linux/jump_label.h>
> >  
> >  struct module;
> >  struct tracepoint;
> > @@ -63,20 +64,22 @@ struct tracepoint {
> >   * not add unwanted padding between the beginning of the section and the
> >   * structure. Force alignment to the same alignment as the section start.
> >   */
> > -#define DECLARE_TRACE(name, proto, args)				\
> > -	extern struct tracepoint __tracepoint_##name;			\
> > -	static inline void trace_##name(proto)				\
> > -	{								\
> > -		if (unlikely(__tracepoint_##name.state))		\
> > -			__DO_TRACE(&__tracepoint_##name,		\
> > -				TP_PROTO(proto), TP_ARGS(args));	\
> > -	}								\
> > -	static inline int register_trace_##name(void (*probe)(proto))	\
> > -	{								\
> > -		return tracepoint_probe_register(#name, (void *)probe);	\
> > -	}								\
> > -	static inline int unregister_trace_##name(void (*probe)(proto))	\
> > -	{								\
> > +#define DECLARE_TRACE(name, proto, args)				 \
> > +	extern struct tracepoint __tracepoint_##name;			 \
> > +	static inline void trace_##name(proto)				 \
> > +	{								 \
> > +		JUMP_LABEL(name, do_trace, __tracepoint_##name.state);   \
> > +		return;							 \
> > +do_trace:								 \
> > +		__DO_TRACE(&__tracepoint_##name,			 \
> > +			   TP_PROTO(proto), TP_ARGS(args));		 \
> 
> Does this still work on all archs, and when jump labels are not
> supported?

yes. See the base patch. If the arch doesn't have jump label support we
do:

#define JUMP_LABEL(tag, label, cond)            \
        if (unlikely(cond))                     \
                goto label;

-Jason

Re: [PATCH 2/5] jump label: base patch

[Avi Kivity]

On 03/22/2010 06:07 PM, Jason Baron wrote:
> base patch to implement 'jump labeling'. Based on a new 'asm goto' inline
> assembly gcc mechanism, we can now branch to labels from an 'asm goto'
> statment. This allows us to create a 'no-op' fastpath, which can subsequently
> be patched with a jump to the slowpath code. This is useful for code which
> might be rarely used, but which we'd like to be able to call, if needed.
> Tracepoints are the current usecase that these are being implemented for.
>
>
> +
> +#define JUMP_LABEL(tag, label, cond)		\
> +	if (unlikely(cond))			\
> +		goto label;
>    

Suggest wrapping with do { } while (0) to avoid problems with the 
dangling semicolon or a trailing 'else' from an outer if ().

-- 
Do not meddle in the internals of kernels, for they are subtle and quick to panic.

Re: [PATCH 2/5] jump label: base patch

[Masami Hiramatsu]

Jason Baron wrote:
> base patch to implement 'jump labeling'. Based on a new 'asm goto' inline
> assembly gcc mechanism, we can now branch to labels from an 'asm goto'
> statment. This allows us to create a 'no-op' fastpath, which can subsequently
> be patched with a jump to the slowpath code. This is useful for code which
> might be rarely used, but which we'd like to be able to call, if needed.
> Tracepoints are the current usecase that these are being implemented for.
> 
> Signed-off-by: Jason Baron <jbaron@redhat.com>
> ---
>  include/asm-generic/vmlinux.lds.h |   10 ++-
>  include/linux/jump_label.h        |   57 +++++++++++++
>  kernel/Makefile                   |    2 +-
>  kernel/jump_label.c               |  165 +++++++++++++++++++++++++++++++++++++
>  4 files changed, 232 insertions(+), 2 deletions(-)
>  create mode 100644 include/linux/jump_label.h
>  create mode 100644 kernel/jump_label.c
> 
> diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
> index 67e6520..83a469d 100644
> --- a/include/asm-generic/vmlinux.lds.h
> +++ b/include/asm-generic/vmlinux.lds.h
> @@ -167,7 +167,8 @@
>  	BRANCH_PROFILE()						\
>  	TRACE_PRINTKS()							\
>  	FTRACE_EVENTS()							\
> -	TRACE_SYSCALLS()
> +	TRACE_SYSCALLS()						\
> +	JUMP_TABLE()							\

Just a minor style issue: no need to add '\' on the last line.

>  
>  /*
>   * Data section helpers
> @@ -206,6 +207,7 @@
>  		*(__vermagic)		/* Kernel version magic */	\
>  		*(__markers_strings)	/* Markers: strings */		\
>  		*(__tracepoints_strings)/* Tracepoints: strings */	\
> +		*(__jump_strings)/* Jump: strings */	\
>  	}								\
>  									\

Just a minor style issue: please align the tab.

[...]
> +static struct jump_label_entry *add_jump_label_entry(const char *name, int nr_entries, struct jump_entry *table)
> +{
> +	struct hlist_head *head;
> +	struct hlist_node *node;
> +	struct jump_label_entry *e;
> +	size_t name_len = strlen(name) + 1;
> +	u32 hash = jhash(name, name_len-1, 0);
> +
> +	head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
> +	hlist_for_each_entry(e, node, head, hlist) {
> +		if (!strcmp(name, e->name))
> +			return ERR_PTR(-EEXIST);
> +	}
> +	e = kmalloc(sizeof(struct jump_label_entry) + name_len, GFP_KERNEL);
> +	if (!e)
> +		return ERR_PTR(-ENOMEM);
> +	memcpy(&e->name[0], name, name_len);

Hmm, why don't you just have a pointer for e->name which points name?
Or, maybe you can find it easily from e->table[0].name (so you can save
some memory).

Thank you,


-- 
Masami Hiramatsu
e-mail: mhiramat@redhat.com