[PATCHv2 1/2] jump_label,x86: use text_poke_smp_batch for entries update

[Jiri Olsa <jolsa@redhat.com>]

Changing the jump label update code to use batch processing
for x86 architectures. 

Currently each jump label update calls text_poke_smp for each
jump label key entry. Thus one key update ends up calling stop
machine multiple times.

This patch is using text_poke_smp_batch, which is called for
all the key's entries. Thus ensuring the stop machine is called
only once per jump_label key.

Added jump_label_update_end function which is paired with
the key's entries update.

The jump_label_update_end calls arch_jump_label_update_end
(with generic weak definition) which is overloaded by x86
arch and makes the batch update of all the entries queued
by arch_jump_label_transform function.

Added dynamic array/handling for queueing jump_label entries
with the same key.

Tested this on x86 and s390 archs.


Signed-off-by: Jiri Olsa <jolsa@redhat.com>
---
 arch/x86/kernel/jump_label.c |  128 +++++++++++++++++++++++++++++++++++++++---
 include/linux/jump_label.h   |    1 +
 kernel/jump_label.c          |   16 +++++-
 3 files changed, 134 insertions(+), 11 deletions(-)

diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c
index 3fee346..7a4dd32 100644
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -11,6 +11,7 @@
 #include <linux/list.h>
 #include <linux/jhash.h>
 #include <linux/cpu.h>
+#include <linux/slab.h>
 #include <asm/kprobes.h>
 #include <asm/alternative.h>
 
@@ -24,24 +25,133 @@ union jump_code_union {
 	} __attribute__((packed));
 };
 
-void arch_jump_label_transform(struct jump_entry *entry,
-			       enum jump_label_type type)
+struct text_poke_buffer {
+	u8 code[JUMP_LABEL_NOP_SIZE];
+};
+
+#define POKE_ALLOC_CNT 30
+
+static struct text_poke_param  *poke_pars;
+static struct text_poke_buffer *poke_bufs;
+static int poke_cnt, poke_size;
+
+static void poke_setup(struct text_poke_param *param, u8 *buf,
+		       int enable,
+		       struct jump_entry *entry)
 {
-	union jump_code_union code;
+	union jump_code_union *code = (union jump_code_union *) buf;
 
-	if (type == JUMP_LABEL_ENABLE) {
-		code.jump = 0xe9;
-		code.offset = entry->target -
-				(entry->code + JUMP_LABEL_NOP_SIZE);
+	if (enable == JUMP_LABEL_ENABLE) {
+		code->jump = 0xe9;
+		code->offset = entry->target -
+			       (entry->code + JUMP_LABEL_NOP_SIZE);
 	} else
-		memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
+		memcpy(code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
+
+	param->addr = (void *) entry->code;
+	param->opcode = code;
+	param->len = JUMP_LABEL_NOP_SIZE;
+}
+
+static int poke_alloc(void)
+{
+	struct text_poke_param  *pars;
+	struct text_poke_buffer *bufs;
+
+	if (poke_cnt % POKE_ALLOC_CNT)
+		return 0;
+
+	poke_size += POKE_ALLOC_CNT;
+
+	pars = krealloc(poke_pars, poke_size * sizeof(*pars),
+			GFP_KERNEL);
+	if (!pars)
+		return -ENOMEM;
+
+	bufs = krealloc(poke_bufs, poke_size * sizeof(*bufs),
+			GFP_KERNEL);
+	if (!bufs) {
+		kfree(pars);
+		return -ENOMEM;
+	}
+
+	poke_pars = pars;
+	poke_bufs = bufs;
+	return 0;
+}
+
+static void poke_free(void)
+{
+	kfree(poke_pars);
+	kfree(poke_bufs);
+
+	poke_cnt = poke_size = 0;
+	poke_pars = NULL;
+	poke_bufs = NULL;
+}
+
+static void poke_process(struct text_poke_param *par, int cnt)
+{
 	get_online_cpus();
 	mutex_lock(&text_mutex);
-	text_poke_smp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
+	text_poke_smp_batch(par, cnt);
 	mutex_unlock(&text_mutex);
 	put_online_cpus();
 }
 
+static void poke_end(void)
+{
+	if (!poke_cnt)
+		return;
+
+	poke_process(poke_pars, poke_cnt);
+	poke_free();
+}
+
+static void process_entry(struct jump_entry *entry,
+			     enum jump_label_type enable)
+{
+	struct text_poke_param  par;
+	struct text_poke_buffer buf;
+
+	poke_setup(&par, buf.code, enable, entry);
+	poke_process(&par, 1);
+}
+
+void arch_jump_label_transform(struct jump_entry *entry,
+			       enum jump_label_type enable)
+{
+	struct text_poke_param  *par;
+	struct text_poke_buffer *buf;
+
+	if (poke_alloc()) {
+		/*
+		 * Allocation failed, let's finish what we
+		 * gather so far and process this entry
+		 * separatelly. Next call we try the allocation
+		 * again.
+		 */
+		poke_end();
+		process_entry(entry, enable);
+		return;
+	}
+
+	par = &poke_pars[poke_cnt];
+	buf = &poke_bufs[poke_cnt];
+
+	poke_setup(par, buf->code, enable, entry);
+	poke_cnt++;
+}
+
+/*
+ * Called after arch_jump_label_transform is called for
+ * all entries of a single key.
+ */
+void arch_jump_label_update_end(void)
+{
+	poke_end();
+}
+
 void arch_jump_label_text_poke_early(jump_label_t addr)
 {
 	text_poke_early((void *)addr, ideal_nops[NOP_ATOMIC5],
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 83e745f..e7a8fa3 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -46,6 +46,7 @@ extern void jump_label_unlock(void);
 extern void arch_jump_label_transform(struct jump_entry *entry,
 				 enum jump_label_type type);
 extern void arch_jump_label_text_poke_early(jump_label_t addr);
+extern void arch_jump_label_update_end(void);
 extern int jump_label_text_reserved(void *start, void *end);
 extern void jump_label_inc(struct jump_label_key *key);
 extern void jump_label_dec(struct jump_label_key *key);
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 74d1c09..6657a37 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -125,6 +125,15 @@ void __weak arch_jump_label_text_poke_early(jump_label_t addr)
 {
 }
 
+void __weak arch_jump_label_update_end(void)
+{
+}
+
+void jump_label_update_end(void)
+{
+	arch_jump_label_update_end();
+}
+
 static __init int jump_label_init(void)
 {
 	struct jump_entry *iter_start = __start___jump_table;
@@ -244,10 +253,11 @@ static int jump_label_add_module(struct module *mod)
 		jlm->next = key->next;
 		key->next = jlm;
 
-		if (jump_label_enabled(key))
+		if (jump_label_enabled(key)) {
 			__jump_label_update(key, iter, JUMP_LABEL_ENABLE);
+			jump_label_update_end();
+		}
 	}
-
 	return 0;
 }
 
@@ -376,6 +386,8 @@ static void jump_label_update(struct jump_label_key *key, int enable)
 #ifdef CONFIG_MODULES
 	__jump_label_mod_update(key, enable);
 #endif
+
+	jump_label_update_end();
 }
 
 #endif
-- 
1.7.1