* [PATCH v2 0/5] static_key: deferred key fixes and improvements
2013-12-07 0:40 [PATCH v2 0/5] static_key: deferred key fixes and improvements Radim Krčmář
@ 2013-12-07 0:40 ` Radim Krčmář
2013-12-07 0:40 ` [PATCH v2 1/5] static_key: add a section for deferred keys Radim Krčmář
` (4 subsequent siblings)
5 siblings, 0 replies; 11+ messages in thread
From: Radim Krčmář @ 2013-12-07 0:40 UTC (permalink / raw)
To: linux-kernel
Cc: kvm, linux-arch, rostedt, pbonzini, tglx, mingo, hpa, x86, arnd,
rusty, Radim Krčmář
This series is made out of three logical parts
- patches 1 and 2 fix panic caused by accessing freed module memory
- patches 3 and 4 fix miscounting by static_key_slow_dec_deferred()
- patch 5 introduces a minor optimization
More jump_label/static_key patches are prepared, but I returned to them
yesterday and implemented a variable jump length on amd64, which
requires some refactoring, porting to remaining architectures, and
retesting, so I'm posting this independent part before it gets overtaken
by higher priority work again
=> This series was tested with additional patches
^ I wrote this on Tuesday and then moved to higher priority work, but
returned with enough courage to post a different first part.
The first part was tested on amd64, s390x and ppc64, the rest also on
armv7.
Applies to next-20131206 and v3.13-rc3.
Radim Krčmář (5):
static_key: add a section for deferred keys
static_key: cancel rate limit timer on rmmod
static_key: add static_key_slow_inc_deferred()
static_key: keep deferred enabled counter debt
static_key: improve deferred inc behavior
arch/x86/kvm/lapic.c | 11 +++++----
arch/x86/kvm/lapic.h | 4 +--
include/asm-generic/vmlinux.lds.h | 1 +
include/linux/jump_label_ratelimit.h | 10 ++++++++
include/linux/module.h | 3 +++
include/linux/perf_event.h | 2 +-
kernel/events/core.c | 8 +++---
kernel/jump_label.c | 47 +++++++++++++++++++++++++++++++-----
kernel/module.c | 4 +++
9 files changed, 72 insertions(+), 18 deletions(-)
--
1.8.4.2
^ permalink raw reply [flat|nested] 11+ messages in thread* [PATCH v2 1/5] static_key: add a section for deferred keys
2013-12-07 0:40 [PATCH v2 0/5] static_key: deferred key fixes and improvements Radim Krčmář
2013-12-07 0:40 ` Radim Krčmář
@ 2013-12-07 0:40 ` Radim Krčmář
2013-12-07 0:40 ` Radim Krčmář
2013-12-07 0:40 ` [PATCH v2 2/5] static_key: cancel rate limit timer on rmmod Radim Krčmář
` (3 subsequent siblings)
5 siblings, 1 reply; 11+ messages in thread
From: Radim Krčmář @ 2013-12-07 0:40 UTC (permalink / raw)
To: linux-kernel
Cc: kvm, linux-arch, rostedt, pbonzini, tglx, mingo, hpa, x86, arnd,
rusty, Radim Krčmář
We need to know about all deferred keys if we want to correctly
- initialize timers on kernel init/module load
- destroy pending timers when unloading a module
We depend on section attribute, so direct definitions of struct
static_key_deferred should be avoided, which is suboptimal.
Signed-off-by: Radim Krčmář <rkrcmar@redhat.com>
---
More general solution would use compile-time magic to generate an array
of pointers to deferred structures, but I am not sure if it is
acceptable and possible.
Worse approach added an unload_callback_list to the struct module.
Callbacks of type void (*)(void *) were registered on
static_key_rate_limit(), to cancel the deferred key work from
module_unload_free().
(I have patches for this)
Jump labels are already notified for module changes, so we could keep track of
deferred keys in modules there, using a static global tree.
(in the worst case)
arch/x86/kvm/lapic.c | 4 ++--
arch/x86/kvm/lapic.h | 4 ++--
include/asm-generic/vmlinux.lds.h | 1 +
include/linux/jump_label_ratelimit.h | 4 ++++
include/linux/module.h | 3 +++
include/linux/perf_event.h | 2 +-
kernel/events/core.c | 2 +-
kernel/module.c | 4 ++++
8 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 5439117..5f01547 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -112,8 +112,8 @@ static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}
-struct static_key_deferred apic_hw_disabled __read_mostly;
-struct static_key_deferred apic_sw_disabled __read_mostly;
+static_key_deferred(apic_hw_disabled);
+static_key_deferred(apic_sw_disabled);
static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
{
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index c730ac9..4ae9a7a 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -108,7 +108,7 @@ static inline bool kvm_vcpu_has_lapic(struct kvm_vcpu *vcpu)
return true;
}
-extern struct static_key_deferred apic_hw_disabled;
+extern static_key_deferred(apic_hw_disabled);
static inline int kvm_apic_hw_enabled(struct kvm_lapic *apic)
{
@@ -117,7 +117,7 @@ static inline int kvm_apic_hw_enabled(struct kvm_lapic *apic)
return MSR_IA32_APICBASE_ENABLE;
}
-extern struct static_key_deferred apic_sw_disabled;
+extern static_key_deferred(apic_sw_disabled);
static inline int kvm_apic_sw_enabled(struct kvm_lapic *apic)
{
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index bc2121f..572e583 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -214,6 +214,7 @@
#define READ_MOSTLY_DATA(align) \
. = ALIGN(align); \
*(.data..read_mostly) \
+ *(__deferred_keys) \
. = ALIGN(align);
#define CACHELINE_ALIGNED_DATA(align) \
diff --git a/include/linux/jump_label_ratelimit.h b/include/linux/jump_label_ratelimit.h
index 089f70f..1216db0 100644
--- a/include/linux/jump_label_ratelimit.h
+++ b/include/linux/jump_label_ratelimit.h
@@ -3,6 +3,10 @@
#include <linux/jump_label.h>
#include <linux/workqueue.h>
+#include <linux/compiler.h>
+
+#define static_key_deferred(name) \
+ struct static_key_deferred name __section(__deferred_keys)
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
struct static_key_deferred {
diff --git a/include/linux/module.h b/include/linux/module.h
index 46e548f..4cc7269 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -347,6 +347,9 @@ struct module
#ifdef HAVE_JUMP_LABEL
struct jump_entry *jump_entries;
unsigned int num_jump_entries;
+
+ struct static_key_deferred *deferred_keys;
+ unsigned int num_deferred_keys;
#endif
#ifdef CONFIG_TRACING
unsigned int num_trace_bprintk_fmt;
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 8f4a70f..8baabca 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -672,7 +672,7 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
}
}
-extern struct static_key_deferred perf_sched_events;
+extern static_key_deferred(perf_sched_events);
static inline void perf_event_task_sched_in(struct task_struct *prev,
struct task_struct *task)
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 403b781..ee64d26 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -138,7 +138,7 @@ enum event_type_t {
* perf_sched_events : >0 events exist
* perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
*/
-struct static_key_deferred perf_sched_events __read_mostly;
+static_key_deferred(perf_sched_events);
static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
diff --git a/kernel/module.c b/kernel/module.c
index f5a3b1e..3e601ef 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -2751,6 +2751,10 @@ static int find_module_sections(struct module *mod, struct load_info *info)
mod->jump_entries = section_objs(info, "__jump_table",
sizeof(*mod->jump_entries),
&mod->num_jump_entries);
+ mod->deferred_keys = section_objs(info, "__deferred_keys",
+ sizeof(*mod->deferred_keys),
+ &mod->num_deferred_keys);
+
#endif
#ifdef CONFIG_EVENT_TRACING
mod->trace_events = section_objs(info, "_ftrace_events",
--
1.8.4.2
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v2 1/5] static_key: add a section for deferred keys
2013-12-07 0:40 ` [PATCH v2 1/5] static_key: add a section for deferred keys Radim Krčmář
@ 2013-12-07 0:40 ` Radim Krčmář
0 siblings, 0 replies; 11+ messages in thread
From: Radim Krčmář @ 2013-12-07 0:40 UTC (permalink / raw)
To: linux-kernel
Cc: kvm, linux-arch, rostedt, pbonzini, tglx, mingo, hpa, x86, arnd,
rusty, Radim Krčmář
We need to know about all deferred keys if we want to correctly
- initialize timers on kernel init/module load
- destroy pending timers when unloading a module
We depend on section attribute, so direct definitions of struct
static_key_deferred should be avoided, which is suboptimal.
Signed-off-by: Radim Krčmář <rkrcmar@redhat.com>
---
More general solution would use compile-time magic to generate an array
of pointers to deferred structures, but I am not sure if it is
acceptable and possible.
Worse approach added an unload_callback_list to the struct module.
Callbacks of type void (*)(void *) were registered on
static_key_rate_limit(), to cancel the deferred key work from
module_unload_free().
(I have patches for this)
Jump labels are already notified for module changes, so we could keep track of
deferred keys in modules there, using a static global tree.
(in the worst case)
arch/x86/kvm/lapic.c | 4 ++--
arch/x86/kvm/lapic.h | 4 ++--
include/asm-generic/vmlinux.lds.h | 1 +
include/linux/jump_label_ratelimit.h | 4 ++++
include/linux/module.h | 3 +++
include/linux/perf_event.h | 2 +-
kernel/events/core.c | 2 +-
kernel/module.c | 4 ++++
8 files changed, 18 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 5439117..5f01547 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -112,8 +112,8 @@ static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
}
-struct static_key_deferred apic_hw_disabled __read_mostly;
-struct static_key_deferred apic_sw_disabled __read_mostly;
+static_key_deferred(apic_hw_disabled);
+static_key_deferred(apic_sw_disabled);
static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
{
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index c730ac9..4ae9a7a 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -108,7 +108,7 @@ static inline bool kvm_vcpu_has_lapic(struct kvm_vcpu *vcpu)
return true;
}
-extern struct static_key_deferred apic_hw_disabled;
+extern static_key_deferred(apic_hw_disabled);
static inline int kvm_apic_hw_enabled(struct kvm_lapic *apic)
{
@@ -117,7 +117,7 @@ static inline int kvm_apic_hw_enabled(struct kvm_lapic *apic)
return MSR_IA32_APICBASE_ENABLE;
}
-extern struct static_key_deferred apic_sw_disabled;
+extern static_key_deferred(apic_sw_disabled);
static inline int kvm_apic_sw_enabled(struct kvm_lapic *apic)
{
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index bc2121f..572e583 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -214,6 +214,7 @@
#define READ_MOSTLY_DATA(align) \
. = ALIGN(align); \
*(.data..read_mostly) \
+ *(__deferred_keys) \
. = ALIGN(align);
#define CACHELINE_ALIGNED_DATA(align) \
diff --git a/include/linux/jump_label_ratelimit.h b/include/linux/jump_label_ratelimit.h
index 089f70f..1216db0 100644
--- a/include/linux/jump_label_ratelimit.h
+++ b/include/linux/jump_label_ratelimit.h
@@ -3,6 +3,10 @@
#include <linux/jump_label.h>
#include <linux/workqueue.h>
+#include <linux/compiler.h>
+
+#define static_key_deferred(name) \
+ struct static_key_deferred name __section(__deferred_keys)
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
struct static_key_deferred {
diff --git a/include/linux/module.h b/include/linux/module.h
index 46e548f..4cc7269 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -347,6 +347,9 @@ struct module
#ifdef HAVE_JUMP_LABEL
struct jump_entry *jump_entries;
unsigned int num_jump_entries;
+
+ struct static_key_deferred *deferred_keys;
+ unsigned int num_deferred_keys;
#endif
#ifdef CONFIG_TRACING
unsigned int num_trace_bprintk_fmt;
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 8f4a70f..8baabca 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -672,7 +672,7 @@ perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
}
}
-extern struct static_key_deferred perf_sched_events;
+extern static_key_deferred(perf_sched_events);
static inline void perf_event_task_sched_in(struct task_struct *prev,
struct task_struct *task)
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 403b781..ee64d26 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -138,7 +138,7 @@ enum event_type_t {
* perf_sched_events : >0 events exist
* perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
*/
-struct static_key_deferred perf_sched_events __read_mostly;
+static_key_deferred(perf_sched_events);
static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
diff --git a/kernel/module.c b/kernel/module.c
index f5a3b1e..3e601ef 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -2751,6 +2751,10 @@ static int find_module_sections(struct module *mod, struct load_info *info)
mod->jump_entries = section_objs(info, "__jump_table",
sizeof(*mod->jump_entries),
&mod->num_jump_entries);
+ mod->deferred_keys = section_objs(info, "__deferred_keys",
+ sizeof(*mod->deferred_keys),
+ &mod->num_deferred_keys);
+
#endif
#ifdef CONFIG_EVENT_TRACING
mod->trace_events = section_objs(info, "_ftrace_events",
--
1.8.4.2
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [PATCH v2 2/5] static_key: cancel rate limit timer on rmmod
2013-12-07 0:40 [PATCH v2 0/5] static_key: deferred key fixes and improvements Radim Krčmář
2013-12-07 0:40 ` Radim Krčmář
2013-12-07 0:40 ` [PATCH v2 1/5] static_key: add a section for deferred keys Radim Krčmář
@ 2013-12-07 0:40 ` Radim Krčmář
2013-12-07 0:40 ` Radim Krčmář
2013-12-07 0:40 ` [PATCH v2 3/5] static_key: add static_key_slow_inc_deferred() Radim Krčmář
` (2 subsequent siblings)
5 siblings, 1 reply; 11+ messages in thread
From: Radim Krčmář @ 2013-12-07 0:40 UTC (permalink / raw)
To: linux-kernel
Cc: kvm, linux-arch, rostedt, pbonzini, tglx, mingo, hpa, x86, arnd,
rusty, Radim Krčmář
Fix a bug when we free module's memory while a timer is pending by
canceling all deferred timers from the unloaded module.
static_key_rate_limit() still can't be called more than once.
Reproducer: (host crasher)
modprobe kvm_intel
(sleep 1; echo quit) \
| qemu-kvm -kernel /dev/null -monitor stdio &
sleep 0.5
until modprobe -rv kvm_intel 2>/dev/null; do :; done
Signed-off-by: Radim Krčmář <rkrcmar@redhat.com>
---
I decided not to post a patch that uses __deferred_key in kernel/module init,
so these three functions might seem like an overkill.
kernel/jump_label.c | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 9019f15..02d610a 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -125,6 +125,27 @@ void jump_label_rate_limit(struct static_key_deferred *key,
}
EXPORT_SYMBOL_GPL(jump_label_rate_limit);
+/* could (should?) be abstracted more */
+static void with_deferred_keys(struct static_key_deferred *entry,
+ struct static_key_deferred *stop,
+ void (*op)(struct static_key_deferred *))
+{
+ struct static_key_deferred *iter;
+
+ for (iter = entry; iter < stop; iter++)
+ op(iter);
+}
+
+#define with_module_deferred_keys(mod, op) \
+ with_deferred_keys(mod->deferred_keys, \
+ mod->deferred_keys + mod->num_deferred_keys, \
+ op)
+
+static void deferred_key_cancel_work(struct static_key_deferred *dkey)
+{
+ cancel_delayed_work_sync(&dkey->work);
+}
+
static int addr_conflict(struct jump_entry *entry, void *start, void *end)
{
if (entry->code <= (unsigned long)end &&
@@ -385,6 +406,7 @@ jump_label_module_notify(struct notifier_block *self, unsigned long val,
jump_label_unlock();
break;
case MODULE_STATE_GOING:
+ with_module_deferred_keys(mod, deferred_key_cancel_work);
jump_label_lock();
jump_label_del_module(mod);
jump_label_unlock();
--
1.8.4.2
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v2 2/5] static_key: cancel rate limit timer on rmmod
2013-12-07 0:40 ` [PATCH v2 2/5] static_key: cancel rate limit timer on rmmod Radim Krčmář
@ 2013-12-07 0:40 ` Radim Krčmář
0 siblings, 0 replies; 11+ messages in thread
From: Radim Krčmář @ 2013-12-07 0:40 UTC (permalink / raw)
To: linux-kernel
Cc: kvm, linux-arch, rostedt, pbonzini, tglx, mingo, hpa, x86, arnd,
rusty, Radim Krčmář
Fix a bug when we free module's memory while a timer is pending by
canceling all deferred timers from the unloaded module.
static_key_rate_limit() still can't be called more than once.
Reproducer: (host crasher)
modprobe kvm_intel
(sleep 1; echo quit) \
| qemu-kvm -kernel /dev/null -monitor stdio &
sleep 0.5
until modprobe -rv kvm_intel 2>/dev/null; do :; done
Signed-off-by: Radim Krčmář <rkrcmar@redhat.com>
---
I decided not to post a patch that uses __deferred_key in kernel/module init,
so these three functions might seem like an overkill.
kernel/jump_label.c | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 9019f15..02d610a 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -125,6 +125,27 @@ void jump_label_rate_limit(struct static_key_deferred *key,
}
EXPORT_SYMBOL_GPL(jump_label_rate_limit);
+/* could (should?) be abstracted more */
+static void with_deferred_keys(struct static_key_deferred *entry,
+ struct static_key_deferred *stop,
+ void (*op)(struct static_key_deferred *))
+{
+ struct static_key_deferred *iter;
+
+ for (iter = entry; iter < stop; iter++)
+ op(iter);
+}
+
+#define with_module_deferred_keys(mod, op) \
+ with_deferred_keys(mod->deferred_keys, \
+ mod->deferred_keys + mod->num_deferred_keys, \
+ op)
+
+static void deferred_key_cancel_work(struct static_key_deferred *dkey)
+{
+ cancel_delayed_work_sync(&dkey->work);
+}
+
static int addr_conflict(struct jump_entry *entry, void *start, void *end)
{
if (entry->code <= (unsigned long)end &&
@@ -385,6 +406,7 @@ jump_label_module_notify(struct notifier_block *self, unsigned long val,
jump_label_unlock();
break;
case MODULE_STATE_GOING:
+ with_module_deferred_keys(mod, deferred_key_cancel_work);
jump_label_lock();
jump_label_del_module(mod);
jump_label_unlock();
--
1.8.4.2
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [PATCH v2 3/5] static_key: add static_key_slow_inc_deferred()
2013-12-07 0:40 [PATCH v2 0/5] static_key: deferred key fixes and improvements Radim Krčmář
` (2 preceding siblings ...)
2013-12-07 0:40 ` [PATCH v2 2/5] static_key: cancel rate limit timer on rmmod Radim Krčmář
@ 2013-12-07 0:40 ` Radim Krčmář
2013-12-07 0:40 ` Radim Krčmář
2013-12-07 0:40 ` [PATCH v2 4/5] static_key: keep deferred enabled counter debt Radim Krčmář
2013-12-07 0:40 ` [PATCH v2 5/5] static_key: improve deferred inc behavior Radim Krčmář
5 siblings, 1 reply; 11+ messages in thread
From: Radim Krčmář @ 2013-12-07 0:40 UTC (permalink / raw)
To: linux-kernel
Cc: kvm, linux-arch, rostedt, pbonzini, tglx, mingo, hpa, x86, arnd,
rusty, Radim Krčmář
Complement the static_key_slow_dec_deferred().
This avoids asymmetrical API, and prepares us for future optimizations
and bug fixes.
Signed-off-by: Radim Krčmář <rkrcmar@redhat.com>
---
arch/x86/kvm/lapic.c | 7 ++++---
include/linux/jump_label_ratelimit.h | 5 +++++
kernel/events/core.c | 6 +++---
kernel/jump_label.c | 7 +++++++
4 files changed, 19 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 5f01547..86973ac 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -121,7 +121,7 @@ static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
if (val & APIC_SPIV_APIC_ENABLED)
static_key_slow_dec_deferred(&apic_sw_disabled);
else
- static_key_slow_inc(&apic_sw_disabled.key);
+ static_key_slow_inc_deferred(&apic_sw_disabled);
}
apic_set_reg(apic, APIC_SPIV, val);
}
@@ -1351,7 +1351,7 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
if (value & MSR_IA32_APICBASE_ENABLE)
static_key_slow_dec_deferred(&apic_hw_disabled);
else
- static_key_slow_inc(&apic_hw_disabled.key);
+ static_key_slow_inc_deferred(&apic_hw_disabled);
recalculate_apic_map(vcpu->kvm);
}
@@ -1546,7 +1546,8 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu)
kvm_lapic_set_base(vcpu,
APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE);
- static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
+ /* sw disabled at reset */
+ static_key_slow_inc_deferred(&apic_sw_disabled);
kvm_lapic_reset(vcpu);
kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
diff --git a/include/linux/jump_label_ratelimit.h b/include/linux/jump_label_ratelimit.h
index 112ba5f..a18aded 100644
--- a/include/linux/jump_label_ratelimit.h
+++ b/include/linux/jump_label_ratelimit.h
@@ -18,6 +18,7 @@ struct static_key_deferred {
#endif
#ifdef HAVE_JUMP_LABEL
+extern void static_key_slow_inc_deferred(struct static_key_deferred *key);
extern void static_key_slow_dec_deferred(struct static_key_deferred *key);
extern void
jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl);
@@ -26,6 +27,10 @@ jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl);
struct static_key_deferred {
struct static_key key;
};
+static inline void static_key_slow_inc_deferred(struct static_key_deferred *key)
+{
+ static_key_slow_inc(&key->key);
+}
static inline void static_key_slow_dec_deferred(struct static_key_deferred *key)
{
STATIC_KEY_CHECK_USE();
diff --git a/kernel/events/core.c b/kernel/events/core.c
index ee64d26..b3fb4c2 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -6597,7 +6597,7 @@ static void account_event(struct perf_event *event)
return;
if (event->attach_state & PERF_ATTACH_TASK)
- static_key_slow_inc(&perf_sched_events.key);
+ static_key_slow_inc_deferred(&perf_sched_events);
if (event->attr.mmap || event->attr.mmap_data)
atomic_inc(&nr_mmap_events);
if (event->attr.comm)
@@ -6609,9 +6609,9 @@ static void account_event(struct perf_event *event)
tick_nohz_full_kick_all();
}
if (has_branch_stack(event))
- static_key_slow_inc(&perf_sched_events.key);
+ static_key_slow_inc_deferred(&perf_sched_events);
if (is_cgroup_event(event))
- static_key_slow_inc(&perf_sched_events.key);
+ static_key_slow_inc_deferred(&perf_sched_events);
account_event_cpu(event, event->cpu);
}
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 02d610a..41592ba 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -74,6 +74,13 @@ void static_key_slow_inc(struct static_key *key)
}
EXPORT_SYMBOL_GPL(static_key_slow_inc);
+void static_key_slow_inc_deferred(struct static_key_deferred *key)
+{
+ STATIC_KEY_CHECK_USE();
+ static_key_slow_inc(&key->key);
+}
+EXPORT_SYMBOL_GPL(static_key_slow_inc_deferred);
+
static void __static_key_slow_dec(struct static_key *key,
unsigned long rate_limit, struct delayed_work *work)
{
--
1.8.4.2
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v2 3/5] static_key: add static_key_slow_inc_deferred()
2013-12-07 0:40 ` [PATCH v2 3/5] static_key: add static_key_slow_inc_deferred() Radim Krčmář
@ 2013-12-07 0:40 ` Radim Krčmář
0 siblings, 0 replies; 11+ messages in thread
From: Radim Krčmář @ 2013-12-07 0:40 UTC (permalink / raw)
To: linux-kernel
Cc: kvm, linux-arch, rostedt, pbonzini, tglx, mingo, hpa, x86, arnd,
rusty, Radim Krčmář
Complement the static_key_slow_dec_deferred().
This avoids asymmetrical API, and prepares us for future optimizations
and bug fixes.
Signed-off-by: Radim Krčmář <rkrcmar@redhat.com>
---
arch/x86/kvm/lapic.c | 7 ++++---
include/linux/jump_label_ratelimit.h | 5 +++++
kernel/events/core.c | 6 +++---
kernel/jump_label.c | 7 +++++++
4 files changed, 19 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index 5f01547..86973ac 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -121,7 +121,7 @@ static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
if (val & APIC_SPIV_APIC_ENABLED)
static_key_slow_dec_deferred(&apic_sw_disabled);
else
- static_key_slow_inc(&apic_sw_disabled.key);
+ static_key_slow_inc_deferred(&apic_sw_disabled);
}
apic_set_reg(apic, APIC_SPIV, val);
}
@@ -1351,7 +1351,7 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
if (value & MSR_IA32_APICBASE_ENABLE)
static_key_slow_dec_deferred(&apic_hw_disabled);
else
- static_key_slow_inc(&apic_hw_disabled.key);
+ static_key_slow_inc_deferred(&apic_hw_disabled);
recalculate_apic_map(vcpu->kvm);
}
@@ -1546,7 +1546,8 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu)
kvm_lapic_set_base(vcpu,
APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE);
- static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
+ /* sw disabled at reset */
+ static_key_slow_inc_deferred(&apic_sw_disabled);
kvm_lapic_reset(vcpu);
kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
diff --git a/include/linux/jump_label_ratelimit.h b/include/linux/jump_label_ratelimit.h
index 112ba5f..a18aded 100644
--- a/include/linux/jump_label_ratelimit.h
+++ b/include/linux/jump_label_ratelimit.h
@@ -18,6 +18,7 @@ struct static_key_deferred {
#endif
#ifdef HAVE_JUMP_LABEL
+extern void static_key_slow_inc_deferred(struct static_key_deferred *key);
extern void static_key_slow_dec_deferred(struct static_key_deferred *key);
extern void
jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl);
@@ -26,6 +27,10 @@ jump_label_rate_limit(struct static_key_deferred *key, unsigned long rl);
struct static_key_deferred {
struct static_key key;
};
+static inline void static_key_slow_inc_deferred(struct static_key_deferred *key)
+{
+ static_key_slow_inc(&key->key);
+}
static inline void static_key_slow_dec_deferred(struct static_key_deferred *key)
{
STATIC_KEY_CHECK_USE();
diff --git a/kernel/events/core.c b/kernel/events/core.c
index ee64d26..b3fb4c2 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -6597,7 +6597,7 @@ static void account_event(struct perf_event *event)
return;
if (event->attach_state & PERF_ATTACH_TASK)
- static_key_slow_inc(&perf_sched_events.key);
+ static_key_slow_inc_deferred(&perf_sched_events);
if (event->attr.mmap || event->attr.mmap_data)
atomic_inc(&nr_mmap_events);
if (event->attr.comm)
@@ -6609,9 +6609,9 @@ static void account_event(struct perf_event *event)
tick_nohz_full_kick_all();
}
if (has_branch_stack(event))
- static_key_slow_inc(&perf_sched_events.key);
+ static_key_slow_inc_deferred(&perf_sched_events);
if (is_cgroup_event(event))
- static_key_slow_inc(&perf_sched_events.key);
+ static_key_slow_inc_deferred(&perf_sched_events);
account_event_cpu(event, event->cpu);
}
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 02d610a..41592ba 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -74,6 +74,13 @@ void static_key_slow_inc(struct static_key *key)
}
EXPORT_SYMBOL_GPL(static_key_slow_inc);
+void static_key_slow_inc_deferred(struct static_key_deferred *key)
+{
+ STATIC_KEY_CHECK_USE();
+ static_key_slow_inc(&key->key);
+}
+EXPORT_SYMBOL_GPL(static_key_slow_inc_deferred);
+
static void __static_key_slow_dec(struct static_key *key,
unsigned long rate_limit, struct delayed_work *work)
{
--
1.8.4.2
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [PATCH v2 4/5] static_key: keep deferred enabled counter debt
2013-12-07 0:40 [PATCH v2 0/5] static_key: deferred key fixes and improvements Radim Krčmář
` (3 preceding siblings ...)
2013-12-07 0:40 ` [PATCH v2 3/5] static_key: add static_key_slow_inc_deferred() Radim Krčmář
@ 2013-12-07 0:40 ` Radim Krčmář
2013-12-07 0:40 ` Radim Krčmář
2013-12-07 0:40 ` [PATCH v2 5/5] static_key: improve deferred inc behavior Radim Krčmář
5 siblings, 1 reply; 11+ messages in thread
From: Radim Krčmář @ 2013-12-07 0:40 UTC (permalink / raw)
To: linux-kernel
Cc: kvm, linux-arch, rostedt, pbonzini, tglx, mingo, hpa, x86, arnd,
rusty, Radim Krčmář
When '.enabled.counter == 1', static_key_slow_dec_deferred() gets
silently dropped if the decrease is already pending.
We print a warning if this happens and because .enabled.counter cannot
go below 1 before the decrease has finished, the number of ignored
static_key_slow_dec_deferred() is kept and we skip an equal amount of
static_key_slow_inc_deferred().
Signed-off-by: Radim Krčmář <rkrcmar@redhat.com>
---
include/linux/jump_label_ratelimit.h | 1 +
kernel/jump_label.c | 17 +++++++++++------
2 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/include/linux/jump_label_ratelimit.h b/include/linux/jump_label_ratelimit.h
index a18aded..2d5fa1a 100644
--- a/include/linux/jump_label_ratelimit.h
+++ b/include/linux/jump_label_ratelimit.h
@@ -14,6 +14,7 @@ struct static_key_deferred {
struct static_key key;
unsigned long timeout;
struct delayed_work work;
+ atomic_t enabled_debt;
};
#endif
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 41592ba..bd7ad31 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -77,12 +77,14 @@ EXPORT_SYMBOL_GPL(static_key_slow_inc);
void static_key_slow_inc_deferred(struct static_key_deferred *key)
{
STATIC_KEY_CHECK_USE();
+ if (atomic_dec_if_positive(&key->enabled_debt) >= 0)
+ return;
static_key_slow_inc(&key->key);
}
EXPORT_SYMBOL_GPL(static_key_slow_inc_deferred);
static void __static_key_slow_dec(struct static_key *key,
- unsigned long rate_limit, struct delayed_work *work)
+ struct static_key_deferred *dkey)
{
if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
WARN(atomic_read(&key->enabled) < 0,
@@ -90,9 +92,12 @@ static void __static_key_slow_dec(struct static_key *key,
return;
}
- if (rate_limit) {
+ if (dkey && dkey->timeout) {
atomic_inc(&key->enabled);
- schedule_delayed_work(work, rate_limit);
+ if (!schedule_delayed_work(&dkey->work, dkey->timeout)) {
+ atomic_inc(&dkey->enabled_debt);
+ WARN(1, "jump label: negative deferred count!\n");
+ }
} else {
if (!jump_label_get_branch_default(key))
jump_label_update(key, JUMP_LABEL_DISABLE);
@@ -106,20 +111,20 @@ static void jump_label_update_timeout(struct work_struct *work)
{
struct static_key_deferred *key =
container_of(work, struct static_key_deferred, work.work);
- __static_key_slow_dec(&key->key, 0, NULL);
+ __static_key_slow_dec(&key->key, NULL);
}
void static_key_slow_dec(struct static_key *key)
{
STATIC_KEY_CHECK_USE();
- __static_key_slow_dec(key, 0, NULL);
+ __static_key_slow_dec(key, NULL);
}
EXPORT_SYMBOL_GPL(static_key_slow_dec);
void static_key_slow_dec_deferred(struct static_key_deferred *key)
{
STATIC_KEY_CHECK_USE();
- __static_key_slow_dec(&key->key, key->timeout, &key->work);
+ __static_key_slow_dec(&key->key, key);
}
EXPORT_SYMBOL_GPL(static_key_slow_dec_deferred);
--
1.8.4.2
^ permalink raw reply related [flat|nested] 11+ messages in thread* [PATCH v2 4/5] static_key: keep deferred enabled counter debt
2013-12-07 0:40 ` [PATCH v2 4/5] static_key: keep deferred enabled counter debt Radim Krčmář
@ 2013-12-07 0:40 ` Radim Krčmář
0 siblings, 0 replies; 11+ messages in thread
From: Radim Krčmář @ 2013-12-07 0:40 UTC (permalink / raw)
To: linux-kernel
Cc: kvm, linux-arch, rostedt, pbonzini, tglx, mingo, hpa, x86, arnd,
rusty, Radim Krčmář
When '.enabled.counter == 1', static_key_slow_dec_deferred() gets
silently dropped if the decrease is already pending.
We print a warning if this happens and because .enabled.counter cannot
go below 1 before the decrease has finished, the number of ignored
static_key_slow_dec_deferred() is kept and we skip an equal amount of
static_key_slow_inc_deferred().
Signed-off-by: Radim Krčmář <rkrcmar@redhat.com>
---
include/linux/jump_label_ratelimit.h | 1 +
kernel/jump_label.c | 17 +++++++++++------
2 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/include/linux/jump_label_ratelimit.h b/include/linux/jump_label_ratelimit.h
index a18aded..2d5fa1a 100644
--- a/include/linux/jump_label_ratelimit.h
+++ b/include/linux/jump_label_ratelimit.h
@@ -14,6 +14,7 @@ struct static_key_deferred {
struct static_key key;
unsigned long timeout;
struct delayed_work work;
+ atomic_t enabled_debt;
};
#endif
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index 41592ba..bd7ad31 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -77,12 +77,14 @@ EXPORT_SYMBOL_GPL(static_key_slow_inc);
void static_key_slow_inc_deferred(struct static_key_deferred *key)
{
STATIC_KEY_CHECK_USE();
+ if (atomic_dec_if_positive(&key->enabled_debt) >= 0)
+ return;
static_key_slow_inc(&key->key);
}
EXPORT_SYMBOL_GPL(static_key_slow_inc_deferred);
static void __static_key_slow_dec(struct static_key *key,
- unsigned long rate_limit, struct delayed_work *work)
+ struct static_key_deferred *dkey)
{
if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
WARN(atomic_read(&key->enabled) < 0,
@@ -90,9 +92,12 @@ static void __static_key_slow_dec(struct static_key *key,
return;
}
- if (rate_limit) {
+ if (dkey && dkey->timeout) {
atomic_inc(&key->enabled);
- schedule_delayed_work(work, rate_limit);
+ if (!schedule_delayed_work(&dkey->work, dkey->timeout)) {
+ atomic_inc(&dkey->enabled_debt);
+ WARN(1, "jump label: negative deferred count!\n");
+ }
} else {
if (!jump_label_get_branch_default(key))
jump_label_update(key, JUMP_LABEL_DISABLE);
@@ -106,20 +111,20 @@ static void jump_label_update_timeout(struct work_struct *work)
{
struct static_key_deferred *key =
container_of(work, struct static_key_deferred, work.work);
- __static_key_slow_dec(&key->key, 0, NULL);
+ __static_key_slow_dec(&key->key, NULL);
}
void static_key_slow_dec(struct static_key *key)
{
STATIC_KEY_CHECK_USE();
- __static_key_slow_dec(key, 0, NULL);
+ __static_key_slow_dec(key, NULL);
}
EXPORT_SYMBOL_GPL(static_key_slow_dec);
void static_key_slow_dec_deferred(struct static_key_deferred *key)
{
STATIC_KEY_CHECK_USE();
- __static_key_slow_dec(&key->key, key->timeout, &key->work);
+ __static_key_slow_dec(&key->key, key);
}
EXPORT_SYMBOL_GPL(static_key_slow_dec_deferred);
--
1.8.4.2
^ permalink raw reply related [flat|nested] 11+ messages in thread
* [PATCH v2 5/5] static_key: improve deferred inc behavior
2013-12-07 0:40 [PATCH v2 0/5] static_key: deferred key fixes and improvements Radim Krčmář
` (4 preceding siblings ...)
2013-12-07 0:40 ` [PATCH v2 4/5] static_key: keep deferred enabled counter debt Radim Krčmář
@ 2013-12-07 0:40 ` Radim Krčmář
5 siblings, 0 replies; 11+ messages in thread
From: Radim Krčmář @ 2013-12-07 0:40 UTC (permalink / raw)
To: linux-kernel
Cc: kvm, linux-arch, rostedt, pbonzini, tglx, mingo, hpa, x86, arnd,
rusty, Radim Krčmář
We can cancel deferred static_key_slow_dec() instead of increasing
.enabled.counter.
Timer now won't fire before 'timeout' since the last increase, so this patch
further stabilizes the case of frequent switching.
Signed-off-by: Radim Krčmář <rkrcmar@redhat.com>
---
kernel/jump_label.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/kernel/jump_label.c b/kernel/jump_label.c
index bd7ad31..9b57261 100644
--- a/kernel/jump_label.c
+++ b/kernel/jump_label.c
@@ -79,7 +79,8 @@ void static_key_slow_inc_deferred(struct static_key_deferred *key)
STATIC_KEY_CHECK_USE();
if (atomic_dec_if_positive(&key->enabled_debt) >= 0)
return;
- static_key_slow_inc(&key->key);
+ if (!cancel_delayed_work(&key->work))
+ static_key_slow_inc(&key->key);
}
EXPORT_SYMBOL_GPL(static_key_slow_inc_deferred);
--
1.8.4.2
^ permalink raw reply related [flat|nested] 11+ messages in thread