* [PATCH] Code clean up for percpu_xxx() functions
@ 2011-10-10 9:10 Alex,Shi
2011-10-10 17:08 ` Christoph Lameter
0 siblings, 1 reply; 42+ messages in thread
From: Alex,Shi @ 2011-10-10 9:10 UTC (permalink / raw)
To: tj, linux-kernel@vger.kernel.org, eric.dumazet, cl@linux.com; +Cc: Huang, Ying
Since percpu_xxx() serial are duplicated with this_cpu_xxx().
I transfer percpu_xxx() to this_cpu_xxx(). This patch is just
do code clean up, no function change.
Could you like to review this patch?
Signed-off-by: Alex Shi <alex.shi@intel.com>
---
arch/x86/include/asm/current.h | 2 +-
arch/x86/include/asm/hardirq.h | 8 ++--
arch/x86/include/asm/irq_regs.h | 4 +-
arch/x86/include/asm/mmu_context.h | 12 ++++----
arch/x86/include/asm/percpu.h | 28 +++++++++---------
arch/x86/include/asm/smp.h | 4 +-
arch/x86/include/asm/stackprotector.h | 4 +-
arch/x86/include/asm/thread_info.h | 2 +-
arch/x86/include/asm/tlbflush.h | 4 +-
arch/x86/kernel/cpu/common.c | 2 +-
arch/x86/kernel/cpu/mcheck/mce.c | 4 +-
arch/x86/kernel/cpu/perf_event.c | 4 +-
arch/x86/kernel/paravirt.c | 12 ++++----
arch/x86/kernel/process_32.c | 2 +-
arch/x86/kernel/process_64.c | 12 ++++----
arch/x86/kvm/x86.c | 14 ++++----
arch/x86/mm/tlb.c | 10 +++---
arch/x86/xen/enlighten.c | 6 ++--
arch/x86/xen/irq.c | 8 ++--
arch/x86/xen/mmu.c | 20 ++++++------
arch/x86/xen/multicalls.h | 2 +-
arch/x86/xen/smp.c | 2 +-
include/linux/percpu.h | 53 ---------------------------------
include/linux/topology.h | 4 +-
net/netfilter/xt_TEE.c | 12 ++++----
net/socket.c | 4 +-
26 files changed, 93 insertions(+), 146 deletions(-)
diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index 4d447b7..9476c04 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task);
static __always_inline struct task_struct *get_current(void)
{
- return percpu_read_stable(current_task);
+ return this_cpu_read_stable(current_task);
}
#define current get_current()
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 55e4de6..fc594da 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -35,14 +35,14 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
#define __ARCH_IRQ_STAT
-#define inc_irq_stat(member) percpu_inc(irq_stat.member)
+#define inc_irq_stat(member) this_cpu_inc(irq_stat.member)
-#define local_softirq_pending() percpu_read(irq_stat.__softirq_pending)
+#define local_softirq_pending() this_cpu_read(irq_stat.__softirq_pending)
#define __ARCH_SET_SOFTIRQ_PENDING
-#define set_softirq_pending(x) percpu_write(irq_stat.__softirq_pending, (x))
-#define or_softirq_pending(x) percpu_or(irq_stat.__softirq_pending, (x))
+#define set_softirq_pending(x) this_cpu_write(irq_stat.__softirq_pending, (x))
+#define or_softirq_pending(x) this_cpu_or(irq_stat.__softirq_pending, (x))
extern void ack_bad_irq(unsigned int irq);
diff --git a/arch/x86/include/asm/irq_regs.h b/arch/x86/include/asm/irq_regs.h
index 7784322..d82250b 100644
--- a/arch/x86/include/asm/irq_regs.h
+++ b/arch/x86/include/asm/irq_regs.h
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);
static inline struct pt_regs *get_irq_regs(void)
{
- return percpu_read(irq_regs);
+ return this_cpu_read(irq_regs);
}
static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
struct pt_regs *old_regs;
old_regs = get_irq_regs();
- percpu_write(irq_regs, new_regs);
+ this_cpu_write(irq_regs, new_regs);
return old_regs;
}
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 6902152..cdbf367 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);
static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
{
#ifdef CONFIG_SMP
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
- percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
+ if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
#endif
}
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
if (likely(prev != next)) {
#ifdef CONFIG_SMP
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- percpu_write(cpu_tlbstate.active_mm, next);
+ this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ this_cpu_write(cpu_tlbstate.active_mm, next);
#endif
cpumask_set_cpu(cpu, mm_cpumask(next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
}
#ifdef CONFIG_SMP
else {
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
+ this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ BUG_ON(this_cpu_read(cpu_tlbstate.active_mm) != next);
if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next))) {
/* We were in lazy tlb mode and leave_mm disabled
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 3470c9d..d1de56c 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -46,7 +46,7 @@
#ifdef CONFIG_SMP
#define __percpu_prefix "%%"__stringify(__percpu_seg)":"
-#define __my_cpu_offset percpu_read(this_cpu_off)
+#define __my_cpu_offset this_cpu_read(this_cpu_off)
/*
* Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,23 @@ do { \
})
/*
- * percpu_read() makes gcc load the percpu variable every time it is
- * accessed while percpu_read_stable() allows the value to be cached.
- * percpu_read_stable() is more efficient and can be used if its value
+ * this_cpu_read() makes gcc load the percpu variable every time it is
+ * accessed while this_cpu_read_stable() allows the value to be cached.
+ * this_cpu_read_stable() is more efficient and can be used if its value
* is guaranteed to be valid across cpus. The current users include
* get_current() and get_thread_info() both of which are actually
* per-thread variables implemented as per-cpu variables and thus
* stable for the duration of the respective task.
*/
-#define percpu_read(var) percpu_from_op("mov", var, "m" (var))
-#define percpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
-#define percpu_write(var, val) percpu_to_op("mov", var, val)
-#define percpu_add(var, val) percpu_add_op(var, val)
-#define percpu_sub(var, val) percpu_add_op(var, -(val))
-#define percpu_and(var, val) percpu_to_op("and", var, val)
-#define percpu_or(var, val) percpu_to_op("or", var, val)
-#define percpu_xor(var, val) percpu_to_op("xor", var, val)
-#define percpu_inc(var) percpu_unary_op("inc", var)
+#define this_cpu_read(var) percpu_from_op("mov", var, "m" (var))
+#define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
+#define this_cpu_write(var, val) percpu_to_op("mov", var, val)
+#define this_cpu_add(var, val) percpu_add_op(var, val)
+#define this_cpu_sub(var, val) percpu_add_op(var, -(val))
+#define this_cpu_and(var, val) percpu_to_op("and", var, val)
+#define this_cpu_or(var, val) percpu_to_op("or", var, val)
+#define this_cpu_xor(var, val) percpu_to_op("xor", var, val)
+#define this_cpu_inc(var) percpu_unary_op("inc", var)
#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +551,7 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr,
{
unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG;
- return ((1UL << (nr % BITS_PER_LONG)) & percpu_read(*a)) != 0;
+ return ((1UL << (nr % BITS_PER_LONG)) & this_cpu_read(*a)) != 0;
}
static inline int x86_this_cpu_variable_test_bit(int nr,
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 73b11bc..dd5749b 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;
* from the initial startup. We map APIC_BASE very early in page_setup(),
* so this is correct in the x86 case.
*/
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
extern int safe_smp_processor_id(void);
#elif defined(CONFIG_X86_64_SMP)
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
#define stack_smp_processor_id() \
({ \
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
index 1575177..3d7a009 100644
--- a/arch/x86/include/asm/stackprotector.h
+++ b/arch/x86/include/asm/stackprotector.h
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void)
current->stack_canary = canary;
#ifdef CONFIG_X86_64
- percpu_write(irq_stack_union.stack_canary, canary);
+ this_cpu_write(irq_stack_union.stack_canary, canary);
#else
- percpu_write(stack_canary.canary, canary);
+ this_cpu_write(stack_canary.canary, canary);
#endif
}
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index a1fe5c1..8b19667 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);
static inline struct thread_info *current_thread_info(void)
{
struct thread_info *ti;
- ti = (void *)(percpu_read_stable(kernel_stack) +
+ ti = (void *)(this_cpu_read_stable(kernel_stack) +
KERNEL_STACK_OFFSET - THREAD_SIZE);
return ti;
}
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 169be89..200c09c 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
static inline void reset_lazy_tlbstate(void)
{
- percpu_write(cpu_tlbstate.state, 0);
- percpu_write(cpu_tlbstate.active_mm, &init_mm);
+ this_cpu_write(cpu_tlbstate.state, 0);
+ this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
}
#endif /* SMP */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 6218439..10513ed 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1158,7 +1158,7 @@ void __cpuinit cpu_init(void)
oist = &per_cpu(orig_ist, cpu);
#ifdef CONFIG_NUMA
- if (cpu != 0 && percpu_read(numa_node) == 0 &&
+ if (cpu != 0 && this_cpu_read(numa_node) == 0 &&
early_cpu_to_node(cpu) != NUMA_NO_NODE)
set_numa_node(early_cpu_to_node(cpu));
#endif
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 08363b0..adbb598 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -512,7 +512,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
struct mce m;
int i;
- percpu_inc(mce_poll_count);
+ this_cpu_inc(mce_poll_count);
mce_gather_info(&m, NULL);
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
atomic_inc(&mce_entry);
- percpu_inc(mce_exception_count);
+ this_cpu_inc(mce_exception_count);
if (notify_die(DIE_NMI, "machine check", regs, error_code,
18, SIGKILL) == NOTIFY_STOP)
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index cfa62ec..ca037c0 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -1389,7 +1389,7 @@ perf_event_nmi_handler(struct notifier_block *self,
case DIE_NMI:
break;
case DIE_NMIUNKNOWN:
- this_nmi = percpu_read(irq_stat.__nmi_count);
+ this_nmi = this_cpu_read(irq_stat.__nmi_count);
if (this_nmi != __this_cpu_read(pmu_nmi.marked))
/* let the kernel handle the unknown nmi */
return NOTIFY_DONE;
@@ -1409,7 +1409,7 @@ perf_event_nmi_handler(struct notifier_block *self,
if (!handled)
return NOTIFY_DONE;
- this_nmi = percpu_read(irq_stat.__nmi_count);
+ this_nmi = this_cpu_read(irq_stat.__nmi_count);
if ((handled > 1) ||
/* the next nmi could be a back-to-back nmi */
((__this_cpu_read(pmu_nmi.marked) == this_nmi) &&
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index d90272e..b098092 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LA
static inline void enter_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
+ BUG_ON(this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
- percpu_write(paravirt_lazy_mode, mode);
+ this_cpu_write(paravirt_lazy_mode, mode);
}
static void leave_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != mode);
+ BUG_ON(this_cpu_read(paravirt_lazy_mode) != mode);
- percpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
+ this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
}
void paravirt_enter_lazy_mmu(void)
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev)
{
BUG_ON(preemptible());
- if (percpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
+ if (this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
arch_leave_lazy_mmu_mode();
set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES);
}
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
if (in_interrupt())
return PARAVIRT_LAZY_NONE;
- return percpu_read(paravirt_lazy_mode);
+ return this_cpu_read(paravirt_lazy_mode);
}
void arch_flush_lazy_mmu_mode(void)
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 7a3b651..f286be1 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -373,7 +373,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
if (prev->gs | next->gs)
lazy_load_gs(next->gs);
- percpu_write(current_task, next_p);
+ this_cpu_write(current_task, next_p);
return prev_p;
}
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index f693e44..35fdc14 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -73,7 +73,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);
void enter_idle(void)
{
- percpu_write(is_idle, 1);
+ this_cpu_write(is_idle, 1);
atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
}
@@ -336,7 +336,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,
load_gs_index(0);
regs->ip = new_ip;
regs->sp = new_sp;
- percpu_write(old_rsp, new_sp);
+ this_cpu_write(old_rsp, new_sp);
regs->cs = _cs;
regs->ss = _ss;
regs->flags = X86_EFLAGS_IF;
@@ -470,11 +470,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
/*
* Switch the PDA and FPU contexts.
*/
- prev->usersp = percpu_read(old_rsp);
- percpu_write(old_rsp, next->usersp);
- percpu_write(current_task, next_p);
+ prev->usersp = this_cpu_read(old_rsp);
+ this_cpu_write(old_rsp, next->usersp);
+ this_cpu_write(current_task, next_p);
- percpu_write(kernel_stack,
+ this_cpu_write(kernel_stack,
(unsigned long)task_stack_page(next_p) +
THREAD_SIZE - KERNEL_STACK_OFFSET);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 84a28ea..f46cb15 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
static int kvm_is_in_guest(void)
{
- return percpu_read(current_vcpu) != NULL;
+ return this_cpu_read(current_vcpu) != NULL;
}
static int kvm_is_user_mode(void)
{
int user_mode = 3;
- if (percpu_read(current_vcpu))
- user_mode = kvm_x86_ops->get_cpl(percpu_read(current_vcpu));
+ if (this_cpu_read(current_vcpu))
+ user_mode = kvm_x86_ops->get_cpl(this_cpu_read(current_vcpu));
return user_mode != 0;
}
@@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void)
{
unsigned long ip = 0;
- if (percpu_read(current_vcpu))
- ip = kvm_rip_read(percpu_read(current_vcpu));
+ if (this_cpu_read(current_vcpu))
+ ip = kvm_rip_read(this_cpu_read(current_vcpu));
return ip;
}
@@ -5055,13 +5055,13 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {
void kvm_before_handle_nmi(struct kvm_vcpu *vcpu)
{
- percpu_write(current_vcpu, vcpu);
+ this_cpu_write(current_vcpu, vcpu);
}
EXPORT_SYMBOL_GPL(kvm_before_handle_nmi);
void kvm_after_handle_nmi(struct kvm_vcpu *vcpu)
{
- percpu_write(current_vcpu, NULL);
+ this_cpu_write(current_vcpu, NULL);
}
EXPORT_SYMBOL_GPL(kvm_after_handle_nmi);
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index d6c0418..3804471 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);
*/
void leave_mm(int cpu)
{
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
BUG();
cpumask_clear_cpu(cpu,
- mm_cpumask(percpu_read(cpu_tlbstate.active_mm)));
+ mm_cpumask(this_cpu_read(cpu_tlbstate.active_mm)));
load_cr3(swapper_pg_dir);
}
EXPORT_SYMBOL_GPL(leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)
* BUG();
*/
- if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
+ if (f->flush_mm == this_cpu_read(cpu_tlbstate.active_mm)) {
+ if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
if (f->flush_va == TLB_FLUSH_ALL)
local_flush_tlb();
else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
static void do_flush_tlb_all(void *info)
{
__flush_tlb_all();
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
+ if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
leave_mm(smp_processor_id());
}
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 2d69617..c4e362a 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -776,11 +776,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);
static unsigned long xen_read_cr0(void)
{
- unsigned long cr0 = percpu_read(xen_cr0_value);
+ unsigned long cr0 = this_cpu_read(xen_cr0_value);
if (unlikely(cr0 == 0)) {
cr0 = native_read_cr0();
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
}
return cr0;
@@ -790,7 +790,7 @@ static void xen_write_cr0(unsigned long cr0)
{
struct multicall_space mcs;
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
/* Only pay attention to cr0.TS; everything else is
ignored. */
diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c
index 8bbb465..1573376 100644
--- a/arch/x86/xen/irq.c
+++ b/arch/x86/xen/irq.c
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)
struct vcpu_info *vcpu;
unsigned long flags;
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
/* flag has opposite sense of mask */
flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = flags;
preempt_enable_no_resched();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- percpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
+ this_cpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
preempt_enable_no_resched();
}
PV_CALLEE_SAVE_REGS_THUNK(xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)
the caller is confused and is trying to re-enable interrupts
on an indeterminate processor. */
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = 0;
/* Doesn't matter if we get preempted here, because any
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 3dd53f9..6a7aaac 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1106,14 +1106,14 @@ static void drop_other_mm_ref(void *info)
struct mm_struct *mm = info;
struct mm_struct *active_mm;
- active_mm = percpu_read(cpu_tlbstate.active_mm);
+ active_mm = this_cpu_read(cpu_tlbstate.active_mm);
- if (active_mm == mm && percpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
+ if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
leave_mm(smp_processor_id());
/* If this cpu still has a stale cr3 reference, then make sure
it has been flushed. */
- if (percpu_read(xen_current_cr3) == __pa(mm->pgd))
+ if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd))
load_cr3(swapper_pg_dir);
}
@@ -1220,17 +1220,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)
static void xen_write_cr2(unsigned long cr2)
{
- percpu_read(xen_vcpu)->arch.cr2 = cr2;
+ this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
}
static unsigned long xen_read_cr2(void)
{
- return percpu_read(xen_vcpu)->arch.cr2;
+ return this_cpu_read(xen_vcpu)->arch.cr2;
}
unsigned long xen_read_cr2_direct(void)
{
- return percpu_read(xen_vcpu_info.arch.cr2);
+ return this_cpu_read(xen_vcpu_info.arch.cr2);
}
static void xen_flush_tlb(void)
@@ -1313,12 +1313,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
static unsigned long xen_read_cr3(void)
{
- return percpu_read(xen_cr3);
+ return this_cpu_read(xen_cr3);
}
static void set_current_cr3(void *v)
{
- percpu_write(xen_current_cr3, (unsigned long)v);
+ this_cpu_write(xen_current_cr3, (unsigned long)v);
}
static void __xen_write_cr3(bool kernel, unsigned long cr3)
@@ -1341,7 +1341,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)
xen_extend_mmuext_op(&op);
if (kernel) {
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
/* Update xen_current_cr3 once the batch has actually
been submitted. */
@@ -1357,7 +1357,7 @@ static void xen_write_cr3(unsigned long cr3)
/* Update while interrupts are disabled, so its atomic with
respect to ipis */
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
__xen_write_cr3(true, cr3);
diff --git a/arch/x86/xen/multicalls.h b/arch/x86/xen/multicalls.h
index dee79b7..9c2e74f 100644
--- a/arch/x86/xen/multicalls.h
+++ b/arch/x86/xen/multicalls.h
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)
xen_mc_flush();
/* restore flags saved in xen_mc_batch */
- local_irq_restore(percpu_read(xen_mc_irq_flags));
+ local_irq_restore(this_cpu_read(xen_mc_irq_flags));
}
/* Set up a callback to be called when the current batch is flushed */
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 041d4fe..449f868 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)
xen_setup_cpu_clockevents();
set_cpu_online(cpu, true);
- percpu_write(cpu_state, CPU_ONLINE);
+ this_cpu_write(cpu_state, CPU_ONLINE);
wmb();
/* We can take interrupts now: we're officially "up". */
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 9ca008f..aad7e1c 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -165,59 +165,6 @@ extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
#define alloc_percpu(type) \
(typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
-/*
- * Optional methods for optimized non-lvalue per-cpu variable access.
- *
- * @var can be a percpu variable or a field of it and its size should
- * equal char, int or long. percpu_read() evaluates to a lvalue and
- * all others to void.
- *
- * These operations are guaranteed to be atomic w.r.t. preemption.
- * The generic versions use plain get/put_cpu_var(). Archs are
- * encouraged to implement single-instruction alternatives which don't
- * require preemption protection.
- */
-#ifndef percpu_read
-# define percpu_read(var) \
- ({ \
- typeof(var) *pr_ptr__ = &(var); \
- typeof(var) pr_ret__; \
- pr_ret__ = get_cpu_var(*pr_ptr__); \
- put_cpu_var(*pr_ptr__); \
- pr_ret__; \
- })
-#endif
-
-#define __percpu_generic_to_op(var, val, op) \
-do { \
- typeof(var) *pgto_ptr__ = &(var); \
- get_cpu_var(*pgto_ptr__) op val; \
- put_cpu_var(*pgto_ptr__); \
-} while (0)
-
-#ifndef percpu_write
-# define percpu_write(var, val) __percpu_generic_to_op(var, (val), =)
-#endif
-
-#ifndef percpu_add
-# define percpu_add(var, val) __percpu_generic_to_op(var, (val), +=)
-#endif
-
-#ifndef percpu_sub
-# define percpu_sub(var, val) __percpu_generic_to_op(var, (val), -=)
-#endif
-
-#ifndef percpu_and
-# define percpu_and(var, val) __percpu_generic_to_op(var, (val), &=)
-#endif
-
-#ifndef percpu_or
-# define percpu_or(var, val) __percpu_generic_to_op(var, (val), |=)
-#endif
-
-#ifndef percpu_xor
-# define percpu_xor(var, val) __percpu_generic_to_op(var, (val), ^=)
-#endif
/*
* Branching function to split up a function into a set of functions that
diff --git a/include/linux/topology.h b/include/linux/topology.h
index fc839bf..bf64eac 100644
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -235,7 +235,7 @@ static inline int cpu_to_node(int cpu)
#ifndef set_numa_node
static inline void set_numa_node(int node)
{
- percpu_write(numa_node, node);
+ this_cpu_write(numa_node, node);
}
#endif
@@ -270,7 +270,7 @@ DECLARE_PER_CPU(int, _numa_mem_);
#ifndef set_numa_mem
static inline void set_numa_mem(int node)
{
- percpu_write(_numa_mem_, node);
+ this_cpu_write(_numa_mem_, node);
}
#endif
diff --git a/net/netfilter/xt_TEE.c b/net/netfilter/xt_TEE.c
index 5f054a0..e78103d 100644
--- a/net/netfilter/xt_TEE.c
+++ b/net/netfilter/xt_TEE.c
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
const struct xt_tee_tginfo *info = par->targinfo;
struct iphdr *iph;
- if (percpu_read(tee_active))
+ if (this_cpu_read(tee_active))
return XT_CONTINUE;
/*
* Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
ip_send_check(iph);
if (tee_tg_route4(skb, info)) {
- percpu_write(tee_active, true);
+ this_cpu_write(tee_active, true);
ip_local_out(skb);
- percpu_write(tee_active, false);
+ this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
{
const struct xt_tee_tginfo *info = par->targinfo;
- if (percpu_read(tee_active))
+ if (this_cpu_read(tee_active))
return XT_CONTINUE;
skb = pskb_copy(skb, GFP_ATOMIC);
if (skb == NULL)
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
--iph->hop_limit;
}
if (tee_tg_route6(skb, info)) {
- percpu_write(tee_active, true);
+ this_cpu_write(tee_active, true);
ip6_local_out(skb);
- percpu_write(tee_active, false);
+ this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
diff --git a/net/socket.c b/net/socket.c
index ffe92ca..4b62ca9 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -479,7 +479,7 @@ static struct socket *sock_alloc(void)
inode->i_uid = current_fsuid();
inode->i_gid = current_fsgid();
- percpu_add(sockets_in_use, 1);
+ this_cpu_add(sockets_in_use, 1);
return sock;
}
@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)
if (rcu_dereference_protected(sock->wq, 1)->fasync_list)
printk(KERN_ERR "sock_release: fasync list not empty!\n");
- percpu_sub(sockets_in_use, 1);
+ this_cpu_sub(sockets_in_use, 1);
if (!sock->file) {
iput(SOCK_INODE(sock));
return;
--
1.7.5.1
^ permalink raw reply related [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-10 9:10 [PATCH] Code clean up for percpu_xxx() functions Alex,Shi
@ 2011-10-10 17:08 ` Christoph Lameter
2011-10-11 9:19 ` Alex,Shi
0 siblings, 1 reply; 42+ messages in thread
From: Christoph Lameter @ 2011-10-10 17:08 UTC (permalink / raw)
To: Alex,Shi; +Cc: tj, linux-kernel@vger.kernel.org, eric.dumazet, Huang, Ying
On Mon, 10 Oct 2011, Alex,Shi wrote:
> index 3470c9d..d1de56c 100644
> --- a/arch/x86/include/asm/percpu.h
> +++ b/arch/x86/include/asm/percpu.h
> @@ -46,7 +46,7 @@
>
> #ifdef CONFIG_SMP
> #define __percpu_prefix "%%"__stringify(__percpu_seg)":"
> -#define __my_cpu_offset percpu_read(this_cpu_off)
> +#define __my_cpu_offset this_cpu_read(this_cpu_off)
Should this not be __this_cpu_read() instead?
> * per-thread variables implemented as per-cpu variables and thus
> * stable for the duration of the respective task.
> */
> -#define percpu_read(var) percpu_from_op("mov", var, "m" (var))
> -#define percpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
> -#define percpu_write(var, val) percpu_to_op("mov", var, val)
> -#define percpu_add(var, val) percpu_add_op(var, val)
> -#define percpu_sub(var, val) percpu_add_op(var, -(val))
> -#define percpu_and(var, val) percpu_to_op("and", var, val)
> -#define percpu_or(var, val) percpu_to_op("or", var, val)
> -#define percpu_xor(var, val) percpu_to_op("xor", var, val)
> -#define percpu_inc(var) percpu_unary_op("inc", var)
> +#define this_cpu_read(var) percpu_from_op("mov", var, "m" (var))
> +#define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
> +#define this_cpu_write(var, val) percpu_to_op("mov", var, val)
> +#define this_cpu_add(var, val) percpu_add_op(var, val)
> +#define this_cpu_sub(var, val) percpu_add_op(var, -(val))
> +#define this_cpu_and(var, val) percpu_to_op("and", var, val)
> +#define this_cpu_or(var, val) percpu_to_op("or", var, val)
> +#define this_cpu_xor(var, val) percpu_to_op("xor", var, val)
> +#define this_cpu_inc(var) percpu_unary_op("inc", var)
Why define these operations? They are already defined in the form of
this_cpu_add_1/2/4/8 and include/linux/percpu.h branches out to them.
> @@ -522,7 +522,7 @@ void sock_release(struct socket *sock)
> if (rcu_dereference_protected(sock->wq, 1)->fasync_list)
> printk(KERN_ERR "sock_release: fasync list not empty!\n");
>
> - percpu_sub(sockets_in_use, 1);
> + this_cpu_sub(sockets_in_use, 1);
> if (!sock->file) {
> iput(SOCK_INODE(sock));
> return;
In general __this_cpu_xx operations can be used instead of this_cpu_xx
operations if the context is already preempt safe. There is no change of
the code generated for x86 in those cases. But the fallback code on other
platforms can generate more efficient code.
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-10 17:08 ` Christoph Lameter
@ 2011-10-11 9:19 ` Alex,Shi
2011-10-11 14:13 ` Christoph Lameter
0 siblings, 1 reply; 42+ messages in thread
From: Alex,Shi @ 2011-10-11 9:19 UTC (permalink / raw)
To: Christoph Lameter
Cc: tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying
On Tue, 2011-10-11 at 01:08 +0800, Christoph Lameter wrote:
> On Mon, 10 Oct 2011, Alex,Shi wrote:
>
> > index 3470c9d..d1de56c 100644
> > --- a/arch/x86/include/asm/percpu.h
> > +++ b/arch/x86/include/asm/percpu.h
> > @@ -46,7 +46,7 @@
> >
> > #ifdef CONFIG_SMP
> > #define __percpu_prefix "%%"__stringify(__percpu_seg)":"
> > -#define __my_cpu_offset percpu_read(this_cpu_off)
> > +#define __my_cpu_offset this_cpu_read(this_cpu_off)
>
> Should this not be __this_cpu_read() instead?
OK.
>
> > * per-thread variables implemented as per-cpu variables and thus
> > * stable for the duration of the respective task.
> > */
> > -#define percpu_read(var) percpu_from_op("mov", var, "m" (var))
> > -#define percpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
> > -#define percpu_write(var, val) percpu_to_op("mov", var, val)
> > -#define percpu_add(var, val) percpu_add_op(var, val)
> > -#define percpu_sub(var, val) percpu_add_op(var, -(val))
> > -#define percpu_and(var, val) percpu_to_op("and", var, val)
> > -#define percpu_or(var, val) percpu_to_op("or", var, val)
> > -#define percpu_xor(var, val) percpu_to_op("xor", var, val)
> > -#define percpu_inc(var) percpu_unary_op("inc", var)
> > +#define this_cpu_read(var) percpu_from_op("mov", var, "m" (var))
> > +#define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
> > +#define this_cpu_write(var, val) percpu_to_op("mov", var, val)
> > +#define this_cpu_add(var, val) percpu_add_op(var, val)
> > +#define this_cpu_sub(var, val) percpu_add_op(var, -(val))
> > +#define this_cpu_and(var, val) percpu_to_op("and", var, val)
> > +#define this_cpu_or(var, val) percpu_to_op("or", var, val)
> > +#define this_cpu_xor(var, val) percpu_to_op("xor", var, val)
> > +#define this_cpu_inc(var) percpu_unary_op("inc", var)
>
> Why define these operations? They are already defined in the form of
> this_cpu_add_1/2/4/8 and include/linux/percpu.h branches out to them.
percpu_read() used in asm/percpu.h yet. Anyway, I removed definitions
here and do a work around for the function as following:
@@ -551,7 +544,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr,
{
unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG;
- return ((1UL << (nr % BITS_PER_LONG)) & percpu_read(*a)) != 0;
+#ifdef CONFIG_X86_64
+ return ((1UL << (nr % BITS_PER_LONG)) & this_cpu_read_8(*a)) != 0;
+#else
+ return ((1UL << (nr % BITS_PER_LONG)) & this_cpu_read_4(*a)) != 0;
+#endif
>
> > @@ -522,7 +522,7 @@ void sock_release(struct socket *sock)
> > if (rcu_dereference_protected(sock->wq, 1)->fasync_list)
> > printk(KERN_ERR "sock_release: fasync list not empty!\n");
> >
> > - percpu_sub(sockets_in_use, 1);
> > + this_cpu_sub(sockets_in_use, 1);
> > if (!sock->file) {
> > iput(SOCK_INODE(sock));
> > return;
>
> In general __this_cpu_xx operations can be used instead of this_cpu_xx
> operations if the context is already preempt safe. There is no change of
> the code generated for x86 in those cases. But the fallback code on other
> platforms can generate more efficient code.
>
Good reminder!
Currently, only x86 has specific percpu_xxx() definition. Other
platforms use generic version. And for this function, it is not preempt
safe. So, it should be fine here.
=========
Since percpu_xxx() serial functions are duplicate with this_cpu_xxx().
Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in
using. This patch is just do code clean up, no function change.
Signed-off-by: Alex Shi <alex.shi@intel.com>
---
arch/x86/include/asm/current.h | 2 +-
arch/x86/include/asm/hardirq.h | 8 ++--
arch/x86/include/asm/irq_regs.h | 4 +-
arch/x86/include/asm/mmu_context.h | 12 ++++----
arch/x86/include/asm/percpu.h | 25 +++++++---------
arch/x86/include/asm/smp.h | 4 +-
arch/x86/include/asm/stackprotector.h | 4 +-
arch/x86/include/asm/thread_info.h | 2 +-
arch/x86/include/asm/tlbflush.h | 4 +-
arch/x86/kernel/cpu/common.c | 2 +-
arch/x86/kernel/cpu/mcheck/mce.c | 4 +-
arch/x86/kernel/cpu/perf_event.c | 4 +-
arch/x86/kernel/paravirt.c | 12 ++++----
arch/x86/kernel/process_32.c | 2 +-
arch/x86/kernel/process_64.c | 12 ++++----
arch/x86/kvm/x86.c | 14 ++++----
arch/x86/mm/tlb.c | 10 +++---
arch/x86/xen/enlighten.c | 6 ++--
arch/x86/xen/irq.c | 8 ++--
arch/x86/xen/mmu.c | 20 ++++++------
arch/x86/xen/multicalls.h | 2 +-
arch/x86/xen/smp.c | 2 +-
include/linux/percpu.h | 53 ---------------------------------
include/linux/topology.h | 4 +-
net/netfilter/xt_TEE.c | 12 ++++----
net/socket.c | 4 +-
26 files changed, 90 insertions(+), 146 deletions(-)
diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index 4d447b7..9476c04 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task);
static __always_inline struct task_struct *get_current(void)
{
- return percpu_read_stable(current_task);
+ return this_cpu_read_stable(current_task);
}
#define current get_current()
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 55e4de6..fc594da 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -35,14 +35,14 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
#define __ARCH_IRQ_STAT
-#define inc_irq_stat(member) percpu_inc(irq_stat.member)
+#define inc_irq_stat(member) this_cpu_inc(irq_stat.member)
-#define local_softirq_pending() percpu_read(irq_stat.__softirq_pending)
+#define local_softirq_pending() this_cpu_read(irq_stat.__softirq_pending)
#define __ARCH_SET_SOFTIRQ_PENDING
-#define set_softirq_pending(x) percpu_write(irq_stat.__softirq_pending, (x))
-#define or_softirq_pending(x) percpu_or(irq_stat.__softirq_pending, (x))
+#define set_softirq_pending(x) this_cpu_write(irq_stat.__softirq_pending, (x))
+#define or_softirq_pending(x) this_cpu_or(irq_stat.__softirq_pending, (x))
extern void ack_bad_irq(unsigned int irq);
diff --git a/arch/x86/include/asm/irq_regs.h b/arch/x86/include/asm/irq_regs.h
index 7784322..d82250b 100644
--- a/arch/x86/include/asm/irq_regs.h
+++ b/arch/x86/include/asm/irq_regs.h
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);
static inline struct pt_regs *get_irq_regs(void)
{
- return percpu_read(irq_regs);
+ return this_cpu_read(irq_regs);
}
static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
struct pt_regs *old_regs;
old_regs = get_irq_regs();
- percpu_write(irq_regs, new_regs);
+ this_cpu_write(irq_regs, new_regs);
return old_regs;
}
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 6902152..cdbf367 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);
static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
{
#ifdef CONFIG_SMP
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
- percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
+ if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
#endif
}
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
if (likely(prev != next)) {
#ifdef CONFIG_SMP
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- percpu_write(cpu_tlbstate.active_mm, next);
+ this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ this_cpu_write(cpu_tlbstate.active_mm, next);
#endif
cpumask_set_cpu(cpu, mm_cpumask(next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
}
#ifdef CONFIG_SMP
else {
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
+ this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ BUG_ON(this_cpu_read(cpu_tlbstate.active_mm) != next);
if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next))) {
/* We were in lazy tlb mode and leave_mm disabled
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 3470c9d..66b50cb 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -46,7 +46,7 @@
#ifdef CONFIG_SMP
#define __percpu_prefix "%%"__stringify(__percpu_seg)":"
-#define __my_cpu_offset percpu_read(this_cpu_off)
+#define __my_cpu_offset __this_cpu_read(this_cpu_off)
/*
* Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,16 @@ do { \
})
/*
- * percpu_read() makes gcc load the percpu variable every time it is
- * accessed while percpu_read_stable() allows the value to be cached.
- * percpu_read_stable() is more efficient and can be used if its value
+ * this_cpu_read() makes gcc load the percpu variable every time it is
+ * accessed while this_cpu_read_stable() allows the value to be cached.
+ * this_cpu_read_stable() is more efficient and can be used if its value
* is guaranteed to be valid across cpus. The current users include
* get_current() and get_thread_info() both of which are actually
* per-thread variables implemented as per-cpu variables and thus
* stable for the duration of the respective task.
*/
-#define percpu_read(var) percpu_from_op("mov", var, "m" (var))
-#define percpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
-#define percpu_write(var, val) percpu_to_op("mov", var, val)
-#define percpu_add(var, val) percpu_add_op(var, val)
-#define percpu_sub(var, val) percpu_add_op(var, -(val))
-#define percpu_and(var, val) percpu_to_op("and", var, val)
-#define percpu_or(var, val) percpu_to_op("or", var, val)
-#define percpu_xor(var, val) percpu_to_op("xor", var, val)
-#define percpu_inc(var) percpu_unary_op("inc", var)
+#define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
+#define this_cpu_inc(var) percpu_unary_op("inc", var)
#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +544,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr,
{
unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG;
- return ((1UL << (nr % BITS_PER_LONG)) & percpu_read(*a)) != 0;
+#ifdef CONFIG_X86_64
+ return ((1UL << (nr % BITS_PER_LONG)) & this_cpu_read_8(*a)) != 0;
+#else
+ return ((1UL << (nr % BITS_PER_LONG)) & this_cpu_read_4(*a)) != 0;
+#endif
}
static inline int x86_this_cpu_variable_test_bit(int nr,
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 73b11bc..dd5749b 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;
* from the initial startup. We map APIC_BASE very early in page_setup(),
* so this is correct in the x86 case.
*/
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
extern int safe_smp_processor_id(void);
#elif defined(CONFIG_X86_64_SMP)
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
#define stack_smp_processor_id() \
({ \
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
index 1575177..3d7a009 100644
--- a/arch/x86/include/asm/stackprotector.h
+++ b/arch/x86/include/asm/stackprotector.h
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void)
current->stack_canary = canary;
#ifdef CONFIG_X86_64
- percpu_write(irq_stack_union.stack_canary, canary);
+ this_cpu_write(irq_stack_union.stack_canary, canary);
#else
- percpu_write(stack_canary.canary, canary);
+ this_cpu_write(stack_canary.canary, canary);
#endif
}
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index a1fe5c1..8b19667 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);
static inline struct thread_info *current_thread_info(void)
{
struct thread_info *ti;
- ti = (void *)(percpu_read_stable(kernel_stack) +
+ ti = (void *)(this_cpu_read_stable(kernel_stack) +
KERNEL_STACK_OFFSET - THREAD_SIZE);
return ti;
}
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 169be89..200c09c 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
static inline void reset_lazy_tlbstate(void)
{
- percpu_write(cpu_tlbstate.state, 0);
- percpu_write(cpu_tlbstate.active_mm, &init_mm);
+ this_cpu_write(cpu_tlbstate.state, 0);
+ this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
}
#endif /* SMP */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 6218439..10513ed 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1158,7 +1158,7 @@ void __cpuinit cpu_init(void)
oist = &per_cpu(orig_ist, cpu);
#ifdef CONFIG_NUMA
- if (cpu != 0 && percpu_read(numa_node) == 0 &&
+ if (cpu != 0 && this_cpu_read(numa_node) == 0 &&
early_cpu_to_node(cpu) != NUMA_NO_NODE)
set_numa_node(early_cpu_to_node(cpu));
#endif
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 08363b0..adbb598 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -512,7 +512,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
struct mce m;
int i;
- percpu_inc(mce_poll_count);
+ this_cpu_inc(mce_poll_count);
mce_gather_info(&m, NULL);
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
atomic_inc(&mce_entry);
- percpu_inc(mce_exception_count);
+ this_cpu_inc(mce_exception_count);
if (notify_die(DIE_NMI, "machine check", regs, error_code,
18, SIGKILL) == NOTIFY_STOP)
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index cfa62ec..ca037c0 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -1389,7 +1389,7 @@ perf_event_nmi_handler(struct notifier_block *self,
case DIE_NMI:
break;
case DIE_NMIUNKNOWN:
- this_nmi = percpu_read(irq_stat.__nmi_count);
+ this_nmi = this_cpu_read(irq_stat.__nmi_count);
if (this_nmi != __this_cpu_read(pmu_nmi.marked))
/* let the kernel handle the unknown nmi */
return NOTIFY_DONE;
@@ -1409,7 +1409,7 @@ perf_event_nmi_handler(struct notifier_block *self,
if (!handled)
return NOTIFY_DONE;
- this_nmi = percpu_read(irq_stat.__nmi_count);
+ this_nmi = this_cpu_read(irq_stat.__nmi_count);
if ((handled > 1) ||
/* the next nmi could be a back-to-back nmi */
((__this_cpu_read(pmu_nmi.marked) == this_nmi) &&
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index d90272e..b098092 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LA
static inline void enter_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
+ BUG_ON(this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
- percpu_write(paravirt_lazy_mode, mode);
+ this_cpu_write(paravirt_lazy_mode, mode);
}
static void leave_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != mode);
+ BUG_ON(this_cpu_read(paravirt_lazy_mode) != mode);
- percpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
+ this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
}
void paravirt_enter_lazy_mmu(void)
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev)
{
BUG_ON(preemptible());
- if (percpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
+ if (this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
arch_leave_lazy_mmu_mode();
set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES);
}
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
if (in_interrupt())
return PARAVIRT_LAZY_NONE;
- return percpu_read(paravirt_lazy_mode);
+ return this_cpu_read(paravirt_lazy_mode);
}
void arch_flush_lazy_mmu_mode(void)
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 7a3b651..f286be1 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -373,7 +373,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
if (prev->gs | next->gs)
lazy_load_gs(next->gs);
- percpu_write(current_task, next_p);
+ this_cpu_write(current_task, next_p);
return prev_p;
}
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index f693e44..35fdc14 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -73,7 +73,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);
void enter_idle(void)
{
- percpu_write(is_idle, 1);
+ this_cpu_write(is_idle, 1);
atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
}
@@ -336,7 +336,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,
load_gs_index(0);
regs->ip = new_ip;
regs->sp = new_sp;
- percpu_write(old_rsp, new_sp);
+ this_cpu_write(old_rsp, new_sp);
regs->cs = _cs;
regs->ss = _ss;
regs->flags = X86_EFLAGS_IF;
@@ -470,11 +470,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
/*
* Switch the PDA and FPU contexts.
*/
- prev->usersp = percpu_read(old_rsp);
- percpu_write(old_rsp, next->usersp);
- percpu_write(current_task, next_p);
+ prev->usersp = this_cpu_read(old_rsp);
+ this_cpu_write(old_rsp, next->usersp);
+ this_cpu_write(current_task, next_p);
- percpu_write(kernel_stack,
+ this_cpu_write(kernel_stack,
(unsigned long)task_stack_page(next_p) +
THREAD_SIZE - KERNEL_STACK_OFFSET);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 84a28ea..f46cb15 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
static int kvm_is_in_guest(void)
{
- return percpu_read(current_vcpu) != NULL;
+ return this_cpu_read(current_vcpu) != NULL;
}
static int kvm_is_user_mode(void)
{
int user_mode = 3;
- if (percpu_read(current_vcpu))
- user_mode = kvm_x86_ops->get_cpl(percpu_read(current_vcpu));
+ if (this_cpu_read(current_vcpu))
+ user_mode = kvm_x86_ops->get_cpl(this_cpu_read(current_vcpu));
return user_mode != 0;
}
@@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void)
{
unsigned long ip = 0;
- if (percpu_read(current_vcpu))
- ip = kvm_rip_read(percpu_read(current_vcpu));
+ if (this_cpu_read(current_vcpu))
+ ip = kvm_rip_read(this_cpu_read(current_vcpu));
return ip;
}
@@ -5055,13 +5055,13 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {
void kvm_before_handle_nmi(struct kvm_vcpu *vcpu)
{
- percpu_write(current_vcpu, vcpu);
+ this_cpu_write(current_vcpu, vcpu);
}
EXPORT_SYMBOL_GPL(kvm_before_handle_nmi);
void kvm_after_handle_nmi(struct kvm_vcpu *vcpu)
{
- percpu_write(current_vcpu, NULL);
+ this_cpu_write(current_vcpu, NULL);
}
EXPORT_SYMBOL_GPL(kvm_after_handle_nmi);
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index d6c0418..3804471 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);
*/
void leave_mm(int cpu)
{
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
BUG();
cpumask_clear_cpu(cpu,
- mm_cpumask(percpu_read(cpu_tlbstate.active_mm)));
+ mm_cpumask(this_cpu_read(cpu_tlbstate.active_mm)));
load_cr3(swapper_pg_dir);
}
EXPORT_SYMBOL_GPL(leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)
* BUG();
*/
- if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
+ if (f->flush_mm == this_cpu_read(cpu_tlbstate.active_mm)) {
+ if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
if (f->flush_va == TLB_FLUSH_ALL)
local_flush_tlb();
else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
static void do_flush_tlb_all(void *info)
{
__flush_tlb_all();
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
+ if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
leave_mm(smp_processor_id());
}
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 2d69617..c4e362a 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -776,11 +776,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);
static unsigned long xen_read_cr0(void)
{
- unsigned long cr0 = percpu_read(xen_cr0_value);
+ unsigned long cr0 = this_cpu_read(xen_cr0_value);
if (unlikely(cr0 == 0)) {
cr0 = native_read_cr0();
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
}
return cr0;
@@ -790,7 +790,7 @@ static void xen_write_cr0(unsigned long cr0)
{
struct multicall_space mcs;
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
/* Only pay attention to cr0.TS; everything else is
ignored. */
diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c
index 8bbb465..1573376 100644
--- a/arch/x86/xen/irq.c
+++ b/arch/x86/xen/irq.c
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)
struct vcpu_info *vcpu;
unsigned long flags;
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
/* flag has opposite sense of mask */
flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = flags;
preempt_enable_no_resched();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- percpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
+ this_cpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
preempt_enable_no_resched();
}
PV_CALLEE_SAVE_REGS_THUNK(xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)
the caller is confused and is trying to re-enable interrupts
on an indeterminate processor. */
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = 0;
/* Doesn't matter if we get preempted here, because any
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 3dd53f9..6a7aaac 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1106,14 +1106,14 @@ static void drop_other_mm_ref(void *info)
struct mm_struct *mm = info;
struct mm_struct *active_mm;
- active_mm = percpu_read(cpu_tlbstate.active_mm);
+ active_mm = this_cpu_read(cpu_tlbstate.active_mm);
- if (active_mm == mm && percpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
+ if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
leave_mm(smp_processor_id());
/* If this cpu still has a stale cr3 reference, then make sure
it has been flushed. */
- if (percpu_read(xen_current_cr3) == __pa(mm->pgd))
+ if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd))
load_cr3(swapper_pg_dir);
}
@@ -1220,17 +1220,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)
static void xen_write_cr2(unsigned long cr2)
{
- percpu_read(xen_vcpu)->arch.cr2 = cr2;
+ this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
}
static unsigned long xen_read_cr2(void)
{
- return percpu_read(xen_vcpu)->arch.cr2;
+ return this_cpu_read(xen_vcpu)->arch.cr2;
}
unsigned long xen_read_cr2_direct(void)
{
- return percpu_read(xen_vcpu_info.arch.cr2);
+ return this_cpu_read(xen_vcpu_info.arch.cr2);
}
static void xen_flush_tlb(void)
@@ -1313,12 +1313,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
static unsigned long xen_read_cr3(void)
{
- return percpu_read(xen_cr3);
+ return this_cpu_read(xen_cr3);
}
static void set_current_cr3(void *v)
{
- percpu_write(xen_current_cr3, (unsigned long)v);
+ this_cpu_write(xen_current_cr3, (unsigned long)v);
}
static void __xen_write_cr3(bool kernel, unsigned long cr3)
@@ -1341,7 +1341,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)
xen_extend_mmuext_op(&op);
if (kernel) {
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
/* Update xen_current_cr3 once the batch has actually
been submitted. */
@@ -1357,7 +1357,7 @@ static void xen_write_cr3(unsigned long cr3)
/* Update while interrupts are disabled, so its atomic with
respect to ipis */
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
__xen_write_cr3(true, cr3);
diff --git a/arch/x86/xen/multicalls.h b/arch/x86/xen/multicalls.h
index dee79b7..9c2e74f 100644
--- a/arch/x86/xen/multicalls.h
+++ b/arch/x86/xen/multicalls.h
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)
xen_mc_flush();
/* restore flags saved in xen_mc_batch */
- local_irq_restore(percpu_read(xen_mc_irq_flags));
+ local_irq_restore(this_cpu_read(xen_mc_irq_flags));
}
/* Set up a callback to be called when the current batch is flushed */
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 041d4fe..449f868 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)
xen_setup_cpu_clockevents();
set_cpu_online(cpu, true);
- percpu_write(cpu_state, CPU_ONLINE);
+ this_cpu_write(cpu_state, CPU_ONLINE);
wmb();
/* We can take interrupts now: we're officially "up". */
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 9ca008f..aad7e1c 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -165,59 +165,6 @@ extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
#define alloc_percpu(type) \
(typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
-/*
- * Optional methods for optimized non-lvalue per-cpu variable access.
- *
- * @var can be a percpu variable or a field of it and its size should
- * equal char, int or long. percpu_read() evaluates to a lvalue and
- * all others to void.
- *
- * These operations are guaranteed to be atomic w.r.t. preemption.
- * The generic versions use plain get/put_cpu_var(). Archs are
- * encouraged to implement single-instruction alternatives which don't
- * require preemption protection.
- */
-#ifndef percpu_read
-# define percpu_read(var) \
- ({ \
- typeof(var) *pr_ptr__ = &(var); \
- typeof(var) pr_ret__; \
- pr_ret__ = get_cpu_var(*pr_ptr__); \
- put_cpu_var(*pr_ptr__); \
- pr_ret__; \
- })
-#endif
-
-#define __percpu_generic_to_op(var, val, op) \
-do { \
- typeof(var) *pgto_ptr__ = &(var); \
- get_cpu_var(*pgto_ptr__) op val; \
- put_cpu_var(*pgto_ptr__); \
-} while (0)
-
-#ifndef percpu_write
-# define percpu_write(var, val) __percpu_generic_to_op(var, (val), =)
-#endif
-
-#ifndef percpu_add
-# define percpu_add(var, val) __percpu_generic_to_op(var, (val), +=)
-#endif
-
-#ifndef percpu_sub
-# define percpu_sub(var, val) __percpu_generic_to_op(var, (val), -=)
-#endif
-
-#ifndef percpu_and
-# define percpu_and(var, val) __percpu_generic_to_op(var, (val), &=)
-#endif
-
-#ifndef percpu_or
-# define percpu_or(var, val) __percpu_generic_to_op(var, (val), |=)
-#endif
-
-#ifndef percpu_xor
-# define percpu_xor(var, val) __percpu_generic_to_op(var, (val), ^=)
-#endif
/*
* Branching function to split up a function into a set of functions that
diff --git a/include/linux/topology.h b/include/linux/topology.h
index fc839bf..bf64eac 100644
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -235,7 +235,7 @@ static inline int cpu_to_node(int cpu)
#ifndef set_numa_node
static inline void set_numa_node(int node)
{
- percpu_write(numa_node, node);
+ this_cpu_write(numa_node, node);
}
#endif
@@ -270,7 +270,7 @@ DECLARE_PER_CPU(int, _numa_mem_);
#ifndef set_numa_mem
static inline void set_numa_mem(int node)
{
- percpu_write(_numa_mem_, node);
+ this_cpu_write(_numa_mem_, node);
}
#endif
diff --git a/net/netfilter/xt_TEE.c b/net/netfilter/xt_TEE.c
index 5f054a0..e78103d 100644
--- a/net/netfilter/xt_TEE.c
+++ b/net/netfilter/xt_TEE.c
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
const struct xt_tee_tginfo *info = par->targinfo;
struct iphdr *iph;
- if (percpu_read(tee_active))
+ if (this_cpu_read(tee_active))
return XT_CONTINUE;
/*
* Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
ip_send_check(iph);
if (tee_tg_route4(skb, info)) {
- percpu_write(tee_active, true);
+ this_cpu_write(tee_active, true);
ip_local_out(skb);
- percpu_write(tee_active, false);
+ this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
{
const struct xt_tee_tginfo *info = par->targinfo;
- if (percpu_read(tee_active))
+ if (this_cpu_read(tee_active))
return XT_CONTINUE;
skb = pskb_copy(skb, GFP_ATOMIC);
if (skb == NULL)
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
--iph->hop_limit;
}
if (tee_tg_route6(skb, info)) {
- percpu_write(tee_active, true);
+ this_cpu_write(tee_active, true);
ip6_local_out(skb);
- percpu_write(tee_active, false);
+ this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
diff --git a/net/socket.c b/net/socket.c
index ffe92ca..4b62ca9 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -479,7 +479,7 @@ static struct socket *sock_alloc(void)
inode->i_uid = current_fsuid();
inode->i_gid = current_fsgid();
- percpu_add(sockets_in_use, 1);
+ this_cpu_add(sockets_in_use, 1);
return sock;
}
@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)
if (rcu_dereference_protected(sock->wq, 1)->fasync_list)
printk(KERN_ERR "sock_release: fasync list not empty!\n");
- percpu_sub(sockets_in_use, 1);
+ this_cpu_sub(sockets_in_use, 1);
if (!sock->file) {
iput(SOCK_INODE(sock));
return;
--
1.7.5.1
^ permalink raw reply related [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-11 9:19 ` Alex,Shi
@ 2011-10-11 14:13 ` Christoph Lameter
2011-10-12 1:29 ` Alex,Shi
0 siblings, 1 reply; 42+ messages in thread
From: Christoph Lameter @ 2011-10-11 14:13 UTC (permalink / raw)
To: Alex,Shi
Cc: tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying
On Tue, 11 Oct 2011, Alex,Shi wrote:
> percpu_read() used in asm/percpu.h yet. Anyway, I removed definitions
> here and do a work around for the function as following:
>
> @@ -551,7 +544,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr,
> {
> unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG;
>
> - return ((1UL << (nr % BITS_PER_LONG)) & percpu_read(*a)) != 0;
> +#ifdef CONFIG_X86_64
> + return ((1UL << (nr % BITS_PER_LONG)) & this_cpu_read_8(*a)) != 0;
> +#else
> + return ((1UL << (nr % BITS_PER_LONG)) & this_cpu_read_4(*a)) != 0;
> +#endif
OK that looks okay.
> Signed-off-by: Alex Shi <alex.shi@intel.com>
Is this a revised patch? I do not see a use of the __this_cpu_xx ops.
> -#define percpu_write(var, val) percpu_to_op("mov", var, val)
> -#define percpu_add(var, val) percpu_add_op(var, val)
> -#define percpu_sub(var, val) percpu_add_op(var, -(val))
> -#define percpu_and(var, val) percpu_to_op("and", var, val)
> -#define percpu_or(var, val) percpu_to_op("or", var, val)
> -#define percpu_xor(var, val) percpu_to_op("xor", var, val)
> -#define percpu_inc(var) percpu_unary_op("inc", var)
> +#define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
> +#define this_cpu_inc(var) percpu_unary_op("inc", var)
Why redefine this_cpu_inc? include/linux/percpu.h already defines it.
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-11 14:13 ` Christoph Lameter
@ 2011-10-12 1:29 ` Alex,Shi
2011-10-12 13:59 ` Christoph Lameter
2011-10-12 14:11 ` Alex,Shi
0 siblings, 2 replies; 42+ messages in thread
From: Alex,Shi @ 2011-10-12 1:29 UTC (permalink / raw)
To: Christoph Lameter
Cc: tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying
On Tue, 2011-10-11 at 22:13 +0800, Christoph Lameter wrote:
> On Tue, 11 Oct 2011, Alex,Shi wrote:
>
> > percpu_read() used in asm/percpu.h yet. Anyway, I removed definitions
> > here and do a work around for the function as following:
> >
> > @@ -551,7 +544,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr,
> > {
> > unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG;
> >
> > - return ((1UL << (nr % BITS_PER_LONG)) & percpu_read(*a)) != 0;
> > +#ifdef CONFIG_X86_64
> > + return ((1UL << (nr % BITS_PER_LONG)) & this_cpu_read_8(*a)) != 0;
> > +#else
> > + return ((1UL << (nr % BITS_PER_LONG)) & this_cpu_read_4(*a)) != 0;
> > +#endif
>
> OK that looks okay.
>
> > Signed-off-by: Alex Shi <alex.shi@intel.com>
>
> Is this a revised patch? I do not see a use of the __this_cpu_xx ops.
I just changed where you pointed as below at the patch line 146:
#ifdef CONFIG_SMP
#define __percpu_prefix "%%"__stringify(__percpu_seg)":"
-#define __my_cpu_offset percpu_read(this_cpu_off)
+#define __my_cpu_offset __this_cpu_read(this_cpu_off)
I do not quite understand you here. Do you mean to change every possible
this_cpu_xxx() to __this_cpu_xxx()?
For x86 platform code, this_cpu_xxx are equal to __this_cpu_xxx.
For other platform, all of them don't enable specific percpu_xxx, they
are using generic percpu_xxx(), and generic percpu_xxx imply
preempt_enable and preempt_disable. So, change to this_cpu_xxx do not
impact performance on them.
>
> > -#define percpu_write(var, val) percpu_to_op("mov", var, val)
> > -#define percpu_add(var, val) percpu_add_op(var, val)
> > -#define percpu_sub(var, val) percpu_add_op(var, -(val))
> > -#define percpu_and(var, val) percpu_to_op("and", var, val)
> > -#define percpu_or(var, val) percpu_to_op("or", var, val)
> > -#define percpu_xor(var, val) percpu_to_op("xor", var, val)
> > -#define percpu_inc(var) percpu_unary_op("inc", var)
> > +#define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
> > +#define this_cpu_inc(var) percpu_unary_op("inc", var)
>
> Why redefine this_cpu_inc? include/linux/percpu.h already defines it.
Ops, My mistaken, don't need this at all. revise this in the following
patch.
===
>From c6f6c18ed79a1cc97cd2b1f86c36f6d5ed3f8665 Mon Sep 17 00:00:00 2001
From: Alex Shi <alex.shi@intel.com>
Date: Wed, 12 Oct 2011 16:55:49 +0800
Subject: [PATCH] Code clean up for percpu_xxx() functions
Since percpu_xxx() serial functions are duplicate with this_cpu_xxx().
Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in
using. This patch is just do code clean up, no function change.
Signed-off-by: Alex Shi <alex.shi@intel.com>
---
arch/x86/include/asm/current.h | 2 +-
arch/x86/include/asm/hardirq.h | 8 ++--
arch/x86/include/asm/irq_regs.h | 4 +-
arch/x86/include/asm/mmu_context.h | 12 ++++----
arch/x86/include/asm/percpu.h | 24 ++++++---------
arch/x86/include/asm/smp.h | 4 +-
arch/x86/include/asm/stackprotector.h | 4 +-
arch/x86/include/asm/thread_info.h | 2 +-
arch/x86/include/asm/tlbflush.h | 4 +-
arch/x86/kernel/cpu/common.c | 2 +-
arch/x86/kernel/cpu/mcheck/mce.c | 4 +-
arch/x86/kernel/cpu/perf_event.c | 4 +-
arch/x86/kernel/paravirt.c | 12 ++++----
arch/x86/kernel/process_32.c | 2 +-
arch/x86/kernel/process_64.c | 12 ++++----
arch/x86/kvm/x86.c | 14 ++++----
arch/x86/mm/tlb.c | 10 +++---
arch/x86/xen/enlighten.c | 6 ++--
arch/x86/xen/irq.c | 8 ++--
arch/x86/xen/mmu.c | 20 ++++++------
arch/x86/xen/multicalls.h | 2 +-
arch/x86/xen/smp.c | 2 +-
include/linux/percpu.h | 53 ---------------------------------
include/linux/topology.h | 4 +-
net/netfilter/xt_TEE.c | 12 ++++----
net/socket.c | 4 +-
26 files changed, 89 insertions(+), 146 deletions(-)
diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index 4d447b7..9476c04 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task);
static __always_inline struct task_struct *get_current(void)
{
- return percpu_read_stable(current_task);
+ return this_cpu_read_stable(current_task);
}
#define current get_current()
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 55e4de6..fc594da 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -35,14 +35,14 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
#define __ARCH_IRQ_STAT
-#define inc_irq_stat(member) percpu_inc(irq_stat.member)
+#define inc_irq_stat(member) this_cpu_inc(irq_stat.member)
-#define local_softirq_pending() percpu_read(irq_stat.__softirq_pending)
+#define local_softirq_pending() this_cpu_read(irq_stat.__softirq_pending)
#define __ARCH_SET_SOFTIRQ_PENDING
-#define set_softirq_pending(x) percpu_write(irq_stat.__softirq_pending, (x))
-#define or_softirq_pending(x) percpu_or(irq_stat.__softirq_pending, (x))
+#define set_softirq_pending(x) this_cpu_write(irq_stat.__softirq_pending, (x))
+#define or_softirq_pending(x) this_cpu_or(irq_stat.__softirq_pending, (x))
extern void ack_bad_irq(unsigned int irq);
diff --git a/arch/x86/include/asm/irq_regs.h b/arch/x86/include/asm/irq_regs.h
index 7784322..d82250b 100644
--- a/arch/x86/include/asm/irq_regs.h
+++ b/arch/x86/include/asm/irq_regs.h
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);
static inline struct pt_regs *get_irq_regs(void)
{
- return percpu_read(irq_regs);
+ return this_cpu_read(irq_regs);
}
static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
struct pt_regs *old_regs;
old_regs = get_irq_regs();
- percpu_write(irq_regs, new_regs);
+ this_cpu_write(irq_regs, new_regs);
return old_regs;
}
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 6902152..cdbf367 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);
static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
{
#ifdef CONFIG_SMP
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
- percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
+ if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
#endif
}
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
if (likely(prev != next)) {
#ifdef CONFIG_SMP
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- percpu_write(cpu_tlbstate.active_mm, next);
+ this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ this_cpu_write(cpu_tlbstate.active_mm, next);
#endif
cpumask_set_cpu(cpu, mm_cpumask(next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
}
#ifdef CONFIG_SMP
else {
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
+ this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ BUG_ON(this_cpu_read(cpu_tlbstate.active_mm) != next);
if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next))) {
/* We were in lazy tlb mode and leave_mm disabled
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 3470c9d..69c8970 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -46,7 +46,7 @@
#ifdef CONFIG_SMP
#define __percpu_prefix "%%"__stringify(__percpu_seg)":"
-#define __my_cpu_offset percpu_read(this_cpu_off)
+#define __my_cpu_offset __this_cpu_read(this_cpu_off)
/*
* Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,15 @@ do { \
})
/*
- * percpu_read() makes gcc load the percpu variable every time it is
- * accessed while percpu_read_stable() allows the value to be cached.
- * percpu_read_stable() is more efficient and can be used if its value
+ * this_cpu_read() makes gcc load the percpu variable every time it is
+ * accessed while this_cpu_read_stable() allows the value to be cached.
+ * this_cpu_read_stable() is more efficient and can be used if its value
* is guaranteed to be valid across cpus. The current users include
* get_current() and get_thread_info() both of which are actually
* per-thread variables implemented as per-cpu variables and thus
* stable for the duration of the respective task.
*/
-#define percpu_read(var) percpu_from_op("mov", var, "m" (var))
-#define percpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
-#define percpu_write(var, val) percpu_to_op("mov", var, val)
-#define percpu_add(var, val) percpu_add_op(var, val)
-#define percpu_sub(var, val) percpu_add_op(var, -(val))
-#define percpu_and(var, val) percpu_to_op("and", var, val)
-#define percpu_or(var, val) percpu_to_op("or", var, val)
-#define percpu_xor(var, val) percpu_to_op("xor", var, val)
-#define percpu_inc(var) percpu_unary_op("inc", var)
+#define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +543,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr,
{
unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG;
- return ((1UL << (nr % BITS_PER_LONG)) & percpu_read(*a)) != 0;
+#ifdef CONFIG_X86_64
+ return ((1UL << (nr % BITS_PER_LONG)) & this_cpu_read_8(*a)) != 0;
+#else
+ return ((1UL << (nr % BITS_PER_LONG)) & this_cpu_read_4(*a)) != 0;
+#endif
}
static inline int x86_this_cpu_variable_test_bit(int nr,
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 73b11bc..dd5749b 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;
* from the initial startup. We map APIC_BASE very early in page_setup(),
* so this is correct in the x86 case.
*/
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
extern int safe_smp_processor_id(void);
#elif defined(CONFIG_X86_64_SMP)
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
#define stack_smp_processor_id() \
({ \
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
index 1575177..3d7a009 100644
--- a/arch/x86/include/asm/stackprotector.h
+++ b/arch/x86/include/asm/stackprotector.h
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void)
current->stack_canary = canary;
#ifdef CONFIG_X86_64
- percpu_write(irq_stack_union.stack_canary, canary);
+ this_cpu_write(irq_stack_union.stack_canary, canary);
#else
- percpu_write(stack_canary.canary, canary);
+ this_cpu_write(stack_canary.canary, canary);
#endif
}
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index a1fe5c1..8b19667 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);
static inline struct thread_info *current_thread_info(void)
{
struct thread_info *ti;
- ti = (void *)(percpu_read_stable(kernel_stack) +
+ ti = (void *)(this_cpu_read_stable(kernel_stack) +
KERNEL_STACK_OFFSET - THREAD_SIZE);
return ti;
}
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 169be89..200c09c 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
static inline void reset_lazy_tlbstate(void)
{
- percpu_write(cpu_tlbstate.state, 0);
- percpu_write(cpu_tlbstate.active_mm, &init_mm);
+ this_cpu_write(cpu_tlbstate.state, 0);
+ this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
}
#endif /* SMP */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 6218439..10513ed 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1158,7 +1158,7 @@ void __cpuinit cpu_init(void)
oist = &per_cpu(orig_ist, cpu);
#ifdef CONFIG_NUMA
- if (cpu != 0 && percpu_read(numa_node) == 0 &&
+ if (cpu != 0 && this_cpu_read(numa_node) == 0 &&
early_cpu_to_node(cpu) != NUMA_NO_NODE)
set_numa_node(early_cpu_to_node(cpu));
#endif
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 08363b0..adbb598 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -512,7 +512,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
struct mce m;
int i;
- percpu_inc(mce_poll_count);
+ this_cpu_inc(mce_poll_count);
mce_gather_info(&m, NULL);
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
atomic_inc(&mce_entry);
- percpu_inc(mce_exception_count);
+ this_cpu_inc(mce_exception_count);
if (notify_die(DIE_NMI, "machine check", regs, error_code,
18, SIGKILL) == NOTIFY_STOP)
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index cfa62ec..ca037c0 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -1389,7 +1389,7 @@ perf_event_nmi_handler(struct notifier_block *self,
case DIE_NMI:
break;
case DIE_NMIUNKNOWN:
- this_nmi = percpu_read(irq_stat.__nmi_count);
+ this_nmi = this_cpu_read(irq_stat.__nmi_count);
if (this_nmi != __this_cpu_read(pmu_nmi.marked))
/* let the kernel handle the unknown nmi */
return NOTIFY_DONE;
@@ -1409,7 +1409,7 @@ perf_event_nmi_handler(struct notifier_block *self,
if (!handled)
return NOTIFY_DONE;
- this_nmi = percpu_read(irq_stat.__nmi_count);
+ this_nmi = this_cpu_read(irq_stat.__nmi_count);
if ((handled > 1) ||
/* the next nmi could be a back-to-back nmi */
((__this_cpu_read(pmu_nmi.marked) == this_nmi) &&
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index d90272e..b098092 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LA
static inline void enter_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
+ BUG_ON(this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
- percpu_write(paravirt_lazy_mode, mode);
+ this_cpu_write(paravirt_lazy_mode, mode);
}
static void leave_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != mode);
+ BUG_ON(this_cpu_read(paravirt_lazy_mode) != mode);
- percpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
+ this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
}
void paravirt_enter_lazy_mmu(void)
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev)
{
BUG_ON(preemptible());
- if (percpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
+ if (this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
arch_leave_lazy_mmu_mode();
set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES);
}
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
if (in_interrupt())
return PARAVIRT_LAZY_NONE;
- return percpu_read(paravirt_lazy_mode);
+ return this_cpu_read(paravirt_lazy_mode);
}
void arch_flush_lazy_mmu_mode(void)
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 7a3b651..f286be1 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -373,7 +373,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
if (prev->gs | next->gs)
lazy_load_gs(next->gs);
- percpu_write(current_task, next_p);
+ this_cpu_write(current_task, next_p);
return prev_p;
}
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index f693e44..35fdc14 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -73,7 +73,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);
void enter_idle(void)
{
- percpu_write(is_idle, 1);
+ this_cpu_write(is_idle, 1);
atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
}
@@ -336,7 +336,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,
load_gs_index(0);
regs->ip = new_ip;
regs->sp = new_sp;
- percpu_write(old_rsp, new_sp);
+ this_cpu_write(old_rsp, new_sp);
regs->cs = _cs;
regs->ss = _ss;
regs->flags = X86_EFLAGS_IF;
@@ -470,11 +470,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
/*
* Switch the PDA and FPU contexts.
*/
- prev->usersp = percpu_read(old_rsp);
- percpu_write(old_rsp, next->usersp);
- percpu_write(current_task, next_p);
+ prev->usersp = this_cpu_read(old_rsp);
+ this_cpu_write(old_rsp, next->usersp);
+ this_cpu_write(current_task, next_p);
- percpu_write(kernel_stack,
+ this_cpu_write(kernel_stack,
(unsigned long)task_stack_page(next_p) +
THREAD_SIZE - KERNEL_STACK_OFFSET);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 84a28ea..f46cb15 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
static int kvm_is_in_guest(void)
{
- return percpu_read(current_vcpu) != NULL;
+ return this_cpu_read(current_vcpu) != NULL;
}
static int kvm_is_user_mode(void)
{
int user_mode = 3;
- if (percpu_read(current_vcpu))
- user_mode = kvm_x86_ops->get_cpl(percpu_read(current_vcpu));
+ if (this_cpu_read(current_vcpu))
+ user_mode = kvm_x86_ops->get_cpl(this_cpu_read(current_vcpu));
return user_mode != 0;
}
@@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void)
{
unsigned long ip = 0;
- if (percpu_read(current_vcpu))
- ip = kvm_rip_read(percpu_read(current_vcpu));
+ if (this_cpu_read(current_vcpu))
+ ip = kvm_rip_read(this_cpu_read(current_vcpu));
return ip;
}
@@ -5055,13 +5055,13 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {
void kvm_before_handle_nmi(struct kvm_vcpu *vcpu)
{
- percpu_write(current_vcpu, vcpu);
+ this_cpu_write(current_vcpu, vcpu);
}
EXPORT_SYMBOL_GPL(kvm_before_handle_nmi);
void kvm_after_handle_nmi(struct kvm_vcpu *vcpu)
{
- percpu_write(current_vcpu, NULL);
+ this_cpu_write(current_vcpu, NULL);
}
EXPORT_SYMBOL_GPL(kvm_after_handle_nmi);
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index d6c0418..3804471 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);
*/
void leave_mm(int cpu)
{
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
BUG();
cpumask_clear_cpu(cpu,
- mm_cpumask(percpu_read(cpu_tlbstate.active_mm)));
+ mm_cpumask(this_cpu_read(cpu_tlbstate.active_mm)));
load_cr3(swapper_pg_dir);
}
EXPORT_SYMBOL_GPL(leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)
* BUG();
*/
- if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
+ if (f->flush_mm == this_cpu_read(cpu_tlbstate.active_mm)) {
+ if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
if (f->flush_va == TLB_FLUSH_ALL)
local_flush_tlb();
else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
static void do_flush_tlb_all(void *info)
{
__flush_tlb_all();
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
+ if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
leave_mm(smp_processor_id());
}
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 2d69617..c4e362a 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -776,11 +776,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);
static unsigned long xen_read_cr0(void)
{
- unsigned long cr0 = percpu_read(xen_cr0_value);
+ unsigned long cr0 = this_cpu_read(xen_cr0_value);
if (unlikely(cr0 == 0)) {
cr0 = native_read_cr0();
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
}
return cr0;
@@ -790,7 +790,7 @@ static void xen_write_cr0(unsigned long cr0)
{
struct multicall_space mcs;
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
/* Only pay attention to cr0.TS; everything else is
ignored. */
diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c
index 8bbb465..1573376 100644
--- a/arch/x86/xen/irq.c
+++ b/arch/x86/xen/irq.c
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)
struct vcpu_info *vcpu;
unsigned long flags;
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
/* flag has opposite sense of mask */
flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = flags;
preempt_enable_no_resched();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- percpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
+ this_cpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
preempt_enable_no_resched();
}
PV_CALLEE_SAVE_REGS_THUNK(xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)
the caller is confused and is trying to re-enable interrupts
on an indeterminate processor. */
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = 0;
/* Doesn't matter if we get preempted here, because any
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 3dd53f9..6a7aaac 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1106,14 +1106,14 @@ static void drop_other_mm_ref(void *info)
struct mm_struct *mm = info;
struct mm_struct *active_mm;
- active_mm = percpu_read(cpu_tlbstate.active_mm);
+ active_mm = this_cpu_read(cpu_tlbstate.active_mm);
- if (active_mm == mm && percpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
+ if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
leave_mm(smp_processor_id());
/* If this cpu still has a stale cr3 reference, then make sure
it has been flushed. */
- if (percpu_read(xen_current_cr3) == __pa(mm->pgd))
+ if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd))
load_cr3(swapper_pg_dir);
}
@@ -1220,17 +1220,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)
static void xen_write_cr2(unsigned long cr2)
{
- percpu_read(xen_vcpu)->arch.cr2 = cr2;
+ this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
}
static unsigned long xen_read_cr2(void)
{
- return percpu_read(xen_vcpu)->arch.cr2;
+ return this_cpu_read(xen_vcpu)->arch.cr2;
}
unsigned long xen_read_cr2_direct(void)
{
- return percpu_read(xen_vcpu_info.arch.cr2);
+ return this_cpu_read(xen_vcpu_info.arch.cr2);
}
static void xen_flush_tlb(void)
@@ -1313,12 +1313,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
static unsigned long xen_read_cr3(void)
{
- return percpu_read(xen_cr3);
+ return this_cpu_read(xen_cr3);
}
static void set_current_cr3(void *v)
{
- percpu_write(xen_current_cr3, (unsigned long)v);
+ this_cpu_write(xen_current_cr3, (unsigned long)v);
}
static void __xen_write_cr3(bool kernel, unsigned long cr3)
@@ -1341,7 +1341,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)
xen_extend_mmuext_op(&op);
if (kernel) {
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
/* Update xen_current_cr3 once the batch has actually
been submitted. */
@@ -1357,7 +1357,7 @@ static void xen_write_cr3(unsigned long cr3)
/* Update while interrupts are disabled, so its atomic with
respect to ipis */
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
__xen_write_cr3(true, cr3);
diff --git a/arch/x86/xen/multicalls.h b/arch/x86/xen/multicalls.h
index dee79b7..9c2e74f 100644
--- a/arch/x86/xen/multicalls.h
+++ b/arch/x86/xen/multicalls.h
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)
xen_mc_flush();
/* restore flags saved in xen_mc_batch */
- local_irq_restore(percpu_read(xen_mc_irq_flags));
+ local_irq_restore(this_cpu_read(xen_mc_irq_flags));
}
/* Set up a callback to be called when the current batch is flushed */
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 041d4fe..449f868 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)
xen_setup_cpu_clockevents();
set_cpu_online(cpu, true);
- percpu_write(cpu_state, CPU_ONLINE);
+ this_cpu_write(cpu_state, CPU_ONLINE);
wmb();
/* We can take interrupts now: we're officially "up". */
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 9ca008f..aad7e1c 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -165,59 +165,6 @@ extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
#define alloc_percpu(type) \
(typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
-/*
- * Optional methods for optimized non-lvalue per-cpu variable access.
- *
- * @var can be a percpu variable or a field of it and its size should
- * equal char, int or long. percpu_read() evaluates to a lvalue and
- * all others to void.
- *
- * These operations are guaranteed to be atomic w.r.t. preemption.
- * The generic versions use plain get/put_cpu_var(). Archs are
- * encouraged to implement single-instruction alternatives which don't
- * require preemption protection.
- */
-#ifndef percpu_read
-# define percpu_read(var) \
- ({ \
- typeof(var) *pr_ptr__ = &(var); \
- typeof(var) pr_ret__; \
- pr_ret__ = get_cpu_var(*pr_ptr__); \
- put_cpu_var(*pr_ptr__); \
- pr_ret__; \
- })
-#endif
-
-#define __percpu_generic_to_op(var, val, op) \
-do { \
- typeof(var) *pgto_ptr__ = &(var); \
- get_cpu_var(*pgto_ptr__) op val; \
- put_cpu_var(*pgto_ptr__); \
-} while (0)
-
-#ifndef percpu_write
-# define percpu_write(var, val) __percpu_generic_to_op(var, (val), =)
-#endif
-
-#ifndef percpu_add
-# define percpu_add(var, val) __percpu_generic_to_op(var, (val), +=)
-#endif
-
-#ifndef percpu_sub
-# define percpu_sub(var, val) __percpu_generic_to_op(var, (val), -=)
-#endif
-
-#ifndef percpu_and
-# define percpu_and(var, val) __percpu_generic_to_op(var, (val), &=)
-#endif
-
-#ifndef percpu_or
-# define percpu_or(var, val) __percpu_generic_to_op(var, (val), |=)
-#endif
-
-#ifndef percpu_xor
-# define percpu_xor(var, val) __percpu_generic_to_op(var, (val), ^=)
-#endif
/*
* Branching function to split up a function into a set of functions that
diff --git a/include/linux/topology.h b/include/linux/topology.h
index fc839bf..bf64eac 100644
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -235,7 +235,7 @@ static inline int cpu_to_node(int cpu)
#ifndef set_numa_node
static inline void set_numa_node(int node)
{
- percpu_write(numa_node, node);
+ this_cpu_write(numa_node, node);
}
#endif
@@ -270,7 +270,7 @@ DECLARE_PER_CPU(int, _numa_mem_);
#ifndef set_numa_mem
static inline void set_numa_mem(int node)
{
- percpu_write(_numa_mem_, node);
+ this_cpu_write(_numa_mem_, node);
}
#endif
diff --git a/net/netfilter/xt_TEE.c b/net/netfilter/xt_TEE.c
index 5f054a0..e78103d 100644
--- a/net/netfilter/xt_TEE.c
+++ b/net/netfilter/xt_TEE.c
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
const struct xt_tee_tginfo *info = par->targinfo;
struct iphdr *iph;
- if (percpu_read(tee_active))
+ if (this_cpu_read(tee_active))
return XT_CONTINUE;
/*
* Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
ip_send_check(iph);
if (tee_tg_route4(skb, info)) {
- percpu_write(tee_active, true);
+ this_cpu_write(tee_active, true);
ip_local_out(skb);
- percpu_write(tee_active, false);
+ this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
{
const struct xt_tee_tginfo *info = par->targinfo;
- if (percpu_read(tee_active))
+ if (this_cpu_read(tee_active))
return XT_CONTINUE;
skb = pskb_copy(skb, GFP_ATOMIC);
if (skb == NULL)
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
--iph->hop_limit;
}
if (tee_tg_route6(skb, info)) {
- percpu_write(tee_active, true);
+ this_cpu_write(tee_active, true);
ip6_local_out(skb);
- percpu_write(tee_active, false);
+ this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
diff --git a/net/socket.c b/net/socket.c
index ffe92ca..4b62ca9 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -479,7 +479,7 @@ static struct socket *sock_alloc(void)
inode->i_uid = current_fsuid();
inode->i_gid = current_fsgid();
- percpu_add(sockets_in_use, 1);
+ this_cpu_add(sockets_in_use, 1);
return sock;
}
@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)
if (rcu_dereference_protected(sock->wq, 1)->fasync_list)
printk(KERN_ERR "sock_release: fasync list not empty!\n");
- percpu_sub(sockets_in_use, 1);
+ this_cpu_sub(sockets_in_use, 1);
if (!sock->file) {
iput(SOCK_INODE(sock));
return;
--
1.7.5.1
^ permalink raw reply related [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-12 1:29 ` Alex,Shi
@ 2011-10-12 13:59 ` Christoph Lameter
2011-10-12 14:11 ` Alex,Shi
1 sibling, 0 replies; 42+ messages in thread
From: Christoph Lameter @ 2011-10-12 13:59 UTC (permalink / raw)
To: Alex,Shi
Cc: tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying
On Wed, 12 Oct 2011, Alex,Shi wrote:
> #ifdef CONFIG_SMP
> #define __percpu_prefix "%%"__stringify(__percpu_seg)":"
> -#define __my_cpu_offset percpu_read(this_cpu_off)
> +#define __my_cpu_offset __this_cpu_read(this_cpu_off)
>
> I do not quite understand you here. Do you mean to change every possible
> this_cpu_xxx() to __this_cpu_xxx()?
Change only those locations where the kernel thread cannot be preempted.
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-12 14:11 ` Alex,Shi
@ 2011-10-12 14:11 ` Christoph Lameter
2011-10-18 3:21 ` Alex,Shi
0 siblings, 1 reply; 42+ messages in thread
From: Christoph Lameter @ 2011-10-12 14:11 UTC (permalink / raw)
To: Alex,Shi
Cc: tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying
On Wed, 12 Oct 2011, Alex,Shi wrote:
> I thought again of this_cpu_xxx function, yes, some of them are better
> to be replaced by __this_cpu__xxx if preemption is safe on the scenario.
> But this change is better to be done in another patch. I did not
> finished all changed function check today. May try again in next week.
> What's you pinion of this?
I think its good to have it all in a single patch.
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-12 1:29 ` Alex,Shi
2011-10-12 13:59 ` Christoph Lameter
@ 2011-10-12 14:11 ` Alex,Shi
2011-10-12 14:11 ` Christoph Lameter
1 sibling, 1 reply; 42+ messages in thread
From: Alex,Shi @ 2011-10-12 14:11 UTC (permalink / raw)
To: Christoph Lameter
Cc: tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying
> > Is this a revised patch? I do not see a use of the __this_cpu_xx ops.
>
> I just changed where you pointed as below at the patch line 146:
>
> #ifdef CONFIG_SMP
> #define __percpu_prefix "%%"__stringify(__percpu_seg)":"
> -#define __my_cpu_offset percpu_read(this_cpu_off)
> +#define __my_cpu_offset __this_cpu_read(this_cpu_off)
>
> I do not quite understand you here. Do you mean to change every possible
> this_cpu_xxx() to __this_cpu_xxx()?
I thought again of this_cpu_xxx function, yes, some of them are better
to be replaced by __this_cpu__xxx if preemption is safe on the scenario.
But this change is better to be done in another patch. I did not
finished all changed function check today. May try again in next week.
What's you pinion of this?
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-12 14:11 ` Christoph Lameter
@ 2011-10-18 3:21 ` Alex,Shi
2011-10-18 14:33 ` Christoph Lameter
0 siblings, 1 reply; 42+ messages in thread
From: Alex,Shi @ 2011-10-18 3:21 UTC (permalink / raw)
To: Christoph Lameter
Cc: tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying
On Wed, 2011-10-12 at 22:11 +0800, Christoph Lameter wrote:
> On Wed, 12 Oct 2011, Alex,Shi wrote:
>
> > I thought again of this_cpu_xxx function, yes, some of them are better
> > to be replaced by __this_cpu__xxx if preemption is safe on the scenario.
> > But this change is better to be done in another patch. I did not
> > finished all changed function check today. May try again in next week.
> > What's you pinion of this?
>
> I think its good to have it all in a single patch.
>
I thought over your requirements, but it is difficult for me to know
lots of fields in kernel, like for xen/kvm/network etc. So, to avoid
introduce bugs here, could we do this blind changes, and then change
each of fields and get owner's ack separately.
Anyway if you still want to keep all changes in one patch, the patch may
needs long time to wait for each of reviews/acks one by one.
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-18 3:21 ` Alex,Shi
@ 2011-10-18 14:33 ` Christoph Lameter
2011-10-19 9:23 ` Alex,Shi
0 siblings, 1 reply; 42+ messages in thread
From: Christoph Lameter @ 2011-10-18 14:33 UTC (permalink / raw)
To: Alex,Shi
Cc: tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying
On Tue, 18 Oct 2011, Alex,Shi wrote:
> > I think its good to have it all in a single patch.
> >
>
> I thought over your requirements, but it is difficult for me to know
> lots of fields in kernel, like for xen/kvm/network etc. So, to avoid
> introduce bugs here, could we do this blind changes, and then change
> each of fields and get owner's ack separately.
Well do the obvious (where you can see that interrupts or preemption are
disabled in the function or in a calling function). Leave the rest as is
and provide separate patches for them?
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-18 14:33 ` Christoph Lameter
@ 2011-10-19 9:23 ` Alex,Shi
2011-10-19 9:26 ` Avi Kivity
2011-10-19 15:34 ` [PATCH] Code clean up for percpu_xxx() functions Christoph Lameter
0 siblings, 2 replies; 42+ messages in thread
From: Alex,Shi @ 2011-10-19 9:23 UTC (permalink / raw)
To: Christoph Lameter
Cc: tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying, tglx, mingo, avi,
akpm@linux-foundation.org, akpm@linux-foundation.org, davem,
kaber, a.p.zijlstra, kvm, jeremy
> Well do the obvious (where you can see that interrupts or preemption are
> disabled in the function or in a calling function). Leave the rest as is
> and provide separate patches for them?
Thanks for comments! I initialized the patch as following accordingly,
And cc to more maintainers for review. I checked all code except xen/kvm
part, totally a idiot for them.
==========
Since percpu_xxx() serial functions are duplicate with this_cpu_xxx().
Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in
code.
And further more, as Christoph Lameter's requirement, I try to use
__this_cpu_xx to replace this_cpu_xxx if it is in preempt safe scenario.
The preempt safe scenarios include:
1, in irq/softirq/nmi handler
2, protected by preempt_disable
3, protected by spin_lock
4, if the code context imply that it is preempt safe, like the code is
follows or be followed a preempt safe code.
I left the xen/kvm part code unchanged, since no any idea of them.
BTW, In fact, this_cpu_xxx are same as __this_cpu_xxx since all funcs
implement in a single instruction for x86 machine. But it maybe
different for other platforms, so, doing this distinguish is helpful for
other platforms' performance.
Signed-off-by: Alex Shi <alex.shi@intel.com>
---
arch/x86/include/asm/current.h | 2 +-
arch/x86/include/asm/hardirq.h | 8 ++--
arch/x86/include/asm/irq_regs.h | 4 +-
arch/x86/include/asm/mmu_context.h | 12 ++++----
arch/x86/include/asm/percpu.h | 24 ++++++---------
arch/x86/include/asm/smp.h | 4 +-
arch/x86/include/asm/stackprotector.h | 4 +-
arch/x86/include/asm/thread_info.h | 2 +-
arch/x86/include/asm/tlbflush.h | 4 +-
arch/x86/kernel/cpu/common.c | 2 +-
arch/x86/kernel/cpu/mcheck/mce.c | 4 +-
arch/x86/kernel/cpu/perf_event.c | 4 +-
arch/x86/kernel/paravirt.c | 12 ++++----
arch/x86/kernel/process_32.c | 2 +-
arch/x86/kernel/process_64.c | 12 ++++----
arch/x86/kvm/x86.c | 14 ++++----
arch/x86/mm/tlb.c | 10 +++---
arch/x86/xen/enlighten.c | 6 ++--
arch/x86/xen/irq.c | 8 ++--
arch/x86/xen/mmu.c | 20 ++++++------
arch/x86/xen/multicalls.h | 2 +-
arch/x86/xen/smp.c | 2 +-
include/linux/percpu.h | 53 ---------------------------------
include/linux/topology.h | 4 +-
net/netfilter/xt_TEE.c | 12 ++++----
net/socket.c | 4 +-
26 files changed, 89 insertions(+), 146 deletions(-)
diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index 4d447b7..9476c04 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task);
static __always_inline struct task_struct *get_current(void)
{
- return percpu_read_stable(current_task);
+ return this_cpu_read_stable(current_task);
}
#define current get_current()
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 55e4de6..a08145f 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -35,14 +35,14 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
#define __ARCH_IRQ_STAT
-#define inc_irq_stat(member) percpu_inc(irq_stat.member)
+#define inc_irq_stat(member) __this_cpu_inc(irq_stat.member)
-#define local_softirq_pending() percpu_read(irq_stat.__softirq_pending)
+#define local_softirq_pending() __this_cpu_read(irq_stat.__softirq_pending)
#define __ARCH_SET_SOFTIRQ_PENDING
-#define set_softirq_pending(x) percpu_write(irq_stat.__softirq_pending, (x))
-#define or_softirq_pending(x) percpu_or(irq_stat.__softirq_pending, (x))
+#define set_softirq_pending(x) __this_cpu_write(irq_stat.__softirq_pending, (x))
+#define or_softirq_pending(x) __this_cpu_or(irq_stat.__softirq_pending, (x))
extern void ack_bad_irq(unsigned int irq);
diff --git a/arch/x86/include/asm/irq_regs.h b/arch/x86/include/asm/irq_regs.h
index 7784322..4c297e1 100644
--- a/arch/x86/include/asm/irq_regs.h
+++ b/arch/x86/include/asm/irq_regs.h
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);
static inline struct pt_regs *get_irq_regs(void)
{
- return percpu_read(irq_regs);
+ return __this_cpu_read(irq_regs);
}
static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
struct pt_regs *old_regs;
old_regs = get_irq_regs();
- percpu_write(irq_regs, new_regs);
+ __this_cpu_write(irq_regs, new_regs);
return old_regs;
}
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 6902152..02ca533 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);
static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
{
#ifdef CONFIG_SMP
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
- percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
#endif
}
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
if (likely(prev != next)) {
#ifdef CONFIG_SMP
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- percpu_write(cpu_tlbstate.active_mm, next);
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ __this_cpu_write(cpu_tlbstate.active_mm, next);
#endif
cpumask_set_cpu(cpu, mm_cpumask(next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
}
#ifdef CONFIG_SMP
else {
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ BUG_ON(__this_cpu_read(cpu_tlbstate.active_mm) != next);
if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next))) {
/* We were in lazy tlb mode and leave_mm disabled
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 3470c9d..520005e 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -46,7 +46,7 @@
#ifdef CONFIG_SMP
#define __percpu_prefix "%%"__stringify(__percpu_seg)":"
-#define __my_cpu_offset percpu_read(this_cpu_off)
+#define __my_cpu_offset __this_cpu_read(this_cpu_off)
/*
* Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,15 @@ do { \
})
/*
- * percpu_read() makes gcc load the percpu variable every time it is
- * accessed while percpu_read_stable() allows the value to be cached.
- * percpu_read_stable() is more efficient and can be used if its value
+ * this_cpu_read() makes gcc load the percpu variable every time it is
+ * accessed while this_cpu_read_stable() allows the value to be cached.
+ * this_cpu_read_stable() is more efficient and can be used if its value
* is guaranteed to be valid across cpus. The current users include
* get_current() and get_thread_info() both of which are actually
* per-thread variables implemented as per-cpu variables and thus
* stable for the duration of the respective task.
*/
-#define percpu_read(var) percpu_from_op("mov", var, "m" (var))
-#define percpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
-#define percpu_write(var, val) percpu_to_op("mov", var, val)
-#define percpu_add(var, val) percpu_add_op(var, val)
-#define percpu_sub(var, val) percpu_add_op(var, -(val))
-#define percpu_and(var, val) percpu_to_op("and", var, val)
-#define percpu_or(var, val) percpu_to_op("or", var, val)
-#define percpu_xor(var, val) percpu_to_op("xor", var, val)
-#define percpu_inc(var) percpu_unary_op("inc", var)
+#define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +543,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr,
{
unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG;
- return ((1UL << (nr % BITS_PER_LONG)) & percpu_read(*a)) != 0;
+#ifdef CONFIG_X86_64
+ return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_8(*a)) != 0;
+#else
+ return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_4(*a)) != 0;
+#endif
}
static inline int x86_this_cpu_variable_test_bit(int nr,
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 73b11bc..dd5749b 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;
* from the initial startup. We map APIC_BASE very early in page_setup(),
* so this is correct in the x86 case.
*/
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
extern int safe_smp_processor_id(void);
#elif defined(CONFIG_X86_64_SMP)
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
#define stack_smp_processor_id() \
({ \
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
index 1575177..e8a60c9 100644
--- a/arch/x86/include/asm/stackprotector.h
+++ b/arch/x86/include/asm/stackprotector.h
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void)
current->stack_canary = canary;
#ifdef CONFIG_X86_64
- percpu_write(irq_stack_union.stack_canary, canary);
+ __this_cpu_write(irq_stack_union.stack_canary, canary);
#else
- percpu_write(stack_canary.canary, canary);
+ __this_cpu_write(stack_canary.canary, canary);
#endif
}
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index a1fe5c1..8b19667 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);
static inline struct thread_info *current_thread_info(void)
{
struct thread_info *ti;
- ti = (void *)(percpu_read_stable(kernel_stack) +
+ ti = (void *)(this_cpu_read_stable(kernel_stack) +
KERNEL_STACK_OFFSET - THREAD_SIZE);
return ti;
}
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 169be89..e90eec0 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
static inline void reset_lazy_tlbstate(void)
{
- percpu_write(cpu_tlbstate.state, 0);
- percpu_write(cpu_tlbstate.active_mm, &init_mm);
+ __this_cpu_write(cpu_tlbstate.state, 0);
+ __this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
}
#endif /* SMP */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 6218439..780d9db 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1158,7 +1158,7 @@ void __cpuinit cpu_init(void)
oist = &per_cpu(orig_ist, cpu);
#ifdef CONFIG_NUMA
- if (cpu != 0 && percpu_read(numa_node) == 0 &&
+ if (cpu != 0 && __this_cpu_read(numa_node) == 0 &&
early_cpu_to_node(cpu) != NUMA_NO_NODE)
set_numa_node(early_cpu_to_node(cpu));
#endif
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 08363b0..5b0ebce 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -512,7 +512,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
struct mce m;
int i;
- percpu_inc(mce_poll_count);
+ __this_cpu_inc(mce_poll_count);
mce_gather_info(&m, NULL);
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
atomic_inc(&mce_entry);
- percpu_inc(mce_exception_count);
+ __this_cpu_inc(mce_exception_count);
if (notify_die(DIE_NMI, "machine check", regs, error_code,
18, SIGKILL) == NOTIFY_STOP)
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index cfa62ec..2e22701 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -1389,7 +1389,7 @@ perf_event_nmi_handler(struct notifier_block *self,
case DIE_NMI:
break;
case DIE_NMIUNKNOWN:
- this_nmi = percpu_read(irq_stat.__nmi_count);
+ this_nmi = __this_cpu_read(irq_stat.__nmi_count);
if (this_nmi != __this_cpu_read(pmu_nmi.marked))
/* let the kernel handle the unknown nmi */
return NOTIFY_DONE;
@@ -1409,7 +1409,7 @@ perf_event_nmi_handler(struct notifier_block *self,
if (!handled)
return NOTIFY_DONE;
- this_nmi = percpu_read(irq_stat.__nmi_count);
+ this_nmi = __this_cpu_read(irq_stat.__nmi_count);
if ((handled > 1) ||
/* the next nmi could be a back-to-back nmi */
((__this_cpu_read(pmu_nmi.marked) == this_nmi) &&
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index d90272e..2f0c1d1 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LA
static inline void enter_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
+ BUG_ON(__this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
- percpu_write(paravirt_lazy_mode, mode);
+ __this_cpu_write(paravirt_lazy_mode, mode);
}
static void leave_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != mode);
+ BUG_ON(__this_cpu_read(paravirt_lazy_mode) != mode);
- percpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
+ __this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
}
void paravirt_enter_lazy_mmu(void)
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev)
{
BUG_ON(preemptible());
- if (percpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
+ if (__this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
arch_leave_lazy_mmu_mode();
set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES);
}
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
if (in_interrupt())
return PARAVIRT_LAZY_NONE;
- return percpu_read(paravirt_lazy_mode);
+ return __this_cpu_read(paravirt_lazy_mode);
}
void arch_flush_lazy_mmu_mode(void)
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 7a3b651..55feac1 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -373,7 +373,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
if (prev->gs | next->gs)
lazy_load_gs(next->gs);
- percpu_write(current_task, next_p);
+ __this_cpu_write(current_task, next_p);
return prev_p;
}
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index f693e44..c3617c3 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -73,7 +73,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);
void enter_idle(void)
{
- percpu_write(is_idle, 1);
+ __this_cpu_write(is_idle, 1);
atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
}
@@ -336,7 +336,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,
load_gs_index(0);
regs->ip = new_ip;
regs->sp = new_sp;
- percpu_write(old_rsp, new_sp);
+ this_cpu_write(old_rsp, new_sp);
regs->cs = _cs;
regs->ss = _ss;
regs->flags = X86_EFLAGS_IF;
@@ -470,11 +470,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
/*
* Switch the PDA and FPU contexts.
*/
- prev->usersp = percpu_read(old_rsp);
- percpu_write(old_rsp, next->usersp);
- percpu_write(current_task, next_p);
+ prev->usersp = __this_cpu_read(old_rsp);
+ __this_cpu_write(old_rsp, next->usersp);
+ __this_cpu_write(current_task, next_p);
- percpu_write(kernel_stack,
+ __this_cpu_write(kernel_stack,
(unsigned long)task_stack_page(next_p) +
THREAD_SIZE - KERNEL_STACK_OFFSET);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 84a28ea..5ff2516 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
static int kvm_is_in_guest(void)
{
- return percpu_read(current_vcpu) != NULL;
+ return this_cpu_read(current_vcpu) != NULL;
}
static int kvm_is_user_mode(void)
{
int user_mode = 3;
- if (percpu_read(current_vcpu))
- user_mode = kvm_x86_ops->get_cpl(percpu_read(current_vcpu));
+ if (this_cpu_read(current_vcpu))
+ user_mode = kvm_x86_ops->get_cpl(this_cpu_read(current_vcpu));
return user_mode != 0;
}
@@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void)
{
unsigned long ip = 0;
- if (percpu_read(current_vcpu))
- ip = kvm_rip_read(percpu_read(current_vcpu));
+ if (this_cpu_read(current_vcpu))
+ ip = kvm_rip_read(this_cpu_read(current_vcpu));
return ip;
}
@@ -5055,13 +5055,13 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {
void kvm_before_handle_nmi(struct kvm_vcpu *vcpu)
{
- percpu_write(current_vcpu, vcpu);
+ __this_cpu_write(current_vcpu, vcpu);
}
EXPORT_SYMBOL_GPL(kvm_before_handle_nmi);
void kvm_after_handle_nmi(struct kvm_vcpu *vcpu)
{
- percpu_write(current_vcpu, NULL);
+ __this_cpu_write(current_vcpu, NULL);
}
EXPORT_SYMBOL_GPL(kvm_after_handle_nmi);
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index d6c0418..e931db0 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);
*/
void leave_mm(int cpu)
{
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
BUG();
cpumask_clear_cpu(cpu,
- mm_cpumask(percpu_read(cpu_tlbstate.active_mm)));
+ mm_cpumask(__this_cpu_read(cpu_tlbstate.active_mm)));
load_cr3(swapper_pg_dir);
}
EXPORT_SYMBOL_GPL(leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)
* BUG();
*/
- if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
+ if (f->flush_mm == __this_cpu_read(cpu_tlbstate.active_mm)) {
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
if (f->flush_va == TLB_FLUSH_ALL)
local_flush_tlb();
else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
static void do_flush_tlb_all(void *info)
{
__flush_tlb_all();
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
leave_mm(smp_processor_id());
}
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 2d69617..c4e362a 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -776,11 +776,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);
static unsigned long xen_read_cr0(void)
{
- unsigned long cr0 = percpu_read(xen_cr0_value);
+ unsigned long cr0 = this_cpu_read(xen_cr0_value);
if (unlikely(cr0 == 0)) {
cr0 = native_read_cr0();
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
}
return cr0;
@@ -790,7 +790,7 @@ static void xen_write_cr0(unsigned long cr0)
{
struct multicall_space mcs;
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
/* Only pay attention to cr0.TS; everything else is
ignored. */
diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c
index 8bbb465..1573376 100644
--- a/arch/x86/xen/irq.c
+++ b/arch/x86/xen/irq.c
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)
struct vcpu_info *vcpu;
unsigned long flags;
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
/* flag has opposite sense of mask */
flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = flags;
preempt_enable_no_resched();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- percpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
+ this_cpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
preempt_enable_no_resched();
}
PV_CALLEE_SAVE_REGS_THUNK(xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)
the caller is confused and is trying to re-enable interrupts
on an indeterminate processor. */
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = 0;
/* Doesn't matter if we get preempted here, because any
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 3dd53f9..6a7aaac 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1106,14 +1106,14 @@ static void drop_other_mm_ref(void *info)
struct mm_struct *mm = info;
struct mm_struct *active_mm;
- active_mm = percpu_read(cpu_tlbstate.active_mm);
+ active_mm = this_cpu_read(cpu_tlbstate.active_mm);
- if (active_mm == mm && percpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
+ if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
leave_mm(smp_processor_id());
/* If this cpu still has a stale cr3 reference, then make sure
it has been flushed. */
- if (percpu_read(xen_current_cr3) == __pa(mm->pgd))
+ if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd))
load_cr3(swapper_pg_dir);
}
@@ -1220,17 +1220,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)
static void xen_write_cr2(unsigned long cr2)
{
- percpu_read(xen_vcpu)->arch.cr2 = cr2;
+ this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
}
static unsigned long xen_read_cr2(void)
{
- return percpu_read(xen_vcpu)->arch.cr2;
+ return this_cpu_read(xen_vcpu)->arch.cr2;
}
unsigned long xen_read_cr2_direct(void)
{
- return percpu_read(xen_vcpu_info.arch.cr2);
+ return this_cpu_read(xen_vcpu_info.arch.cr2);
}
static void xen_flush_tlb(void)
@@ -1313,12 +1313,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
static unsigned long xen_read_cr3(void)
{
- return percpu_read(xen_cr3);
+ return this_cpu_read(xen_cr3);
}
static void set_current_cr3(void *v)
{
- percpu_write(xen_current_cr3, (unsigned long)v);
+ this_cpu_write(xen_current_cr3, (unsigned long)v);
}
static void __xen_write_cr3(bool kernel, unsigned long cr3)
@@ -1341,7 +1341,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)
xen_extend_mmuext_op(&op);
if (kernel) {
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
/* Update xen_current_cr3 once the batch has actually
been submitted. */
@@ -1357,7 +1357,7 @@ static void xen_write_cr3(unsigned long cr3)
/* Update while interrupts are disabled, so its atomic with
respect to ipis */
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
__xen_write_cr3(true, cr3);
diff --git a/arch/x86/xen/multicalls.h b/arch/x86/xen/multicalls.h
index dee79b7..9c2e74f 100644
--- a/arch/x86/xen/multicalls.h
+++ b/arch/x86/xen/multicalls.h
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)
xen_mc_flush();
/* restore flags saved in xen_mc_batch */
- local_irq_restore(percpu_read(xen_mc_irq_flags));
+ local_irq_restore(this_cpu_read(xen_mc_irq_flags));
}
/* Set up a callback to be called when the current batch is flushed */
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 041d4fe..449f868 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)
xen_setup_cpu_clockevents();
set_cpu_online(cpu, true);
- percpu_write(cpu_state, CPU_ONLINE);
+ this_cpu_write(cpu_state, CPU_ONLINE);
wmb();
/* We can take interrupts now: we're officially "up". */
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 9ca008f..aad7e1c 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -165,59 +165,6 @@ extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
#define alloc_percpu(type) \
(typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
-/*
- * Optional methods for optimized non-lvalue per-cpu variable access.
- *
- * @var can be a percpu variable or a field of it and its size should
- * equal char, int or long. percpu_read() evaluates to a lvalue and
- * all others to void.
- *
- * These operations are guaranteed to be atomic w.r.t. preemption.
- * The generic versions use plain get/put_cpu_var(). Archs are
- * encouraged to implement single-instruction alternatives which don't
- * require preemption protection.
- */
-#ifndef percpu_read
-# define percpu_read(var) \
- ({ \
- typeof(var) *pr_ptr__ = &(var); \
- typeof(var) pr_ret__; \
- pr_ret__ = get_cpu_var(*pr_ptr__); \
- put_cpu_var(*pr_ptr__); \
- pr_ret__; \
- })
-#endif
-
-#define __percpu_generic_to_op(var, val, op) \
-do { \
- typeof(var) *pgto_ptr__ = &(var); \
- get_cpu_var(*pgto_ptr__) op val; \
- put_cpu_var(*pgto_ptr__); \
-} while (0)
-
-#ifndef percpu_write
-# define percpu_write(var, val) __percpu_generic_to_op(var, (val), =)
-#endif
-
-#ifndef percpu_add
-# define percpu_add(var, val) __percpu_generic_to_op(var, (val), +=)
-#endif
-
-#ifndef percpu_sub
-# define percpu_sub(var, val) __percpu_generic_to_op(var, (val), -=)
-#endif
-
-#ifndef percpu_and
-# define percpu_and(var, val) __percpu_generic_to_op(var, (val), &=)
-#endif
-
-#ifndef percpu_or
-# define percpu_or(var, val) __percpu_generic_to_op(var, (val), |=)
-#endif
-
-#ifndef percpu_xor
-# define percpu_xor(var, val) __percpu_generic_to_op(var, (val), ^=)
-#endif
/*
* Branching function to split up a function into a set of functions that
diff --git a/include/linux/topology.h b/include/linux/topology.h
index fc839bf..aede2d9 100644
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -235,7 +235,7 @@ static inline int cpu_to_node(int cpu)
#ifndef set_numa_node
static inline void set_numa_node(int node)
{
- percpu_write(numa_node, node);
+ __this_cpu_write(numa_node, node);
}
#endif
@@ -270,7 +270,7 @@ DECLARE_PER_CPU(int, _numa_mem_);
#ifndef set_numa_mem
static inline void set_numa_mem(int node)
{
- percpu_write(_numa_mem_, node);
+ __this_cpu_write(_numa_mem_, node);
}
#endif
diff --git a/net/netfilter/xt_TEE.c b/net/netfilter/xt_TEE.c
index 5f054a0..678084c 100644
--- a/net/netfilter/xt_TEE.c
+++ b/net/netfilter/xt_TEE.c
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
const struct xt_tee_tginfo *info = par->targinfo;
struct iphdr *iph;
- if (percpu_read(tee_active))
+ if (__this_cpu_read(tee_active))
return XT_CONTINUE;
/*
* Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
ip_send_check(iph);
if (tee_tg_route4(skb, info)) {
- percpu_write(tee_active, true);
+ __this_cpu_write(tee_active, true);
ip_local_out(skb);
- percpu_write(tee_active, false);
+ __this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
{
const struct xt_tee_tginfo *info = par->targinfo;
- if (percpu_read(tee_active))
+ if (__this_cpu_read(tee_active))
return XT_CONTINUE;
skb = pskb_copy(skb, GFP_ATOMIC);
if (skb == NULL)
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
--iph->hop_limit;
}
if (tee_tg_route6(skb, info)) {
- percpu_write(tee_active, true);
+ __this_cpu_write(tee_active, true);
ip6_local_out(skb);
- percpu_write(tee_active, false);
+ __this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
diff --git a/net/socket.c b/net/socket.c
index ffe92ca..2845d38 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -479,7 +479,7 @@ static struct socket *sock_alloc(void)
inode->i_uid = current_fsuid();
inode->i_gid = current_fsgid();
- percpu_add(sockets_in_use, 1);
+ __this_cpu_add(sockets_in_use, 1);
return sock;
}
@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)
if (rcu_dereference_protected(sock->wq, 1)->fasync_list)
printk(KERN_ERR "sock_release: fasync list not empty!\n");
- percpu_sub(sockets_in_use, 1);
+ __this_cpu_sub(sockets_in_use, 1);
if (!sock->file) {
iput(SOCK_INODE(sock));
return;
--
1.7.5.1
^ permalink raw reply related [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-19 9:23 ` Alex,Shi
@ 2011-10-19 9:26 ` Avi Kivity
2011-10-20 2:45 ` Alex,Shi
2011-10-20 7:34 ` [PATCH 2/3] kvm: use this_cpu_xxx replace percpu_xxx funcs Alex,Shi
2011-10-19 15:34 ` [PATCH] Code clean up for percpu_xxx() functions Christoph Lameter
1 sibling, 2 replies; 42+ messages in thread
From: Avi Kivity @ 2011-10-19 9:26 UTC (permalink / raw)
To: Alex,Shi
Cc: Christoph Lameter, tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying, tglx, mingo,
akpm@linux-foundation.org, davem, kaber, a.p.zijlstra, kvm,
jeremy
On 10/19/2011 11:23 AM, Alex,Shi wrote:
> > Well do the obvious (where you can see that interrupts or preemption are
> > disabled in the function or in a calling function). Leave the rest as is
> > and provide separate patches for them?
>
> Thanks for comments! I initialized the patch as following accordingly,
> And cc to more maintainers for review. I checked all code except xen/kvm
> part, totally a idiot for them.
>
>
>
> ==========
>
> Since percpu_xxx() serial functions are duplicate with this_cpu_xxx().
> Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in
> code.
>
> And further more, as Christoph Lameter's requirement, I try to use
> __this_cpu_xx to replace this_cpu_xxx if it is in preempt safe scenario.
> The preempt safe scenarios include:
> 1, in irq/softirq/nmi handler
> 2, protected by preempt_disable
> 3, protected by spin_lock
> 4, if the code context imply that it is preempt safe, like the code is
> follows or be followed a preempt safe code.
>
> I left the xen/kvm part code unchanged, since no any idea of them.
>
>
All of the kvm usage is in preemption disabled contexts.
--
I have a truly marvellous patch that fixes the bug which this
signature is too narrow to contain.
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-19 9:23 ` Alex,Shi
2011-10-19 9:26 ` Avi Kivity
@ 2011-10-19 15:34 ` Christoph Lameter
2011-10-20 2:44 ` Alex,Shi
2011-10-20 7:35 ` [PATCH 3/3] Code clean up for percpu_xxx() functions Alex,Shi
1 sibling, 2 replies; 42+ messages in thread
From: Christoph Lameter @ 2011-10-19 15:34 UTC (permalink / raw)
To: Alex,Shi
Cc: tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying, Thomas Gleixner, mingo, avi,
akpm@linux-foundation.org, David Miller, kaber, a.p.zijlstra, kvm,
jeremy
On Wed, 19 Oct 2011, Alex,Shi wrote:
> Thanks for comments! I initialized the patch as following accordingly,
> And cc to more maintainers for review. I checked all code except xen/kvm
> part, totally a idiot for them.
Acked-by: Christoph Lameter <cl@gentwo.org>
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-19 15:34 ` [PATCH] Code clean up for percpu_xxx() functions Christoph Lameter
@ 2011-10-20 2:44 ` Alex,Shi
2011-10-20 5:08 ` Eric Dumazet
2011-10-20 7:35 ` [PATCH 3/3] Code clean up for percpu_xxx() functions Alex,Shi
1 sibling, 1 reply; 42+ messages in thread
From: Alex,Shi @ 2011-10-20 2:44 UTC (permalink / raw)
To: Christoph Lameter
Cc: tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying, Thomas Gleixner,
mingo@redhat.com, avi@redhat.com, akpm@linux-foundation.org,
David Miller, kaber@trash.net, a.p.zijlstra@chello.nl,
kvm@vger.kernel.org, jeremy@xensource.com
> Acked-by: Christoph Lameter <cl@gentwo.org>
Thanks, Christoph!
and resend for code style problem correction.
========
>From b8d595063259a636778b14c43f3cf024e908fd25 Mon Sep 17 00:00:00 2001
From: Alex Shi <alex.shi@intel.com>
Date: Thu, 20 Oct 2011 10:25:50 +0800
Subject: [PATCH 1/2] Code clean up for percpu_xxx() functions
Since percpu_xxx() serial functions are duplicate with this_cpu_xxx().
Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in
code.
And further more, as Christoph Lameter's requirement, I try to use
__this_cpu_xx to replace this_cpu_xxx if it is in preempt safe scenario.
The preempt safe scenarios include:
1, in irq/softirq/nmi handler
2, protected by preempt_disable
3, protected by spin_lock
4, if the code context imply that it is preempt safe, like the code is
follows or be followed a preempt safe code.
I left the xen/kvm part code unchanged, since no any idea of them.
BTW, In fact, this_cpu_xxx are same as __this_cpu_xxx since all funcs
implement in a single instruction for x86 machine. But it maybe
different for other platforms, so, doing this distinguish is helpful for
other platforms' performance.
Signed-off-by: Alex Shi <alex.shi@intel.com>
Acked-by: Christoph Lameter <cl@gentwo.org>
---
arch/x86/include/asm/current.h | 2 +-
arch/x86/include/asm/hardirq.h | 9 +++--
arch/x86/include/asm/irq_regs.h | 4 +-
arch/x86/include/asm/mmu_context.h | 12 ++++----
arch/x86/include/asm/percpu.h | 24 ++++++---------
arch/x86/include/asm/smp.h | 4 +-
arch/x86/include/asm/stackprotector.h | 4 +-
arch/x86/include/asm/thread_info.h | 2 +-
arch/x86/include/asm/tlbflush.h | 4 +-
arch/x86/kernel/cpu/common.c | 2 +-
arch/x86/kernel/cpu/mcheck/mce.c | 4 +-
arch/x86/kernel/cpu/perf_event.c | 4 +-
arch/x86/kernel/paravirt.c | 12 ++++----
arch/x86/kernel/process_32.c | 2 +-
arch/x86/kernel/process_64.c | 12 ++++----
arch/x86/kvm/x86.c | 14 ++++----
arch/x86/mm/tlb.c | 10 +++---
arch/x86/xen/enlighten.c | 6 ++--
arch/x86/xen/irq.c | 8 ++--
arch/x86/xen/mmu.c | 20 ++++++------
arch/x86/xen/multicalls.h | 2 +-
arch/x86/xen/smp.c | 2 +-
include/linux/percpu.h | 53 ---------------------------------
include/linux/topology.h | 4 +-
net/netfilter/xt_TEE.c | 12 ++++----
net/socket.c | 4 +-
26 files changed, 90 insertions(+), 146 deletions(-)
diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index 4d447b7..9476c04 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task);
static __always_inline struct task_struct *get_current(void)
{
- return percpu_read_stable(current_task);
+ return this_cpu_read_stable(current_task);
}
#define current get_current()
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 55e4de6..2890444 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -35,14 +35,15 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
#define __ARCH_IRQ_STAT
-#define inc_irq_stat(member) percpu_inc(irq_stat.member)
+#define inc_irq_stat(member) __this_cpu_inc(irq_stat.member)
-#define local_softirq_pending() percpu_read(irq_stat.__softirq_pending)
+#define local_softirq_pending() __this_cpu_read(irq_stat.__softirq_pending)
#define __ARCH_SET_SOFTIRQ_PENDING
-#define set_softirq_pending(x) percpu_write(irq_stat.__softirq_pending, (x))
-#define or_softirq_pending(x) percpu_or(irq_stat.__softirq_pending, (x))
+#define set_softirq_pending(x) \
+ __this_cpu_write(irq_stat.__softirq_pending, (x))
+#define or_softirq_pending(x) __this_cpu_or(irq_stat.__softirq_pending, (x))
extern void ack_bad_irq(unsigned int irq);
diff --git a/arch/x86/include/asm/irq_regs.h b/arch/x86/include/asm/irq_regs.h
index 7784322..15639ed 100644
--- a/arch/x86/include/asm/irq_regs.h
+++ b/arch/x86/include/asm/irq_regs.h
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);
static inline struct pt_regs *get_irq_regs(void)
{
- return percpu_read(irq_regs);
+ return __this_cpu_read(irq_regs);
}
static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
struct pt_regs *old_regs;
old_regs = get_irq_regs();
- percpu_write(irq_regs, new_regs);
+ __this_cpu_write(irq_regs, new_regs);
return old_regs;
}
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 6902152..02ca533 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);
static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
{
#ifdef CONFIG_SMP
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
- percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
#endif
}
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
if (likely(prev != next)) {
#ifdef CONFIG_SMP
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- percpu_write(cpu_tlbstate.active_mm, next);
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ __this_cpu_write(cpu_tlbstate.active_mm, next);
#endif
cpumask_set_cpu(cpu, mm_cpumask(next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
}
#ifdef CONFIG_SMP
else {
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ BUG_ON(__this_cpu_read(cpu_tlbstate.active_mm) != next);
if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next))) {
/* We were in lazy tlb mode and leave_mm disabled
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 3470c9d..520005e 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -46,7 +46,7 @@
#ifdef CONFIG_SMP
#define __percpu_prefix "%%"__stringify(__percpu_seg)":"
-#define __my_cpu_offset percpu_read(this_cpu_off)
+#define __my_cpu_offset __this_cpu_read(this_cpu_off)
/*
* Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,15 @@ do { \
})
/*
- * percpu_read() makes gcc load the percpu variable every time it is
- * accessed while percpu_read_stable() allows the value to be cached.
- * percpu_read_stable() is more efficient and can be used if its value
+ * this_cpu_read() makes gcc load the percpu variable every time it is
+ * accessed while this_cpu_read_stable() allows the value to be cached.
+ * this_cpu_read_stable() is more efficient and can be used if its value
* is guaranteed to be valid across cpus. The current users include
* get_current() and get_thread_info() both of which are actually
* per-thread variables implemented as per-cpu variables and thus
* stable for the duration of the respective task.
*/
-#define percpu_read(var) percpu_from_op("mov", var, "m" (var))
-#define percpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
-#define percpu_write(var, val) percpu_to_op("mov", var, val)
-#define percpu_add(var, val) percpu_add_op(var, val)
-#define percpu_sub(var, val) percpu_add_op(var, -(val))
-#define percpu_and(var, val) percpu_to_op("and", var, val)
-#define percpu_or(var, val) percpu_to_op("or", var, val)
-#define percpu_xor(var, val) percpu_to_op("xor", var, val)
-#define percpu_inc(var) percpu_unary_op("inc", var)
+#define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +543,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr,
{
unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG;
- return ((1UL << (nr % BITS_PER_LONG)) & percpu_read(*a)) != 0;
+#ifdef CONFIG_X86_64
+ return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_8(*a)) != 0;
+#else
+ return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_4(*a)) != 0;
+#endif
}
static inline int x86_this_cpu_variable_test_bit(int nr,
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 73b11bc..dd5749b 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;
* from the initial startup. We map APIC_BASE very early in page_setup(),
* so this is correct in the x86 case.
*/
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
extern int safe_smp_processor_id(void);
#elif defined(CONFIG_X86_64_SMP)
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
#define stack_smp_processor_id() \
({ \
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
index 1575177..e8a60c9 100644
--- a/arch/x86/include/asm/stackprotector.h
+++ b/arch/x86/include/asm/stackprotector.h
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void)
current->stack_canary = canary;
#ifdef CONFIG_X86_64
- percpu_write(irq_stack_union.stack_canary, canary);
+ __this_cpu_write(irq_stack_union.stack_canary, canary);
#else
- percpu_write(stack_canary.canary, canary);
+ __this_cpu_write(stack_canary.canary, canary);
#endif
}
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index a1fe5c1..8b19667 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);
static inline struct thread_info *current_thread_info(void)
{
struct thread_info *ti;
- ti = (void *)(percpu_read_stable(kernel_stack) +
+ ti = (void *)(this_cpu_read_stable(kernel_stack) +
KERNEL_STACK_OFFSET - THREAD_SIZE);
return ti;
}
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 169be89..e90eec0 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
static inline void reset_lazy_tlbstate(void)
{
- percpu_write(cpu_tlbstate.state, 0);
- percpu_write(cpu_tlbstate.active_mm, &init_mm);
+ __this_cpu_write(cpu_tlbstate.state, 0);
+ __this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
}
#endif /* SMP */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 6218439..780d9db 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1158,7 +1158,7 @@ void __cpuinit cpu_init(void)
oist = &per_cpu(orig_ist, cpu);
#ifdef CONFIG_NUMA
- if (cpu != 0 && percpu_read(numa_node) == 0 &&
+ if (cpu != 0 && __this_cpu_read(numa_node) == 0 &&
early_cpu_to_node(cpu) != NUMA_NO_NODE)
set_numa_node(early_cpu_to_node(cpu));
#endif
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 08363b0..5b0ebce 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -512,7 +512,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
struct mce m;
int i;
- percpu_inc(mce_poll_count);
+ __this_cpu_inc(mce_poll_count);
mce_gather_info(&m, NULL);
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
atomic_inc(&mce_entry);
- percpu_inc(mce_exception_count);
+ __this_cpu_inc(mce_exception_count);
if (notify_die(DIE_NMI, "machine check", regs, error_code,
18, SIGKILL) == NOTIFY_STOP)
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index cfa62ec..2e22701 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -1389,7 +1389,7 @@ perf_event_nmi_handler(struct notifier_block *self,
case DIE_NMI:
break;
case DIE_NMIUNKNOWN:
- this_nmi = percpu_read(irq_stat.__nmi_count);
+ this_nmi = __this_cpu_read(irq_stat.__nmi_count);
if (this_nmi != __this_cpu_read(pmu_nmi.marked))
/* let the kernel handle the unknown nmi */
return NOTIFY_DONE;
@@ -1409,7 +1409,7 @@ perf_event_nmi_handler(struct notifier_block *self,
if (!handled)
return NOTIFY_DONE;
- this_nmi = percpu_read(irq_stat.__nmi_count);
+ this_nmi = __this_cpu_read(irq_stat.__nmi_count);
if ((handled > 1) ||
/* the next nmi could be a back-to-back nmi */
((__this_cpu_read(pmu_nmi.marked) == this_nmi) &&
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index d90272e..2f0c1d1 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LA
static inline void enter_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
+ BUG_ON(__this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
- percpu_write(paravirt_lazy_mode, mode);
+ __this_cpu_write(paravirt_lazy_mode, mode);
}
static void leave_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != mode);
+ BUG_ON(__this_cpu_read(paravirt_lazy_mode) != mode);
- percpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
+ __this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
}
void paravirt_enter_lazy_mmu(void)
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev)
{
BUG_ON(preemptible());
- if (percpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
+ if (__this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
arch_leave_lazy_mmu_mode();
set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES);
}
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
if (in_interrupt())
return PARAVIRT_LAZY_NONE;
- return percpu_read(paravirt_lazy_mode);
+ return __this_cpu_read(paravirt_lazy_mode);
}
void arch_flush_lazy_mmu_mode(void)
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 7a3b651..55feac1 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -373,7 +373,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
if (prev->gs | next->gs)
lazy_load_gs(next->gs);
- percpu_write(current_task, next_p);
+ __this_cpu_write(current_task, next_p);
return prev_p;
}
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index f693e44..c3617c3 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -73,7 +73,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);
void enter_idle(void)
{
- percpu_write(is_idle, 1);
+ __this_cpu_write(is_idle, 1);
atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
}
@@ -336,7 +336,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,
load_gs_index(0);
regs->ip = new_ip;
regs->sp = new_sp;
- percpu_write(old_rsp, new_sp);
+ this_cpu_write(old_rsp, new_sp);
regs->cs = _cs;
regs->ss = _ss;
regs->flags = X86_EFLAGS_IF;
@@ -470,11 +470,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
/*
* Switch the PDA and FPU contexts.
*/
- prev->usersp = percpu_read(old_rsp);
- percpu_write(old_rsp, next->usersp);
- percpu_write(current_task, next_p);
+ prev->usersp = __this_cpu_read(old_rsp);
+ __this_cpu_write(old_rsp, next->usersp);
+ __this_cpu_write(current_task, next_p);
- percpu_write(kernel_stack,
+ __this_cpu_write(kernel_stack,
(unsigned long)task_stack_page(next_p) +
THREAD_SIZE - KERNEL_STACK_OFFSET);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 84a28ea..5ff2516 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
static int kvm_is_in_guest(void)
{
- return percpu_read(current_vcpu) != NULL;
+ return this_cpu_read(current_vcpu) != NULL;
}
static int kvm_is_user_mode(void)
{
int user_mode = 3;
- if (percpu_read(current_vcpu))
- user_mode = kvm_x86_ops->get_cpl(percpu_read(current_vcpu));
+ if (this_cpu_read(current_vcpu))
+ user_mode = kvm_x86_ops->get_cpl(this_cpu_read(current_vcpu));
return user_mode != 0;
}
@@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void)
{
unsigned long ip = 0;
- if (percpu_read(current_vcpu))
- ip = kvm_rip_read(percpu_read(current_vcpu));
+ if (this_cpu_read(current_vcpu))
+ ip = kvm_rip_read(this_cpu_read(current_vcpu));
return ip;
}
@@ -5055,13 +5055,13 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {
void kvm_before_handle_nmi(struct kvm_vcpu *vcpu)
{
- percpu_write(current_vcpu, vcpu);
+ __this_cpu_write(current_vcpu, vcpu);
}
EXPORT_SYMBOL_GPL(kvm_before_handle_nmi);
void kvm_after_handle_nmi(struct kvm_vcpu *vcpu)
{
- percpu_write(current_vcpu, NULL);
+ __this_cpu_write(current_vcpu, NULL);
}
EXPORT_SYMBOL_GPL(kvm_after_handle_nmi);
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index d6c0418..e931db0 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);
*/
void leave_mm(int cpu)
{
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
BUG();
cpumask_clear_cpu(cpu,
- mm_cpumask(percpu_read(cpu_tlbstate.active_mm)));
+ mm_cpumask(__this_cpu_read(cpu_tlbstate.active_mm)));
load_cr3(swapper_pg_dir);
}
EXPORT_SYMBOL_GPL(leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)
* BUG();
*/
- if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
+ if (f->flush_mm == __this_cpu_read(cpu_tlbstate.active_mm)) {
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
if (f->flush_va == TLB_FLUSH_ALL)
local_flush_tlb();
else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
static void do_flush_tlb_all(void *info)
{
__flush_tlb_all();
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
leave_mm(smp_processor_id());
}
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 2d69617..c4e362a 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -776,11 +776,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);
static unsigned long xen_read_cr0(void)
{
- unsigned long cr0 = percpu_read(xen_cr0_value);
+ unsigned long cr0 = this_cpu_read(xen_cr0_value);
if (unlikely(cr0 == 0)) {
cr0 = native_read_cr0();
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
}
return cr0;
@@ -790,7 +790,7 @@ static void xen_write_cr0(unsigned long cr0)
{
struct multicall_space mcs;
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
/* Only pay attention to cr0.TS; everything else is
ignored. */
diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c
index 8bbb465..1573376 100644
--- a/arch/x86/xen/irq.c
+++ b/arch/x86/xen/irq.c
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)
struct vcpu_info *vcpu;
unsigned long flags;
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
/* flag has opposite sense of mask */
flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = flags;
preempt_enable_no_resched();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- percpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
+ this_cpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
preempt_enable_no_resched();
}
PV_CALLEE_SAVE_REGS_THUNK(xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)
the caller is confused and is trying to re-enable interrupts
on an indeterminate processor. */
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = 0;
/* Doesn't matter if we get preempted here, because any
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 3dd53f9..6a7aaac 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1106,14 +1106,14 @@ static void drop_other_mm_ref(void *info)
struct mm_struct *mm = info;
struct mm_struct *active_mm;
- active_mm = percpu_read(cpu_tlbstate.active_mm);
+ active_mm = this_cpu_read(cpu_tlbstate.active_mm);
- if (active_mm == mm && percpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
+ if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
leave_mm(smp_processor_id());
/* If this cpu still has a stale cr3 reference, then make sure
it has been flushed. */
- if (percpu_read(xen_current_cr3) == __pa(mm->pgd))
+ if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd))
load_cr3(swapper_pg_dir);
}
@@ -1220,17 +1220,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)
static void xen_write_cr2(unsigned long cr2)
{
- percpu_read(xen_vcpu)->arch.cr2 = cr2;
+ this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
}
static unsigned long xen_read_cr2(void)
{
- return percpu_read(xen_vcpu)->arch.cr2;
+ return this_cpu_read(xen_vcpu)->arch.cr2;
}
unsigned long xen_read_cr2_direct(void)
{
- return percpu_read(xen_vcpu_info.arch.cr2);
+ return this_cpu_read(xen_vcpu_info.arch.cr2);
}
static void xen_flush_tlb(void)
@@ -1313,12 +1313,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
static unsigned long xen_read_cr3(void)
{
- return percpu_read(xen_cr3);
+ return this_cpu_read(xen_cr3);
}
static void set_current_cr3(void *v)
{
- percpu_write(xen_current_cr3, (unsigned long)v);
+ this_cpu_write(xen_current_cr3, (unsigned long)v);
}
static void __xen_write_cr3(bool kernel, unsigned long cr3)
@@ -1341,7 +1341,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)
xen_extend_mmuext_op(&op);
if (kernel) {
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
/* Update xen_current_cr3 once the batch has actually
been submitted. */
@@ -1357,7 +1357,7 @@ static void xen_write_cr3(unsigned long cr3)
/* Update while interrupts are disabled, so its atomic with
respect to ipis */
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
__xen_write_cr3(true, cr3);
diff --git a/arch/x86/xen/multicalls.h b/arch/x86/xen/multicalls.h
index dee79b7..9c2e74f 100644
--- a/arch/x86/xen/multicalls.h
+++ b/arch/x86/xen/multicalls.h
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)
xen_mc_flush();
/* restore flags saved in xen_mc_batch */
- local_irq_restore(percpu_read(xen_mc_irq_flags));
+ local_irq_restore(this_cpu_read(xen_mc_irq_flags));
}
/* Set up a callback to be called when the current batch is flushed */
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 041d4fe..449f868 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)
xen_setup_cpu_clockevents();
set_cpu_online(cpu, true);
- percpu_write(cpu_state, CPU_ONLINE);
+ this_cpu_write(cpu_state, CPU_ONLINE);
wmb();
/* We can take interrupts now: we're officially "up". */
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 9ca008f..aad7e1c 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -165,59 +165,6 @@ extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
#define alloc_percpu(type) \
(typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
-/*
- * Optional methods for optimized non-lvalue per-cpu variable access.
- *
- * @var can be a percpu variable or a field of it and its size should
- * equal char, int or long. percpu_read() evaluates to a lvalue and
- * all others to void.
- *
- * These operations are guaranteed to be atomic w.r.t. preemption.
- * The generic versions use plain get/put_cpu_var(). Archs are
- * encouraged to implement single-instruction alternatives which don't
- * require preemption protection.
- */
-#ifndef percpu_read
-# define percpu_read(var) \
- ({ \
- typeof(var) *pr_ptr__ = &(var); \
- typeof(var) pr_ret__; \
- pr_ret__ = get_cpu_var(*pr_ptr__); \
- put_cpu_var(*pr_ptr__); \
- pr_ret__; \
- })
-#endif
-
-#define __percpu_generic_to_op(var, val, op) \
-do { \
- typeof(var) *pgto_ptr__ = &(var); \
- get_cpu_var(*pgto_ptr__) op val; \
- put_cpu_var(*pgto_ptr__); \
-} while (0)
-
-#ifndef percpu_write
-# define percpu_write(var, val) __percpu_generic_to_op(var, (val), =)
-#endif
-
-#ifndef percpu_add
-# define percpu_add(var, val) __percpu_generic_to_op(var, (val), +=)
-#endif
-
-#ifndef percpu_sub
-# define percpu_sub(var, val) __percpu_generic_to_op(var, (val), -=)
-#endif
-
-#ifndef percpu_and
-# define percpu_and(var, val) __percpu_generic_to_op(var, (val), &=)
-#endif
-
-#ifndef percpu_or
-# define percpu_or(var, val) __percpu_generic_to_op(var, (val), |=)
-#endif
-
-#ifndef percpu_xor
-# define percpu_xor(var, val) __percpu_generic_to_op(var, (val), ^=)
-#endif
/*
* Branching function to split up a function into a set of functions that
diff --git a/include/linux/topology.h b/include/linux/topology.h
index fc839bf..aede2d9 100644
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -235,7 +235,7 @@ static inline int cpu_to_node(int cpu)
#ifndef set_numa_node
static inline void set_numa_node(int node)
{
- percpu_write(numa_node, node);
+ __this_cpu_write(numa_node, node);
}
#endif
@@ -270,7 +270,7 @@ DECLARE_PER_CPU(int, _numa_mem_);
#ifndef set_numa_mem
static inline void set_numa_mem(int node)
{
- percpu_write(_numa_mem_, node);
+ __this_cpu_write(_numa_mem_, node);
}
#endif
diff --git a/net/netfilter/xt_TEE.c b/net/netfilter/xt_TEE.c
index 5f054a0..678084c 100644
--- a/net/netfilter/xt_TEE.c
+++ b/net/netfilter/xt_TEE.c
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
const struct xt_tee_tginfo *info = par->targinfo;
struct iphdr *iph;
- if (percpu_read(tee_active))
+ if (__this_cpu_read(tee_active))
return XT_CONTINUE;
/*
* Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
ip_send_check(iph);
if (tee_tg_route4(skb, info)) {
- percpu_write(tee_active, true);
+ __this_cpu_write(tee_active, true);
ip_local_out(skb);
- percpu_write(tee_active, false);
+ __this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
{
const struct xt_tee_tginfo *info = par->targinfo;
- if (percpu_read(tee_active))
+ if (__this_cpu_read(tee_active))
return XT_CONTINUE;
skb = pskb_copy(skb, GFP_ATOMIC);
if (skb == NULL)
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
--iph->hop_limit;
}
if (tee_tg_route6(skb, info)) {
- percpu_write(tee_active, true);
+ __this_cpu_write(tee_active, true);
ip6_local_out(skb);
- percpu_write(tee_active, false);
+ __this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
diff --git a/net/socket.c b/net/socket.c
index ffe92ca..2845d38 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -479,7 +479,7 @@ static struct socket *sock_alloc(void)
inode->i_uid = current_fsuid();
inode->i_gid = current_fsgid();
- percpu_add(sockets_in_use, 1);
+ __this_cpu_add(sockets_in_use, 1);
return sock;
}
@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)
if (rcu_dereference_protected(sock->wq, 1)->fasync_list)
printk(KERN_ERR "sock_release: fasync list not empty!\n");
- percpu_sub(sockets_in_use, 1);
+ __this_cpu_sub(sockets_in_use, 1);
if (!sock->file) {
iput(SOCK_INODE(sock));
return;
--
1.6.3.3
^ permalink raw reply related [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-19 9:26 ` Avi Kivity
@ 2011-10-20 2:45 ` Alex,Shi
2011-10-20 7:34 ` [PATCH 2/3] kvm: use this_cpu_xxx replace percpu_xxx funcs Alex,Shi
1 sibling, 0 replies; 42+ messages in thread
From: Alex,Shi @ 2011-10-20 2:45 UTC (permalink / raw)
To: Avi Kivity
Cc: Christoph Lameter, tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying, tglx@linutronix.de,
mingo@redhat.com, akpm@linux-foundation.org, davem@davemloft.net,
kaber@trash.net, a.p.zijlstra@chello.nl, kvm@vger.kernel.org,
jeremy@xensource.com
> All of the kvm usage is in preemption disabled contexts.
Thanks! So would you like to ack this patch?
===
>From 8ca7e6e40d75eab598c4fb4f8d555299b747adfa Mon Sep 17 00:00:00 2001
From: Alex Shi <alex.shi@intel.com>
Date: Thu, 20 Oct 2011 10:31:51 +0800
Subject: [PATCH 2/2] Change this_cpu_xxx to __this_cpu_xxx funcs
According to Avi, all of this_cpu_xx used in preempt disable context,
So change them to __this_cpu_xxx that without abundant preempt
protection.
Signed-off-by: Alex Shi <alex.shi@intel.com>
---
arch/x86/kvm/x86.c | 10 +++++-----
1 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 5ff2516..432c4bc 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
static int kvm_is_in_guest(void)
{
- return this_cpu_read(current_vcpu) != NULL;
+ return __this_cpu_read(current_vcpu) != NULL;
}
static int kvm_is_user_mode(void)
{
int user_mode = 3;
- if (this_cpu_read(current_vcpu))
- user_mode = kvm_x86_ops->get_cpl(this_cpu_read(current_vcpu));
+ if (__this_cpu_read(current_vcpu))
+ user_mode = kvm_x86_ops->get_cpl(__this_cpu_read(current_vcpu));
return user_mode != 0;
}
@@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void)
{
unsigned long ip = 0;
- if (this_cpu_read(current_vcpu))
- ip = kvm_rip_read(this_cpu_read(current_vcpu));
+ if (__this_cpu_read(current_vcpu))
+ ip = kvm_rip_read(__this_cpu_read(current_vcpu));
return ip;
}
--
1.6.3.3
^ permalink raw reply related [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-20 2:44 ` Alex,Shi
@ 2011-10-20 5:08 ` Eric Dumazet
2011-10-20 6:04 ` Alex,Shi
` (3 more replies)
0 siblings, 4 replies; 42+ messages in thread
From: Eric Dumazet @ 2011-10-20 5:08 UTC (permalink / raw)
To: Alex,Shi
Cc: Christoph Lameter, tj@kernel.org, linux-kernel@vger.kernel.org,
Huang, Ying, Thomas Gleixner, mingo@redhat.com, avi@redhat.com,
akpm@linux-foundation.org, David Miller, kaber@trash.net,
a.p.zijlstra@chello.nl, kvm@vger.kernel.org, jeremy@xensource.com
Le jeudi 20 octobre 2011 à 10:44 +0800, Alex,Shi a écrit :
> > Acked-by: Christoph Lameter <cl@gentwo.org>
>
> Thanks, Christoph!
> and resend for code style problem correction.
>
Sorry this huge patch brings too many potential bugs.
I ask you a separate patch for the networking part, because I dont want
to Ack all the other parts.
> diff --git a/net/socket.c b/net/socket.c
> index ffe92ca..2845d38 100644
> --- a/net/socket.c
> +++ b/net/socket.c
> @@ -479,7 +479,7 @@ static struct socket *sock_alloc(void)
> inode->i_uid = current_fsuid();
> inode->i_gid = current_fsgid();
>
> - percpu_add(sockets_in_use, 1);
> + __this_cpu_add(sockets_in_use, 1);
No, we are in process context, you need
this_cpu_add(sockets_in_use, 1);
> return sock;
> }
>
> @@ -522,7 +522,7 @@ void sock_release(struct socket *sock)
> if (rcu_dereference_protected(sock->wq, 1)->fasync_list)
> printk(KERN_ERR "sock_release: fasync list not empty!\n");
>
> - percpu_sub(sockets_in_use, 1);
> + __this_cpu_sub(sockets_in_use, 1);
this_cpu_sub();
> if (!sock->file) {
> iput(SOCK_INODE(sock));
> return;
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-20 5:08 ` Eric Dumazet
@ 2011-10-20 6:04 ` Alex,Shi
2011-10-20 6:13 ` Alex,Shi
2011-10-20 6:04 ` Alex,Shi
` (2 subsequent siblings)
3 siblings, 1 reply; 42+ messages in thread
From: Alex,Shi @ 2011-10-20 6:04 UTC (permalink / raw)
To: Eric Dumazet
Cc: Christoph Lameter, tj@kernel.org, linux-kernel@vger.kernel.org,
Huang, Ying, Thomas Gleixner, mingo@redhat.com, avi@redhat.com,
akpm@linux-foundation.org, David Miller, kaber@trash.net,
a.p.zijlstra@chello.nl, kvm@vger.kernel.org, jeremy@xensource.com
> Sorry this huge patch brings too many potential bugs.
>
> I ask you a separate patch for the networking part, because I dont want
> to Ack all the other parts.
>
Thanks a lot for review! I split the first patch into 2 according your
suggestion. The second one for network here:
>From d5d71127ad22890f6f13fa382736c0df7f5d0231 Mon Sep 17 00:00:00 2001
From: Alex Shi <alex.shi@intel.com>
Date: Thu, 20 Oct 2011 13:44:37 +0800
Subject: [PATCH 2/2] change this_cpu_xxx funcs for preempt safe scenarios
If context is preempt safe, using __this_cpu_xxx to replace this_cpu_xxx
have performance benefit, since the later has a abundant preempt_disable
instruction.
Signed-off-by: Alex Shi <alex.shi@intel.com>
---
net/netfilter/xt_TEE.c | 12 ++++++------
1 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/net/netfilter/xt_TEE.c b/net/netfilter/xt_TEE.c
index e78103d..678084c 100644
--- a/net/netfilter/xt_TEE.c
+++ b/net/netfilter/xt_TEE.c
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
const struct xt_tee_tginfo *info = par->targinfo;
struct iphdr *iph;
- if (this_cpu_read(tee_active))
+ if (__this_cpu_read(tee_active))
return XT_CONTINUE;
/*
* Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
ip_send_check(iph);
if (tee_tg_route4(skb, info)) {
- this_cpu_write(tee_active, true);
+ __this_cpu_write(tee_active, true);
ip_local_out(skb);
- this_cpu_write(tee_active, false);
+ __this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
{
const struct xt_tee_tginfo *info = par->targinfo;
- if (this_cpu_read(tee_active))
+ if (__this_cpu_read(tee_active))
return XT_CONTINUE;
skb = pskb_copy(skb, GFP_ATOMIC);
if (skb == NULL)
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
--iph->hop_limit;
}
if (tee_tg_route6(skb, info)) {
- this_cpu_write(tee_active, true);
+ __this_cpu_write(tee_active, true);
ip6_local_out(skb);
- this_cpu_write(tee_active, false);
+ __this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
--
1.6.3.3
^ permalink raw reply related [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-20 5:08 ` Eric Dumazet
2011-10-20 6:04 ` Alex,Shi
@ 2011-10-20 6:04 ` Alex,Shi
2011-10-20 7:20 ` [PATCH 0/3] code clean and optimzed for percpu_xxx serials funcs Alex,Shi
2011-10-20 7:32 ` [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs Alex,Shi
3 siblings, 0 replies; 42+ messages in thread
From: Alex,Shi @ 2011-10-20 6:04 UTC (permalink / raw)
To: Eric Dumazet
Cc: Christoph Lameter, tj@kernel.org, linux-kernel@vger.kernel.org,
Huang, Ying, Thomas Gleixner, mingo@redhat.com, avi@redhat.com,
akpm@linux-foundation.org, David Miller, kaber@trash.net,
a.p.zijlstra@chello.nl, kvm@vger.kernel.org, jeremy@xensource.com
> Sorry this huge patch brings too many potential bugs.
>
> I ask you a separate patch for the networking part, because I dont want
> to Ack all the other parts.
Thanks a lot for review! I split the first patch into 2 according your
suggestion. The first one is here:
>From 516756ad094c49f8ddc4022157db9a5ce8f86578 Mon Sep 17 00:00:00 2001
From: Alex Shi <alex.shi@intel.com>
Date: Thu, 20 Oct 2011 13:30:11 +0800
Subject: [PATCH 1/2] Code clean up for percpu_xxx() functions
Since percpu_xxx() serial functions are duplicate with this_cpu_xxx().
Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in
code.
And further more, as Christoph Lameter's requirement, I try to use
__this_cpu_xx to replace this_cpu_xxx if it is in preempt safe scenario.
The preempt safe scenarios include:
1, in irq/softirq/nmi handler
2, protected by preempt_disable
3, protected by spin_lock
4, if the code context imply that it is preempt safe, like the code is
follows or be followed a preempt safe code.
I left the xen/kvm/network part code unchanged, since no idea of them.
BTW, In fact, this_cpu_xxx are same as __this_cpu_xxx since all funcs
implement in a single instruction for x86 machine. But it maybe
different for other platforms, so, doing this distinguish is helpful for
other platforms' performance.
Signed-off-by: Alex Shi <alex.shi@intel.com>
Acked-by: Christoph Lameter <cl@gentwo.org>
---
arch/x86/include/asm/current.h | 2 +-
arch/x86/include/asm/hardirq.h | 9 +++--
arch/x86/include/asm/irq_regs.h | 4 +-
arch/x86/include/asm/mmu_context.h | 12 ++++----
arch/x86/include/asm/percpu.h | 24 ++++++---------
arch/x86/include/asm/smp.h | 4 +-
arch/x86/include/asm/stackprotector.h | 4 +-
arch/x86/include/asm/thread_info.h | 2 +-
arch/x86/include/asm/tlbflush.h | 4 +-
arch/x86/kernel/cpu/common.c | 2 +-
arch/x86/kernel/cpu/mcheck/mce.c | 4 +-
arch/x86/kernel/cpu/perf_event.c | 4 +-
arch/x86/kernel/paravirt.c | 12 ++++----
arch/x86/kernel/process_32.c | 2 +-
arch/x86/kernel/process_64.c | 12 ++++----
arch/x86/kvm/x86.c | 14 ++++----
arch/x86/mm/tlb.c | 10 +++---
arch/x86/xen/enlighten.c | 6 ++--
arch/x86/xen/irq.c | 8 ++--
arch/x86/xen/mmu.c | 20 ++++++------
arch/x86/xen/multicalls.h | 2 +-
arch/x86/xen/smp.c | 2 +-
include/linux/percpu.h | 53 ---------------------------------
include/linux/topology.h | 4 +-
net/netfilter/xt_TEE.c | 12 ++++----
net/socket.c | 4 +-
26 files changed, 90 insertions(+), 146 deletions(-)
diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index 4d447b7..9476c04 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task);
static __always_inline struct task_struct *get_current(void)
{
- return percpu_read_stable(current_task);
+ return this_cpu_read_stable(current_task);
}
#define current get_current()
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 55e4de6..2890444 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -35,14 +35,15 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
#define __ARCH_IRQ_STAT
-#define inc_irq_stat(member) percpu_inc(irq_stat.member)
+#define inc_irq_stat(member) __this_cpu_inc(irq_stat.member)
-#define local_softirq_pending() percpu_read(irq_stat.__softirq_pending)
+#define local_softirq_pending() __this_cpu_read(irq_stat.__softirq_pending)
#define __ARCH_SET_SOFTIRQ_PENDING
-#define set_softirq_pending(x) percpu_write(irq_stat.__softirq_pending, (x))
-#define or_softirq_pending(x) percpu_or(irq_stat.__softirq_pending, (x))
+#define set_softirq_pending(x) \
+ __this_cpu_write(irq_stat.__softirq_pending, (x))
+#define or_softirq_pending(x) __this_cpu_or(irq_stat.__softirq_pending, (x))
extern void ack_bad_irq(unsigned int irq);
diff --git a/arch/x86/include/asm/irq_regs.h b/arch/x86/include/asm/irq_regs.h
index 7784322..15639ed 100644
--- a/arch/x86/include/asm/irq_regs.h
+++ b/arch/x86/include/asm/irq_regs.h
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);
static inline struct pt_regs *get_irq_regs(void)
{
- return percpu_read(irq_regs);
+ return __this_cpu_read(irq_regs);
}
static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
struct pt_regs *old_regs;
old_regs = get_irq_regs();
- percpu_write(irq_regs, new_regs);
+ __this_cpu_write(irq_regs, new_regs);
return old_regs;
}
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 6902152..02ca533 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);
static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
{
#ifdef CONFIG_SMP
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
- percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
#endif
}
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
if (likely(prev != next)) {
#ifdef CONFIG_SMP
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- percpu_write(cpu_tlbstate.active_mm, next);
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ __this_cpu_write(cpu_tlbstate.active_mm, next);
#endif
cpumask_set_cpu(cpu, mm_cpumask(next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
}
#ifdef CONFIG_SMP
else {
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ BUG_ON(__this_cpu_read(cpu_tlbstate.active_mm) != next);
if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next))) {
/* We were in lazy tlb mode and leave_mm disabled
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 3470c9d..520005e 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -46,7 +46,7 @@
#ifdef CONFIG_SMP
#define __percpu_prefix "%%"__stringify(__percpu_seg)":"
-#define __my_cpu_offset percpu_read(this_cpu_off)
+#define __my_cpu_offset __this_cpu_read(this_cpu_off)
/*
* Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,15 @@ do { \
})
/*
- * percpu_read() makes gcc load the percpu variable every time it is
- * accessed while percpu_read_stable() allows the value to be cached.
- * percpu_read_stable() is more efficient and can be used if its value
+ * this_cpu_read() makes gcc load the percpu variable every time it is
+ * accessed while this_cpu_read_stable() allows the value to be cached.
+ * this_cpu_read_stable() is more efficient and can be used if its value
* is guaranteed to be valid across cpus. The current users include
* get_current() and get_thread_info() both of which are actually
* per-thread variables implemented as per-cpu variables and thus
* stable for the duration of the respective task.
*/
-#define percpu_read(var) percpu_from_op("mov", var, "m" (var))
-#define percpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
-#define percpu_write(var, val) percpu_to_op("mov", var, val)
-#define percpu_add(var, val) percpu_add_op(var, val)
-#define percpu_sub(var, val) percpu_add_op(var, -(val))
-#define percpu_and(var, val) percpu_to_op("and", var, val)
-#define percpu_or(var, val) percpu_to_op("or", var, val)
-#define percpu_xor(var, val) percpu_to_op("xor", var, val)
-#define percpu_inc(var) percpu_unary_op("inc", var)
+#define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +543,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr,
{
unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG;
- return ((1UL << (nr % BITS_PER_LONG)) & percpu_read(*a)) != 0;
+#ifdef CONFIG_X86_64
+ return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_8(*a)) != 0;
+#else
+ return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_4(*a)) != 0;
+#endif
}
static inline int x86_this_cpu_variable_test_bit(int nr,
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 73b11bc..dd5749b 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;
* from the initial startup. We map APIC_BASE very early in page_setup(),
* so this is correct in the x86 case.
*/
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
extern int safe_smp_processor_id(void);
#elif defined(CONFIG_X86_64_SMP)
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
#define stack_smp_processor_id() \
({ \
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
index 1575177..e8a60c9 100644
--- a/arch/x86/include/asm/stackprotector.h
+++ b/arch/x86/include/asm/stackprotector.h
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void)
current->stack_canary = canary;
#ifdef CONFIG_X86_64
- percpu_write(irq_stack_union.stack_canary, canary);
+ __this_cpu_write(irq_stack_union.stack_canary, canary);
#else
- percpu_write(stack_canary.canary, canary);
+ __this_cpu_write(stack_canary.canary, canary);
#endif
}
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index a1fe5c1..8b19667 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);
static inline struct thread_info *current_thread_info(void)
{
struct thread_info *ti;
- ti = (void *)(percpu_read_stable(kernel_stack) +
+ ti = (void *)(this_cpu_read_stable(kernel_stack) +
KERNEL_STACK_OFFSET - THREAD_SIZE);
return ti;
}
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 169be89..e90eec0 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
static inline void reset_lazy_tlbstate(void)
{
- percpu_write(cpu_tlbstate.state, 0);
- percpu_write(cpu_tlbstate.active_mm, &init_mm);
+ __this_cpu_write(cpu_tlbstate.state, 0);
+ __this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
}
#endif /* SMP */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 6218439..780d9db 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1158,7 +1158,7 @@ void __cpuinit cpu_init(void)
oist = &per_cpu(orig_ist, cpu);
#ifdef CONFIG_NUMA
- if (cpu != 0 && percpu_read(numa_node) == 0 &&
+ if (cpu != 0 && __this_cpu_read(numa_node) == 0 &&
early_cpu_to_node(cpu) != NUMA_NO_NODE)
set_numa_node(early_cpu_to_node(cpu));
#endif
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 08363b0..5b0ebce 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -512,7 +512,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
struct mce m;
int i;
- percpu_inc(mce_poll_count);
+ __this_cpu_inc(mce_poll_count);
mce_gather_info(&m, NULL);
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
atomic_inc(&mce_entry);
- percpu_inc(mce_exception_count);
+ __this_cpu_inc(mce_exception_count);
if (notify_die(DIE_NMI, "machine check", regs, error_code,
18, SIGKILL) == NOTIFY_STOP)
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index cfa62ec..2e22701 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -1389,7 +1389,7 @@ perf_event_nmi_handler(struct notifier_block *self,
case DIE_NMI:
break;
case DIE_NMIUNKNOWN:
- this_nmi = percpu_read(irq_stat.__nmi_count);
+ this_nmi = __this_cpu_read(irq_stat.__nmi_count);
if (this_nmi != __this_cpu_read(pmu_nmi.marked))
/* let the kernel handle the unknown nmi */
return NOTIFY_DONE;
@@ -1409,7 +1409,7 @@ perf_event_nmi_handler(struct notifier_block *self,
if (!handled)
return NOTIFY_DONE;
- this_nmi = percpu_read(irq_stat.__nmi_count);
+ this_nmi = __this_cpu_read(irq_stat.__nmi_count);
if ((handled > 1) ||
/* the next nmi could be a back-to-back nmi */
((__this_cpu_read(pmu_nmi.marked) == this_nmi) &&
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index d90272e..2f0c1d1 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LA
static inline void enter_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
+ BUG_ON(__this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
- percpu_write(paravirt_lazy_mode, mode);
+ __this_cpu_write(paravirt_lazy_mode, mode);
}
static void leave_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != mode);
+ BUG_ON(__this_cpu_read(paravirt_lazy_mode) != mode);
- percpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
+ __this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
}
void paravirt_enter_lazy_mmu(void)
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev)
{
BUG_ON(preemptible());
- if (percpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
+ if (__this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
arch_leave_lazy_mmu_mode();
set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES);
}
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
if (in_interrupt())
return PARAVIRT_LAZY_NONE;
- return percpu_read(paravirt_lazy_mode);
+ return __this_cpu_read(paravirt_lazy_mode);
}
void arch_flush_lazy_mmu_mode(void)
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 7a3b651..55feac1 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -373,7 +373,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
if (prev->gs | next->gs)
lazy_load_gs(next->gs);
- percpu_write(current_task, next_p);
+ __this_cpu_write(current_task, next_p);
return prev_p;
}
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index f693e44..c3617c3 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -73,7 +73,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);
void enter_idle(void)
{
- percpu_write(is_idle, 1);
+ __this_cpu_write(is_idle, 1);
atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
}
@@ -336,7 +336,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,
load_gs_index(0);
regs->ip = new_ip;
regs->sp = new_sp;
- percpu_write(old_rsp, new_sp);
+ this_cpu_write(old_rsp, new_sp);
regs->cs = _cs;
regs->ss = _ss;
regs->flags = X86_EFLAGS_IF;
@@ -470,11 +470,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
/*
* Switch the PDA and FPU contexts.
*/
- prev->usersp = percpu_read(old_rsp);
- percpu_write(old_rsp, next->usersp);
- percpu_write(current_task, next_p);
+ prev->usersp = __this_cpu_read(old_rsp);
+ __this_cpu_write(old_rsp, next->usersp);
+ __this_cpu_write(current_task, next_p);
- percpu_write(kernel_stack,
+ __this_cpu_write(kernel_stack,
(unsigned long)task_stack_page(next_p) +
THREAD_SIZE - KERNEL_STACK_OFFSET);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 84a28ea..5ff2516 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
static int kvm_is_in_guest(void)
{
- return percpu_read(current_vcpu) != NULL;
+ return this_cpu_read(current_vcpu) != NULL;
}
static int kvm_is_user_mode(void)
{
int user_mode = 3;
- if (percpu_read(current_vcpu))
- user_mode = kvm_x86_ops->get_cpl(percpu_read(current_vcpu));
+ if (this_cpu_read(current_vcpu))
+ user_mode = kvm_x86_ops->get_cpl(this_cpu_read(current_vcpu));
return user_mode != 0;
}
@@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void)
{
unsigned long ip = 0;
- if (percpu_read(current_vcpu))
- ip = kvm_rip_read(percpu_read(current_vcpu));
+ if (this_cpu_read(current_vcpu))
+ ip = kvm_rip_read(this_cpu_read(current_vcpu));
return ip;
}
@@ -5055,13 +5055,13 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {
void kvm_before_handle_nmi(struct kvm_vcpu *vcpu)
{
- percpu_write(current_vcpu, vcpu);
+ __this_cpu_write(current_vcpu, vcpu);
}
EXPORT_SYMBOL_GPL(kvm_before_handle_nmi);
void kvm_after_handle_nmi(struct kvm_vcpu *vcpu)
{
- percpu_write(current_vcpu, NULL);
+ __this_cpu_write(current_vcpu, NULL);
}
EXPORT_SYMBOL_GPL(kvm_after_handle_nmi);
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index d6c0418..e931db0 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);
*/
void leave_mm(int cpu)
{
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
BUG();
cpumask_clear_cpu(cpu,
- mm_cpumask(percpu_read(cpu_tlbstate.active_mm)));
+ mm_cpumask(__this_cpu_read(cpu_tlbstate.active_mm)));
load_cr3(swapper_pg_dir);
}
EXPORT_SYMBOL_GPL(leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)
* BUG();
*/
- if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
+ if (f->flush_mm == __this_cpu_read(cpu_tlbstate.active_mm)) {
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
if (f->flush_va == TLB_FLUSH_ALL)
local_flush_tlb();
else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
static void do_flush_tlb_all(void *info)
{
__flush_tlb_all();
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
leave_mm(smp_processor_id());
}
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 2d69617..c4e362a 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -776,11 +776,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);
static unsigned long xen_read_cr0(void)
{
- unsigned long cr0 = percpu_read(xen_cr0_value);
+ unsigned long cr0 = this_cpu_read(xen_cr0_value);
if (unlikely(cr0 == 0)) {
cr0 = native_read_cr0();
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
}
return cr0;
@@ -790,7 +790,7 @@ static void xen_write_cr0(unsigned long cr0)
{
struct multicall_space mcs;
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
/* Only pay attention to cr0.TS; everything else is
ignored. */
diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c
index 8bbb465..1573376 100644
--- a/arch/x86/xen/irq.c
+++ b/arch/x86/xen/irq.c
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)
struct vcpu_info *vcpu;
unsigned long flags;
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
/* flag has opposite sense of mask */
flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = flags;
preempt_enable_no_resched();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- percpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
+ this_cpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
preempt_enable_no_resched();
}
PV_CALLEE_SAVE_REGS_THUNK(xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)
the caller is confused and is trying to re-enable interrupts
on an indeterminate processor. */
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = 0;
/* Doesn't matter if we get preempted here, because any
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 3dd53f9..6a7aaac 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1106,14 +1106,14 @@ static void drop_other_mm_ref(void *info)
struct mm_struct *mm = info;
struct mm_struct *active_mm;
- active_mm = percpu_read(cpu_tlbstate.active_mm);
+ active_mm = this_cpu_read(cpu_tlbstate.active_mm);
- if (active_mm == mm && percpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
+ if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
leave_mm(smp_processor_id());
/* If this cpu still has a stale cr3 reference, then make sure
it has been flushed. */
- if (percpu_read(xen_current_cr3) == __pa(mm->pgd))
+ if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd))
load_cr3(swapper_pg_dir);
}
@@ -1220,17 +1220,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)
static void xen_write_cr2(unsigned long cr2)
{
- percpu_read(xen_vcpu)->arch.cr2 = cr2;
+ this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
}
static unsigned long xen_read_cr2(void)
{
- return percpu_read(xen_vcpu)->arch.cr2;
+ return this_cpu_read(xen_vcpu)->arch.cr2;
}
unsigned long xen_read_cr2_direct(void)
{
- return percpu_read(xen_vcpu_info.arch.cr2);
+ return this_cpu_read(xen_vcpu_info.arch.cr2);
}
static void xen_flush_tlb(void)
@@ -1313,12 +1313,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
static unsigned long xen_read_cr3(void)
{
- return percpu_read(xen_cr3);
+ return this_cpu_read(xen_cr3);
}
static void set_current_cr3(void *v)
{
- percpu_write(xen_current_cr3, (unsigned long)v);
+ this_cpu_write(xen_current_cr3, (unsigned long)v);
}
static void __xen_write_cr3(bool kernel, unsigned long cr3)
@@ -1341,7 +1341,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)
xen_extend_mmuext_op(&op);
if (kernel) {
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
/* Update xen_current_cr3 once the batch has actually
been submitted. */
@@ -1357,7 +1357,7 @@ static void xen_write_cr3(unsigned long cr3)
/* Update while interrupts are disabled, so its atomic with
respect to ipis */
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
__xen_write_cr3(true, cr3);
diff --git a/arch/x86/xen/multicalls.h b/arch/x86/xen/multicalls.h
index dee79b7..9c2e74f 100644
--- a/arch/x86/xen/multicalls.h
+++ b/arch/x86/xen/multicalls.h
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)
xen_mc_flush();
/* restore flags saved in xen_mc_batch */
- local_irq_restore(percpu_read(xen_mc_irq_flags));
+ local_irq_restore(this_cpu_read(xen_mc_irq_flags));
}
/* Set up a callback to be called when the current batch is flushed */
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 041d4fe..449f868 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)
xen_setup_cpu_clockevents();
set_cpu_online(cpu, true);
- percpu_write(cpu_state, CPU_ONLINE);
+ this_cpu_write(cpu_state, CPU_ONLINE);
wmb();
/* We can take interrupts now: we're officially "up". */
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 9ca008f..aad7e1c 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -165,59 +165,6 @@ extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
#define alloc_percpu(type) \
(typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
-/*
- * Optional methods for optimized non-lvalue per-cpu variable access.
- *
- * @var can be a percpu variable or a field of it and its size should
- * equal char, int or long. percpu_read() evaluates to a lvalue and
- * all others to void.
- *
- * These operations are guaranteed to be atomic w.r.t. preemption.
- * The generic versions use plain get/put_cpu_var(). Archs are
- * encouraged to implement single-instruction alternatives which don't
- * require preemption protection.
- */
-#ifndef percpu_read
-# define percpu_read(var) \
- ({ \
- typeof(var) *pr_ptr__ = &(var); \
- typeof(var) pr_ret__; \
- pr_ret__ = get_cpu_var(*pr_ptr__); \
- put_cpu_var(*pr_ptr__); \
- pr_ret__; \
- })
-#endif
-
-#define __percpu_generic_to_op(var, val, op) \
-do { \
- typeof(var) *pgto_ptr__ = &(var); \
- get_cpu_var(*pgto_ptr__) op val; \
- put_cpu_var(*pgto_ptr__); \
-} while (0)
-
-#ifndef percpu_write
-# define percpu_write(var, val) __percpu_generic_to_op(var, (val), =)
-#endif
-
-#ifndef percpu_add
-# define percpu_add(var, val) __percpu_generic_to_op(var, (val), +=)
-#endif
-
-#ifndef percpu_sub
-# define percpu_sub(var, val) __percpu_generic_to_op(var, (val), -=)
-#endif
-
-#ifndef percpu_and
-# define percpu_and(var, val) __percpu_generic_to_op(var, (val), &=)
-#endif
-
-#ifndef percpu_or
-# define percpu_or(var, val) __percpu_generic_to_op(var, (val), |=)
-#endif
-
-#ifndef percpu_xor
-# define percpu_xor(var, val) __percpu_generic_to_op(var, (val), ^=)
-#endif
/*
* Branching function to split up a function into a set of functions that
diff --git a/include/linux/topology.h b/include/linux/topology.h
index fc839bf..aede2d9 100644
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -235,7 +235,7 @@ static inline int cpu_to_node(int cpu)
#ifndef set_numa_node
static inline void set_numa_node(int node)
{
- percpu_write(numa_node, node);
+ __this_cpu_write(numa_node, node);
}
#endif
@@ -270,7 +270,7 @@ DECLARE_PER_CPU(int, _numa_mem_);
#ifndef set_numa_mem
static inline void set_numa_mem(int node)
{
- percpu_write(_numa_mem_, node);
+ __this_cpu_write(_numa_mem_, node);
}
#endif
diff --git a/net/netfilter/xt_TEE.c b/net/netfilter/xt_TEE.c
index 5f054a0..e78103d 100644
--- a/net/netfilter/xt_TEE.c
+++ b/net/netfilter/xt_TEE.c
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
const struct xt_tee_tginfo *info = par->targinfo;
struct iphdr *iph;
- if (percpu_read(tee_active))
+ if (this_cpu_read(tee_active))
return XT_CONTINUE;
/*
* Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
ip_send_check(iph);
if (tee_tg_route4(skb, info)) {
- percpu_write(tee_active, true);
+ this_cpu_write(tee_active, true);
ip_local_out(skb);
- percpu_write(tee_active, false);
+ this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
{
const struct xt_tee_tginfo *info = par->targinfo;
- if (percpu_read(tee_active))
+ if (this_cpu_read(tee_active))
return XT_CONTINUE;
skb = pskb_copy(skb, GFP_ATOMIC);
if (skb == NULL)
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
--iph->hop_limit;
}
if (tee_tg_route6(skb, info)) {
- percpu_write(tee_active, true);
+ this_cpu_write(tee_active, true);
ip6_local_out(skb);
- percpu_write(tee_active, false);
+ this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
diff --git a/net/socket.c b/net/socket.c
index ffe92ca..4b62ca9 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -479,7 +479,7 @@ static struct socket *sock_alloc(void)
inode->i_uid = current_fsuid();
inode->i_gid = current_fsgid();
- percpu_add(sockets_in_use, 1);
+ this_cpu_add(sockets_in_use, 1);
return sock;
}
@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)
if (rcu_dereference_protected(sock->wq, 1)->fasync_list)
printk(KERN_ERR "sock_release: fasync list not empty!\n");
- percpu_sub(sockets_in_use, 1);
+ this_cpu_sub(sockets_in_use, 1);
if (!sock->file) {
iput(SOCK_INODE(sock));
return;
--
1.6.3.3
^ permalink raw reply related [flat|nested] 42+ messages in thread
* Re: [PATCH] Code clean up for percpu_xxx() functions
2011-10-20 6:04 ` Alex,Shi
@ 2011-10-20 6:13 ` Alex,Shi
0 siblings, 0 replies; 42+ messages in thread
From: Alex,Shi @ 2011-10-20 6:13 UTC (permalink / raw)
To: Eric Dumazet
Cc: Christoph Lameter, tj@kernel.org, linux-kernel@vger.kernel.org,
Huang, Ying, Thomas Gleixner, mingo@redhat.com, avi@redhat.com,
akpm@linux-foundation.org, David Miller, kaber@trash.net,
a.p.zijlstra@chello.nl, kvm@vger.kernel.org, jeremy@xensource.com
On Thu, 2011-10-20 at 14:04 +0800, Alex,Shi wrote:
> > Sorry this huge patch brings too many potential bugs.
> >
> > I ask you a separate patch for the networking part, because I dont want
> > to Ack all the other parts.
> >
>
> Thanks a lot for review! I split the first patch into 2 according your
> suggestion. The second one for network here:
Sorry for too big patch, any other split requirements?
^ permalink raw reply [flat|nested] 42+ messages in thread
* [PATCH 0/3] code clean and optimzed for percpu_xxx serials funcs
2011-10-20 5:08 ` Eric Dumazet
2011-10-20 6:04 ` Alex,Shi
2011-10-20 6:04 ` Alex,Shi
@ 2011-10-20 7:20 ` Alex,Shi
2011-10-20 7:32 ` [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs Alex,Shi
3 siblings, 0 replies; 42+ messages in thread
From: Alex,Shi @ 2011-10-20 7:20 UTC (permalink / raw)
To: Eric Dumazet
Cc: Christoph Lameter, tj@kernel.org, linux-kernel@vger.kernel.org,
Huang, Ying, Thomas Gleixner, mingo@redhat.com, avi@redhat.com,
akpm@linux-foundation.org, David Miller, kaber@trash.net,
a.p.zijlstra@chello.nl, kvm@vger.kernel.org, jeremy@xensource.com
Think over Eric's requirement, I split my old patches and reorganized
them into 3 small patches, include net, kvm and a final clean up patch.
Since the clean up contents has no change beside move kvm/net part into
seprated patches. I keep Christoph's ack. Hope these are what you
want. :)
Regards
Alex
^ permalink raw reply [flat|nested] 42+ messages in thread
* [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs
2011-10-20 5:08 ` Eric Dumazet
` (2 preceding siblings ...)
2011-10-20 7:20 ` [PATCH 0/3] code clean and optimzed for percpu_xxx serials funcs Alex,Shi
@ 2011-10-20 7:32 ` Alex,Shi
2011-10-20 8:38 ` Eric Dumazet
3 siblings, 1 reply; 42+ messages in thread
From: Alex,Shi @ 2011-10-20 7:32 UTC (permalink / raw)
To: Eric Dumazet
Cc: Christoph Lameter, tj@kernel.org, linux-kernel@vger.kernel.org,
Huang, Ying, Thomas Gleixner, mingo@redhat.com, avi@redhat.com,
akpm@linux-foundation.org, David Miller, kaber@trash.net,
a.p.zijlstra@chello.nl, kvm@vger.kernel.org, jeremy@xensource.com
percpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them
for further code clean up.
And in preempt safe scenario, __this_cpu_xxx funcs has a bit better
performance since __this_cpu_xxx has no redundant preempt_disable()
Signed-off-by: Alex Shi <alex.shi@intel.com>
---
net/netfilter/xt_TEE.c | 12 ++++++------
net/socket.c | 4 ++--
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/net/netfilter/xt_TEE.c b/net/netfilter/xt_TEE.c
index 5f054a0..678084c 100644
--- a/net/netfilter/xt_TEE.c
+++ b/net/netfilter/xt_TEE.c
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
const struct xt_tee_tginfo *info = par->targinfo;
struct iphdr *iph;
- if (percpu_read(tee_active))
+ if (__this_cpu_read(tee_active))
return XT_CONTINUE;
/*
* Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
ip_send_check(iph);
if (tee_tg_route4(skb, info)) {
- percpu_write(tee_active, true);
+ __this_cpu_write(tee_active, true);
ip_local_out(skb);
- percpu_write(tee_active, false);
+ __this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
{
const struct xt_tee_tginfo *info = par->targinfo;
- if (percpu_read(tee_active))
+ if (__this_cpu_read(tee_active))
return XT_CONTINUE;
skb = pskb_copy(skb, GFP_ATOMIC);
if (skb == NULL)
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
--iph->hop_limit;
}
if (tee_tg_route6(skb, info)) {
- percpu_write(tee_active, true);
+ __this_cpu_write(tee_active, true);
ip6_local_out(skb);
- percpu_write(tee_active, false);
+ __this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
diff --git a/net/socket.c b/net/socket.c
index ffe92ca..4b62ca9 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -479,7 +479,7 @@ static struct socket *sock_alloc(void)
inode->i_uid = current_fsuid();
inode->i_gid = current_fsgid();
- percpu_add(sockets_in_use, 1);
+ this_cpu_add(sockets_in_use, 1);
return sock;
}
@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)
if (rcu_dereference_protected(sock->wq, 1)->fasync_list)
printk(KERN_ERR "sock_release: fasync list not empty!\n");
- percpu_sub(sockets_in_use, 1);
+ this_cpu_sub(sockets_in_use, 1);
if (!sock->file) {
iput(SOCK_INODE(sock));
return;
--
1.6.3.3
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [PATCH 2/3] kvm: use this_cpu_xxx replace percpu_xxx funcs
2011-10-19 9:26 ` Avi Kivity
2011-10-20 2:45 ` Alex,Shi
@ 2011-10-20 7:34 ` Alex,Shi
2011-10-24 2:50 ` Alex,Shi
1 sibling, 1 reply; 42+ messages in thread
From: Alex,Shi @ 2011-10-20 7:34 UTC (permalink / raw)
To: Avi Kivity
Cc: Christoph Lameter, tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying, tglx@linutronix.de,
mingo@redhat.com, akpm@linux-foundation.org, davem@davemloft.net,
kaber@trash.net, a.p.zijlstra@chello.nl, kvm@vger.kernel.org,
jeremy@xensource.com
percpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them
for further code clean up.
And in preempt safe scenario, __this_cpu_xxx funcs has a bit better
performance since __this_cpu_xxx has no redundant preempt_disable()
Signed-off-by: Alex Shi <alex.shi@intel.com>
---
arch/x86/kvm/x86.c | 14 +++++++-------
1 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 84a28ea..432c4bc 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
static int kvm_is_in_guest(void)
{
- return percpu_read(current_vcpu) != NULL;
+ return __this_cpu_read(current_vcpu) != NULL;
}
static int kvm_is_user_mode(void)
{
int user_mode = 3;
- if (percpu_read(current_vcpu))
- user_mode = kvm_x86_ops->get_cpl(percpu_read(current_vcpu));
+ if (__this_cpu_read(current_vcpu))
+ user_mode = kvm_x86_ops->get_cpl(__this_cpu_read(current_vcpu));
return user_mode != 0;
}
@@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void)
{
unsigned long ip = 0;
- if (percpu_read(current_vcpu))
- ip = kvm_rip_read(percpu_read(current_vcpu));
+ if (__this_cpu_read(current_vcpu))
+ ip = kvm_rip_read(__this_cpu_read(current_vcpu));
return ip;
}
@@ -5055,13 +5055,13 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {
void kvm_before_handle_nmi(struct kvm_vcpu *vcpu)
{
- percpu_write(current_vcpu, vcpu);
+ __this_cpu_write(current_vcpu, vcpu);
}
EXPORT_SYMBOL_GPL(kvm_before_handle_nmi);
void kvm_after_handle_nmi(struct kvm_vcpu *vcpu)
{
- percpu_write(current_vcpu, NULL);
+ __this_cpu_write(current_vcpu, NULL);
}
EXPORT_SYMBOL_GPL(kvm_after_handle_nmi);
--
1.6.3.3
^ permalink raw reply related [flat|nested] 42+ messages in thread
* [PATCH 3/3] Code clean up for percpu_xxx() functions
2011-10-19 15:34 ` [PATCH] Code clean up for percpu_xxx() functions Christoph Lameter
2011-10-20 2:44 ` Alex,Shi
@ 2011-10-20 7:35 ` Alex,Shi
2011-11-21 9:10 ` Alex,Shi
1 sibling, 1 reply; 42+ messages in thread
From: Alex,Shi @ 2011-10-20 7:35 UTC (permalink / raw)
To: Christoph Lameter
Cc: tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying, Thomas Gleixner,
mingo@redhat.com, avi@redhat.com, akpm@linux-foundation.org,
David Miller, kaber@trash.net, a.p.zijlstra@chello.nl,
kvm@vger.kernel.org, jeremy@xensource.com
Since percpu_xxx() serial functions are duplicate with this_cpu_xxx().
Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in
code.
And further more, as Christoph Lameter's requirement, I try to use
__this_cpu_xx to replace this_cpu_xxx if it is in preempt safe scenario.
The preempt safe scenarios include:
1, in irq/softirq/nmi handler
2, protected by preempt_disable
3, protected by spin_lock
4, if the code context imply that it is preempt safe, like the code is
follows or be followed a preempt safe code.
I left the xen code unchanged, since no idea of them.
BTW, In fact, this_cpu_xxx are same as __this_cpu_xxx since all funcs
implement in a single instruction for x86 machine. But it maybe
different for other platforms, so, doing this distinguish is helpful for
other platforms' performance.
Signed-off-by: Alex Shi <alex.shi@intel.com>
Acked-by: Christoph Lameter <cl@gentwo.org>
---
arch/x86/include/asm/current.h | 2 +-
arch/x86/include/asm/hardirq.h | 9 +++--
arch/x86/include/asm/irq_regs.h | 4 +-
arch/x86/include/asm/mmu_context.h | 12 ++++----
arch/x86/include/asm/percpu.h | 24 ++++++---------
arch/x86/include/asm/smp.h | 4 +-
arch/x86/include/asm/stackprotector.h | 4 +-
arch/x86/include/asm/thread_info.h | 2 +-
arch/x86/include/asm/tlbflush.h | 4 +-
arch/x86/kernel/cpu/common.c | 2 +-
arch/x86/kernel/cpu/mcheck/mce.c | 4 +-
arch/x86/kernel/cpu/perf_event.c | 4 +-
arch/x86/kernel/paravirt.c | 12 ++++----
arch/x86/kernel/process_32.c | 2 +-
arch/x86/kernel/process_64.c | 12 ++++----
arch/x86/mm/tlb.c | 10 +++---
arch/x86/xen/enlighten.c | 6 ++--
arch/x86/xen/irq.c | 8 ++--
arch/x86/xen/mmu.c | 20 ++++++------
arch/x86/xen/multicalls.h | 2 +-
arch/x86/xen/smp.c | 2 +-
include/linux/percpu.h | 53 ---------------------------------
include/linux/topology.h | 4 +-
23 files changed, 75 insertions(+), 131 deletions(-)
diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index 4d447b7..9476c04 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task);
static __always_inline struct task_struct *get_current(void)
{
- return percpu_read_stable(current_task);
+ return this_cpu_read_stable(current_task);
}
#define current get_current()
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 55e4de6..2890444 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -35,14 +35,15 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
#define __ARCH_IRQ_STAT
-#define inc_irq_stat(member) percpu_inc(irq_stat.member)
+#define inc_irq_stat(member) __this_cpu_inc(irq_stat.member)
-#define local_softirq_pending() percpu_read(irq_stat.__softirq_pending)
+#define local_softirq_pending() __this_cpu_read(irq_stat.__softirq_pending)
#define __ARCH_SET_SOFTIRQ_PENDING
-#define set_softirq_pending(x) percpu_write(irq_stat.__softirq_pending, (x))
-#define or_softirq_pending(x) percpu_or(irq_stat.__softirq_pending, (x))
+#define set_softirq_pending(x) \
+ __this_cpu_write(irq_stat.__softirq_pending, (x))
+#define or_softirq_pending(x) __this_cpu_or(irq_stat.__softirq_pending, (x))
extern void ack_bad_irq(unsigned int irq);
diff --git a/arch/x86/include/asm/irq_regs.h b/arch/x86/include/asm/irq_regs.h
index 7784322..15639ed 100644
--- a/arch/x86/include/asm/irq_regs.h
+++ b/arch/x86/include/asm/irq_regs.h
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);
static inline struct pt_regs *get_irq_regs(void)
{
- return percpu_read(irq_regs);
+ return __this_cpu_read(irq_regs);
}
static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
struct pt_regs *old_regs;
old_regs = get_irq_regs();
- percpu_write(irq_regs, new_regs);
+ __this_cpu_write(irq_regs, new_regs);
return old_regs;
}
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 6902152..02ca533 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);
static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
{
#ifdef CONFIG_SMP
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
- percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
#endif
}
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
if (likely(prev != next)) {
#ifdef CONFIG_SMP
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- percpu_write(cpu_tlbstate.active_mm, next);
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ __this_cpu_write(cpu_tlbstate.active_mm, next);
#endif
cpumask_set_cpu(cpu, mm_cpumask(next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
}
#ifdef CONFIG_SMP
else {
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ BUG_ON(__this_cpu_read(cpu_tlbstate.active_mm) != next);
if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next))) {
/* We were in lazy tlb mode and leave_mm disabled
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 3470c9d..520005e 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -46,7 +46,7 @@
#ifdef CONFIG_SMP
#define __percpu_prefix "%%"__stringify(__percpu_seg)":"
-#define __my_cpu_offset percpu_read(this_cpu_off)
+#define __my_cpu_offset __this_cpu_read(this_cpu_off)
/*
* Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,15 @@ do { \
})
/*
- * percpu_read() makes gcc load the percpu variable every time it is
- * accessed while percpu_read_stable() allows the value to be cached.
- * percpu_read_stable() is more efficient and can be used if its value
+ * this_cpu_read() makes gcc load the percpu variable every time it is
+ * accessed while this_cpu_read_stable() allows the value to be cached.
+ * this_cpu_read_stable() is more efficient and can be used if its value
* is guaranteed to be valid across cpus. The current users include
* get_current() and get_thread_info() both of which are actually
* per-thread variables implemented as per-cpu variables and thus
* stable for the duration of the respective task.
*/
-#define percpu_read(var) percpu_from_op("mov", var, "m" (var))
-#define percpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
-#define percpu_write(var, val) percpu_to_op("mov", var, val)
-#define percpu_add(var, val) percpu_add_op(var, val)
-#define percpu_sub(var, val) percpu_add_op(var, -(val))
-#define percpu_and(var, val) percpu_to_op("and", var, val)
-#define percpu_or(var, val) percpu_to_op("or", var, val)
-#define percpu_xor(var, val) percpu_to_op("xor", var, val)
-#define percpu_inc(var) percpu_unary_op("inc", var)
+#define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +543,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr,
{
unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG;
- return ((1UL << (nr % BITS_PER_LONG)) & percpu_read(*a)) != 0;
+#ifdef CONFIG_X86_64
+ return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_8(*a)) != 0;
+#else
+ return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_4(*a)) != 0;
+#endif
}
static inline int x86_this_cpu_variable_test_bit(int nr,
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 73b11bc..dd5749b 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;
* from the initial startup. We map APIC_BASE very early in page_setup(),
* so this is correct in the x86 case.
*/
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
extern int safe_smp_processor_id(void);
#elif defined(CONFIG_X86_64_SMP)
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
#define stack_smp_processor_id() \
({ \
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
index 1575177..e8a60c9 100644
--- a/arch/x86/include/asm/stackprotector.h
+++ b/arch/x86/include/asm/stackprotector.h
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void)
current->stack_canary = canary;
#ifdef CONFIG_X86_64
- percpu_write(irq_stack_union.stack_canary, canary);
+ __this_cpu_write(irq_stack_union.stack_canary, canary);
#else
- percpu_write(stack_canary.canary, canary);
+ __this_cpu_write(stack_canary.canary, canary);
#endif
}
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index a1fe5c1..8b19667 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);
static inline struct thread_info *current_thread_info(void)
{
struct thread_info *ti;
- ti = (void *)(percpu_read_stable(kernel_stack) +
+ ti = (void *)(this_cpu_read_stable(kernel_stack) +
KERNEL_STACK_OFFSET - THREAD_SIZE);
return ti;
}
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 169be89..e90eec0 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
static inline void reset_lazy_tlbstate(void)
{
- percpu_write(cpu_tlbstate.state, 0);
- percpu_write(cpu_tlbstate.active_mm, &init_mm);
+ __this_cpu_write(cpu_tlbstate.state, 0);
+ __this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
}
#endif /* SMP */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 6218439..780d9db 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1158,7 +1158,7 @@ void __cpuinit cpu_init(void)
oist = &per_cpu(orig_ist, cpu);
#ifdef CONFIG_NUMA
- if (cpu != 0 && percpu_read(numa_node) == 0 &&
+ if (cpu != 0 && __this_cpu_read(numa_node) == 0 &&
early_cpu_to_node(cpu) != NUMA_NO_NODE)
set_numa_node(early_cpu_to_node(cpu));
#endif
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 08363b0..5b0ebce 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -512,7 +512,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
struct mce m;
int i;
- percpu_inc(mce_poll_count);
+ __this_cpu_inc(mce_poll_count);
mce_gather_info(&m, NULL);
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
atomic_inc(&mce_entry);
- percpu_inc(mce_exception_count);
+ __this_cpu_inc(mce_exception_count);
if (notify_die(DIE_NMI, "machine check", regs, error_code,
18, SIGKILL) == NOTIFY_STOP)
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index cfa62ec..2e22701 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -1389,7 +1389,7 @@ perf_event_nmi_handler(struct notifier_block *self,
case DIE_NMI:
break;
case DIE_NMIUNKNOWN:
- this_nmi = percpu_read(irq_stat.__nmi_count);
+ this_nmi = __this_cpu_read(irq_stat.__nmi_count);
if (this_nmi != __this_cpu_read(pmu_nmi.marked))
/* let the kernel handle the unknown nmi */
return NOTIFY_DONE;
@@ -1409,7 +1409,7 @@ perf_event_nmi_handler(struct notifier_block *self,
if (!handled)
return NOTIFY_DONE;
- this_nmi = percpu_read(irq_stat.__nmi_count);
+ this_nmi = __this_cpu_read(irq_stat.__nmi_count);
if ((handled > 1) ||
/* the next nmi could be a back-to-back nmi */
((__this_cpu_read(pmu_nmi.marked) == this_nmi) &&
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index d90272e..2f0c1d1 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LA
static inline void enter_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
+ BUG_ON(__this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
- percpu_write(paravirt_lazy_mode, mode);
+ __this_cpu_write(paravirt_lazy_mode, mode);
}
static void leave_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != mode);
+ BUG_ON(__this_cpu_read(paravirt_lazy_mode) != mode);
- percpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
+ __this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
}
void paravirt_enter_lazy_mmu(void)
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev)
{
BUG_ON(preemptible());
- if (percpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
+ if (__this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
arch_leave_lazy_mmu_mode();
set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES);
}
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
if (in_interrupt())
return PARAVIRT_LAZY_NONE;
- return percpu_read(paravirt_lazy_mode);
+ return __this_cpu_read(paravirt_lazy_mode);
}
void arch_flush_lazy_mmu_mode(void)
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 7a3b651..55feac1 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -373,7 +373,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
if (prev->gs | next->gs)
lazy_load_gs(next->gs);
- percpu_write(current_task, next_p);
+ __this_cpu_write(current_task, next_p);
return prev_p;
}
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index f693e44..c3617c3 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -73,7 +73,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);
void enter_idle(void)
{
- percpu_write(is_idle, 1);
+ __this_cpu_write(is_idle, 1);
atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
}
@@ -336,7 +336,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,
load_gs_index(0);
regs->ip = new_ip;
regs->sp = new_sp;
- percpu_write(old_rsp, new_sp);
+ this_cpu_write(old_rsp, new_sp);
regs->cs = _cs;
regs->ss = _ss;
regs->flags = X86_EFLAGS_IF;
@@ -470,11 +470,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
/*
* Switch the PDA and FPU contexts.
*/
- prev->usersp = percpu_read(old_rsp);
- percpu_write(old_rsp, next->usersp);
- percpu_write(current_task, next_p);
+ prev->usersp = __this_cpu_read(old_rsp);
+ __this_cpu_write(old_rsp, next->usersp);
+ __this_cpu_write(current_task, next_p);
- percpu_write(kernel_stack,
+ __this_cpu_write(kernel_stack,
(unsigned long)task_stack_page(next_p) +
THREAD_SIZE - KERNEL_STACK_OFFSET);
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index d6c0418..e931db0 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);
*/
void leave_mm(int cpu)
{
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
BUG();
cpumask_clear_cpu(cpu,
- mm_cpumask(percpu_read(cpu_tlbstate.active_mm)));
+ mm_cpumask(__this_cpu_read(cpu_tlbstate.active_mm)));
load_cr3(swapper_pg_dir);
}
EXPORT_SYMBOL_GPL(leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)
* BUG();
*/
- if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
+ if (f->flush_mm == __this_cpu_read(cpu_tlbstate.active_mm)) {
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
if (f->flush_va == TLB_FLUSH_ALL)
local_flush_tlb();
else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
static void do_flush_tlb_all(void *info)
{
__flush_tlb_all();
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
leave_mm(smp_processor_id());
}
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 2d69617..c4e362a 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -776,11 +776,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);
static unsigned long xen_read_cr0(void)
{
- unsigned long cr0 = percpu_read(xen_cr0_value);
+ unsigned long cr0 = this_cpu_read(xen_cr0_value);
if (unlikely(cr0 == 0)) {
cr0 = native_read_cr0();
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
}
return cr0;
@@ -790,7 +790,7 @@ static void xen_write_cr0(unsigned long cr0)
{
struct multicall_space mcs;
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
/* Only pay attention to cr0.TS; everything else is
ignored. */
diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c
index 8bbb465..1573376 100644
--- a/arch/x86/xen/irq.c
+++ b/arch/x86/xen/irq.c
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)
struct vcpu_info *vcpu;
unsigned long flags;
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
/* flag has opposite sense of mask */
flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = flags;
preempt_enable_no_resched();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- percpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
+ this_cpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
preempt_enable_no_resched();
}
PV_CALLEE_SAVE_REGS_THUNK(xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)
the caller is confused and is trying to re-enable interrupts
on an indeterminate processor. */
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = 0;
/* Doesn't matter if we get preempted here, because any
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 3dd53f9..6a7aaac 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1106,14 +1106,14 @@ static void drop_other_mm_ref(void *info)
struct mm_struct *mm = info;
struct mm_struct *active_mm;
- active_mm = percpu_read(cpu_tlbstate.active_mm);
+ active_mm = this_cpu_read(cpu_tlbstate.active_mm);
- if (active_mm == mm && percpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
+ if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
leave_mm(smp_processor_id());
/* If this cpu still has a stale cr3 reference, then make sure
it has been flushed. */
- if (percpu_read(xen_current_cr3) == __pa(mm->pgd))
+ if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd))
load_cr3(swapper_pg_dir);
}
@@ -1220,17 +1220,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)
static void xen_write_cr2(unsigned long cr2)
{
- percpu_read(xen_vcpu)->arch.cr2 = cr2;
+ this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
}
static unsigned long xen_read_cr2(void)
{
- return percpu_read(xen_vcpu)->arch.cr2;
+ return this_cpu_read(xen_vcpu)->arch.cr2;
}
unsigned long xen_read_cr2_direct(void)
{
- return percpu_read(xen_vcpu_info.arch.cr2);
+ return this_cpu_read(xen_vcpu_info.arch.cr2);
}
static void xen_flush_tlb(void)
@@ -1313,12 +1313,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
static unsigned long xen_read_cr3(void)
{
- return percpu_read(xen_cr3);
+ return this_cpu_read(xen_cr3);
}
static void set_current_cr3(void *v)
{
- percpu_write(xen_current_cr3, (unsigned long)v);
+ this_cpu_write(xen_current_cr3, (unsigned long)v);
}
static void __xen_write_cr3(bool kernel, unsigned long cr3)
@@ -1341,7 +1341,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)
xen_extend_mmuext_op(&op);
if (kernel) {
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
/* Update xen_current_cr3 once the batch has actually
been submitted. */
@@ -1357,7 +1357,7 @@ static void xen_write_cr3(unsigned long cr3)
/* Update while interrupts are disabled, so its atomic with
respect to ipis */
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
__xen_write_cr3(true, cr3);
diff --git a/arch/x86/xen/multicalls.h b/arch/x86/xen/multicalls.h
index dee79b7..9c2e74f 100644
--- a/arch/x86/xen/multicalls.h
+++ b/arch/x86/xen/multicalls.h
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)
xen_mc_flush();
/* restore flags saved in xen_mc_batch */
- local_irq_restore(percpu_read(xen_mc_irq_flags));
+ local_irq_restore(this_cpu_read(xen_mc_irq_flags));
}
/* Set up a callback to be called when the current batch is flushed */
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 041d4fe..449f868 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)
xen_setup_cpu_clockevents();
set_cpu_online(cpu, true);
- percpu_write(cpu_state, CPU_ONLINE);
+ this_cpu_write(cpu_state, CPU_ONLINE);
wmb();
/* We can take interrupts now: we're officially "up". */
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 9ca008f..aad7e1c 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -165,59 +165,6 @@ extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
#define alloc_percpu(type) \
(typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
-/*
- * Optional methods for optimized non-lvalue per-cpu variable access.
- *
- * @var can be a percpu variable or a field of it and its size should
- * equal char, int or long. percpu_read() evaluates to a lvalue and
- * all others to void.
- *
- * These operations are guaranteed to be atomic w.r.t. preemption.
- * The generic versions use plain get/put_cpu_var(). Archs are
- * encouraged to implement single-instruction alternatives which don't
- * require preemption protection.
- */
-#ifndef percpu_read
-# define percpu_read(var) \
- ({ \
- typeof(var) *pr_ptr__ = &(var); \
- typeof(var) pr_ret__; \
- pr_ret__ = get_cpu_var(*pr_ptr__); \
- put_cpu_var(*pr_ptr__); \
- pr_ret__; \
- })
-#endif
-
-#define __percpu_generic_to_op(var, val, op) \
-do { \
- typeof(var) *pgto_ptr__ = &(var); \
- get_cpu_var(*pgto_ptr__) op val; \
- put_cpu_var(*pgto_ptr__); \
-} while (0)
-
-#ifndef percpu_write
-# define percpu_write(var, val) __percpu_generic_to_op(var, (val), =)
-#endif
-
-#ifndef percpu_add
-# define percpu_add(var, val) __percpu_generic_to_op(var, (val), +=)
-#endif
-
-#ifndef percpu_sub
-# define percpu_sub(var, val) __percpu_generic_to_op(var, (val), -=)
-#endif
-
-#ifndef percpu_and
-# define percpu_and(var, val) __percpu_generic_to_op(var, (val), &=)
-#endif
-
-#ifndef percpu_or
-# define percpu_or(var, val) __percpu_generic_to_op(var, (val), |=)
-#endif
-
-#ifndef percpu_xor
-# define percpu_xor(var, val) __percpu_generic_to_op(var, (val), ^=)
-#endif
/*
* Branching function to split up a function into a set of functions that
diff --git a/include/linux/topology.h b/include/linux/topology.h
index fc839bf..aede2d9 100644
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -235,7 +235,7 @@ static inline int cpu_to_node(int cpu)
#ifndef set_numa_node
static inline void set_numa_node(int node)
{
- percpu_write(numa_node, node);
+ __this_cpu_write(numa_node, node);
}
#endif
@@ -270,7 +270,7 @@ DECLARE_PER_CPU(int, _numa_mem_);
#ifndef set_numa_mem
static inline void set_numa_mem(int node)
{
- percpu_write(_numa_mem_, node);
+ __this_cpu_write(_numa_mem_, node);
}
#endif
--
1.6.3.3
^ permalink raw reply related [flat|nested] 42+ messages in thread
* Re: [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs
2011-10-20 7:32 ` [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs Alex,Shi
@ 2011-10-20 8:38 ` Eric Dumazet
2011-11-21 9:00 ` Alex,Shi
0 siblings, 1 reply; 42+ messages in thread
From: Eric Dumazet @ 2011-10-20 8:38 UTC (permalink / raw)
To: Alex,Shi
Cc: Christoph Lameter, tj@kernel.org, linux-kernel@vger.kernel.org,
Huang, Ying, Thomas Gleixner, mingo@redhat.com, avi@redhat.com,
akpm@linux-foundation.org, David Miller, kaber@trash.net,
a.p.zijlstra@chello.nl, kvm@vger.kernel.org, jeremy@xensource.com
Le jeudi 20 octobre 2011 à 15:32 +0800, Alex,Shi a écrit :
> percpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them
> for further code clean up.
>
> And in preempt safe scenario, __this_cpu_xxx funcs has a bit better
> performance since __this_cpu_xxx has no redundant preempt_disable()
>
> Signed-off-by: Alex Shi <alex.shi@intel.com>
> ---
> net/netfilter/xt_TEE.c | 12 ++++++------
> net/socket.c | 4 ++--
> 2 files changed, 8 insertions(+), 8 deletions(-)
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
Thanks !
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH 2/3] kvm: use this_cpu_xxx replace percpu_xxx funcs
2011-10-20 7:34 ` [PATCH 2/3] kvm: use this_cpu_xxx replace percpu_xxx funcs Alex,Shi
@ 2011-10-24 2:50 ` Alex,Shi
2011-10-24 11:05 ` Avi Kivity
0 siblings, 1 reply; 42+ messages in thread
From: Alex,Shi @ 2011-10-24 2:50 UTC (permalink / raw)
To: Avi Kivity
Cc: Christoph Lameter, tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying, tglx@linutronix.de,
mingo@redhat.com, akpm@linux-foundation.org, davem@davemloft.net,
kaber@trash.net, a.p.zijlstra@chello.nl, kvm@vger.kernel.org,
jeremy@xensource.com
On Thu, 2011-10-20 at 15:34 +0800, Alex,Shi wrote:
> percpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them
> for further code clean up.
>
> And in preempt safe scenario, __this_cpu_xxx funcs has a bit better
> performance since __this_cpu_xxx has no redundant preempt_disable()
>
Avi:
Would you like to give some comments of this?
> Signed-off-by: Alex Shi <alex.shi@intel.com>
> ---
> arch/x86/kvm/x86.c | 14 +++++++-------
> 1 files changed, 7 insertions(+), 7 deletions(-)
>
> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index 84a28ea..432c4bc 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -5024,15 +5024,15 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
>
> static int kvm_is_in_guest(void)
> {
> - return percpu_read(current_vcpu) != NULL;
> + return __this_cpu_read(current_vcpu) != NULL;
> }
>
> static int kvm_is_user_mode(void)
> {
> int user_mode = 3;
>
> - if (percpu_read(current_vcpu))
> - user_mode = kvm_x86_ops->get_cpl(percpu_read(current_vcpu));
> + if (__this_cpu_read(current_vcpu))
> + user_mode = kvm_x86_ops->get_cpl(__this_cpu_read(current_vcpu));
>
> return user_mode != 0;
> }
> @@ -5041,8 +5041,8 @@ static unsigned long kvm_get_guest_ip(void)
> {
> unsigned long ip = 0;
>
> - if (percpu_read(current_vcpu))
> - ip = kvm_rip_read(percpu_read(current_vcpu));
> + if (__this_cpu_read(current_vcpu))
> + ip = kvm_rip_read(__this_cpu_read(current_vcpu));
>
> return ip;
> }
> @@ -5055,13 +5055,13 @@ static struct perf_guest_info_callbacks kvm_guest_cbs = {
>
> void kvm_before_handle_nmi(struct kvm_vcpu *vcpu)
> {
> - percpu_write(current_vcpu, vcpu);
> + __this_cpu_write(current_vcpu, vcpu);
> }
> EXPORT_SYMBOL_GPL(kvm_before_handle_nmi);
>
> void kvm_after_handle_nmi(struct kvm_vcpu *vcpu)
> {
> - percpu_write(current_vcpu, NULL);
> + __this_cpu_write(current_vcpu, NULL);
> }
> EXPORT_SYMBOL_GPL(kvm_after_handle_nmi);
>
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH 2/3] kvm: use this_cpu_xxx replace percpu_xxx funcs
2011-10-24 2:50 ` Alex,Shi
@ 2011-10-24 11:05 ` Avi Kivity
2011-11-21 9:02 ` Alex,Shi
0 siblings, 1 reply; 42+ messages in thread
From: Avi Kivity @ 2011-10-24 11:05 UTC (permalink / raw)
To: Alex,Shi
Cc: Christoph Lameter, tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying, tglx@linutronix.de,
mingo@redhat.com, akpm@linux-foundation.org, davem@davemloft.net,
kaber@trash.net, a.p.zijlstra@chello.nl, kvm@vger.kernel.org,
jeremy@xensource.com
On 10/24/2011 04:50 AM, Alex,Shi wrote:
> On Thu, 2011-10-20 at 15:34 +0800, Alex,Shi wrote:
> > percpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them
> > for further code clean up.
> >
> > And in preempt safe scenario, __this_cpu_xxx funcs has a bit better
> > performance since __this_cpu_xxx has no redundant preempt_disable()
> >
>
> Avi:
> Would you like to give some comments of this?
>
Sorry, was travelling:
Acked-by: Avi Kivity <avi@redhat.com>
--
I have a truly marvellous patch that fixes the bug which this
signature is too narrow to contain.
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs
2011-10-20 8:38 ` Eric Dumazet
@ 2011-11-21 9:00 ` Alex,Shi
2011-12-02 2:33 ` Alex,Shi
0 siblings, 1 reply; 42+ messages in thread
From: Alex,Shi @ 2011-11-21 9:00 UTC (permalink / raw)
To: Eric Dumazet
Cc: Christoph Lameter, tj@kernel.org, linux-kernel@vger.kernel.org,
Huang, Ying, Thomas Gleixner, mingo@redhat.com, avi@redhat.com,
akpm@linux-foundation.org, David Miller, kaber@trash.net,
a.p.zijlstra@chello.nl, kvm@vger.kernel.org, jeremy@xensource.com
On Thu, 2011-10-20 at 16:38 +0800, Eric Dumazet wrote:
> Le jeudi 20 octobre 2011 à 15:32 +0800, Alex,Shi a écrit :
> > percpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them
> > for further code clean up.
> >
> > And in preempt safe scenario, __this_cpu_xxx funcs has a bit better
> > performance since __this_cpu_xxx has no redundant preempt_disable()
> >
> > Signed-off-by: Alex Shi <alex.shi@intel.com>
> > ---
> > net/netfilter/xt_TEE.c | 12 ++++++------
> > net/socket.c | 4 ++--
> > 2 files changed, 8 insertions(+), 8 deletions(-)
>
> Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
>
> Thanks !
Anyone like to pick up this patch? or more comments for this?
>
>
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH 2/3] kvm: use this_cpu_xxx replace percpu_xxx funcs
2011-10-24 11:05 ` Avi Kivity
@ 2011-11-21 9:02 ` Alex,Shi
2011-11-21 9:20 ` Avi Kivity
0 siblings, 1 reply; 42+ messages in thread
From: Alex,Shi @ 2011-11-21 9:02 UTC (permalink / raw)
To: Avi Kivity
Cc: Christoph Lameter, tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying, tglx@linutronix.de,
mingo@redhat.com, akpm@linux-foundation.org, davem@davemloft.net,
kaber@trash.net, a.p.zijlstra@chello.nl, kvm@vger.kernel.org,
jeremy@xensource.com
On Mon, 2011-10-24 at 19:05 +0800, Avi Kivity wrote:
> On 10/24/2011 04:50 AM, Alex,Shi wrote:
> > On Thu, 2011-10-20 at 15:34 +0800, Alex,Shi wrote:
> > > percpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them
> > > for further code clean up.
> > >
> > > And in preempt safe scenario, __this_cpu_xxx funcs has a bit better
> > > performance since __this_cpu_xxx has no redundant preempt_disable()
> > >
> >
> > Avi:
> > Would you like to give some comments of this?
> >
>
> Sorry, was travelling:
>
> Acked-by: Avi Kivity <avi@redhat.com>
>
And this one, picking up or comments are all appreciated. :)
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH 3/3] Code clean up for percpu_xxx() functions
2011-10-20 7:35 ` [PATCH 3/3] Code clean up for percpu_xxx() functions Alex,Shi
@ 2011-11-21 9:10 ` Alex,Shi
2011-11-22 0:06 ` tj
0 siblings, 1 reply; 42+ messages in thread
From: Alex,Shi @ 2011-11-21 9:10 UTC (permalink / raw)
To: Christoph Lameter
Cc: tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying, Thomas Gleixner,
mingo@redhat.com, avi@redhat.com, akpm@linux-foundation.org,
David Miller, kaber@trash.net, a.p.zijlstra@chello.nl,
kvm@vger.kernel.org, jeremy@xensource.com, Andi Kleen
refreshed the patch on latest upstream kernel. Any comments or picking
up are appreciated.
===
>From 0dce61dc88b8ed2687b4d5c0633aa54d1f66fdc0 Mon Sep 17 00:00:00 2001
From: Alex Shi <alex.shi@intel.com>
Date: Tue, 22 Nov 2011 00:05:37 +0800
Subject: [PATCH 3/3] Code clean up for percpu_xxx() functions
Since percpu_xxx() serial functions are duplicate with this_cpu_xxx().
Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in
code.
And further more, as Christoph Lameter's requirement, I try to use
__this_cpu_xx to replace this_cpu_xxx if it is in preempt safe scenario.
The preempt safe scenarios include:
1, in irq/softirq/nmi handler
2, protected by preempt_disable
3, protected by spin_lock
4, if the code context imply that it is preempt safe, like the code is
follows or be followed a preempt safe code.
I left the xen code unchanged, since no idea of them.
BTW, In fact, this_cpu_xxx are same as __this_cpu_xxx since all funcs
implement in a single instruction for x86 machine. But it maybe
different for other platforms, so, doing this distinguish is helpful for
other platforms' performance.
Signed-off-by: Alex Shi <alex.shi@intel.com>
Acked-by: Christoph Lameter <cl@gentwo.org>
---
arch/x86/include/asm/current.h | 2 +-
arch/x86/include/asm/hardirq.h | 9 +++--
arch/x86/include/asm/irq_regs.h | 4 +-
arch/x86/include/asm/mmu_context.h | 12 ++++----
arch/x86/include/asm/percpu.h | 24 ++++++---------
arch/x86/include/asm/smp.h | 4 +-
arch/x86/include/asm/stackprotector.h | 4 +-
arch/x86/include/asm/thread_info.h | 2 +-
arch/x86/include/asm/tlbflush.h | 4 +-
arch/x86/kernel/cpu/common.c | 2 +-
arch/x86/kernel/cpu/mcheck/mce.c | 4 +-
arch/x86/kernel/paravirt.c | 12 ++++----
arch/x86/kernel/process_32.c | 2 +-
arch/x86/kernel/process_64.c | 12 ++++----
arch/x86/mm/tlb.c | 10 +++---
arch/x86/xen/enlighten.c | 6 ++--
arch/x86/xen/irq.c | 8 ++--
arch/x86/xen/mmu.c | 20 ++++++------
arch/x86/xen/multicalls.h | 2 +-
arch/x86/xen/smp.c | 2 +-
include/linux/percpu.h | 53 ---------------------------------
include/linux/topology.h | 4 +-
22 files changed, 73 insertions(+), 129 deletions(-)
diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index 4d447b7..9476c04 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task);
static __always_inline struct task_struct *get_current(void)
{
- return percpu_read_stable(current_task);
+ return this_cpu_read_stable(current_task);
}
#define current get_current()
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 55e4de6..2890444 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -35,14 +35,15 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
#define __ARCH_IRQ_STAT
-#define inc_irq_stat(member) percpu_inc(irq_stat.member)
+#define inc_irq_stat(member) __this_cpu_inc(irq_stat.member)
-#define local_softirq_pending() percpu_read(irq_stat.__softirq_pending)
+#define local_softirq_pending() __this_cpu_read(irq_stat.__softirq_pending)
#define __ARCH_SET_SOFTIRQ_PENDING
-#define set_softirq_pending(x) percpu_write(irq_stat.__softirq_pending, (x))
-#define or_softirq_pending(x) percpu_or(irq_stat.__softirq_pending, (x))
+#define set_softirq_pending(x) \
+ __this_cpu_write(irq_stat.__softirq_pending, (x))
+#define or_softirq_pending(x) __this_cpu_or(irq_stat.__softirq_pending, (x))
extern void ack_bad_irq(unsigned int irq);
diff --git a/arch/x86/include/asm/irq_regs.h b/arch/x86/include/asm/irq_regs.h
index 7784322..15639ed 100644
--- a/arch/x86/include/asm/irq_regs.h
+++ b/arch/x86/include/asm/irq_regs.h
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);
static inline struct pt_regs *get_irq_regs(void)
{
- return percpu_read(irq_regs);
+ return __this_cpu_read(irq_regs);
}
static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
struct pt_regs *old_regs;
old_regs = get_irq_regs();
- percpu_write(irq_regs, new_regs);
+ __this_cpu_write(irq_regs, new_regs);
return old_regs;
}
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 6902152..02ca533 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);
static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
{
#ifdef CONFIG_SMP
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
- percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
#endif
}
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
if (likely(prev != next)) {
#ifdef CONFIG_SMP
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- percpu_write(cpu_tlbstate.active_mm, next);
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ __this_cpu_write(cpu_tlbstate.active_mm, next);
#endif
cpumask_set_cpu(cpu, mm_cpumask(next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
}
#ifdef CONFIG_SMP
else {
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ BUG_ON(__this_cpu_read(cpu_tlbstate.active_mm) != next);
if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next))) {
/* We were in lazy tlb mode and leave_mm disabled
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 3470c9d..520005e 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -46,7 +46,7 @@
#ifdef CONFIG_SMP
#define __percpu_prefix "%%"__stringify(__percpu_seg)":"
-#define __my_cpu_offset percpu_read(this_cpu_off)
+#define __my_cpu_offset __this_cpu_read(this_cpu_off)
/*
* Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,15 @@ do { \
})
/*
- * percpu_read() makes gcc load the percpu variable every time it is
- * accessed while percpu_read_stable() allows the value to be cached.
- * percpu_read_stable() is more efficient and can be used if its value
+ * this_cpu_read() makes gcc load the percpu variable every time it is
+ * accessed while this_cpu_read_stable() allows the value to be cached.
+ * this_cpu_read_stable() is more efficient and can be used if its value
* is guaranteed to be valid across cpus. The current users include
* get_current() and get_thread_info() both of which are actually
* per-thread variables implemented as per-cpu variables and thus
* stable for the duration of the respective task.
*/
-#define percpu_read(var) percpu_from_op("mov", var, "m" (var))
-#define percpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
-#define percpu_write(var, val) percpu_to_op("mov", var, val)
-#define percpu_add(var, val) percpu_add_op(var, val)
-#define percpu_sub(var, val) percpu_add_op(var, -(val))
-#define percpu_and(var, val) percpu_to_op("and", var, val)
-#define percpu_or(var, val) percpu_to_op("or", var, val)
-#define percpu_xor(var, val) percpu_to_op("xor", var, val)
-#define percpu_inc(var) percpu_unary_op("inc", var)
+#define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +543,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr,
{
unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG;
- return ((1UL << (nr % BITS_PER_LONG)) & percpu_read(*a)) != 0;
+#ifdef CONFIG_X86_64
+ return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_8(*a)) != 0;
+#else
+ return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_4(*a)) != 0;
+#endif
}
static inline int x86_this_cpu_variable_test_bit(int nr,
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 73b11bc..dd5749b 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;
* from the initial startup. We map APIC_BASE very early in page_setup(),
* so this is correct in the x86 case.
*/
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
extern int safe_smp_processor_id(void);
#elif defined(CONFIG_X86_64_SMP)
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
#define stack_smp_processor_id() \
({ \
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
index 1575177..e8a60c9 100644
--- a/arch/x86/include/asm/stackprotector.h
+++ b/arch/x86/include/asm/stackprotector.h
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void)
current->stack_canary = canary;
#ifdef CONFIG_X86_64
- percpu_write(irq_stack_union.stack_canary, canary);
+ __this_cpu_write(irq_stack_union.stack_canary, canary);
#else
- percpu_write(stack_canary.canary, canary);
+ __this_cpu_write(stack_canary.canary, canary);
#endif
}
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index a1fe5c1..8b19667 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);
static inline struct thread_info *current_thread_info(void)
{
struct thread_info *ti;
- ti = (void *)(percpu_read_stable(kernel_stack) +
+ ti = (void *)(this_cpu_read_stable(kernel_stack) +
KERNEL_STACK_OFFSET - THREAD_SIZE);
return ti;
}
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 169be89..e90eec0 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
static inline void reset_lazy_tlbstate(void)
{
- percpu_write(cpu_tlbstate.state, 0);
- percpu_write(cpu_tlbstate.active_mm, &init_mm);
+ __this_cpu_write(cpu_tlbstate.state, 0);
+ __this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
}
#endif /* SMP */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index aa003b1..d9b4fe7 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1163,7 +1163,7 @@ void __cpuinit cpu_init(void)
oist = &per_cpu(orig_ist, cpu);
#ifdef CONFIG_NUMA
- if (cpu != 0 && percpu_read(numa_node) == 0 &&
+ if (cpu != 0 && __this_cpu_read(numa_node) == 0 &&
early_cpu_to_node(cpu) != NUMA_NO_NODE)
set_numa_node(early_cpu_to_node(cpu));
#endif
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 2af127d..b7f3a1e 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -514,7 +514,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
struct mce m;
int i;
- percpu_inc(mce_poll_count);
+ __this_cpu_inc(mce_poll_count);
mce_gather_info(&m, NULL);
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
atomic_inc(&mce_entry);
- percpu_inc(mce_exception_count);
+ __this_cpu_inc(mce_exception_count);
if (!banks)
goto out;
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index d90272e..2f0c1d1 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LA
static inline void enter_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
+ BUG_ON(__this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
- percpu_write(paravirt_lazy_mode, mode);
+ __this_cpu_write(paravirt_lazy_mode, mode);
}
static void leave_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != mode);
+ BUG_ON(__this_cpu_read(paravirt_lazy_mode) != mode);
- percpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
+ __this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
}
void paravirt_enter_lazy_mmu(void)
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev)
{
BUG_ON(preemptible());
- if (percpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
+ if (__this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
arch_leave_lazy_mmu_mode();
set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES);
}
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
if (in_interrupt())
return PARAVIRT_LAZY_NONE;
- return percpu_read(paravirt_lazy_mode);
+ return __this_cpu_read(paravirt_lazy_mode);
}
void arch_flush_lazy_mmu_mode(void)
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 795b79f..b383fe8 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -375,7 +375,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
if (prev->gs | next->gs)
lazy_load_gs(next->gs);
- percpu_write(current_task, next_p);
+ __this_cpu_write(current_task, next_p);
return prev_p;
}
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 3bd7e6e..7b6edbb 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -74,7 +74,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);
void enter_idle(void)
{
- percpu_write(is_idle, 1);
+ __this_cpu_write(is_idle, 1);
atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
}
@@ -338,7 +338,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,
load_gs_index(0);
regs->ip = new_ip;
regs->sp = new_sp;
- percpu_write(old_rsp, new_sp);
+ this_cpu_write(old_rsp, new_sp);
regs->cs = _cs;
regs->ss = _ss;
regs->flags = X86_EFLAGS_IF;
@@ -472,11 +472,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
/*
* Switch the PDA and FPU contexts.
*/
- prev->usersp = percpu_read(old_rsp);
- percpu_write(old_rsp, next->usersp);
- percpu_write(current_task, next_p);
+ prev->usersp = __this_cpu_read(old_rsp);
+ __this_cpu_write(old_rsp, next->usersp);
+ __this_cpu_write(current_task, next_p);
- percpu_write(kernel_stack,
+ __this_cpu_write(kernel_stack,
(unsigned long)task_stack_page(next_p) +
THREAD_SIZE - KERNEL_STACK_OFFSET);
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index d6c0418..e931db0 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);
*/
void leave_mm(int cpu)
{
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
BUG();
cpumask_clear_cpu(cpu,
- mm_cpumask(percpu_read(cpu_tlbstate.active_mm)));
+ mm_cpumask(__this_cpu_read(cpu_tlbstate.active_mm)));
load_cr3(swapper_pg_dir);
}
EXPORT_SYMBOL_GPL(leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)
* BUG();
*/
- if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
+ if (f->flush_mm == __this_cpu_read(cpu_tlbstate.active_mm)) {
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
if (f->flush_va == TLB_FLUSH_ALL)
local_flush_tlb();
else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
static void do_flush_tlb_all(void *info)
{
__flush_tlb_all();
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
leave_mm(smp_processor_id());
}
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 1f92865..3622488 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -777,11 +777,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);
static unsigned long xen_read_cr0(void)
{
- unsigned long cr0 = percpu_read(xen_cr0_value);
+ unsigned long cr0 = this_cpu_read(xen_cr0_value);
if (unlikely(cr0 == 0)) {
cr0 = native_read_cr0();
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
}
return cr0;
@@ -791,7 +791,7 @@ static void xen_write_cr0(unsigned long cr0)
{
struct multicall_space mcs;
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
/* Only pay attention to cr0.TS; everything else is
ignored. */
diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c
index 8bbb465..1573376 100644
--- a/arch/x86/xen/irq.c
+++ b/arch/x86/xen/irq.c
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)
struct vcpu_info *vcpu;
unsigned long flags;
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
/* flag has opposite sense of mask */
flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = flags;
preempt_enable_no_resched();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- percpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
+ this_cpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
preempt_enable_no_resched();
}
PV_CALLEE_SAVE_REGS_THUNK(xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)
the caller is confused and is trying to re-enable interrupts
on an indeterminate processor. */
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = 0;
/* Doesn't matter if we get preempted here, because any
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 87f6673..426ff61 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1071,14 +1071,14 @@ static void drop_other_mm_ref(void *info)
struct mm_struct *mm = info;
struct mm_struct *active_mm;
- active_mm = percpu_read(cpu_tlbstate.active_mm);
+ active_mm = this_cpu_read(cpu_tlbstate.active_mm);
- if (active_mm == mm && percpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
+ if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
leave_mm(smp_processor_id());
/* If this cpu still has a stale cr3 reference, then make sure
it has been flushed. */
- if (percpu_read(xen_current_cr3) == __pa(mm->pgd))
+ if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd))
load_cr3(swapper_pg_dir);
}
@@ -1185,17 +1185,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)
static void xen_write_cr2(unsigned long cr2)
{
- percpu_read(xen_vcpu)->arch.cr2 = cr2;
+ this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
}
static unsigned long xen_read_cr2(void)
{
- return percpu_read(xen_vcpu)->arch.cr2;
+ return this_cpu_read(xen_vcpu)->arch.cr2;
}
unsigned long xen_read_cr2_direct(void)
{
- return percpu_read(xen_vcpu_info.arch.cr2);
+ return this_cpu_read(xen_vcpu_info.arch.cr2);
}
static void xen_flush_tlb(void)
@@ -1278,12 +1278,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
static unsigned long xen_read_cr3(void)
{
- return percpu_read(xen_cr3);
+ return this_cpu_read(xen_cr3);
}
static void set_current_cr3(void *v)
{
- percpu_write(xen_current_cr3, (unsigned long)v);
+ this_cpu_write(xen_current_cr3, (unsigned long)v);
}
static void __xen_write_cr3(bool kernel, unsigned long cr3)
@@ -1306,7 +1306,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)
xen_extend_mmuext_op(&op);
if (kernel) {
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
/* Update xen_current_cr3 once the batch has actually
been submitted. */
@@ -1322,7 +1322,7 @@ static void xen_write_cr3(unsigned long cr3)
/* Update while interrupts are disabled, so its atomic with
respect to ipis */
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
__xen_write_cr3(true, cr3);
diff --git a/arch/x86/xen/multicalls.h b/arch/x86/xen/multicalls.h
index dee79b7..9c2e74f 100644
--- a/arch/x86/xen/multicalls.h
+++ b/arch/x86/xen/multicalls.h
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)
xen_mc_flush();
/* restore flags saved in xen_mc_batch */
- local_irq_restore(percpu_read(xen_mc_irq_flags));
+ local_irq_restore(this_cpu_read(xen_mc_irq_flags));
}
/* Set up a callback to be called when the current batch is flushed */
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 041d4fe..449f868 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)
xen_setup_cpu_clockevents();
set_cpu_online(cpu, true);
- percpu_write(cpu_state, CPU_ONLINE);
+ this_cpu_write(cpu_state, CPU_ONLINE);
wmb();
/* We can take interrupts now: we're officially "up". */
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 9ca008f..aad7e1c 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -165,59 +165,6 @@ extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
#define alloc_percpu(type) \
(typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
-/*
- * Optional methods for optimized non-lvalue per-cpu variable access.
- *
- * @var can be a percpu variable or a field of it and its size should
- * equal char, int or long. percpu_read() evaluates to a lvalue and
- * all others to void.
- *
- * These operations are guaranteed to be atomic w.r.t. preemption.
- * The generic versions use plain get/put_cpu_var(). Archs are
- * encouraged to implement single-instruction alternatives which don't
- * require preemption protection.
- */
-#ifndef percpu_read
-# define percpu_read(var) \
- ({ \
- typeof(var) *pr_ptr__ = &(var); \
- typeof(var) pr_ret__; \
- pr_ret__ = get_cpu_var(*pr_ptr__); \
- put_cpu_var(*pr_ptr__); \
- pr_ret__; \
- })
-#endif
-
-#define __percpu_generic_to_op(var, val, op) \
-do { \
- typeof(var) *pgto_ptr__ = &(var); \
- get_cpu_var(*pgto_ptr__) op val; \
- put_cpu_var(*pgto_ptr__); \
-} while (0)
-
-#ifndef percpu_write
-# define percpu_write(var, val) __percpu_generic_to_op(var, (val), =)
-#endif
-
-#ifndef percpu_add
-# define percpu_add(var, val) __percpu_generic_to_op(var, (val), +=)
-#endif
-
-#ifndef percpu_sub
-# define percpu_sub(var, val) __percpu_generic_to_op(var, (val), -=)
-#endif
-
-#ifndef percpu_and
-# define percpu_and(var, val) __percpu_generic_to_op(var, (val), &=)
-#endif
-
-#ifndef percpu_or
-# define percpu_or(var, val) __percpu_generic_to_op(var, (val), |=)
-#endif
-
-#ifndef percpu_xor
-# define percpu_xor(var, val) __percpu_generic_to_op(var, (val), ^=)
-#endif
/*
* Branching function to split up a function into a set of functions that
diff --git a/include/linux/topology.h b/include/linux/topology.h
index e26db03..b480403 100644
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -239,7 +239,7 @@ static inline int cpu_to_node(int cpu)
#ifndef set_numa_node
static inline void set_numa_node(int node)
{
- percpu_write(numa_node, node);
+ __this_cpu_write(numa_node, node);
}
#endif
@@ -274,7 +274,7 @@ DECLARE_PER_CPU(int, _numa_mem_);
#ifndef set_numa_mem
static inline void set_numa_mem(int node)
{
- percpu_write(_numa_mem_, node);
+ __this_cpu_write(_numa_mem_, node);
}
#endif
--
1.7.5.1
^ permalink raw reply related [flat|nested] 42+ messages in thread
* Re: [PATCH 2/3] kvm: use this_cpu_xxx replace percpu_xxx funcs
2011-11-21 9:02 ` Alex,Shi
@ 2011-11-21 9:20 ` Avi Kivity
2011-11-21 9:52 ` Alex,Shi
2012-01-11 8:48 ` Alex,Shi
0 siblings, 2 replies; 42+ messages in thread
From: Avi Kivity @ 2011-11-21 9:20 UTC (permalink / raw)
To: Alex,Shi, Marcelo Tosatti
Cc: Christoph Lameter, tj@kernel.org, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying, tglx@linutronix.de,
mingo@redhat.com, akpm@linux-foundation.org, davem@davemloft.net,
kaber@trash.net, a.p.zijlstra@chello.nl, kvm@vger.kernel.org,
jeremy@xensource.com
On 11/21/2011 11:02 AM, Alex,Shi wrote:
> On Mon, 2011-10-24 at 19:05 +0800, Avi Kivity wrote:
> > On 10/24/2011 04:50 AM, Alex,Shi wrote:
> > > On Thu, 2011-10-20 at 15:34 +0800, Alex,Shi wrote:
> > > > percpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them
> > > > for further code clean up.
> > > >
> > > > And in preempt safe scenario, __this_cpu_xxx funcs has a bit better
> > > > performance since __this_cpu_xxx has no redundant preempt_disable()
> > > >
> > >
> > > Avi:
> > > Would you like to give some comments of this?
> > >
> >
> > Sorry, was travelling:
> >
> > Acked-by: Avi Kivity <avi@redhat.com>
> >
>
> And this one, picking up or comments are all appreciated. :)
Just to be clear, you want this applied in kvm.git?
--
error compiling committee.c: too many arguments to function
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH 2/3] kvm: use this_cpu_xxx replace percpu_xxx funcs
2011-11-21 9:20 ` Avi Kivity
@ 2011-11-21 9:52 ` Alex,Shi
2012-01-11 8:48 ` Alex,Shi
1 sibling, 0 replies; 42+ messages in thread
From: Alex,Shi @ 2011-11-21 9:52 UTC (permalink / raw)
To: Avi Kivity
Cc: Marcelo Tosatti, Christoph Lameter, tj@kernel.org,
linux-kernel@vger.kernel.org, eric.dumazet@gmail.com, Huang, Ying,
tglx@linutronix.de, mingo@redhat.com, akpm@linux-foundation.org,
davem@davemloft.net, kaber@trash.net, a.p.zijlstra@chello.nl,
kvm@vger.kernel.org, jeremy@xensource.com
> > > > > percpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them
> > > > > for further code clean up.
> > > > >
> > > > > And in preempt safe scenario, __this_cpu_xxx funcs has a bit better
> > > > > performance since __this_cpu_xxx has no redundant preempt_disable()
> > > > >
> > > >
> > > > Avi:
> > > > Would you like to give some comments of this?
> > > >
> > >
> > > Sorry, was travelling:
> > >
> > > Acked-by: Avi Kivity <avi@redhat.com>
> > >
> >
> > And this one, picking up or comments are all appreciated. :)
>
> Just to be clear, you want this applied in kvm.git?
>
Glad to be there! :)
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH 3/3] Code clean up for percpu_xxx() functions
2011-11-21 9:10 ` Alex,Shi
@ 2011-11-22 0:06 ` tj
2012-01-11 9:08 ` Alex,Shi
0 siblings, 1 reply; 42+ messages in thread
From: tj @ 2011-11-22 0:06 UTC (permalink / raw)
To: Alex,Shi, H. Peter Anvin
Cc: Christoph Lameter, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying, Thomas Gleixner,
mingo@redhat.com, avi@redhat.com, akpm@linux-foundation.org,
David Miller, kaber@trash.net, a.p.zijlstra@chello.nl,
kvm@vger.kernel.org, jeremy@xensource.com, Andi Kleen
(cc'ing hpa and quoting whole body)
On Mon, Nov 21, 2011 at 05:10:12PM +0800, Alex,Shi wrote:
> refreshed the patch on latest upstream kernel. Any comments or picking
> up are appreciated.
>
> ===
> >From 0dce61dc88b8ed2687b4d5c0633aa54d1f66fdc0 Mon Sep 17 00:00:00 2001
> From: Alex Shi <alex.shi@intel.com>
> Date: Tue, 22 Nov 2011 00:05:37 +0800
> Subject: [PATCH 3/3] Code clean up for percpu_xxx() functions
>
> Since percpu_xxx() serial functions are duplicate with this_cpu_xxx().
> Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in
> code.
>
> And further more, as Christoph Lameter's requirement, I try to use
> __this_cpu_xx to replace this_cpu_xxx if it is in preempt safe scenario.
> The preempt safe scenarios include:
> 1, in irq/softirq/nmi handler
> 2, protected by preempt_disable
> 3, protected by spin_lock
> 4, if the code context imply that it is preempt safe, like the code is
> follows or be followed a preempt safe code.
>
> I left the xen code unchanged, since no idea of them.
>
> BTW, In fact, this_cpu_xxx are same as __this_cpu_xxx since all funcs
> implement in a single instruction for x86 machine. But it maybe
> different for other platforms, so, doing this distinguish is helpful for
> other platforms' performance.
>
> Signed-off-by: Alex Shi <alex.shi@intel.com>
> Acked-by: Christoph Lameter <cl@gentwo.org>
Acked-by: Tejun Heo <tj@kernel.org>
hpa, I suppose this should go through x86? The original patch can be
accessed at
http://article.gmane.org/gmane.linux.kernel/1218055/raw
Thanks.
> arch/x86/include/asm/current.h | 2 +-
> arch/x86/include/asm/hardirq.h | 9 +++--
> arch/x86/include/asm/irq_regs.h | 4 +-
> arch/x86/include/asm/mmu_context.h | 12 ++++----
> arch/x86/include/asm/percpu.h | 24 ++++++---------
> arch/x86/include/asm/smp.h | 4 +-
> arch/x86/include/asm/stackprotector.h | 4 +-
> arch/x86/include/asm/thread_info.h | 2 +-
> arch/x86/include/asm/tlbflush.h | 4 +-
> arch/x86/kernel/cpu/common.c | 2 +-
> arch/x86/kernel/cpu/mcheck/mce.c | 4 +-
> arch/x86/kernel/paravirt.c | 12 ++++----
> arch/x86/kernel/process_32.c | 2 +-
> arch/x86/kernel/process_64.c | 12 ++++----
> arch/x86/mm/tlb.c | 10 +++---
> arch/x86/xen/enlighten.c | 6 ++--
> arch/x86/xen/irq.c | 8 ++--
> arch/x86/xen/mmu.c | 20 ++++++------
> arch/x86/xen/multicalls.h | 2 +-
> arch/x86/xen/smp.c | 2 +-
> include/linux/percpu.h | 53 ---------------------------------
> include/linux/topology.h | 4 +-
> 22 files changed, 73 insertions(+), 129 deletions(-)
>
> diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
> index 4d447b7..9476c04 100644
> --- a/arch/x86/include/asm/current.h
> +++ b/arch/x86/include/asm/current.h
> @@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task);
>
> static __always_inline struct task_struct *get_current(void)
> {
> - return percpu_read_stable(current_task);
> + return this_cpu_read_stable(current_task);
> }
>
> #define current get_current()
> diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
> index 55e4de6..2890444 100644
> --- a/arch/x86/include/asm/hardirq.h
> +++ b/arch/x86/include/asm/hardirq.h
> @@ -35,14 +35,15 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
>
> #define __ARCH_IRQ_STAT
>
> -#define inc_irq_stat(member) percpu_inc(irq_stat.member)
> +#define inc_irq_stat(member) __this_cpu_inc(irq_stat.member)
>
> -#define local_softirq_pending() percpu_read(irq_stat.__softirq_pending)
> +#define local_softirq_pending() __this_cpu_read(irq_stat.__softirq_pending)
>
> #define __ARCH_SET_SOFTIRQ_PENDING
>
> -#define set_softirq_pending(x) percpu_write(irq_stat.__softirq_pending, (x))
> -#define or_softirq_pending(x) percpu_or(irq_stat.__softirq_pending, (x))
> +#define set_softirq_pending(x) \
> + __this_cpu_write(irq_stat.__softirq_pending, (x))
> +#define or_softirq_pending(x) __this_cpu_or(irq_stat.__softirq_pending, (x))
>
> extern void ack_bad_irq(unsigned int irq);
>
> diff --git a/arch/x86/include/asm/irq_regs.h b/arch/x86/include/asm/irq_regs.h
> index 7784322..15639ed 100644
> --- a/arch/x86/include/asm/irq_regs.h
> +++ b/arch/x86/include/asm/irq_regs.h
> @@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);
>
> static inline struct pt_regs *get_irq_regs(void)
> {
> - return percpu_read(irq_regs);
> + return __this_cpu_read(irq_regs);
> }
>
> static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
> @@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
> struct pt_regs *old_regs;
>
> old_regs = get_irq_regs();
> - percpu_write(irq_regs, new_regs);
> + __this_cpu_write(irq_regs, new_regs);
>
> return old_regs;
> }
> diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
> index 6902152..02ca533 100644
> --- a/arch/x86/include/asm/mmu_context.h
> +++ b/arch/x86/include/asm/mmu_context.h
> @@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);
> static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
> {
> #ifdef CONFIG_SMP
> - if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
> - percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
> + if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
> + __this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
> #endif
> }
>
> @@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
>
> if (likely(prev != next)) {
> #ifdef CONFIG_SMP
> - percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
> - percpu_write(cpu_tlbstate.active_mm, next);
> + __this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
> + __this_cpu_write(cpu_tlbstate.active_mm, next);
> #endif
> cpumask_set_cpu(cpu, mm_cpumask(next));
>
> @@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
> }
> #ifdef CONFIG_SMP
> else {
> - percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
> - BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
> + __this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
> + BUG_ON(__this_cpu_read(cpu_tlbstate.active_mm) != next);
>
> if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next))) {
> /* We were in lazy tlb mode and leave_mm disabled
> diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
> index 3470c9d..520005e 100644
> --- a/arch/x86/include/asm/percpu.h
> +++ b/arch/x86/include/asm/percpu.h
> @@ -46,7 +46,7 @@
>
> #ifdef CONFIG_SMP
> #define __percpu_prefix "%%"__stringify(__percpu_seg)":"
> -#define __my_cpu_offset percpu_read(this_cpu_off)
> +#define __my_cpu_offset __this_cpu_read(this_cpu_off)
>
> /*
> * Compared to the generic __my_cpu_offset version, the following
> @@ -351,23 +351,15 @@ do { \
> })
>
> /*
> - * percpu_read() makes gcc load the percpu variable every time it is
> - * accessed while percpu_read_stable() allows the value to be cached.
> - * percpu_read_stable() is more efficient and can be used if its value
> + * this_cpu_read() makes gcc load the percpu variable every time it is
> + * accessed while this_cpu_read_stable() allows the value to be cached.
> + * this_cpu_read_stable() is more efficient and can be used if its value
> * is guaranteed to be valid across cpus. The current users include
> * get_current() and get_thread_info() both of which are actually
> * per-thread variables implemented as per-cpu variables and thus
> * stable for the duration of the respective task.
> */
> -#define percpu_read(var) percpu_from_op("mov", var, "m" (var))
> -#define percpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
> -#define percpu_write(var, val) percpu_to_op("mov", var, val)
> -#define percpu_add(var, val) percpu_add_op(var, val)
> -#define percpu_sub(var, val) percpu_add_op(var, -(val))
> -#define percpu_and(var, val) percpu_to_op("and", var, val)
> -#define percpu_or(var, val) percpu_to_op("or", var, val)
> -#define percpu_xor(var, val) percpu_to_op("xor", var, val)
> -#define percpu_inc(var) percpu_unary_op("inc", var)
> +#define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
>
> #define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
> #define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
> @@ -551,7 +543,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr,
> {
> unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG;
>
> - return ((1UL << (nr % BITS_PER_LONG)) & percpu_read(*a)) != 0;
> +#ifdef CONFIG_X86_64
> + return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_8(*a)) != 0;
> +#else
> + return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_4(*a)) != 0;
> +#endif
> }
>
> static inline int x86_this_cpu_variable_test_bit(int nr,
> diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
> index 73b11bc..dd5749b 100644
> --- a/arch/x86/include/asm/smp.h
> +++ b/arch/x86/include/asm/smp.h
> @@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;
> * from the initial startup. We map APIC_BASE very early in page_setup(),
> * so this is correct in the x86 case.
> */
> -#define raw_smp_processor_id() (percpu_read(cpu_number))
> +#define raw_smp_processor_id() (this_cpu_read(cpu_number))
> extern int safe_smp_processor_id(void);
>
> #elif defined(CONFIG_X86_64_SMP)
> -#define raw_smp_processor_id() (percpu_read(cpu_number))
> +#define raw_smp_processor_id() (this_cpu_read(cpu_number))
>
> #define stack_smp_processor_id() \
> ({ \
> diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
> index 1575177..e8a60c9 100644
> --- a/arch/x86/include/asm/stackprotector.h
> +++ b/arch/x86/include/asm/stackprotector.h
> @@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void)
>
> current->stack_canary = canary;
> #ifdef CONFIG_X86_64
> - percpu_write(irq_stack_union.stack_canary, canary);
> + __this_cpu_write(irq_stack_union.stack_canary, canary);
> #else
> - percpu_write(stack_canary.canary, canary);
> + __this_cpu_write(stack_canary.canary, canary);
> #endif
> }
>
> diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
> index a1fe5c1..8b19667 100644
> --- a/arch/x86/include/asm/thread_info.h
> +++ b/arch/x86/include/asm/thread_info.h
> @@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);
> static inline struct thread_info *current_thread_info(void)
> {
> struct thread_info *ti;
> - ti = (void *)(percpu_read_stable(kernel_stack) +
> + ti = (void *)(this_cpu_read_stable(kernel_stack) +
> KERNEL_STACK_OFFSET - THREAD_SIZE);
> return ti;
> }
> diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
> index 169be89..e90eec0 100644
> --- a/arch/x86/include/asm/tlbflush.h
> +++ b/arch/x86/include/asm/tlbflush.h
> @@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
>
> static inline void reset_lazy_tlbstate(void)
> {
> - percpu_write(cpu_tlbstate.state, 0);
> - percpu_write(cpu_tlbstate.active_mm, &init_mm);
> + __this_cpu_write(cpu_tlbstate.state, 0);
> + __this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
> }
>
> #endif /* SMP */
> diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
> index aa003b1..d9b4fe7 100644
> --- a/arch/x86/kernel/cpu/common.c
> +++ b/arch/x86/kernel/cpu/common.c
> @@ -1163,7 +1163,7 @@ void __cpuinit cpu_init(void)
> oist = &per_cpu(orig_ist, cpu);
>
> #ifdef CONFIG_NUMA
> - if (cpu != 0 && percpu_read(numa_node) == 0 &&
> + if (cpu != 0 && __this_cpu_read(numa_node) == 0 &&
> early_cpu_to_node(cpu) != NUMA_NO_NODE)
> set_numa_node(early_cpu_to_node(cpu));
> #endif
> diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
> index 2af127d..b7f3a1e 100644
> --- a/arch/x86/kernel/cpu/mcheck/mce.c
> +++ b/arch/x86/kernel/cpu/mcheck/mce.c
> @@ -514,7 +514,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
> struct mce m;
> int i;
>
> - percpu_inc(mce_poll_count);
> + __this_cpu_inc(mce_poll_count);
>
> mce_gather_info(&m, NULL);
>
> @@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
>
> atomic_inc(&mce_entry);
>
> - percpu_inc(mce_exception_count);
> + __this_cpu_inc(mce_exception_count);
>
> if (!banks)
> goto out;
> diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
> index d90272e..2f0c1d1 100644
> --- a/arch/x86/kernel/paravirt.c
> +++ b/arch/x86/kernel/paravirt.c
> @@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LA
>
> static inline void enter_lazy(enum paravirt_lazy_mode mode)
> {
> - BUG_ON(percpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
> + BUG_ON(__this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
>
> - percpu_write(paravirt_lazy_mode, mode);
> + __this_cpu_write(paravirt_lazy_mode, mode);
> }
>
> static void leave_lazy(enum paravirt_lazy_mode mode)
> {
> - BUG_ON(percpu_read(paravirt_lazy_mode) != mode);
> + BUG_ON(__this_cpu_read(paravirt_lazy_mode) != mode);
>
> - percpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
> + __this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
> }
>
> void paravirt_enter_lazy_mmu(void)
> @@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev)
> {
> BUG_ON(preemptible());
>
> - if (percpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
> + if (__this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
> arch_leave_lazy_mmu_mode();
> set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES);
> }
> @@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
> if (in_interrupt())
> return PARAVIRT_LAZY_NONE;
>
> - return percpu_read(paravirt_lazy_mode);
> + return __this_cpu_read(paravirt_lazy_mode);
> }
>
> void arch_flush_lazy_mmu_mode(void)
> diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
> index 795b79f..b383fe8 100644
> --- a/arch/x86/kernel/process_32.c
> +++ b/arch/x86/kernel/process_32.c
> @@ -375,7 +375,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
> if (prev->gs | next->gs)
> lazy_load_gs(next->gs);
>
> - percpu_write(current_task, next_p);
> + __this_cpu_write(current_task, next_p);
>
> return prev_p;
> }
> diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
> index 3bd7e6e..7b6edbb 100644
> --- a/arch/x86/kernel/process_64.c
> +++ b/arch/x86/kernel/process_64.c
> @@ -74,7 +74,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);
>
> void enter_idle(void)
> {
> - percpu_write(is_idle, 1);
> + __this_cpu_write(is_idle, 1);
> atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
> }
>
> @@ -338,7 +338,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,
> load_gs_index(0);
> regs->ip = new_ip;
> regs->sp = new_sp;
> - percpu_write(old_rsp, new_sp);
> + this_cpu_write(old_rsp, new_sp);
> regs->cs = _cs;
> regs->ss = _ss;
> regs->flags = X86_EFLAGS_IF;
> @@ -472,11 +472,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
> /*
> * Switch the PDA and FPU contexts.
> */
> - prev->usersp = percpu_read(old_rsp);
> - percpu_write(old_rsp, next->usersp);
> - percpu_write(current_task, next_p);
> + prev->usersp = __this_cpu_read(old_rsp);
> + __this_cpu_write(old_rsp, next->usersp);
> + __this_cpu_write(current_task, next_p);
>
> - percpu_write(kernel_stack,
> + __this_cpu_write(kernel_stack,
> (unsigned long)task_stack_page(next_p) +
> THREAD_SIZE - KERNEL_STACK_OFFSET);
>
> diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
> index d6c0418..e931db0 100644
> --- a/arch/x86/mm/tlb.c
> +++ b/arch/x86/mm/tlb.c
> @@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);
> */
> void leave_mm(int cpu)
> {
> - if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
> + if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
> BUG();
> cpumask_clear_cpu(cpu,
> - mm_cpumask(percpu_read(cpu_tlbstate.active_mm)));
> + mm_cpumask(__this_cpu_read(cpu_tlbstate.active_mm)));
> load_cr3(swapper_pg_dir);
> }
> EXPORT_SYMBOL_GPL(leave_mm);
> @@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)
> * BUG();
> */
>
> - if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
> - if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
> + if (f->flush_mm == __this_cpu_read(cpu_tlbstate.active_mm)) {
> + if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
> if (f->flush_va == TLB_FLUSH_ALL)
> local_flush_tlb();
> else
> @@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
> static void do_flush_tlb_all(void *info)
> {
> __flush_tlb_all();
> - if (percpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
> + if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
> leave_mm(smp_processor_id());
> }
>
> diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
> index 1f92865..3622488 100644
> --- a/arch/x86/xen/enlighten.c
> +++ b/arch/x86/xen/enlighten.c
> @@ -777,11 +777,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);
>
> static unsigned long xen_read_cr0(void)
> {
> - unsigned long cr0 = percpu_read(xen_cr0_value);
> + unsigned long cr0 = this_cpu_read(xen_cr0_value);
>
> if (unlikely(cr0 == 0)) {
> cr0 = native_read_cr0();
> - percpu_write(xen_cr0_value, cr0);
> + this_cpu_write(xen_cr0_value, cr0);
> }
>
> return cr0;
> @@ -791,7 +791,7 @@ static void xen_write_cr0(unsigned long cr0)
> {
> struct multicall_space mcs;
>
> - percpu_write(xen_cr0_value, cr0);
> + this_cpu_write(xen_cr0_value, cr0);
>
> /* Only pay attention to cr0.TS; everything else is
> ignored. */
> diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c
> index 8bbb465..1573376 100644
> --- a/arch/x86/xen/irq.c
> +++ b/arch/x86/xen/irq.c
> @@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)
> struct vcpu_info *vcpu;
> unsigned long flags;
>
> - vcpu = percpu_read(xen_vcpu);
> + vcpu = this_cpu_read(xen_vcpu);
>
> /* flag has opposite sense of mask */
> flags = !vcpu->evtchn_upcall_mask;
> @@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)
> make sure we're don't switch CPUs between getting the vcpu
> pointer and updating the mask. */
> preempt_disable();
> - vcpu = percpu_read(xen_vcpu);
> + vcpu = this_cpu_read(xen_vcpu);
> vcpu->evtchn_upcall_mask = flags;
> preempt_enable_no_resched();
>
> @@ -72,7 +72,7 @@ static void xen_irq_disable(void)
> make sure we're don't switch CPUs between getting the vcpu
> pointer and updating the mask. */
> preempt_disable();
> - percpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
> + this_cpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
> preempt_enable_no_resched();
> }
> PV_CALLEE_SAVE_REGS_THUNK(xen_irq_disable);
> @@ -86,7 +86,7 @@ static void xen_irq_enable(void)
> the caller is confused and is trying to re-enable interrupts
> on an indeterminate processor. */
>
> - vcpu = percpu_read(xen_vcpu);
> + vcpu = this_cpu_read(xen_vcpu);
> vcpu->evtchn_upcall_mask = 0;
>
> /* Doesn't matter if we get preempted here, because any
> diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
> index 87f6673..426ff61 100644
> --- a/arch/x86/xen/mmu.c
> +++ b/arch/x86/xen/mmu.c
> @@ -1071,14 +1071,14 @@ static void drop_other_mm_ref(void *info)
> struct mm_struct *mm = info;
> struct mm_struct *active_mm;
>
> - active_mm = percpu_read(cpu_tlbstate.active_mm);
> + active_mm = this_cpu_read(cpu_tlbstate.active_mm);
>
> - if (active_mm == mm && percpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
> + if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
> leave_mm(smp_processor_id());
>
> /* If this cpu still has a stale cr3 reference, then make sure
> it has been flushed. */
> - if (percpu_read(xen_current_cr3) == __pa(mm->pgd))
> + if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd))
> load_cr3(swapper_pg_dir);
> }
>
> @@ -1185,17 +1185,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)
>
> static void xen_write_cr2(unsigned long cr2)
> {
> - percpu_read(xen_vcpu)->arch.cr2 = cr2;
> + this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
> }
>
> static unsigned long xen_read_cr2(void)
> {
> - return percpu_read(xen_vcpu)->arch.cr2;
> + return this_cpu_read(xen_vcpu)->arch.cr2;
> }
>
> unsigned long xen_read_cr2_direct(void)
> {
> - return percpu_read(xen_vcpu_info.arch.cr2);
> + return this_cpu_read(xen_vcpu_info.arch.cr2);
> }
>
> static void xen_flush_tlb(void)
> @@ -1278,12 +1278,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
>
> static unsigned long xen_read_cr3(void)
> {
> - return percpu_read(xen_cr3);
> + return this_cpu_read(xen_cr3);
> }
>
> static void set_current_cr3(void *v)
> {
> - percpu_write(xen_current_cr3, (unsigned long)v);
> + this_cpu_write(xen_current_cr3, (unsigned long)v);
> }
>
> static void __xen_write_cr3(bool kernel, unsigned long cr3)
> @@ -1306,7 +1306,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)
> xen_extend_mmuext_op(&op);
>
> if (kernel) {
> - percpu_write(xen_cr3, cr3);
> + this_cpu_write(xen_cr3, cr3);
>
> /* Update xen_current_cr3 once the batch has actually
> been submitted. */
> @@ -1322,7 +1322,7 @@ static void xen_write_cr3(unsigned long cr3)
>
> /* Update while interrupts are disabled, so its atomic with
> respect to ipis */
> - percpu_write(xen_cr3, cr3);
> + this_cpu_write(xen_cr3, cr3);
>
> __xen_write_cr3(true, cr3);
>
> diff --git a/arch/x86/xen/multicalls.h b/arch/x86/xen/multicalls.h
> index dee79b7..9c2e74f 100644
> --- a/arch/x86/xen/multicalls.h
> +++ b/arch/x86/xen/multicalls.h
> @@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)
> xen_mc_flush();
>
> /* restore flags saved in xen_mc_batch */
> - local_irq_restore(percpu_read(xen_mc_irq_flags));
> + local_irq_restore(this_cpu_read(xen_mc_irq_flags));
> }
>
> /* Set up a callback to be called when the current batch is flushed */
> diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
> index 041d4fe..449f868 100644
> --- a/arch/x86/xen/smp.c
> +++ b/arch/x86/xen/smp.c
> @@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)
> xen_setup_cpu_clockevents();
>
> set_cpu_online(cpu, true);
> - percpu_write(cpu_state, CPU_ONLINE);
> + this_cpu_write(cpu_state, CPU_ONLINE);
> wmb();
>
> /* We can take interrupts now: we're officially "up". */
> diff --git a/include/linux/percpu.h b/include/linux/percpu.h
> index 9ca008f..aad7e1c 100644
> --- a/include/linux/percpu.h
> +++ b/include/linux/percpu.h
> @@ -165,59 +165,6 @@ extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
> #define alloc_percpu(type) \
> (typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
>
> -/*
> - * Optional methods for optimized non-lvalue per-cpu variable access.
> - *
> - * @var can be a percpu variable or a field of it and its size should
> - * equal char, int or long. percpu_read() evaluates to a lvalue and
> - * all others to void.
> - *
> - * These operations are guaranteed to be atomic w.r.t. preemption.
> - * The generic versions use plain get/put_cpu_var(). Archs are
> - * encouraged to implement single-instruction alternatives which don't
> - * require preemption protection.
> - */
> -#ifndef percpu_read
> -# define percpu_read(var) \
> - ({ \
> - typeof(var) *pr_ptr__ = &(var); \
> - typeof(var) pr_ret__; \
> - pr_ret__ = get_cpu_var(*pr_ptr__); \
> - put_cpu_var(*pr_ptr__); \
> - pr_ret__; \
> - })
> -#endif
> -
> -#define __percpu_generic_to_op(var, val, op) \
> -do { \
> - typeof(var) *pgto_ptr__ = &(var); \
> - get_cpu_var(*pgto_ptr__) op val; \
> - put_cpu_var(*pgto_ptr__); \
> -} while (0)
> -
> -#ifndef percpu_write
> -# define percpu_write(var, val) __percpu_generic_to_op(var, (val), =)
> -#endif
> -
> -#ifndef percpu_add
> -# define percpu_add(var, val) __percpu_generic_to_op(var, (val), +=)
> -#endif
> -
> -#ifndef percpu_sub
> -# define percpu_sub(var, val) __percpu_generic_to_op(var, (val), -=)
> -#endif
> -
> -#ifndef percpu_and
> -# define percpu_and(var, val) __percpu_generic_to_op(var, (val), &=)
> -#endif
> -
> -#ifndef percpu_or
> -# define percpu_or(var, val) __percpu_generic_to_op(var, (val), |=)
> -#endif
> -
> -#ifndef percpu_xor
> -# define percpu_xor(var, val) __percpu_generic_to_op(var, (val), ^=)
> -#endif
>
> /*
> * Branching function to split up a function into a set of functions that
> diff --git a/include/linux/topology.h b/include/linux/topology.h
> index e26db03..b480403 100644
> --- a/include/linux/topology.h
> +++ b/include/linux/topology.h
> @@ -239,7 +239,7 @@ static inline int cpu_to_node(int cpu)
> #ifndef set_numa_node
> static inline void set_numa_node(int node)
> {
> - percpu_write(numa_node, node);
> + __this_cpu_write(numa_node, node);
> }
> #endif
>
> @@ -274,7 +274,7 @@ DECLARE_PER_CPU(int, _numa_mem_);
> #ifndef set_numa_mem
> static inline void set_numa_mem(int node)
> {
> - percpu_write(_numa_mem_, node);
> + __this_cpu_write(_numa_mem_, node);
> }
> #endif
>
> --
> 1.7.5.1
>
>
>
--
tejun
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs
2011-11-21 9:00 ` Alex,Shi
@ 2011-12-02 2:33 ` Alex,Shi
2011-12-02 11:56 ` Patrick McHardy
0 siblings, 1 reply; 42+ messages in thread
From: Alex,Shi @ 2011-12-02 2:33 UTC (permalink / raw)
To: Eric Dumazet
Cc: Christoph Lameter, tj@kernel.org, linux-kernel@vger.kernel.org,
Huang, Ying, Thomas Gleixner, mingo@redhat.com, avi@redhat.com,
akpm@linux-foundation.org, David Miller, kaber@trash.net,
a.p.zijlstra@chello.nl, kvm@vger.kernel.org, jeremy@xensource.com
On Mon, 2011-11-21 at 17:00 +0800, Alex,Shi wrote:
> On Thu, 2011-10-20 at 16:38 +0800, Eric Dumazet wrote:
> > Le jeudi 20 octobre 2011 à 15:32 +0800, Alex,Shi a écrit :
> > > percpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them
> > > for further code clean up.
> > >
> > > And in preempt safe scenario, __this_cpu_xxx funcs has a bit better
> > > performance since __this_cpu_xxx has no redundant preempt_disable()
> > >
> > > Signed-off-by: Alex Shi <alex.shi@intel.com>
> > > ---
> > > net/netfilter/xt_TEE.c | 12 ++++++------
> > > net/socket.c | 4 ++--
> > > 2 files changed, 8 insertions(+), 8 deletions(-)
> >
> > Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
> >
> > Thanks !
>
> Anyone like to pick up this patch? or more comments for this?
Kaber, David:
I appreciate for your any comments on this. Could you like do me a
favor?
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs
2011-12-02 2:33 ` Alex,Shi
@ 2011-12-02 11:56 ` Patrick McHardy
2012-01-11 8:45 ` Alex,Shi
0 siblings, 1 reply; 42+ messages in thread
From: Patrick McHardy @ 2011-12-02 11:56 UTC (permalink / raw)
To: Alex,Shi
Cc: Eric Dumazet, Christoph Lameter, tj@kernel.org,
linux-kernel@vger.kernel.org, Huang, Ying, Thomas Gleixner,
mingo@redhat.com, avi@redhat.com, akpm@linux-foundation.org,
David Miller, a.p.zijlstra@chello.nl, kvm@vger.kernel.org,
jeremy@xensource.com
On 02.12.2011 03:33, Alex,Shi wrote:
> On Mon, 2011-11-21 at 17:00 +0800, Alex,Shi wrote:
>> On Thu, 2011-10-20 at 16:38 +0800, Eric Dumazet wrote:
>>> Le jeudi 20 octobre 2011 à 15:32 +0800, Alex,Shi a écrit :
>>>> percpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them
>>>> for further code clean up.
>>>>
>>>> And in preempt safe scenario, __this_cpu_xxx funcs has a bit better
>>>> performance since __this_cpu_xxx has no redundant preempt_disable()
>>>>
>>>> Signed-off-by: Alex Shi <alex.shi@intel.com>
>>>> ---
>>>> net/netfilter/xt_TEE.c | 12 ++++++------
>>>> net/socket.c | 4 ++--
>>>> 2 files changed, 8 insertions(+), 8 deletions(-)
>>>
>>> Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
>>>
>>> Thanks !
>>
>> Anyone like to pick up this patch? or more comments for this?
>
> Kaber, David:
> I appreciate for your any comments on this. Could you like do me a
> favor?
No objections from me.
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs
2011-12-02 11:56 ` Patrick McHardy
@ 2012-01-11 8:45 ` Alex,Shi
2012-01-11 9:03 ` David Miller
0 siblings, 1 reply; 42+ messages in thread
From: Alex,Shi @ 2012-01-11 8:45 UTC (permalink / raw)
To: Patrick McHardy, davem
Cc: Eric Dumazet, Christoph Lameter, tj@kernel.org,
linux-kernel@vger.kernel.org, Huang, Ying, Thomas Gleixner,
mingo@redhat.com, avi@redhat.com, akpm@linux-foundation.org,
a.p.zijlstra@chello.nl, kvm@vger.kernel.org, jeremy@xensource.com
> >>>> percpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them
> >>>> for further code clean up.
> >>>>
> >>>> And in preempt safe scenario, __this_cpu_xxx funcs has a bit better
> >>>> performance since __this_cpu_xxx has no redundant preempt_disable()
> >>>>
> >>>> Signed-off-by: Alex Shi <alex.shi@intel.com>
> >>>> ---
> >>>> net/netfilter/xt_TEE.c | 12 ++++++------
> >>>> net/socket.c | 4 ++--
> >>>> 2 files changed, 8 insertions(+), 8 deletions(-)
> >>>
> >>> Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
> >>>
> >>> Thanks !
> >>
> >> Anyone like to pick up this patch? or more comments for this?
> >
> > Kaber, David:
> > I appreciate for your any comments on this. Could you like do me a
> > favor?
>
> No objections from me.
rend this patch for 3.2.0 kernel with Eric's Ack.
David, do you have any concerns for this patch? I will very appreciate
if it can met 3.3 open window.
---------
>From 037bd159fdf52b915e452fac8db2252b1c60297e Mon Sep 17 00:00:00 2001
From: Alex Shi <alex.shi@intel.com>
Date: Thu, 20 Oct 2011 14:52:17 +0800
Subject: [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs
percpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them
for further code clean up.
And in preempt safe scenario, __this_cpu_xxx funcs has a bit better
performance since __this_cpu_xxx has no redundant preempt_disable()
Signed-off-by: Alex Shi <alex.shi@intel.com>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
---
net/netfilter/xt_TEE.c | 12 ++++++------
net/socket.c | 4 ++--
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/net/netfilter/xt_TEE.c b/net/netfilter/xt_TEE.c
index 5f054a0..678084c 100644
--- a/net/netfilter/xt_TEE.c
+++ b/net/netfilter/xt_TEE.c
@@ -90,7 +90,7 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
const struct xt_tee_tginfo *info = par->targinfo;
struct iphdr *iph;
- if (percpu_read(tee_active))
+ if (__this_cpu_read(tee_active))
return XT_CONTINUE;
/*
* Copy the skb, and route the copy. Will later return %XT_CONTINUE for
@@ -127,9 +127,9 @@ tee_tg4(struct sk_buff *skb, const struct xt_action_param *par)
ip_send_check(iph);
if (tee_tg_route4(skb, info)) {
- percpu_write(tee_active, true);
+ __this_cpu_write(tee_active, true);
ip_local_out(skb);
- percpu_write(tee_active, false);
+ __this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
@@ -170,7 +170,7 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
{
const struct xt_tee_tginfo *info = par->targinfo;
- if (percpu_read(tee_active))
+ if (__this_cpu_read(tee_active))
return XT_CONTINUE;
skb = pskb_copy(skb, GFP_ATOMIC);
if (skb == NULL)
@@ -188,9 +188,9 @@ tee_tg6(struct sk_buff *skb, const struct xt_action_param *par)
--iph->hop_limit;
}
if (tee_tg_route6(skb, info)) {
- percpu_write(tee_active, true);
+ __this_cpu_write(tee_active, true);
ip6_local_out(skb);
- percpu_write(tee_active, false);
+ __this_cpu_write(tee_active, false);
} else {
kfree_skb(skb);
}
diff --git a/net/socket.c b/net/socket.c
index ffe92ca..4b62ca9 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -479,7 +479,7 @@ static struct socket *sock_alloc(void)
inode->i_uid = current_fsuid();
inode->i_gid = current_fsgid();
- percpu_add(sockets_in_use, 1);
+ this_cpu_add(sockets_in_use, 1);
return sock;
}
@@ -522,7 +522,7 @@ void sock_release(struct socket *sock)
if (rcu_dereference_protected(sock->wq, 1)->fasync_list)
printk(KERN_ERR "sock_release: fasync list not empty!\n");
- percpu_sub(sockets_in_use, 1);
+ this_cpu_sub(sockets_in_use, 1);
if (!sock->file) {
iput(SOCK_INODE(sock));
return;
--
1.6.3.3
^ permalink raw reply related [flat|nested] 42+ messages in thread
* Re: [PATCH 2/3] kvm: use this_cpu_xxx replace percpu_xxx funcs
2011-11-21 9:20 ` Avi Kivity
2011-11-21 9:52 ` Alex,Shi
@ 2012-01-11 8:48 ` Alex,Shi
1 sibling, 0 replies; 42+ messages in thread
From: Alex,Shi @ 2012-01-11 8:48 UTC (permalink / raw)
To: Avi Kivity
Cc: Marcelo Tosatti, Christoph Lameter, tj@kernel.org,
linux-kernel@vger.kernel.org, eric.dumazet@gmail.com, Huang, Ying,
tglx@linutronix.de, mingo@redhat.com, akpm@linux-foundation.org,
davem@davemloft.net, kaber@trash.net, a.p.zijlstra@chello.nl,
kvm@vger.kernel.org, jeremy@xensource.com
> > >
> > > Acked-by: Avi Kivity <avi@redhat.com>
> > >
> >
> > And this one, picking up or comments are all appreciated. :)
>
> Just to be clear, you want this applied in kvm.git?
>
Thanks Avi!
I saw it is in your 3.3 submit list.
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs
2012-01-11 8:45 ` Alex,Shi
@ 2012-01-11 9:03 ` David Miller
2012-01-11 9:10 ` Alex,Shi
0 siblings, 1 reply; 42+ messages in thread
From: David Miller @ 2012-01-11 9:03 UTC (permalink / raw)
To: alex.shi
Cc: kaber, eric.dumazet, cl, tj, linux-kernel, ying.huang, tglx,
mingo, avi, akpm, a.p.zijlstra, kvm, jeremy
From: "Alex,Shi" <alex.shi@intel.com>
Date: Wed, 11 Jan 2012 16:45:33 +0800
>> >>>> percpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them
>> >>>> for further code clean up.
>> >>>>
>> >>>> And in preempt safe scenario, __this_cpu_xxx funcs has a bit better
>> >>>> performance since __this_cpu_xxx has no redundant preempt_disable()
>> >>>>
>> >>>> Signed-off-by: Alex Shi <alex.shi@intel.com>
>> >>>> ---
>> >>>> net/netfilter/xt_TEE.c | 12 ++++++------
>> >>>> net/socket.c | 4 ++--
>> >>>> 2 files changed, 8 insertions(+), 8 deletions(-)
>> >>>
>> >>> Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
>> >>>
>> >>> Thanks !
>> >>
>> >> Anyone like to pick up this patch? or more comments for this?
>> >
>> > Kaber, David:
>> > I appreciate for your any comments on this. Could you like do me a
>> > favor?
>>
>> No objections from me.
>
> rend this patch for 3.2.0 kernel with Eric's Ack.
>
> David, do you have any concerns for this patch? I will very appreciate
> if it can met 3.3 open window.
Please just submit it directly with the other this_cpu() patches:
Acked-by: David S. Miller <davem@davemloft.net>
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH 3/3] Code clean up for percpu_xxx() functions
2011-11-22 0:06 ` tj
@ 2012-01-11 9:08 ` Alex,Shi
2012-01-11 17:19 ` tj
0 siblings, 1 reply; 42+ messages in thread
From: Alex,Shi @ 2012-01-11 9:08 UTC (permalink / raw)
To: tj@kernel.org
Cc: H. Peter Anvin, Christoph Lameter, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying, Thomas Gleixner,
mingo@redhat.com, avi@redhat.com, akpm@linux-foundation.org,
David Miller, kaber@trash.net, a.p.zijlstra@chello.nl,
kvm@vger.kernel.org, Andi Kleen
On Mon, 2011-11-21 at 17:06 -0700, tj@kernel.org wrote:
> (cc'ing hpa and quoting whole body)
> >
> > Signed-off-by: Alex Shi <alex.shi@intel.com>
> > Acked-by: Christoph Lameter <cl@gentwo.org>
>
> Acked-by: Tejun Heo <tj@kernel.org>
>
> hpa, I suppose this should go through x86? The original patch can be
> accessed at
>
> http://article.gmane.org/gmane.linux.kernel/1218055/raw
Rend for 3.2 kernel, no any change needed to apply on latest Linus'
tree. :)
Actually, this clean up has no performance or security impact for
kernel. On the contrary, removing some potential redundant preempt
disable will bring a slight performance benefit to kernel.
This 3rd patch depends on previous 2 patches, the 2nd one kvm code clean
up was submitted for 3.3 kernel. but the 2st one net code clean up is
waiting for David's comments.
----------
>From 0dce61dc88b8ed2687b4d5c0633aa54d1f66fdc0 Mon Sep 17 00:00:00 2001
From: Alex Shi <alex.shi@intel.com>
Date: Tue, 22 Nov 2011 00:05:37 +0800
Subject: [PATCH 3/3] Code clean up for percpu_xxx() functions
Since percpu_xxx() serial functions are duplicate with this_cpu_xxx().
Removing percpu_xxx() definition and replacing them by this_cpu_xxx() in
code.
And further more, as Christoph Lameter's requirement, I try to use
__this_cpu_xx to replace this_cpu_xxx if it is in preempt safe scenario.
The preempt safe scenarios include:
1, in irq/softirq/nmi handler
2, protected by preempt_disable
3, protected by spin_lock
4, if the code context imply that it is preempt safe, like the code is
follows or be followed a preempt safe code.
I left the xen code unchanged, since no idea of them.
BTW, In fact, this_cpu_xxx are same as __this_cpu_xxx since all funcs
implement in a single instruction for x86 machine. But it maybe
different for other platforms, so, doing this distinguish is helpful for
other platforms' performance.
Signed-off-by: Alex Shi <alex.shi@intel.com>
Acked-by: Christoph Lameter <cl@gentwo.org>
Acked-by: Tejun Heo <tj@kernel.org>
---
arch/x86/include/asm/current.h | 2 +-
arch/x86/include/asm/hardirq.h | 9 +++--
arch/x86/include/asm/irq_regs.h | 4 +-
arch/x86/include/asm/mmu_context.h | 12 ++++----
arch/x86/include/asm/percpu.h | 24 ++++++---------
arch/x86/include/asm/smp.h | 4 +-
arch/x86/include/asm/stackprotector.h | 4 +-
arch/x86/include/asm/thread_info.h | 2 +-
arch/x86/include/asm/tlbflush.h | 4 +-
arch/x86/kernel/cpu/common.c | 2 +-
arch/x86/kernel/cpu/mcheck/mce.c | 4 +-
arch/x86/kernel/paravirt.c | 12 ++++----
arch/x86/kernel/process_32.c | 2 +-
arch/x86/kernel/process_64.c | 12 ++++----
arch/x86/mm/tlb.c | 10 +++---
arch/x86/xen/enlighten.c | 6 ++--
arch/x86/xen/irq.c | 8 ++--
arch/x86/xen/mmu.c | 20 ++++++------
arch/x86/xen/multicalls.h | 2 +-
arch/x86/xen/smp.c | 2 +-
include/linux/percpu.h | 53 ---------------------------------
include/linux/topology.h | 4 +-
22 files changed, 73 insertions(+), 129 deletions(-)
diff --git a/arch/x86/include/asm/current.h b/arch/x86/include/asm/current.h
index 4d447b7..9476c04 100644
--- a/arch/x86/include/asm/current.h
+++ b/arch/x86/include/asm/current.h
@@ -11,7 +11,7 @@ DECLARE_PER_CPU(struct task_struct *, current_task);
static __always_inline struct task_struct *get_current(void)
{
- return percpu_read_stable(current_task);
+ return this_cpu_read_stable(current_task);
}
#define current get_current()
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 55e4de6..2890444 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -35,14 +35,15 @@ DECLARE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
#define __ARCH_IRQ_STAT
-#define inc_irq_stat(member) percpu_inc(irq_stat.member)
+#define inc_irq_stat(member) __this_cpu_inc(irq_stat.member)
-#define local_softirq_pending() percpu_read(irq_stat.__softirq_pending)
+#define local_softirq_pending() __this_cpu_read(irq_stat.__softirq_pending)
#define __ARCH_SET_SOFTIRQ_PENDING
-#define set_softirq_pending(x) percpu_write(irq_stat.__softirq_pending, (x))
-#define or_softirq_pending(x) percpu_or(irq_stat.__softirq_pending, (x))
+#define set_softirq_pending(x) \
+ __this_cpu_write(irq_stat.__softirq_pending, (x))
+#define or_softirq_pending(x) __this_cpu_or(irq_stat.__softirq_pending, (x))
extern void ack_bad_irq(unsigned int irq);
diff --git a/arch/x86/include/asm/irq_regs.h b/arch/x86/include/asm/irq_regs.h
index 7784322..15639ed 100644
--- a/arch/x86/include/asm/irq_regs.h
+++ b/arch/x86/include/asm/irq_regs.h
@@ -15,7 +15,7 @@ DECLARE_PER_CPU(struct pt_regs *, irq_regs);
static inline struct pt_regs *get_irq_regs(void)
{
- return percpu_read(irq_regs);
+ return __this_cpu_read(irq_regs);
}
static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
@@ -23,7 +23,7 @@ static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs)
struct pt_regs *old_regs;
old_regs = get_irq_regs();
- percpu_write(irq_regs, new_regs);
+ __this_cpu_write(irq_regs, new_regs);
return old_regs;
}
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
index 6902152..02ca533 100644
--- a/arch/x86/include/asm/mmu_context.h
+++ b/arch/x86/include/asm/mmu_context.h
@@ -25,8 +25,8 @@ void destroy_context(struct mm_struct *mm);
static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
{
#ifdef CONFIG_SMP
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
- percpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_LAZY);
#endif
}
@@ -37,8 +37,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
if (likely(prev != next)) {
#ifdef CONFIG_SMP
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- percpu_write(cpu_tlbstate.active_mm, next);
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ __this_cpu_write(cpu_tlbstate.active_mm, next);
#endif
cpumask_set_cpu(cpu, mm_cpumask(next));
@@ -56,8 +56,8 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
}
#ifdef CONFIG_SMP
else {
- percpu_write(cpu_tlbstate.state, TLBSTATE_OK);
- BUG_ON(percpu_read(cpu_tlbstate.active_mm) != next);
+ __this_cpu_write(cpu_tlbstate.state, TLBSTATE_OK);
+ BUG_ON(__this_cpu_read(cpu_tlbstate.active_mm) != next);
if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next))) {
/* We were in lazy tlb mode and leave_mm disabled
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index 3470c9d..520005e 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -46,7 +46,7 @@
#ifdef CONFIG_SMP
#define __percpu_prefix "%%"__stringify(__percpu_seg)":"
-#define __my_cpu_offset percpu_read(this_cpu_off)
+#define __my_cpu_offset __this_cpu_read(this_cpu_off)
/*
* Compared to the generic __my_cpu_offset version, the following
@@ -351,23 +351,15 @@ do { \
})
/*
- * percpu_read() makes gcc load the percpu variable every time it is
- * accessed while percpu_read_stable() allows the value to be cached.
- * percpu_read_stable() is more efficient and can be used if its value
+ * this_cpu_read() makes gcc load the percpu variable every time it is
+ * accessed while this_cpu_read_stable() allows the value to be cached.
+ * this_cpu_read_stable() is more efficient and can be used if its value
* is guaranteed to be valid across cpus. The current users include
* get_current() and get_thread_info() both of which are actually
* per-thread variables implemented as per-cpu variables and thus
* stable for the duration of the respective task.
*/
-#define percpu_read(var) percpu_from_op("mov", var, "m" (var))
-#define percpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
-#define percpu_write(var, val) percpu_to_op("mov", var, val)
-#define percpu_add(var, val) percpu_add_op(var, val)
-#define percpu_sub(var, val) percpu_add_op(var, -(val))
-#define percpu_and(var, val) percpu_to_op("and", var, val)
-#define percpu_or(var, val) percpu_to_op("or", var, val)
-#define percpu_xor(var, val) percpu_to_op("xor", var, val)
-#define percpu_inc(var) percpu_unary_op("inc", var)
+#define this_cpu_read_stable(var) percpu_from_op("mov", var, "p" (&(var)))
#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
@@ -551,7 +543,11 @@ static __always_inline int x86_this_cpu_constant_test_bit(unsigned int nr,
{
unsigned long __percpu *a = (unsigned long *)addr + nr / BITS_PER_LONG;
- return ((1UL << (nr % BITS_PER_LONG)) & percpu_read(*a)) != 0;
+#ifdef CONFIG_X86_64
+ return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_8(*a)) != 0;
+#else
+ return ((1UL << (nr % BITS_PER_LONG)) & __this_cpu_read_4(*a)) != 0;
+#endif
}
static inline int x86_this_cpu_variable_test_bit(int nr,
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 73b11bc..dd5749b 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -188,11 +188,11 @@ extern unsigned disabled_cpus __cpuinitdata;
* from the initial startup. We map APIC_BASE very early in page_setup(),
* so this is correct in the x86 case.
*/
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
extern int safe_smp_processor_id(void);
#elif defined(CONFIG_X86_64_SMP)
-#define raw_smp_processor_id() (percpu_read(cpu_number))
+#define raw_smp_processor_id() (this_cpu_read(cpu_number))
#define stack_smp_processor_id() \
({ \
diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h
index 1575177..e8a60c9 100644
--- a/arch/x86/include/asm/stackprotector.h
+++ b/arch/x86/include/asm/stackprotector.h
@@ -76,9 +76,9 @@ static __always_inline void boot_init_stack_canary(void)
current->stack_canary = canary;
#ifdef CONFIG_X86_64
- percpu_write(irq_stack_union.stack_canary, canary);
+ __this_cpu_write(irq_stack_union.stack_canary, canary);
#else
- percpu_write(stack_canary.canary, canary);
+ __this_cpu_write(stack_canary.canary, canary);
#endif
}
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index a1fe5c1..8b19667 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -219,7 +219,7 @@ DECLARE_PER_CPU(unsigned long, kernel_stack);
static inline struct thread_info *current_thread_info(void)
{
struct thread_info *ti;
- ti = (void *)(percpu_read_stable(kernel_stack) +
+ ti = (void *)(this_cpu_read_stable(kernel_stack) +
KERNEL_STACK_OFFSET - THREAD_SIZE);
return ti;
}
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 169be89..e90eec0 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -156,8 +156,8 @@ DECLARE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate);
static inline void reset_lazy_tlbstate(void)
{
- percpu_write(cpu_tlbstate.state, 0);
- percpu_write(cpu_tlbstate.active_mm, &init_mm);
+ __this_cpu_write(cpu_tlbstate.state, 0);
+ __this_cpu_write(cpu_tlbstate.active_mm, &init_mm);
}
#endif /* SMP */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index aa003b1..d9b4fe7 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1163,7 +1163,7 @@ void __cpuinit cpu_init(void)
oist = &per_cpu(orig_ist, cpu);
#ifdef CONFIG_NUMA
- if (cpu != 0 && percpu_read(numa_node) == 0 &&
+ if (cpu != 0 && __this_cpu_read(numa_node) == 0 &&
early_cpu_to_node(cpu) != NUMA_NO_NODE)
set_numa_node(early_cpu_to_node(cpu));
#endif
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 2af127d..b7f3a1e 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -514,7 +514,7 @@ void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
struct mce m;
int i;
- percpu_inc(mce_poll_count);
+ __this_cpu_inc(mce_poll_count);
mce_gather_info(&m, NULL);
@@ -906,7 +906,7 @@ void do_machine_check(struct pt_regs *regs, long error_code)
atomic_inc(&mce_entry);
- percpu_inc(mce_exception_count);
+ __this_cpu_inc(mce_exception_count);
if (!banks)
goto out;
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index d90272e..2f0c1d1 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -239,16 +239,16 @@ static DEFINE_PER_CPU(enum paravirt_lazy_mode, paravirt_lazy_mode) = PARAVIRT_LA
static inline void enter_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
+ BUG_ON(__this_cpu_read(paravirt_lazy_mode) != PARAVIRT_LAZY_NONE);
- percpu_write(paravirt_lazy_mode, mode);
+ __this_cpu_write(paravirt_lazy_mode, mode);
}
static void leave_lazy(enum paravirt_lazy_mode mode)
{
- BUG_ON(percpu_read(paravirt_lazy_mode) != mode);
+ BUG_ON(__this_cpu_read(paravirt_lazy_mode) != mode);
- percpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
+ __this_cpu_write(paravirt_lazy_mode, PARAVIRT_LAZY_NONE);
}
void paravirt_enter_lazy_mmu(void)
@@ -265,7 +265,7 @@ void paravirt_start_context_switch(struct task_struct *prev)
{
BUG_ON(preemptible());
- if (percpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
+ if (__this_cpu_read(paravirt_lazy_mode) == PARAVIRT_LAZY_MMU) {
arch_leave_lazy_mmu_mode();
set_ti_thread_flag(task_thread_info(prev), TIF_LAZY_MMU_UPDATES);
}
@@ -287,7 +287,7 @@ enum paravirt_lazy_mode paravirt_get_lazy_mode(void)
if (in_interrupt())
return PARAVIRT_LAZY_NONE;
- return percpu_read(paravirt_lazy_mode);
+ return __this_cpu_read(paravirt_lazy_mode);
}
void arch_flush_lazy_mmu_mode(void)
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 795b79f..b383fe8 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -375,7 +375,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
if (prev->gs | next->gs)
lazy_load_gs(next->gs);
- percpu_write(current_task, next_p);
+ __this_cpu_write(current_task, next_p);
return prev_p;
}
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 3bd7e6e..7b6edbb 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -74,7 +74,7 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister);
void enter_idle(void)
{
- percpu_write(is_idle, 1);
+ __this_cpu_write(is_idle, 1);
atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
}
@@ -338,7 +338,7 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,
load_gs_index(0);
regs->ip = new_ip;
regs->sp = new_sp;
- percpu_write(old_rsp, new_sp);
+ this_cpu_write(old_rsp, new_sp);
regs->cs = _cs;
regs->ss = _ss;
regs->flags = X86_EFLAGS_IF;
@@ -472,11 +472,11 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
/*
* Switch the PDA and FPU contexts.
*/
- prev->usersp = percpu_read(old_rsp);
- percpu_write(old_rsp, next->usersp);
- percpu_write(current_task, next_p);
+ prev->usersp = __this_cpu_read(old_rsp);
+ __this_cpu_write(old_rsp, next->usersp);
+ __this_cpu_write(current_task, next_p);
- percpu_write(kernel_stack,
+ __this_cpu_write(kernel_stack,
(unsigned long)task_stack_page(next_p) +
THREAD_SIZE - KERNEL_STACK_OFFSET);
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index d6c0418..e931db0 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -61,10 +61,10 @@ static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);
*/
void leave_mm(int cpu)
{
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK)
BUG();
cpumask_clear_cpu(cpu,
- mm_cpumask(percpu_read(cpu_tlbstate.active_mm)));
+ mm_cpumask(__this_cpu_read(cpu_tlbstate.active_mm)));
load_cr3(swapper_pg_dir);
}
EXPORT_SYMBOL_GPL(leave_mm);
@@ -152,8 +152,8 @@ void smp_invalidate_interrupt(struct pt_regs *regs)
* BUG();
*/
- if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
+ if (f->flush_mm == __this_cpu_read(cpu_tlbstate.active_mm)) {
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
if (f->flush_va == TLB_FLUSH_ALL)
local_flush_tlb();
else
@@ -322,7 +322,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
static void do_flush_tlb_all(void *info)
{
__flush_tlb_all();
- if (percpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
+ if (__this_cpu_read(cpu_tlbstate.state) == TLBSTATE_LAZY)
leave_mm(smp_processor_id());
}
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 1f92865..3622488 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -777,11 +777,11 @@ static DEFINE_PER_CPU(unsigned long, xen_cr0_value);
static unsigned long xen_read_cr0(void)
{
- unsigned long cr0 = percpu_read(xen_cr0_value);
+ unsigned long cr0 = this_cpu_read(xen_cr0_value);
if (unlikely(cr0 == 0)) {
cr0 = native_read_cr0();
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
}
return cr0;
@@ -791,7 +791,7 @@ static void xen_write_cr0(unsigned long cr0)
{
struct multicall_space mcs;
- percpu_write(xen_cr0_value, cr0);
+ this_cpu_write(xen_cr0_value, cr0);
/* Only pay attention to cr0.TS; everything else is
ignored. */
diff --git a/arch/x86/xen/irq.c b/arch/x86/xen/irq.c
index 8bbb465..1573376 100644
--- a/arch/x86/xen/irq.c
+++ b/arch/x86/xen/irq.c
@@ -26,7 +26,7 @@ static unsigned long xen_save_fl(void)
struct vcpu_info *vcpu;
unsigned long flags;
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
/* flag has opposite sense of mask */
flags = !vcpu->evtchn_upcall_mask;
@@ -50,7 +50,7 @@ static void xen_restore_fl(unsigned long flags)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = flags;
preempt_enable_no_resched();
@@ -72,7 +72,7 @@ static void xen_irq_disable(void)
make sure we're don't switch CPUs between getting the vcpu
pointer and updating the mask. */
preempt_disable();
- percpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
+ this_cpu_read(xen_vcpu)->evtchn_upcall_mask = 1;
preempt_enable_no_resched();
}
PV_CALLEE_SAVE_REGS_THUNK(xen_irq_disable);
@@ -86,7 +86,7 @@ static void xen_irq_enable(void)
the caller is confused and is trying to re-enable interrupts
on an indeterminate processor. */
- vcpu = percpu_read(xen_vcpu);
+ vcpu = this_cpu_read(xen_vcpu);
vcpu->evtchn_upcall_mask = 0;
/* Doesn't matter if we get preempted here, because any
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 87f6673..426ff61 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1071,14 +1071,14 @@ static void drop_other_mm_ref(void *info)
struct mm_struct *mm = info;
struct mm_struct *active_mm;
- active_mm = percpu_read(cpu_tlbstate.active_mm);
+ active_mm = this_cpu_read(cpu_tlbstate.active_mm);
- if (active_mm == mm && percpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
+ if (active_mm == mm && this_cpu_read(cpu_tlbstate.state) != TLBSTATE_OK)
leave_mm(smp_processor_id());
/* If this cpu still has a stale cr3 reference, then make sure
it has been flushed. */
- if (percpu_read(xen_current_cr3) == __pa(mm->pgd))
+ if (this_cpu_read(xen_current_cr3) == __pa(mm->pgd))
load_cr3(swapper_pg_dir);
}
@@ -1185,17 +1185,17 @@ static void __init xen_pagetable_setup_done(pgd_t *base)
static void xen_write_cr2(unsigned long cr2)
{
- percpu_read(xen_vcpu)->arch.cr2 = cr2;
+ this_cpu_read(xen_vcpu)->arch.cr2 = cr2;
}
static unsigned long xen_read_cr2(void)
{
- return percpu_read(xen_vcpu)->arch.cr2;
+ return this_cpu_read(xen_vcpu)->arch.cr2;
}
unsigned long xen_read_cr2_direct(void)
{
- return percpu_read(xen_vcpu_info.arch.cr2);
+ return this_cpu_read(xen_vcpu_info.arch.cr2);
}
static void xen_flush_tlb(void)
@@ -1278,12 +1278,12 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
static unsigned long xen_read_cr3(void)
{
- return percpu_read(xen_cr3);
+ return this_cpu_read(xen_cr3);
}
static void set_current_cr3(void *v)
{
- percpu_write(xen_current_cr3, (unsigned long)v);
+ this_cpu_write(xen_current_cr3, (unsigned long)v);
}
static void __xen_write_cr3(bool kernel, unsigned long cr3)
@@ -1306,7 +1306,7 @@ static void __xen_write_cr3(bool kernel, unsigned long cr3)
xen_extend_mmuext_op(&op);
if (kernel) {
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
/* Update xen_current_cr3 once the batch has actually
been submitted. */
@@ -1322,7 +1322,7 @@ static void xen_write_cr3(unsigned long cr3)
/* Update while interrupts are disabled, so its atomic with
respect to ipis */
- percpu_write(xen_cr3, cr3);
+ this_cpu_write(xen_cr3, cr3);
__xen_write_cr3(true, cr3);
diff --git a/arch/x86/xen/multicalls.h b/arch/x86/xen/multicalls.h
index dee79b7..9c2e74f 100644
--- a/arch/x86/xen/multicalls.h
+++ b/arch/x86/xen/multicalls.h
@@ -47,7 +47,7 @@ static inline void xen_mc_issue(unsigned mode)
xen_mc_flush();
/* restore flags saved in xen_mc_batch */
- local_irq_restore(percpu_read(xen_mc_irq_flags));
+ local_irq_restore(this_cpu_read(xen_mc_irq_flags));
}
/* Set up a callback to be called when the current batch is flushed */
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 041d4fe..449f868 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -76,7 +76,7 @@ static void __cpuinit cpu_bringup(void)
xen_setup_cpu_clockevents();
set_cpu_online(cpu, true);
- percpu_write(cpu_state, CPU_ONLINE);
+ this_cpu_write(cpu_state, CPU_ONLINE);
wmb();
/* We can take interrupts now: we're officially "up". */
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 9ca008f..aad7e1c 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -165,59 +165,6 @@ extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
#define alloc_percpu(type) \
(typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
-/*
- * Optional methods for optimized non-lvalue per-cpu variable access.
- *
- * @var can be a percpu variable or a field of it and its size should
- * equal char, int or long. percpu_read() evaluates to a lvalue and
- * all others to void.
- *
- * These operations are guaranteed to be atomic w.r.t. preemption.
- * The generic versions use plain get/put_cpu_var(). Archs are
- * encouraged to implement single-instruction alternatives which don't
- * require preemption protection.
- */
-#ifndef percpu_read
-# define percpu_read(var) \
- ({ \
- typeof(var) *pr_ptr__ = &(var); \
- typeof(var) pr_ret__; \
- pr_ret__ = get_cpu_var(*pr_ptr__); \
- put_cpu_var(*pr_ptr__); \
- pr_ret__; \
- })
-#endif
-
-#define __percpu_generic_to_op(var, val, op) \
-do { \
- typeof(var) *pgto_ptr__ = &(var); \
- get_cpu_var(*pgto_ptr__) op val; \
- put_cpu_var(*pgto_ptr__); \
-} while (0)
-
-#ifndef percpu_write
-# define percpu_write(var, val) __percpu_generic_to_op(var, (val), =)
-#endif
-
-#ifndef percpu_add
-# define percpu_add(var, val) __percpu_generic_to_op(var, (val), +=)
-#endif
-
-#ifndef percpu_sub
-# define percpu_sub(var, val) __percpu_generic_to_op(var, (val), -=)
-#endif
-
-#ifndef percpu_and
-# define percpu_and(var, val) __percpu_generic_to_op(var, (val), &=)
-#endif
-
-#ifndef percpu_or
-# define percpu_or(var, val) __percpu_generic_to_op(var, (val), |=)
-#endif
-
-#ifndef percpu_xor
-# define percpu_xor(var, val) __percpu_generic_to_op(var, (val), ^=)
-#endif
/*
* Branching function to split up a function into a set of functions that
diff --git a/include/linux/topology.h b/include/linux/topology.h
index e26db03..b480403 100644
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -239,7 +239,7 @@ static inline int cpu_to_node(int cpu)
#ifndef set_numa_node
static inline void set_numa_node(int node)
{
- percpu_write(numa_node, node);
+ __this_cpu_write(numa_node, node);
}
#endif
@@ -274,7 +274,7 @@ DECLARE_PER_CPU(int, _numa_mem_);
#ifndef set_numa_mem
static inline void set_numa_mem(int node)
{
- percpu_write(_numa_mem_, node);
+ __this_cpu_write(_numa_mem_, node);
}
#endif
--
1.7.5.1
^ permalink raw reply related [flat|nested] 42+ messages in thread
* Re: [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs
2012-01-11 9:03 ` David Miller
@ 2012-01-11 9:10 ` Alex,Shi
0 siblings, 0 replies; 42+ messages in thread
From: Alex,Shi @ 2012-01-11 9:10 UTC (permalink / raw)
To: David Miller
Cc: kaber, eric.dumazet, cl, tj, linux-kernel, ying.huang, tglx,
mingo, avi, akpm, a.p.zijlstra, kvm, jeremy
On Wed, 2012-01-11 at 01:03 -0800, David Miller wrote:
> From: "Alex,Shi" <alex.shi@intel.com>
> Date: Wed, 11 Jan 2012 16:45:33 +0800
>
> >> >>>> percpu_xxx funcs are duplicated with this_cpu_xxx funcs, so replace them
> >> >>>> for further code clean up.
> >> >>>>
> >> >>>> And in preempt safe scenario, __this_cpu_xxx funcs has a bit better
> >> >>>> performance since __this_cpu_xxx has no redundant preempt_disable()
> >> >>>>
> >> >>>> Signed-off-by: Alex Shi <alex.shi@intel.com>
> >> >>>> ---
> >> >>>> net/netfilter/xt_TEE.c | 12 ++++++------
> >> >>>> net/socket.c | 4 ++--
> >> >>>> 2 files changed, 8 insertions(+), 8 deletions(-)
> >> >>>
> >> >>> Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
> >> >>>
> >> >>> Thanks !
> >> >>
> >> >> Anyone like to pick up this patch? or more comments for this?
> >> >
> >> > Kaber, David:
> >> > I appreciate for your any comments on this. Could you like do me a
> >> > favor?
> >>
> >> No objections from me.
> >
> > rend this patch for 3.2.0 kernel with Eric's Ack.
> >
> > David, do you have any concerns for this patch? I will very appreciate
> > if it can met 3.3 open window.
>
> Please just submit it directly with the other this_cpu() patches:
>
> Acked-by: David S. Miller <davem@davemloft.net>
Thanks a lot! :)
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH 3/3] Code clean up for percpu_xxx() functions
2012-01-11 9:08 ` Alex,Shi
@ 2012-01-11 17:19 ` tj
2012-01-12 0:44 ` H. Peter Anvin
0 siblings, 1 reply; 42+ messages in thread
From: tj @ 2012-01-11 17:19 UTC (permalink / raw)
To: Alex,Shi
Cc: H. Peter Anvin, Christoph Lameter, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying, Thomas Gleixner,
mingo@redhat.com, avi@redhat.com, akpm@linux-foundation.org,
David Miller, kaber@trash.net, a.p.zijlstra@chello.nl,
kvm@vger.kernel.org, Andi Kleen
On Wed, Jan 11, 2012 at 05:08:41PM +0800, Alex,Shi wrote:
> On Mon, 2011-11-21 at 17:06 -0700, tj@kernel.org wrote:
> > (cc'ing hpa and quoting whole body)
> > >
> > > Signed-off-by: Alex Shi <alex.shi@intel.com>
> > > Acked-by: Christoph Lameter <cl@gentwo.org>
> >
> > Acked-by: Tejun Heo <tj@kernel.org>
> >
> > hpa, I suppose this should go through x86? The original patch can be
> > accessed at
> >
> > http://article.gmane.org/gmane.linux.kernel/1218055/raw
>
> Rend for 3.2 kernel, no any change needed to apply on latest Linus'
> tree. :)
>
> Actually, this clean up has no performance or security impact for
> kernel. On the contrary, removing some potential redundant preempt
> disable will bring a slight performance benefit to kernel.
>
> This 3rd patch depends on previous 2 patches, the 2nd one kvm code clean
> up was submitted for 3.3 kernel. but the 2st one net code clean up is
> waiting for David's comments.
Alex, can you please collect all patches into a single patchset?
Please split it such that, usage changes are per-system so that they
can be routed through respective subsystems (x86 or net) and updates
to percpu proper which can be applied after other changes have been
applied. It would probably be best to route these patches separately
rather than all through percpu as it touches a lot of different places
and is likely to cause conflicts. I *think* the best way would be,
* Submit per-subsystem patches and get them merged to subsystem trees.
* (Optional) Apply a patch to mark unused interface deprecated in
percpu tree, so that new usages in linux-next can be detected.
* Towards the end of the next merge window, merge a patch to actually
kill the old interface.
Thanks.
--
tejun
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH 3/3] Code clean up for percpu_xxx() functions
2012-01-11 17:19 ` tj
@ 2012-01-12 0:44 ` H. Peter Anvin
2012-01-12 2:05 ` Alex,Shi
0 siblings, 1 reply; 42+ messages in thread
From: H. Peter Anvin @ 2012-01-12 0:44 UTC (permalink / raw)
To: tj@kernel.org
Cc: Alex,Shi, Christoph Lameter, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying, Thomas Gleixner,
mingo@redhat.com, avi@redhat.com, akpm@linux-foundation.org,
David Miller, kaber@trash.net, a.p.zijlstra@chello.nl,
kvm@vger.kernel.org, Andi Kleen
On 01/11/2012 09:19 AM, tj@kernel.org wrote:
>
> Alex, can you please collect all patches into a single patchset?
> Please split it such that, usage changes are per-system so that they
> can be routed through respective subsystems (x86 or net) and updates
> to percpu proper which can be applied after other changes have been
> applied. It would probably be best to route these patches separately
> rather than all through percpu as it touches a lot of different places
> and is likely to cause conflicts. I *think* the best way would be,
>
> * Submit per-subsystem patches and get them merged to subsystem trees.
>
> * (Optional) Apply a patch to mark unused interface deprecated in
> percpu tree, so that new usages in linux-next can be detected.
>
> * Towards the end of the next merge window, merge a patch to actually
> kill the old interface.
>
That sounds like a good idea.
-hpa
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [PATCH 3/3] Code clean up for percpu_xxx() functions
2012-01-12 0:44 ` H. Peter Anvin
@ 2012-01-12 2:05 ` Alex,Shi
0 siblings, 0 replies; 42+ messages in thread
From: Alex,Shi @ 2012-01-12 2:05 UTC (permalink / raw)
To: H. Peter Anvin
Cc: tj@kernel.org, Christoph Lameter, linux-kernel@vger.kernel.org,
eric.dumazet@gmail.com, Huang, Ying, Thomas Gleixner,
mingo@redhat.com, avi@redhat.com, akpm@linux-foundation.org,
David Miller, kaber@trash.net, a.p.zijlstra@chello.nl,
kvm@vger.kernel.org, Andi Kleen
On Wed, 2012-01-11 at 16:44 -0800, H. Peter Anvin wrote:
> On 01/11/2012 09:19 AM, tj@kernel.org wrote:
> >
> > Alex, can you please collect all patches into a single patchset?
> > Please split it such that, usage changes are per-system so that they
> > can be routed through respective subsystems (x86 or net) and updates
> > to percpu proper which can be applied after other changes have been
> > applied. It would probably be best to route these patches separately
> > rather than all through percpu as it touches a lot of different places
> > and is likely to cause conflicts. I *think* the best way would be,
> >
> > * Submit per-subsystem patches and get them merged to subsystem trees.
> >
> > * (Optional) Apply a patch to mark unused interface deprecated in
> > percpu tree, so that new usages in linux-next can be detected.
> >
> > * Towards the end of the next merge window, merge a patch to actually
> > kill the old interface.
> >
>
> That sounds like a good idea.
I will try to do so. Many thanks for the advices!
>
> -hpa
^ permalink raw reply [flat|nested] 42+ messages in thread
end of thread, other threads:[~2012-01-12 2:06 UTC | newest]
Thread overview: 42+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2011-10-10 9:10 [PATCH] Code clean up for percpu_xxx() functions Alex,Shi
2011-10-10 17:08 ` Christoph Lameter
2011-10-11 9:19 ` Alex,Shi
2011-10-11 14:13 ` Christoph Lameter
2011-10-12 1:29 ` Alex,Shi
2011-10-12 13:59 ` Christoph Lameter
2011-10-12 14:11 ` Alex,Shi
2011-10-12 14:11 ` Christoph Lameter
2011-10-18 3:21 ` Alex,Shi
2011-10-18 14:33 ` Christoph Lameter
2011-10-19 9:23 ` Alex,Shi
2011-10-19 9:26 ` Avi Kivity
2011-10-20 2:45 ` Alex,Shi
2011-10-20 7:34 ` [PATCH 2/3] kvm: use this_cpu_xxx replace percpu_xxx funcs Alex,Shi
2011-10-24 2:50 ` Alex,Shi
2011-10-24 11:05 ` Avi Kivity
2011-11-21 9:02 ` Alex,Shi
2011-11-21 9:20 ` Avi Kivity
2011-11-21 9:52 ` Alex,Shi
2012-01-11 8:48 ` Alex,Shi
2011-10-19 15:34 ` [PATCH] Code clean up for percpu_xxx() functions Christoph Lameter
2011-10-20 2:44 ` Alex,Shi
2011-10-20 5:08 ` Eric Dumazet
2011-10-20 6:04 ` Alex,Shi
2011-10-20 6:13 ` Alex,Shi
2011-10-20 6:04 ` Alex,Shi
2011-10-20 7:20 ` [PATCH 0/3] code clean and optimzed for percpu_xxx serials funcs Alex,Shi
2011-10-20 7:32 ` [PATCH 1/3] net: use this_cpu_xxx replace percpu_xxx funcs Alex,Shi
2011-10-20 8:38 ` Eric Dumazet
2011-11-21 9:00 ` Alex,Shi
2011-12-02 2:33 ` Alex,Shi
2011-12-02 11:56 ` Patrick McHardy
2012-01-11 8:45 ` Alex,Shi
2012-01-11 9:03 ` David Miller
2012-01-11 9:10 ` Alex,Shi
2011-10-20 7:35 ` [PATCH 3/3] Code clean up for percpu_xxx() functions Alex,Shi
2011-11-21 9:10 ` Alex,Shi
2011-11-22 0:06 ` tj
2012-01-11 9:08 ` Alex,Shi
2012-01-11 17:19 ` tj
2012-01-12 0:44 ` H. Peter Anvin
2012-01-12 2:05 ` Alex,Shi
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).