* [PATCH 12/45] sched/fair: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:40 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, zhong, linux-pm, Peter Zijlstra, fweisbec,
linux-kernel, rostedt, xiaoguangrong, sbw, Ingo Molnar, wangyun,
srivatsa.bhat, netdev, linuxppc-dev
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
kernel/sched/fair.c | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index c0ac2c3..88f056e 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -3338,7 +3338,8 @@ done:
*
* Returns the target CPU number, or the same CPU if no balancing is needed.
*
- * preempt must be disabled.
+ * Must be called within get/put_online_cpus_atomic(), to prevent CPUs
+ * from going offline from under us.
*/
static int
select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags)
@@ -5267,6 +5268,8 @@ void idle_balance(int this_cpu, struct rq *this_rq)
raw_spin_unlock(&this_rq->lock);
update_blocked_averages(this_cpu);
+
+ get_online_cpus_atomic();
rcu_read_lock();
for_each_domain(this_cpu, sd) {
unsigned long interval;
@@ -5290,6 +5293,7 @@ void idle_balance(int this_cpu, struct rq *this_rq)
}
}
rcu_read_unlock();
+ put_online_cpus_atomic();
raw_spin_lock(&this_rq->lock);
@@ -5316,6 +5320,7 @@ static int active_load_balance_cpu_stop(void *data)
struct rq *target_rq = cpu_rq(target_cpu);
struct sched_domain *sd;
+ get_online_cpus_atomic();
raw_spin_lock_irq(&busiest_rq->lock);
/* make sure the requested cpu hasn't gone down in the meantime */
@@ -5367,6 +5372,7 @@ static int active_load_balance_cpu_stop(void *data)
out_unlock:
busiest_rq->active_balance = 0;
raw_spin_unlock_irq(&busiest_rq->lock);
+ put_online_cpus_atomic();
return 0;
}
@@ -5527,6 +5533,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
update_blocked_averages(cpu);
+ get_online_cpus_atomic();
rcu_read_lock();
for_each_domain(cpu, sd) {
if (!(sd->flags & SD_LOAD_BALANCE))
@@ -5575,6 +5582,7 @@ out:
break;
}
rcu_read_unlock();
+ put_online_cpus_atomic();
/*
* next_balance will be updated only when there is a need.
@@ -5706,6 +5714,7 @@ static void run_rebalance_domains(struct softirq_action *h)
enum cpu_idle_type idle = this_rq->idle_balance ?
CPU_IDLE : CPU_NOT_IDLE;
+ get_online_cpus_atomic();
rebalance_domains(this_cpu, idle);
/*
@@ -5714,6 +5723,7 @@ static void run_rebalance_domains(struct softirq_action *h)
* stopped.
*/
nohz_idle_balance(this_cpu, idle);
+ put_online_cpus_atomic();
}
static inline int on_null_domain(int cpu)
@@ -5731,8 +5741,10 @@ void trigger_load_balance(struct rq *rq, int cpu)
likely(!on_null_domain(cpu)))
raise_softirq(SCHED_SOFTIRQ);
#ifdef CONFIG_NO_HZ_COMMON
+ get_online_cpus_atomic();
if (nohz_kick_needed(rq, cpu) && likely(!on_null_domain(cpu)))
nohz_balancer_kick(cpu);
+ put_online_cpus_atomic();
#endif
}
^ permalink raw reply related
* [PATCH 13/45] timer: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:40 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, zhong, linux-pm, fweisbec, linux-kernel,
rostedt, xiaoguangrong, sbw, wangyun, srivatsa.bhat, netdev,
Thomas Gleixner, linuxppc-dev
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
kernel/timer.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/kernel/timer.c b/kernel/timer.c
index 15ffdb3..5db594c 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -729,6 +729,7 @@ __mod_timer(struct timer_list *timer, unsigned long expires,
timer_stats_timer_set_start_info(timer);
BUG_ON(!timer->function);
+ get_online_cpus_atomic();
base = lock_timer_base(timer, &flags);
ret = detach_if_pending(timer, base, false);
@@ -768,6 +769,7 @@ __mod_timer(struct timer_list *timer, unsigned long expires,
out_unlock:
spin_unlock_irqrestore(&base->lock, flags);
+ put_online_cpus_atomic();
return ret;
}
@@ -926,6 +928,7 @@ void add_timer_on(struct timer_list *timer, int cpu)
timer_stats_timer_set_start_info(timer);
BUG_ON(timer_pending(timer) || !timer->function);
+ get_online_cpus_atomic();
spin_lock_irqsave(&base->lock, flags);
timer_set_base(timer, base);
debug_activate(timer, timer->expires);
@@ -940,6 +943,7 @@ void add_timer_on(struct timer_list *timer, int cpu)
*/
wake_up_nohz_cpu(cpu);
spin_unlock_irqrestore(&base->lock, flags);
+ put_online_cpus_atomic();
}
EXPORT_SYMBOL_GPL(add_timer_on);
^ permalink raw reply related
* [PATCH 14/45] sched/rt: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:41 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, zhong, linux-pm, Peter Zijlstra, fweisbec,
linux-kernel, rostedt, xiaoguangrong, sbw, Ingo Molnar, wangyun,
srivatsa.bhat, netdev, linuxppc-dev
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
kernel/sched/rt.c | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 01970c8..03d9f38 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -6,6 +6,7 @@
#include "sched.h"
#include <linux/slab.h>
+#include <linux/cpu.h>
int sched_rr_timeslice = RR_TIMESLICE;
@@ -28,7 +29,9 @@ static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
if (!overrun)
break;
+ get_online_cpus_atomic();
idle = do_sched_rt_period_timer(rt_b, overrun);
+ put_online_cpus_atomic();
}
return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
@@ -547,6 +550,7 @@ static int do_balance_runtime(struct rt_rq *rt_rq)
int i, weight, more = 0;
u64 rt_period;
+ get_online_cpus_atomic();
weight = cpumask_weight(rd->span);
raw_spin_lock(&rt_b->rt_runtime_lock);
@@ -588,6 +592,7 @@ next:
raw_spin_unlock(&iter->rt_runtime_lock);
}
raw_spin_unlock(&rt_b->rt_runtime_lock);
+ put_online_cpus_atomic();
return more;
}
@@ -1168,6 +1173,10 @@ static void yield_task_rt(struct rq *rq)
#ifdef CONFIG_SMP
static int find_lowest_rq(struct task_struct *task);
+/*
+ * Must be called within get/put_online_cpus_atomic(), to prevent CPUs
+ * from going offline from under us.
+ */
static int
select_task_rq_rt(struct task_struct *p, int sd_flag, int flags)
{
@@ -1561,6 +1570,8 @@ retry:
return 0;
}
+ get_online_cpus_atomic();
+
/* We might release rq lock */
get_task_struct(next_task);
@@ -1611,6 +1622,7 @@ retry:
out:
put_task_struct(next_task);
+ put_online_cpus_atomic();
return ret;
}
@@ -1630,6 +1642,7 @@ static int pull_rt_task(struct rq *this_rq)
if (likely(!rt_overloaded(this_rq)))
return 0;
+ get_online_cpus_atomic();
for_each_cpu(cpu, this_rq->rd->rto_mask) {
if (this_cpu == cpu)
continue;
@@ -1695,6 +1708,7 @@ skip:
double_unlock_balance(this_rq, src_rq);
}
+ put_online_cpus_atomic();
return ret;
}
^ permalink raw reply related
* [PATCH 15/45] rcu: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:41 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, zhong, linux-pm, fweisbec, linux-kernel,
rostedt, xiaoguangrong, sbw, wangyun, srivatsa.bhat, netdev,
Paul E. McKenney, linuxppc-dev
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
In RCU code, rcu_implicit_dynticks_qs() checks if a CPU is offline,
while being protected by a spinlock. Use the get/put_online_cpus_atomic()
APIs to prevent CPUs from going offline, while invoking from atomic context.
Cc: Dipankar Sarma <dipankar@in.ibm.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
kernel/rcutree.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index cf3adc6..caeed1a 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -2107,6 +2107,8 @@ static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *))
rcu_initiate_boost(rnp, flags); /* releases rnp->lock */
continue;
}
+
+ get_online_cpus_atomic();
cpu = rnp->grplo;
bit = 1;
for (; cpu <= rnp->grphi; cpu++, bit <<= 1) {
@@ -2114,6 +2116,8 @@ static void force_qs_rnp(struct rcu_state *rsp, int (*f)(struct rcu_data *))
f(per_cpu_ptr(rsp->rda, cpu)))
mask |= bit;
}
+ put_online_cpus_atomic();
+
if (mask != 0) {
/* rcu_report_qs_rnp() releases rnp->lock. */
^ permalink raw reply related
* [PATCH 16/45] tick-broadcast: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:41 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, zhong, linux-pm, fweisbec, linux-kernel,
rostedt, xiaoguangrong, sbw, wangyun, srivatsa.bhat, netdev,
Thomas Gleixner, linuxppc-dev
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
kernel/time/tick-broadcast.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index d66f554..53493a6 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -227,12 +227,14 @@ static void tick_do_broadcast(struct cpumask *mask)
*/
static void tick_do_periodic_broadcast(void)
{
+ get_online_cpus_atomic();
raw_spin_lock(&tick_broadcast_lock);
cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask);
tick_do_broadcast(tmpmask);
raw_spin_unlock(&tick_broadcast_lock);
+ put_online_cpus_atomic();
}
/*
@@ -335,11 +337,13 @@ out:
*/
void tick_broadcast_on_off(unsigned long reason, int *oncpu)
{
+ get_online_cpus_atomic();
if (!cpumask_test_cpu(*oncpu, cpu_online_mask))
printk(KERN_ERR "tick-broadcast: ignoring broadcast for "
"offline CPU #%d\n", *oncpu);
else
tick_do_broadcast_on_off(&reason);
+ put_online_cpus_atomic();
}
/*
@@ -505,6 +509,7 @@ static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
ktime_t now, next_event;
int cpu, next_cpu = 0;
+ get_online_cpus_atomic();
raw_spin_lock(&tick_broadcast_lock);
again:
dev->next_event.tv64 = KTIME_MAX;
@@ -562,6 +567,7 @@ again:
goto again;
}
raw_spin_unlock(&tick_broadcast_lock);
+ put_online_cpus_atomic();
}
/*
@@ -753,6 +759,7 @@ void tick_broadcast_switch_to_oneshot(void)
struct clock_event_device *bc;
unsigned long flags;
+ get_online_cpus_atomic();
raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
@@ -761,6 +768,7 @@ void tick_broadcast_switch_to_oneshot(void)
tick_broadcast_setup_oneshot(bc);
raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
+ put_online_cpus_atomic();
}
^ permalink raw reply related
* [PATCH 17/45] time/clocksource: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:41 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, zhong, linux-pm, fweisbec, linux-kernel,
rostedt, xiaoguangrong, sbw, John Stultz, wangyun, srivatsa.bhat,
netdev, Thomas Gleixner, linuxppc-dev
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: John Stultz <john.stultz@linaro.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
kernel/time/clocksource.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index e713ef7..c4bbc25 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -30,6 +30,7 @@
#include <linux/sched.h> /* for spin_unlock_irq() using preempt_count() m68k */
#include <linux/tick.h>
#include <linux/kthread.h>
+#include <linux/cpu.h>
#include "tick-internal.h"
@@ -252,6 +253,7 @@ static void clocksource_watchdog(unsigned long data)
int64_t wd_nsec, cs_nsec;
int next_cpu, reset_pending;
+ get_online_cpus_atomic();
spin_lock(&watchdog_lock);
if (!watchdog_running)
goto out;
@@ -329,6 +331,7 @@ static void clocksource_watchdog(unsigned long data)
add_timer_on(&watchdog_timer, next_cpu);
out:
spin_unlock(&watchdog_lock);
+ put_online_cpus_atomic();
}
static inline void clocksource_start_watchdog(void)
@@ -367,6 +370,7 @@ static void clocksource_enqueue_watchdog(struct clocksource *cs)
{
unsigned long flags;
+ get_online_cpus_atomic();
spin_lock_irqsave(&watchdog_lock, flags);
if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) {
/* cs is a clocksource to be watched. */
@@ -386,6 +390,7 @@ static void clocksource_enqueue_watchdog(struct clocksource *cs)
/* Check if the watchdog timer needs to be started. */
clocksource_start_watchdog();
spin_unlock_irqrestore(&watchdog_lock, flags);
+ put_online_cpus_atomic();
}
static void clocksource_dequeue_watchdog(struct clocksource *cs)
^ permalink raw reply related
* [PATCH 18/45] softirq: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:42 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, zhong, linux-pm, Frederic Weisbecker,
linux-kernel, rostedt, xiaoguangrong, sbw, wangyun, srivatsa.bhat,
netdev, Sedat Dilek, Thomas Gleixner, Paul E. McKenney,
linuxppc-dev, Andrew Morton
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Sedat Dilek <sedat.dilek@gmail.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
kernel/softirq.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 3d6833f..c289722 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -644,14 +644,17 @@ static void remote_softirq_receive(void *data)
static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
{
+ get_online_cpus_atomic();
if (cpu_online(cpu)) {
cp->func = remote_softirq_receive;
cp->info = &softirq;
cp->flags = 0;
__smp_call_function_single(cpu, cp, 0);
+ put_online_cpus_atomic();
return 0;
}
+ put_online_cpus_atomic();
return 1;
}
#else /* CONFIG_USE_GENERIC_SMP_HELPERS */
^ permalink raw reply related
* [PATCH 19/45] irq: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:42 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, zhong, linux-pm, fweisbec, linux-kernel,
rostedt, xiaoguangrong, sbw, wangyun, srivatsa.bhat, netdev,
Thomas Gleixner, linuxppc-dev
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
kernel/irq/manage.c | 7 +++++++
kernel/irq/proc.c | 3 +++
2 files changed, 10 insertions(+)
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index e16caa8..4d89f19 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -18,6 +18,7 @@
#include <linux/sched.h>
#include <linux/sched/rt.h>
#include <linux/task_work.h>
+#include <linux/cpu.h>
#include "internals.h"
@@ -202,9 +203,11 @@ int irq_set_affinity(unsigned int irq, const struct cpumask *mask)
if (!desc)
return -EINVAL;
+ get_online_cpus_atomic();
raw_spin_lock_irqsave(&desc->lock, flags);
ret = __irq_set_affinity_locked(irq_desc_get_irq_data(desc), mask);
raw_spin_unlock_irqrestore(&desc->lock, flags);
+ put_online_cpus_atomic();
return ret;
}
@@ -343,9 +346,11 @@ int irq_select_affinity_usr(unsigned int irq, struct cpumask *mask)
unsigned long flags;
int ret;
+ get_online_cpus_atomic();
raw_spin_lock_irqsave(&desc->lock, flags);
ret = setup_affinity(irq, desc, mask);
raw_spin_unlock_irqrestore(&desc->lock, flags);
+ put_online_cpus_atomic();
return ret;
}
@@ -1128,7 +1133,9 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
}
/* Set default affinity mask once everything is setup */
+ get_online_cpus_atomic();
setup_affinity(irq, desc, mask);
+ put_online_cpus_atomic();
} else if (new->flags & IRQF_TRIGGER_MASK) {
unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK;
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c
index 19ed5c4..47f9a74 100644
--- a/kernel/irq/proc.c
+++ b/kernel/irq/proc.c
@@ -7,6 +7,7 @@
*/
#include <linux/irq.h>
+#include <linux/cpu.h>
#include <linux/gfp.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
@@ -441,6 +442,7 @@ int show_interrupts(struct seq_file *p, void *v)
if (!desc)
return 0;
+ get_online_cpus_atomic();
raw_spin_lock_irqsave(&desc->lock, flags);
for_each_online_cpu(j)
any_count |= kstat_irqs_cpu(i, j);
@@ -477,6 +479,7 @@ int show_interrupts(struct seq_file *p, void *v)
seq_putc(p, '\n');
out:
raw_spin_unlock_irqrestore(&desc->lock, flags);
+ put_online_cpus_atomic();
return 0;
}
#endif
^ permalink raw reply related
* [PATCH 20/45] net: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:42 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, Alexander Duyck, nikunj, zhong, linux-pm, fweisbec,
linux-kernel, rostedt, xiaoguangrong, sbw, Eric Dumazet, wangyun,
srivatsa.bhat, netdev, Ben Hutchings, Cong Wang, linuxppc-dev,
David S. Miller
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Alexander Duyck <alexander.h.duyck@intel.com>
Cc: Cong Wang <xiyou.wangcong@gmail.com>
Cc: Ben Hutchings <bhutchings@solarflare.com>
Cc: netdev@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
net/core/dev.c | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/net/core/dev.c b/net/core/dev.c
index fc1e289..90519e9 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3141,7 +3141,7 @@ int netif_rx(struct sk_buff *skb)
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu;
- preempt_disable();
+ get_online_cpus_atomic();
rcu_read_lock();
cpu = get_rps_cpu(skb->dev, skb, &rflow);
@@ -3151,7 +3151,7 @@ int netif_rx(struct sk_buff *skb)
ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
rcu_read_unlock();
- preempt_enable();
+ put_online_cpus_atomic();
} else
#endif
{
@@ -3570,6 +3570,7 @@ int netif_receive_skb(struct sk_buff *skb)
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu, ret;
+ get_online_cpus_atomic();
rcu_read_lock();
cpu = get_rps_cpu(skb->dev, skb, &rflow);
@@ -3577,9 +3578,11 @@ int netif_receive_skb(struct sk_buff *skb)
if (cpu >= 0) {
ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
rcu_read_unlock();
+ put_online_cpus_atomic();
return ret;
}
rcu_read_unlock();
+ put_online_cpus_atomic();
}
#endif
return __netif_receive_skb(skb);
@@ -3957,6 +3960,7 @@ static void net_rps_action_and_irq_enable(struct softnet_data *sd)
local_irq_enable();
/* Send pending IPI's to kick RPS processing on remote cpus. */
+ get_online_cpus_atomic();
while (remsd) {
struct softnet_data *next = remsd->rps_ipi_next;
@@ -3965,6 +3969,7 @@ static void net_rps_action_and_irq_enable(struct softnet_data *sd)
&remsd->csd, 0);
remsd = next;
}
+ put_online_cpus_atomic();
} else
#endif
local_irq_enable();
^ permalink raw reply related
* [PATCH 21/45] block: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:42 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, Jens Axboe, nikunj, zhong, linux-pm, fweisbec,
linux-kernel, rostedt, xiaoguangrong, sbw, wangyun, srivatsa.bhat,
netdev, linuxppc-dev
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
block/blk-softirq.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/block/blk-softirq.c b/block/blk-softirq.c
index 467c8de..bbab3d3 100644
--- a/block/blk-softirq.c
+++ b/block/blk-softirq.c
@@ -58,6 +58,7 @@ static void trigger_softirq(void *data)
*/
static int raise_blk_irq(int cpu, struct request *rq)
{
+ get_online_cpus_atomic();
if (cpu_online(cpu)) {
struct call_single_data *data = &rq->csd;
@@ -66,8 +67,10 @@ static int raise_blk_irq(int cpu, struct request *rq)
data->flags = 0;
__smp_call_function_single(cpu, data, 0);
+ put_online_cpus_atomic();
return 0;
}
+ put_online_cpus_atomic();
return 1;
}
^ permalink raw reply related
* [PATCH 22/45] percpu_counter: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:42 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, zhong, linux-pm, fweisbec, linux-kernel,
rostedt, xiaoguangrong, sbw, wangyun, srivatsa.bhat, netdev,
linuxppc-dev, Al Viro
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Al Viro <viro@zeniv.linux.org.uk>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
lib/percpu_counter.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c
index ba6085d..9cf9086 100644
--- a/lib/percpu_counter.c
+++ b/lib/percpu_counter.c
@@ -98,6 +98,7 @@ s64 __percpu_counter_sum(struct percpu_counter *fbc)
s64 ret;
int cpu;
+ get_online_cpus_atomic();
raw_spin_lock(&fbc->lock);
ret = fbc->count;
for_each_online_cpu(cpu) {
@@ -105,6 +106,7 @@ s64 __percpu_counter_sum(struct percpu_counter *fbc)
ret += *pcount;
}
raw_spin_unlock(&fbc->lock);
+ put_online_cpus_atomic();
return ret;
}
EXPORT_SYMBOL(__percpu_counter_sum);
^ permalink raw reply related
* [PATCH 23/45] infiniband: ehca: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:43 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, Roland Dreier, nikunj, zhong, linux-pm, linux-rdma,
fweisbec, linux-kernel, rostedt, xiaoguangrong, sbw,
Christoph Raisch, Hoang-Nam Nguyen, wangyun, srivatsa.bhat,
netdev, Sean Hefty, linuxppc-dev, Hal Rosenstock
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Hoang-Nam Nguyen <hnguyen@de.ibm.com>
Cc: Christoph Raisch <raisch@de.ibm.com>
Cc: Roland Dreier <roland@kernel.org>
Cc: Sean Hefty <sean.hefty@intel.com>
Cc: Hal Rosenstock <hal.rosenstock@gmail.com>
Cc: linux-rdma@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
drivers/infiniband/hw/ehca/ehca_irq.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/drivers/infiniband/hw/ehca/ehca_irq.c b/drivers/infiniband/hw/ehca/ehca_irq.c
index 8615d7c..ace901e 100644
--- a/drivers/infiniband/hw/ehca/ehca_irq.c
+++ b/drivers/infiniband/hw/ehca/ehca_irq.c
@@ -43,6 +43,7 @@
#include <linux/slab.h>
#include <linux/smpboot.h>
+#include <linux/cpu.h>
#include "ehca_classes.h"
#include "ehca_irq.h"
@@ -703,6 +704,7 @@ static void queue_comp_task(struct ehca_cq *__cq)
int cq_jobs;
unsigned long flags;
+ get_online_cpus_atomic();
cpu_id = find_next_online_cpu(pool);
BUG_ON(!cpu_online(cpu_id));
@@ -720,6 +722,7 @@ static void queue_comp_task(struct ehca_cq *__cq)
BUG_ON(!cct || !thread);
}
__queue_comp_task(__cq, cct, thread);
+ put_online_cpus_atomic();
}
static void run_comp_task(struct ehca_cpu_comp_task *cct)
@@ -759,6 +762,7 @@ static void comp_task_park(unsigned int cpu)
list_splice_init(&cct->cq_list, &list);
spin_unlock_irq(&cct->task_lock);
+ get_online_cpus_atomic();
cpu = find_next_online_cpu(pool);
target = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu);
@@ -768,6 +772,7 @@ static void comp_task_park(unsigned int cpu)
__queue_comp_task(cq, target, thread);
}
spin_unlock_irq(&target->task_lock);
+ put_online_cpus_atomic();
}
static void comp_task_stop(unsigned int cpu, bool online)
^ permalink raw reply related
* [PATCH 24/45] [SCSI] fcoe: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:43 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, James E.J. Bottomley, nikunj, zhong, linux-pm,
fweisbec, linux-kernel, rostedt, xiaoguangrong, linux-scsi, sbw,
wangyun, srivatsa.bhat, netdev, Robert Love, linuxppc-dev, devel
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Robert Love <robert.w.love@intel.com>
Cc: "James E.J. Bottomley" <JBottomley@parallels.com>
Cc: devel@open-fcoe.org
Cc: linux-scsi@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
drivers/scsi/fcoe/fcoe.c | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/drivers/scsi/fcoe/fcoe.c b/drivers/scsi/fcoe/fcoe.c
index 292b24f..a107d3c 100644
--- a/drivers/scsi/fcoe/fcoe.c
+++ b/drivers/scsi/fcoe/fcoe.c
@@ -1484,6 +1484,7 @@ static int fcoe_rcv(struct sk_buff *skb, struct net_device *netdev,
* was originated, otherwise select cpu using rx exchange id
* or fcoe_select_cpu().
*/
+ get_online_cpus_atomic();
if (ntoh24(fh->fh_f_ctl) & FC_FC_EX_CTX)
cpu = ntohs(fh->fh_ox_id) & fc_cpu_mask;
else {
@@ -1493,8 +1494,10 @@ static int fcoe_rcv(struct sk_buff *skb, struct net_device *netdev,
cpu = ntohs(fh->fh_rx_id) & fc_cpu_mask;
}
- if (cpu >= nr_cpu_ids)
+ if (cpu >= nr_cpu_ids) {
+ put_online_cpus_atomic();
goto err;
+ }
fps = &per_cpu(fcoe_percpu, cpu);
spin_lock(&fps->fcoe_rx_list.lock);
@@ -1514,6 +1517,7 @@ static int fcoe_rcv(struct sk_buff *skb, struct net_device *netdev,
spin_lock(&fps->fcoe_rx_list.lock);
if (!fps->thread) {
spin_unlock(&fps->fcoe_rx_list.lock);
+ put_online_cpus_atomic();
goto err;
}
}
@@ -1535,6 +1539,7 @@ static int fcoe_rcv(struct sk_buff *skb, struct net_device *netdev,
if (fps->thread->state == TASK_INTERRUPTIBLE)
wake_up_process(fps->thread);
spin_unlock(&fps->fcoe_rx_list.lock);
+ put_online_cpus_atomic();
return 0;
err:
^ permalink raw reply related
* [PATCH 25/45] staging/octeon: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:43 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, devel, nikunj, zhong, linux-pm, fweisbec,
linux-kernel, rostedt, xiaoguangrong, sbw, Greg Kroah-Hartman,
wangyun, srivatsa.bhat, netdev, linuxppc-dev
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: devel@driverdev.osuosl.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
drivers/staging/octeon/ethernet-rx.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/drivers/staging/octeon/ethernet-rx.c b/drivers/staging/octeon/ethernet-rx.c
index 34afc16..8588b4d 100644
--- a/drivers/staging/octeon/ethernet-rx.c
+++ b/drivers/staging/octeon/ethernet-rx.c
@@ -36,6 +36,7 @@
#include <linux/prefetch.h>
#include <linux/ratelimit.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/interrupt.h>
#include <net/dst.h>
#ifdef CONFIG_XFRM
@@ -97,6 +98,7 @@ static void cvm_oct_enable_one_cpu(void)
return;
/* ... if a CPU is available, Turn on NAPI polling for that CPU. */
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
if (!cpu_test_and_set(cpu, core_state.cpu_state)) {
v = smp_call_function_single(cpu, cvm_oct_enable_napi,
@@ -106,6 +108,7 @@ static void cvm_oct_enable_one_cpu(void)
break;
}
}
+ put_online_cpus_atomic();
}
static void cvm_oct_no_more_work(void)
^ permalink raw reply related
* [PATCH 26/45] x86: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:43 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: fweisbec, Sebastian Andrzej Siewior, Jan Beulich, H. Peter Anvin,
linux-arch, Joonsoo Kim, Joerg Roedel, x86, xiaoguangrong,
Ingo Molnar, wangyun, nikunj, linux-pm, rostedt,
Konrad Rzeszutek Wilk, Borislav Petkov, Thomas Gleixner,
linux-edac, Tony Luck, zhong, netdev, linux-kernel, sbw,
srivatsa.bhat, linuxppc-dev
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Joerg Roedel <joro@8bytes.org>
Cc: Jan Beulich <jbeulich@suse.com>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: linux-edac@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/x86/kernel/apic/io_apic.c | 21 ++++++++++++++++++---
arch/x86/kernel/cpu/mcheck/therm_throt.c | 4 ++--
arch/x86/mm/tlb.c | 14 +++++++-------
3 files changed, 27 insertions(+), 12 deletions(-)
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 9ed796c..4c71c1e 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -25,6 +25,7 @@
#include <linux/init.h>
#include <linux/delay.h>
#include <linux/sched.h>
+#include <linux/cpu.h>
#include <linux/pci.h>
#include <linux/mc146818rtc.h>
#include <linux/compiler.h>
@@ -1169,9 +1170,11 @@ int assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
int err;
unsigned long flags;
+ get_online_cpus_atomic();
raw_spin_lock_irqsave(&vector_lock, flags);
err = __assign_irq_vector(irq, cfg, mask);
raw_spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
return err;
}
@@ -1757,13 +1760,13 @@ __apicdebuginit(void) print_local_APICs(int maxcpu)
if (!maxcpu)
return;
- preempt_disable();
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
if (cpu >= maxcpu)
break;
smp_call_function_single(cpu, print_local_APIC, NULL, 1);
}
- preempt_enable();
+ put_online_cpus_atomic();
}
__apicdebuginit(void) print_PIC(void)
@@ -2153,10 +2156,12 @@ static int ioapic_retrigger_irq(struct irq_data *data)
unsigned long flags;
int cpu;
+ get_online_cpus_atomic();
raw_spin_lock_irqsave(&vector_lock, flags);
cpu = cpumask_first_and(cfg->domain, cpu_online_mask);
apic->send_IPI_mask(cpumask_of(cpu), cfg->vector);
raw_spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
return 1;
}
@@ -2175,6 +2180,7 @@ void send_cleanup_vector(struct irq_cfg *cfg)
{
cpumask_var_t cleanup_mask;
+ get_online_cpus_atomic();
if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
unsigned int i;
for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
@@ -2185,6 +2191,7 @@ void send_cleanup_vector(struct irq_cfg *cfg)
free_cpumask_var(cleanup_mask);
}
cfg->move_in_progress = 0;
+ put_online_cpus_atomic();
}
asmlinkage void smp_irq_move_cleanup_interrupt(void)
@@ -2939,11 +2946,13 @@ unsigned int __create_irqs(unsigned int from, unsigned int count, int node)
goto out_irqs;
}
+ get_online_cpus_atomic();
raw_spin_lock_irqsave(&vector_lock, flags);
for (i = 0; i < count; i++)
if (__assign_irq_vector(irq + i, cfg[i], apic->target_cpus()))
goto out_vecs;
raw_spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
for (i = 0; i < count; i++) {
irq_set_chip_data(irq + i, cfg[i]);
@@ -2957,6 +2966,7 @@ out_vecs:
for (i--; i >= 0; i--)
__clear_irq_vector(irq + i, cfg[i]);
raw_spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
out_irqs:
for (i = 0; i < count; i++)
free_irq_at(irq + i, cfg[i]);
@@ -2994,9 +3004,11 @@ void destroy_irq(unsigned int irq)
free_remapped_irq(irq);
+ get_online_cpus_atomic();
raw_spin_lock_irqsave(&vector_lock, flags);
__clear_irq_vector(irq, cfg);
raw_spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
free_irq_at(irq, cfg);
}
@@ -3365,8 +3377,11 @@ io_apic_setup_irq_pin(unsigned int irq, int node, struct io_apic_irq_attr *attr)
if (!cfg)
return -EINVAL;
ret = __add_pin_to_irq_node(cfg, node, attr->ioapic, attr->ioapic_pin);
- if (!ret)
+ if (!ret) {
+ get_online_cpus_atomic();
setup_ioapic_irq(irq, cfg, attr);
+ put_online_cpus_atomic();
+ }
return ret;
}
diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c
index 47a1870..d128ba4 100644
--- a/arch/x86/kernel/cpu/mcheck/therm_throt.c
+++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c
@@ -82,13 +82,13 @@ static ssize_t therm_throt_device_show_##event##_##name( \
unsigned int cpu = dev->id; \
ssize_t ret; \
\
- preempt_disable(); /* CPU hotplug */ \
+ get_online_cpus_atomic(); /* CPU hotplug */ \
if (cpu_online(cpu)) { \
ret = sprintf(buf, "%lu\n", \
per_cpu(thermal_state, cpu).event.name); \
} else \
ret = 0; \
- preempt_enable(); \
+ put_online_cpus_atomic(); \
\
return ret; \
}
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 282375f..8126374 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -147,12 +147,12 @@ void flush_tlb_current_task(void)
{
struct mm_struct *mm = current->mm;
- preempt_disable();
+ get_online_cpus_atomic();
local_flush_tlb();
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
- preempt_enable();
+ put_online_cpus_atomic();
}
/*
@@ -187,7 +187,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
unsigned long addr;
unsigned act_entries, tlb_entries = 0;
- preempt_disable();
+ get_online_cpus_atomic();
if (current->active_mm != mm)
goto flush_all;
@@ -225,21 +225,21 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
if (cpumask_any_but(mm_cpumask(mm),
smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, start, end);
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
flush_all:
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
- preempt_enable();
+ put_online_cpus_atomic();
}
void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
{
struct mm_struct *mm = vma->vm_mm;
- preempt_disable();
+ get_online_cpus_atomic();
if (current->active_mm == mm) {
if (current->mm)
@@ -251,7 +251,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, start, 0UL);
- preempt_enable();
+ put_online_cpus_atomic();
}
static void do_flush_tlb_all(void *info)
^ permalink raw reply related
* [PATCH 27/45] perf/x86: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:43 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, x86, nikunj, zhong, linux-pm, fweisbec,
H. Peter Anvin, linux-kernel, rostedt, xiaoguangrong, sbw,
Ingo Molnar, Paul Mackerras, wangyun, srivatsa.bhat, netdev,
Thomas Gleixner, linuxppc-dev, Arnaldo Carvalho de Melo,
Peter Zijlstra
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
The CPU_DYING notifier modifies the per-cpu pointer pmu->box, and this can
race with functions such as uncore_pmu_to_box() and uncore_pci_remove() when
we remove stop_machine() from the CPU offline path. So protect them using
get/put_online_cpus_atomic().
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Arnaldo Carvalho de Melo <acme@ghostprotocols.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/x86/kernel/cpu/perf_event_intel_uncore.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/arch/x86/kernel/cpu/perf_event_intel_uncore.c b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
index 9dd9975..7c2a064 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_uncore.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
@@ -1,3 +1,4 @@
+#include <linux/cpu.h>
#include "perf_event_intel_uncore.h"
static struct intel_uncore_type *empty_uncore[] = { NULL, };
@@ -2630,6 +2631,7 @@ uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
if (box)
return box;
+ get_online_cpus_atomic();
raw_spin_lock(&uncore_box_lock);
list_for_each_entry(box, &pmu->box_list, list) {
if (box->phys_id == topology_physical_package_id(cpu)) {
@@ -2639,6 +2641,7 @@ uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
}
}
raw_spin_unlock(&uncore_box_lock);
+ put_online_cpus_atomic();
return *per_cpu_ptr(pmu->box, cpu);
}
@@ -3229,6 +3232,7 @@ static void uncore_pci_remove(struct pci_dev *pdev)
list_del(&box->list);
raw_spin_unlock(&uncore_box_lock);
+ get_online_cpus_atomic();
for_each_possible_cpu(cpu) {
if (*per_cpu_ptr(pmu->box, cpu) == box) {
*per_cpu_ptr(pmu->box, cpu) = NULL;
@@ -3237,6 +3241,8 @@ static void uncore_pci_remove(struct pci_dev *pdev)
}
WARN_ON_ONCE(atomic_read(&box->refcnt) != 1);
+ put_online_cpus_atomic();
+
kfree(box);
}
^ permalink raw reply related
* [PATCH 28/45] KVM: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:44 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, kvm, Gleb Natapov, nikunj, zhong, linux-pm, fweisbec,
linux-kernel, rostedt, xiaoguangrong, sbw, wangyun, srivatsa.bhat,
netdev, Paolo Bonzini, linuxppc-dev
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Gleb Natapov <gleb@redhat.com>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: kvm@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
virt/kvm/kvm_main.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 302681c..5bbfa30 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -174,7 +174,7 @@ static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
zalloc_cpumask_var(&cpus, GFP_ATOMIC);
- me = get_cpu();
+ me = get_online_cpus_atomic();
kvm_for_each_vcpu(i, vcpu, kvm) {
kvm_make_request(req, vcpu);
cpu = vcpu->cpu;
@@ -192,7 +192,7 @@ static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
smp_call_function_many(cpus, ack_flush, NULL, 1);
else
called = false;
- put_cpu();
+ put_online_cpus_atomic();
free_cpumask_var(cpus);
return called;
}
@@ -1707,11 +1707,11 @@ void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
++vcpu->stat.halt_wakeup;
}
- me = get_cpu();
+ me = get_online_cpus_atomic();
if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
if (kvm_arch_vcpu_should_kick(vcpu))
smp_send_reschedule(cpu);
- put_cpu();
+ put_online_cpus_atomic();
}
EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
#endif /* !CONFIG_S390 */
^ permalink raw reply related
* [PATCH 29/45] kvm/vmx: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:44 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, kvm, Gleb Natapov, x86, nikunj, zhong, linux-pm,
fweisbec, H. Peter Anvin, linux-kernel, rostedt, xiaoguangrong,
sbw, Ingo Molnar, wangyun, srivatsa.bhat, netdev, Paolo Bonzini,
Thomas Gleixner, linuxppc-dev
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Gleb Natapov <gleb@redhat.com>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Cc: kvm@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/x86/kvm/vmx.c | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 260a919..4e1e966 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -26,6 +26,7 @@
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/sched.h>
+#include <linux/cpu.h>
#include <linux/moduleparam.h>
#include <linux/mod_devicetable.h>
#include <linux/ftrace_event.h>
@@ -7164,12 +7165,12 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
if (!vmm_exclusive)
kvm_cpu_vmxoff();
- cpu = get_cpu();
+ cpu = get_online_cpus_atomic();
vmx_vcpu_load(&vmx->vcpu, cpu);
vmx->vcpu.cpu = cpu;
err = vmx_vcpu_setup(vmx);
vmx_vcpu_put(&vmx->vcpu);
- put_cpu();
+ put_online_cpus_atomic();
if (err)
goto free_vmcs;
if (vm_need_virtualize_apic_accesses(kvm)) {
@@ -7706,12 +7707,12 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
vmx->nested.vmcs01_tsc_offset = vmcs_read64(TSC_OFFSET);
- cpu = get_cpu();
+ cpu = get_online_cpus_atomic();
vmx->loaded_vmcs = vmcs02;
vmx_vcpu_put(vcpu);
vmx_vcpu_load(vcpu, cpu);
vcpu->cpu = cpu;
- put_cpu();
+ put_online_cpus_atomic();
vmx_segment_cache_clear(vmx);
@@ -8023,12 +8024,12 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu)
leave_guest_mode(vcpu);
prepare_vmcs12(vcpu, vmcs12);
- cpu = get_cpu();
+ cpu = get_online_cpus_atomic();
vmx->loaded_vmcs = &vmx->vmcs01;
vmx_vcpu_put(vcpu);
vmx_vcpu_load(vcpu, cpu);
vcpu->cpu = cpu;
- put_cpu();
+ put_online_cpus_atomic();
vmx_segment_cache_clear(vmx);
^ permalink raw reply related
* [PATCH 30/45] x86/xen: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:44 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, Jeremy Fitzhardinge, x86, H. Peter Anvin, nikunj,
zhong, linux-pm, fweisbec, Konrad Rzeszutek Wilk, linux-kernel,
rostedt, xiaoguangrong, sbw, Ingo Molnar, wangyun, srivatsa.bhat,
netdev, Thomas Gleixner, virtualization, linuxppc-dev, xen-devel
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Cc: xen-devel@lists.xensource.com
Cc: virtualization@lists.linux-foundation.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/x86/xen/mmu.c | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index fdc3ba2..3229c4f 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -39,6 +39,7 @@
* Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
*/
#include <linux/sched.h>
+#include <linux/cpu.h>
#include <linux/highmem.h>
#include <linux/debugfs.h>
#include <linux/bug.h>
@@ -1163,9 +1164,13 @@ static void xen_drop_mm_ref(struct mm_struct *mm)
*/
static void xen_exit_mmap(struct mm_struct *mm)
{
- get_cpu(); /* make sure we don't move around */
+ /*
+ * Make sure we don't move around, and also prevent CPUs from
+ * going offline.
+ */
+ get_online_cpus_atomic();
xen_drop_mm_ref(mm);
- put_cpu();
+ put_online_cpus_atomic();
spin_lock(&mm->page_table_lock);
^ permalink raw reply related
* [PATCH 31/45] alpha/smp: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:45 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, zhong, linux-pm, fweisbec, linux-alpha,
linux-kernel, rostedt, xiaoguangrong, sbw, Ivan Kokshaysky,
wangyun, srivatsa.bhat, netdev, Matt Turner, linuxppc-dev,
Thomas Gleixner, Richard Henderson
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Also, remove the non-ASCII character present in this file!
Cc: Richard Henderson <rth@twiddle.net>
Cc: Ivan Kokshaysky <ink@jurassic.park.msu.ru>
Cc: Matt Turner <mattst88@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-alpha@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/alpha/kernel/smp.c | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
diff --git a/arch/alpha/kernel/smp.c b/arch/alpha/kernel/smp.c
index 7b60834..e147268 100644
--- a/arch/alpha/kernel/smp.c
+++ b/arch/alpha/kernel/smp.c
@@ -497,7 +497,6 @@ smp_cpus_done(unsigned int max_cpus)
((bogosum + 2500) / (5000/HZ)) % 100);
}
-\f
void
smp_percpu_timer_interrupt(struct pt_regs *regs)
{
@@ -681,7 +680,7 @@ ipi_flush_tlb_mm(void *x)
void
flush_tlb_mm(struct mm_struct *mm)
{
- preempt_disable();
+ get_online_cpus_atomic();
if (mm == current->active_mm) {
flush_tlb_current(mm);
@@ -693,7 +692,7 @@ flush_tlb_mm(struct mm_struct *mm)
if (mm->context[cpu])
mm->context[cpu] = 0;
}
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
}
@@ -702,7 +701,7 @@ flush_tlb_mm(struct mm_struct *mm)
printk(KERN_CRIT "flush_tlb_mm: timed out\n");
}
- preempt_enable();
+ put_online_cpus_atomic();
}
EXPORT_SYMBOL(flush_tlb_mm);
@@ -730,7 +729,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
struct flush_tlb_page_struct data;
struct mm_struct *mm = vma->vm_mm;
- preempt_disable();
+ get_online_cpus_atomic();
if (mm == current->active_mm) {
flush_tlb_current_page(mm, vma, addr);
@@ -742,7 +741,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
if (mm->context[cpu])
mm->context[cpu] = 0;
}
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
}
@@ -755,7 +754,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
printk(KERN_CRIT "flush_tlb_page: timed out\n");
}
- preempt_enable();
+ put_online_cpus_atomic();
}
EXPORT_SYMBOL(flush_tlb_page);
@@ -786,7 +785,7 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
if ((vma->vm_flags & VM_EXEC) == 0)
return;
- preempt_disable();
+ get_online_cpus_atomic();
if (mm == current->active_mm) {
__load_new_mm_context(mm);
@@ -798,7 +797,7 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
if (mm->context[cpu])
mm->context[cpu] = 0;
}
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
}
@@ -807,5 +806,5 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
printk(KERN_CRIT "flush_icache_page: timed out\n");
}
- preempt_enable();
+ put_online_cpus_atomic();
}
^ permalink raw reply related
* [PATCH 32/45] blackfin/smp: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:45 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, Bob Liu, Mike Frysinger, nikunj, zhong, linux-pm,
fweisbec, Steven Miao, linux-kernel, rostedt, xiaoguangrong, sbw,
wangyun, srivatsa.bhat, netdev, uclinux-dist-devel,
Thomas Gleixner, linuxppc-dev
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Mike Frysinger <vapier@gentoo.org>
Cc: Bob Liu <lliubbo@gmail.com>
Cc: Steven Miao <realmz6@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: uclinux-dist-devel@blackfin.uclinux.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/blackfin/mach-common/smp.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/arch/blackfin/mach-common/smp.c b/arch/blackfin/mach-common/smp.c
index 1bc2ce6..11496cd 100644
--- a/arch/blackfin/mach-common/smp.c
+++ b/arch/blackfin/mach-common/smp.c
@@ -238,13 +238,13 @@ void smp_send_stop(void)
{
cpumask_t callmap;
- preempt_disable();
+ get_online_cpus_atomic();
cpumask_copy(&callmap, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &callmap);
if (!cpumask_empty(&callmap))
send_ipi(&callmap, BFIN_IPI_CPU_STOP);
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
^ permalink raw reply related
* [PATCH 33/45] cris/smp: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:45 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, Jesper Nilsson, nikunj, zhong, linux-pm,
linux-cris-kernel, fweisbec, linux-kernel, rostedt, xiaoguangrong,
Mikael Starvik, sbw, wangyun, srivatsa.bhat, netdev,
Thomas Gleixner, linuxppc-dev
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Mikael Starvik <starvik@axis.com>
Cc: Jesper Nilsson <jesper.nilsson@axis.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-cris-kernel@axis.com
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/cris/arch-v32/kernel/smp.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/arch/cris/arch-v32/kernel/smp.c b/arch/cris/arch-v32/kernel/smp.c
index cdd1202..b2d4612 100644
--- a/arch/cris/arch-v32/kernel/smp.c
+++ b/arch/cris/arch-v32/kernel/smp.c
@@ -13,6 +13,7 @@
#include <linux/init.h>
#include <linux/timex.h>
#include <linux/sched.h>
+#include <linux/cpu.h>
#include <linux/kernel.h>
#include <linux/cpumask.h>
#include <linux/interrupt.h>
@@ -222,6 +223,7 @@ void flush_tlb_common(struct mm_struct* mm, struct vm_area_struct* vma, unsigned
unsigned long flags;
cpumask_t cpu_mask;
+ get_online_cpus_atomic();
spin_lock_irqsave(&tlbstate_lock, flags);
cpu_mask = (mm == FLUSH_ALL ? cpu_all_mask : *mm_cpumask(mm));
cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
@@ -230,6 +232,7 @@ void flush_tlb_common(struct mm_struct* mm, struct vm_area_struct* vma, unsigned
flush_addr = addr;
send_ipi(IPI_FLUSH_TLB, 1, cpu_mask);
spin_unlock_irqrestore(&tlbstate_lock, flags);
+ put_online_cpus_atomic();
}
void flush_tlb_all(void)
@@ -319,10 +322,12 @@ int smp_call_function(void (*func)(void *info), void *info, int wait)
data.info = info;
data.wait = wait;
+ get_online_cpus_atomic();
spin_lock(&call_lock);
call_data = &data;
ret = send_ipi(IPI_CALL, wait, cpu_mask);
spin_unlock(&call_lock);
+ put_online_cpus_atomic();
return ret;
}
^ permalink raw reply related
* [PATCH 34/45] hexagon/smp: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:45 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, zhong, linux-pm, fweisbec, linux-kernel,
rostedt, xiaoguangrong, sbw, wangyun, srivatsa.bhat, netdev,
linux-hexagon, Thomas Gleixner, Richard Kuo, linuxppc-dev
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Richard Kuo <rkuo@codeaurora.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-hexagon@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/hexagon/kernel/smp.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/arch/hexagon/kernel/smp.c b/arch/hexagon/kernel/smp.c
index 0e364ca..30d4318 100644
--- a/arch/hexagon/kernel/smp.c
+++ b/arch/hexagon/kernel/smp.c
@@ -241,9 +241,12 @@ void smp_send_reschedule(int cpu)
void smp_send_stop(void)
{
struct cpumask targets;
+
+ get_online_cpus_atomic();
cpumask_copy(&targets, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &targets);
send_ipi(&targets, IPI_CPU_STOP);
+ put_online_cpus_atomic();
}
void arch_send_call_function_single_ipi(int cpu)
^ permalink raw reply related
* [PATCH 35/45] ia64: irq, perfmon: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:45 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, Fenghua Yu, Tony Luck, linux-ia64, nikunj, zhong,
linux-pm, fweisbec, linux-kernel, rostedt, xiaoguangrong, sbw,
wangyun, srivatsa.bhat, netdev, Andrew Morton, linuxppc-dev,
Thomas Gleixner, Eric W. Biederman
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Tony Luck <tony.luck@intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-ia64@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/ia64/kernel/irq_ia64.c | 15 +++++++++++++++
arch/ia64/kernel/perfmon.c | 8 +++++++-
2 files changed, 22 insertions(+), 1 deletion(-)
diff --git a/arch/ia64/kernel/irq_ia64.c b/arch/ia64/kernel/irq_ia64.c
index 1034884..f58b162 100644
--- a/arch/ia64/kernel/irq_ia64.c
+++ b/arch/ia64/kernel/irq_ia64.c
@@ -25,6 +25,7 @@
#include <linux/ptrace.h>
#include <linux/signal.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/threads.h>
#include <linux/bitops.h>
#include <linux/irq.h>
@@ -160,9 +161,11 @@ int bind_irq_vector(int irq, int vector, cpumask_t domain)
unsigned long flags;
int ret;
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
ret = __bind_irq_vector(irq, vector, domain);
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
return ret;
}
@@ -190,9 +193,11 @@ static void clear_irq_vector(int irq)
{
unsigned long flags;
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
__clear_irq_vector(irq);
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
}
int
@@ -204,6 +209,7 @@ ia64_native_assign_irq_vector (int irq)
vector = -ENOSPC;
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
for_each_online_cpu(cpu) {
domain = vector_allocation_domain(cpu);
@@ -218,6 +224,7 @@ ia64_native_assign_irq_vector (int irq)
BUG_ON(__bind_irq_vector(irq, vector, domain));
out:
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
return vector;
}
@@ -302,9 +309,11 @@ int irq_prepare_move(int irq, int cpu)
unsigned long flags;
int ret;
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
ret = __irq_prepare_move(irq, cpu);
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
return ret;
}
@@ -320,11 +329,13 @@ void irq_complete_move(unsigned irq)
if (unlikely(cpu_isset(smp_processor_id(), cfg->old_domain)))
return;
+ get_online_cpus_atomic();
cpumask_and(&cleanup_mask, &cfg->old_domain, cpu_online_mask);
cfg->move_cleanup_count = cpus_weight(cleanup_mask);
for_each_cpu_mask(i, cleanup_mask)
platform_send_ipi(i, IA64_IRQ_MOVE_VECTOR, IA64_IPI_DM_INT, 0);
cfg->move_in_progress = 0;
+ put_online_cpus_atomic();
}
static irqreturn_t smp_irq_move_cleanup_interrupt(int irq, void *dev_id)
@@ -393,10 +404,12 @@ void destroy_and_reserve_irq(unsigned int irq)
dynamic_irq_cleanup(irq);
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
__clear_irq_vector(irq);
irq_status[irq] = IRQ_RSVD;
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
}
/*
@@ -409,6 +422,7 @@ int create_irq(void)
cpumask_t domain = CPU_MASK_NONE;
irq = vector = -ENOSPC;
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
for_each_online_cpu(cpu) {
domain = vector_allocation_domain(cpu);
@@ -424,6 +438,7 @@ int create_irq(void)
BUG_ON(__bind_irq_vector(irq, vector, domain));
out:
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
if (irq >= 0)
dynamic_irq_init(irq);
return irq;
diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c
index 9ea25fc..16c8303 100644
--- a/arch/ia64/kernel/perfmon.c
+++ b/arch/ia64/kernel/perfmon.c
@@ -6476,9 +6476,12 @@ pfm_install_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
/* do the easy test first */
if (pfm_alt_intr_handler) return -EBUSY;
+ get_online_cpus_atomic();
+
/* one at a time in the install or remove, just fail the others */
if (!spin_trylock(&pfm_alt_install_check)) {
- return -EBUSY;
+ ret = -EBUSY;
+ goto out;
}
/* reserve our session */
@@ -6498,6 +6501,7 @@ pfm_install_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
pfm_alt_intr_handler = hdl;
spin_unlock(&pfm_alt_install_check);
+ put_online_cpus_atomic();
return 0;
@@ -6510,6 +6514,8 @@ cleanup_reserve:
}
spin_unlock(&pfm_alt_install_check);
+out:
+ put_online_cpus_atomic();
return ret;
}
^ permalink raw reply related
* [PATCH 36/45] ia64: smp, tlb: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-23 13:46 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, Fenghua Yu, Tony Luck, linux-ia64, nikunj, zhong,
linux-pm, fweisbec, linux-kernel, rostedt, xiaoguangrong, sbw,
wangyun, srivatsa.bhat, netdev, linuxppc-dev
In-Reply-To: <20130623133642.19094.16038.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Tony Luck <tony.luck@intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: linux-ia64@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/ia64/kernel/smp.c | 12 ++++++------
arch/ia64/mm/tlb.c | 4 ++--
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/arch/ia64/kernel/smp.c b/arch/ia64/kernel/smp.c
index 9fcd4e6..25991ba 100644
--- a/arch/ia64/kernel/smp.c
+++ b/arch/ia64/kernel/smp.c
@@ -24,6 +24,7 @@
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/kernel_stat.h>
#include <linux/mm.h>
#include <linux/cache.h>
@@ -259,8 +260,7 @@ smp_flush_tlb_cpumask(cpumask_t xcpumask)
cpumask_t cpumask = xcpumask;
int mycpu, cpu, flush_mycpu = 0;
- preempt_disable();
- mycpu = smp_processor_id();
+ mycpu = get_online_cpus_atomic();
for_each_cpu_mask(cpu, cpumask)
counts[cpu] = local_tlb_flush_counts[cpu].count & 0xffff;
@@ -280,7 +280,7 @@ smp_flush_tlb_cpumask(cpumask_t xcpumask)
while(counts[cpu] == (local_tlb_flush_counts[cpu].count & 0xffff))
udelay(FLUSH_DELAY);
- preempt_enable();
+ put_online_cpus_atomic();
}
void
@@ -293,12 +293,12 @@ void
smp_flush_tlb_mm (struct mm_struct *mm)
{
cpumask_var_t cpus;
- preempt_disable();
+ get_online_cpus_atomic();
/* this happens for the common case of a single-threaded fork(): */
if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1))
{
local_finish_flush_tlb_mm(mm);
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
if (!alloc_cpumask_var(&cpus, GFP_ATOMIC)) {
@@ -313,7 +313,7 @@ smp_flush_tlb_mm (struct mm_struct *mm)
local_irq_disable();
local_finish_flush_tlb_mm(mm);
local_irq_enable();
- preempt_enable();
+ put_online_cpus_atomic();
}
void arch_send_call_function_single_ipi(int cpu)
diff --git a/arch/ia64/mm/tlb.c b/arch/ia64/mm/tlb.c
index ed61297..8c55ef5 100644
--- a/arch/ia64/mm/tlb.c
+++ b/arch/ia64/mm/tlb.c
@@ -87,11 +87,11 @@ wrap_mmu_context (struct mm_struct *mm)
* can't call flush_tlb_all() here because of race condition
* with O(1) scheduler [EF]
*/
- cpu = get_cpu(); /* prevent preemption/migration */
+ cpu = get_online_cpus_atomic(); /* prevent preemption/migration */
for_each_online_cpu(i)
if (i != cpu)
per_cpu(ia64_need_tlb_flush, i) = 1;
- put_cpu();
+ put_online_cpus_atomic();
local_flush_tlb_all();
}
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox