* [PATCH v5 13/45] sched/rt: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:36 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
kernel/sched/rt.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 418feb0..2a637be 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -6,6 +6,7 @@
#include "sched.h"
#include <linux/slab.h>
+#include <linux/cpu.h>
static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
@@ -26,7 +27,9 @@ static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
if (!overrun)
break;
+ get_online_cpus_atomic();
idle = do_sched_rt_period_timer(rt_b, overrun);
+ put_online_cpus_atomic();
}
return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
^ permalink raw reply related
* [PATCH v5 14/45] rcu, CPU hotplug: Fix comment referring to stop_machine()
From: Srivatsa S. Bhat @ 2013-01-22 7:36 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Don't refer to stop_machine() in the CPU hotplug path, since we are going
to get rid of it. Also, move the comment referring to callback adoption
to the CPU_DEAD case, because that's where it happens now.
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
kernel/rcutree.c | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index e441b77..ac94474 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -2827,11 +2827,6 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
break;
case CPU_DYING:
case CPU_DYING_FROZEN:
- /*
- * The whole machine is "stopped" except this CPU, so we can
- * touch any data without introducing corruption. We send the
- * dying CPU's callbacks to an arbitrarily chosen online CPU.
- */
for_each_rcu_flavor(rsp)
rcu_cleanup_dying_cpu(rsp);
rcu_cleanup_after_idle(cpu);
@@ -2840,6 +2835,10 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
case CPU_DEAD_FROZEN:
case CPU_UP_CANCELED:
case CPU_UP_CANCELED_FROZEN:
+ /*
+ * We send the dead CPU's callbacks to an arbitrarily chosen
+ * online CPU.
+ */
for_each_rcu_flavor(rsp)
rcu_cleanup_dead_cpu(cpu, rsp);
break;
^ permalink raw reply related
* [PATCH v5 15/45] tick: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:36 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
kernel/time/tick-broadcast.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index f113755..d123a2c 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -160,6 +160,7 @@ static void tick_do_broadcast(struct cpumask *mask)
*/
static void tick_do_periodic_broadcast(void)
{
+ get_online_cpus_atomic();
raw_spin_lock(&tick_broadcast_lock);
cpumask_and(to_cpumask(tmpmask),
@@ -167,6 +168,7 @@ static void tick_do_periodic_broadcast(void)
tick_do_broadcast(to_cpumask(tmpmask));
raw_spin_unlock(&tick_broadcast_lock);
+ put_online_cpus_atomic();
}
/*
^ permalink raw reply related
* [PATCH v5 16/45] time/clocksource: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:37 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from going
offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: John Stultz <john.stultz@linaro.org>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
kernel/time/clocksource.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index c958338..1c8d735 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -30,6 +30,7 @@
#include <linux/sched.h> /* for spin_unlock_irq() using preempt_count() m68k */
#include <linux/tick.h>
#include <linux/kthread.h>
+#include <linux/cpu.h>
void timecounter_init(struct timecounter *tc,
const struct cyclecounter *cc,
@@ -320,11 +321,13 @@ static void clocksource_watchdog(unsigned long data)
* Cycle through CPUs to check if the CPUs stay synchronized
* to each other.
*/
+ get_online_cpus_atomic();
next_cpu = cpumask_next(raw_smp_processor_id(), cpu_online_mask);
if (next_cpu >= nr_cpu_ids)
next_cpu = cpumask_first(cpu_online_mask);
watchdog_timer.expires += WATCHDOG_INTERVAL;
add_timer_on(&watchdog_timer, next_cpu);
+ put_online_cpus_atomic();
out:
spin_unlock(&watchdog_lock);
}
@@ -336,7 +339,9 @@ static inline void clocksource_start_watchdog(void)
init_timer(&watchdog_timer);
watchdog_timer.function = clocksource_watchdog;
watchdog_timer.expires = jiffies + WATCHDOG_INTERVAL;
+ get_online_cpus_atomic();
add_timer_on(&watchdog_timer, cpumask_first(cpu_online_mask));
+ put_online_cpus_atomic();
watchdog_running = 1;
}
^ permalink raw reply related
* [PATCH v5 17/45] softirq: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:37 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
kernel/softirq.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/kernel/softirq.c b/kernel/softirq.c
index ed567ba..98c3e27 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -631,6 +631,7 @@ static void remote_softirq_receive(void *data)
static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq)
{
+ get_online_cpus_atomic();
if (cpu_online(cpu)) {
cp->func = remote_softirq_receive;
cp->info = cp;
@@ -638,8 +639,10 @@ static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softir
cp->priv = softirq;
__smp_call_function_single(cpu, cp, 0);
+ put_online_cpus_atomic();
return 0;
}
+ put_online_cpus_atomic();
return 1;
}
#else /* CONFIG_USE_GENERIC_SMP_HELPERS */
^ permalink raw reply related
* [PATCH v5 18/45] irq: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:38 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
kernel/irq/manage.c | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index e49a288..b4240b9 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -16,6 +16,7 @@
#include <linux/interrupt.h>
#include <linux/slab.h>
#include <linux/sched.h>
+#include <linux/cpu.h>
#include <linux/task_work.h>
#include "internals.h"
@@ -202,7 +203,9 @@ int irq_set_affinity(unsigned int irq, const struct cpumask *mask)
return -EINVAL;
raw_spin_lock_irqsave(&desc->lock, flags);
+ get_online_cpus_atomic();
ret = __irq_set_affinity_locked(irq_desc_get_irq_data(desc), mask);
+ put_online_cpus_atomic();
raw_spin_unlock_irqrestore(&desc->lock, flags);
return ret;
}
@@ -343,7 +346,9 @@ int irq_select_affinity_usr(unsigned int irq, struct cpumask *mask)
int ret;
raw_spin_lock_irqsave(&desc->lock, flags);
+ get_online_cpus_atomic();
ret = setup_affinity(irq, desc, mask);
+ put_online_cpus_atomic();
raw_spin_unlock_irqrestore(&desc->lock, flags);
return ret;
}
@@ -1126,7 +1131,9 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
}
/* Set default affinity mask once everything is setup */
+ get_online_cpus_atomic();
setup_affinity(irq, desc, mask);
+ put_online_cpus_atomic();
} else if (new->flags & IRQF_TRIGGER_MASK) {
unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK;
^ permalink raw reply related
* [PATCH v5 19/45] net: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:38 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Eric Dumazet <edumazet@google.com>
Cc: netdev at vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
net/core/dev.c | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/net/core/dev.c b/net/core/dev.c
index f64e439..5421f96 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3089,7 +3089,7 @@ int netif_rx(struct sk_buff *skb)
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu;
- preempt_disable();
+ get_online_cpus_atomic();
rcu_read_lock();
cpu = get_rps_cpu(skb->dev, skb, &rflow);
@@ -3099,7 +3099,7 @@ int netif_rx(struct sk_buff *skb)
ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
rcu_read_unlock();
- preempt_enable();
+ put_online_cpus_atomic();
} else
#endif
{
@@ -3498,6 +3498,7 @@ int netif_receive_skb(struct sk_buff *skb)
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu, ret;
+ get_online_cpus_atomic();
rcu_read_lock();
cpu = get_rps_cpu(skb->dev, skb, &rflow);
@@ -3505,9 +3506,11 @@ int netif_receive_skb(struct sk_buff *skb)
if (cpu >= 0) {
ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
rcu_read_unlock();
+ put_online_cpus_atomic();
return ret;
}
rcu_read_unlock();
+ put_online_cpus_atomic();
}
#endif
return __netif_receive_skb(skb);
@@ -3887,6 +3890,7 @@ static void net_rps_action_and_irq_enable(struct softnet_data *sd)
local_irq_enable();
/* Send pending IPI's to kick RPS processing on remote cpus. */
+ get_online_cpus_atomic();
while (remsd) {
struct softnet_data *next = remsd->rps_ipi_next;
@@ -3895,6 +3899,7 @@ static void net_rps_action_and_irq_enable(struct softnet_data *sd)
&remsd->csd, 0);
remsd = next;
}
+ put_online_cpus_atomic();
} else
#endif
local_irq_enable();
^ permalink raw reply related
* [PATCH v5 20/45] block: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:38 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
block/blk-softirq.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/block/blk-softirq.c b/block/blk-softirq.c
index 467c8de..448f9a9 100644
--- a/block/blk-softirq.c
+++ b/block/blk-softirq.c
@@ -58,6 +58,8 @@ static void trigger_softirq(void *data)
*/
static int raise_blk_irq(int cpu, struct request *rq)
{
+ get_online_cpus_atomic();
+
if (cpu_online(cpu)) {
struct call_single_data *data = &rq->csd;
@@ -66,9 +68,11 @@ static int raise_blk_irq(int cpu, struct request *rq)
data->flags = 0;
__smp_call_function_single(cpu, data, 0);
+ put_online_cpus_atomic();
return 0;
}
+ put_online_cpus_atomic();
return 1;
}
#else /* CONFIG_SMP && CONFIG_USE_GENERIC_SMP_HELPERS */
^ permalink raw reply related
* [PATCH v5 21/45] crypto: pcrypt - Protect access to cpu_online_mask with get/put_online_cpus()
From: Srivatsa S. Bhat @ 2013-01-22 7:38 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
The pcrypt_aead_init_tfm() function access the cpu_online_mask without
disabling CPU hotplug. And it looks like it can afford to sleep, so use
the get/put_online_cpus() APIs to protect against CPU hotplug.
Cc: Steffen Klassert <steffen.klassert@secunet.com>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: linux-crypto at vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
crypto/pcrypt.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c
index b2c99dc..10f64e2 100644
--- a/crypto/pcrypt.c
+++ b/crypto/pcrypt.c
@@ -280,12 +280,16 @@ static int pcrypt_aead_init_tfm(struct crypto_tfm *tfm)
ictx->tfm_count++;
+ get_online_cpus();
+
cpu_index = ictx->tfm_count % cpumask_weight(cpu_online_mask);
ctx->cb_cpu = cpumask_first(cpu_online_mask);
for (cpu = 0; cpu < cpu_index; cpu++)
ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_online_mask);
+ put_online_cpus();
+
cipher = crypto_spawn_aead(crypto_instance_ctx(inst));
if (IS_ERR(cipher))
^ permalink raw reply related
* [PATCH v5 22/45] infiniband: ehca: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Roland Dreier <roland@kernel.org>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
drivers/infiniband/hw/ehca/ehca_irq.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/drivers/infiniband/hw/ehca/ehca_irq.c b/drivers/infiniband/hw/ehca/ehca_irq.c
index 8615d7c..d61936c 100644
--- a/drivers/infiniband/hw/ehca/ehca_irq.c
+++ b/drivers/infiniband/hw/ehca/ehca_irq.c
@@ -43,6 +43,7 @@
#include <linux/slab.h>
#include <linux/smpboot.h>
+#include <linux/cpu.h>
#include "ehca_classes.h"
#include "ehca_irq.h"
@@ -653,6 +654,9 @@ void ehca_tasklet_eq(unsigned long data)
ehca_process_eq((struct ehca_shca*)data, 1);
}
+/*
+ * Must be called under get_online_cpus_atomic() and put_online_cpus_atomic().
+ */
static int find_next_online_cpu(struct ehca_comp_pool *pool)
{
int cpu;
@@ -703,6 +707,7 @@ static void queue_comp_task(struct ehca_cq *__cq)
int cq_jobs;
unsigned long flags;
+ get_online_cpus_atomic();
cpu_id = find_next_online_cpu(pool);
BUG_ON(!cpu_online(cpu_id));
@@ -720,6 +725,7 @@ static void queue_comp_task(struct ehca_cq *__cq)
BUG_ON(!cct || !thread);
}
__queue_comp_task(__cq, cct, thread);
+ put_online_cpus_atomic();
}
static void run_comp_task(struct ehca_cpu_comp_task *cct)
@@ -759,6 +765,7 @@ static void comp_task_park(unsigned int cpu)
list_splice_init(&cct->cq_list, &list);
spin_unlock_irq(&cct->task_lock);
+ get_online_cpus_atomic();
cpu = find_next_online_cpu(pool);
target = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu);
@@ -768,6 +775,7 @@ static void comp_task_park(unsigned int cpu)
__queue_comp_task(cq, target, thread);
}
spin_unlock_irq(&target->task_lock);
+ put_online_cpus_atomic();
}
static void comp_task_stop(unsigned int cpu, bool online)
^ permalink raw reply related
* [PATCH v5 23/45] [SCSI] fcoe: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Robert Love <robert.w.love@intel.com>
Cc: "James E.J. Bottomley" <JBottomley@parallels.com>
Cc: devel at open-fcoe.org
Cc: linux-scsi at vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
drivers/scsi/fcoe/fcoe.c | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/drivers/scsi/fcoe/fcoe.c b/drivers/scsi/fcoe/fcoe.c
index 666b7ac..c971a17 100644
--- a/drivers/scsi/fcoe/fcoe.c
+++ b/drivers/scsi/fcoe/fcoe.c
@@ -1475,6 +1475,7 @@ static int fcoe_rcv(struct sk_buff *skb, struct net_device *netdev,
* was originated, otherwise select cpu using rx exchange id
* or fcoe_select_cpu().
*/
+ get_online_cpus_atomic();
if (ntoh24(fh->fh_f_ctl) & FC_FC_EX_CTX)
cpu = ntohs(fh->fh_ox_id) & fc_cpu_mask;
else {
@@ -1484,8 +1485,10 @@ static int fcoe_rcv(struct sk_buff *skb, struct net_device *netdev,
cpu = ntohs(fh->fh_rx_id) & fc_cpu_mask;
}
- if (cpu >= nr_cpu_ids)
+ if (cpu >= nr_cpu_ids) {
+ put_online_cpus_atomic();
goto err;
+ }
fps = &per_cpu(fcoe_percpu, cpu);
spin_lock(&fps->fcoe_rx_list.lock);
@@ -1505,6 +1508,7 @@ static int fcoe_rcv(struct sk_buff *skb, struct net_device *netdev,
spin_lock(&fps->fcoe_rx_list.lock);
if (!fps->thread) {
spin_unlock(&fps->fcoe_rx_list.lock);
+ put_online_cpus_atomic();
goto err;
}
}
@@ -1526,6 +1530,7 @@ static int fcoe_rcv(struct sk_buff *skb, struct net_device *netdev,
if (fps->thread->state == TASK_INTERRUPTIBLE)
wake_up_process(fps->thread);
spin_unlock(&fps->fcoe_rx_list.lock);
+ put_online_cpus_atomic();
return 0;
err:
^ permalink raw reply related
* [PATCH v5 24/45] staging: octeon: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: David Daney <david.daney@cavium.com>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
drivers/staging/octeon/ethernet-rx.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/drivers/staging/octeon/ethernet-rx.c b/drivers/staging/octeon/ethernet-rx.c
index 34afc16..8588b4d 100644
--- a/drivers/staging/octeon/ethernet-rx.c
+++ b/drivers/staging/octeon/ethernet-rx.c
@@ -36,6 +36,7 @@
#include <linux/prefetch.h>
#include <linux/ratelimit.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/interrupt.h>
#include <net/dst.h>
#ifdef CONFIG_XFRM
@@ -97,6 +98,7 @@ static void cvm_oct_enable_one_cpu(void)
return;
/* ... if a CPU is available, Turn on NAPI polling for that CPU. */
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
if (!cpu_test_and_set(cpu, core_state.cpu_state)) {
v = smp_call_function_single(cpu, cvm_oct_enable_napi,
@@ -106,6 +108,7 @@ static void cvm_oct_enable_one_cpu(void)
break;
}
}
+ put_online_cpus_atomic();
}
static void cvm_oct_no_more_work(void)
^ permalink raw reply related
* [PATCH v5 25/45] x86: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86 at kernel.org
Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Yinghai Lu <yinghai@kernel.org>
Cc: Daniel J Blueman <daniel@numascale-asia.com>
Cc: Steffen Persvold <sp@numascale.com>
Cc: Joerg Roedel <joerg.roedel@amd.com>
Cc: linux-edac at vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/x86/include/asm/ipi.h | 5 +++++
arch/x86/kernel/apic/apic_flat_64.c | 10 ++++++++++
arch/x86/kernel/apic/apic_numachip.c | 5 +++++
arch/x86/kernel/apic/es7000_32.c | 5 +++++
arch/x86/kernel/apic/io_apic.c | 7 +++++--
arch/x86/kernel/apic/ipi.c | 10 ++++++++++
arch/x86/kernel/apic/x2apic_cluster.c | 4 ++++
arch/x86/kernel/apic/x2apic_uv_x.c | 4 ++++
arch/x86/kernel/cpu/mcheck/therm_throt.c | 4 ++--
arch/x86/mm/tlb.c | 14 +++++++-------
10 files changed, 57 insertions(+), 11 deletions(-)
diff --git a/arch/x86/include/asm/ipi.h b/arch/x86/include/asm/ipi.h
index 615fa90..112249c 100644
--- a/arch/x86/include/asm/ipi.h
+++ b/arch/x86/include/asm/ipi.h
@@ -20,6 +20,7 @@
* Subject to the GNU Public License, v.2
*/
+#include <linux/cpu.h>
#include <asm/hw_irq.h>
#include <asm/apic.h>
#include <asm/smp.h>
@@ -131,18 +132,22 @@ extern int no_broadcast;
static inline void __default_local_send_IPI_allbutself(int vector)
{
+ get_online_cpus_atomic();
if (no_broadcast || vector == NMI_VECTOR)
apic->send_IPI_mask_allbutself(cpu_online_mask, vector);
else
__default_send_IPI_shortcut(APIC_DEST_ALLBUT, vector, apic->dest_logical);
+ put_online_cpus_atomic();
}
static inline void __default_local_send_IPI_all(int vector)
{
+ get_online_cpus_atomic();
if (no_broadcast || vector == NMI_VECTOR)
apic->send_IPI_mask(cpu_online_mask, vector);
else
__default_send_IPI_shortcut(APIC_DEST_ALLINC, vector, apic->dest_logical);
+ put_online_cpus_atomic();
}
#ifdef CONFIG_X86_32
diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c
index 00c77cf..8207ade 100644
--- a/arch/x86/kernel/apic/apic_flat_64.c
+++ b/arch/x86/kernel/apic/apic_flat_64.c
@@ -11,6 +11,7 @@
#include <linux/errno.h>
#include <linux/threads.h>
#include <linux/cpumask.h>
+#include <linux/cpu.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <linux/ctype.h>
@@ -92,6 +93,8 @@ static void flat_send_IPI_allbutself(int vector)
#else
int hotplug = 0;
#endif
+
+ get_online_cpus_atomic();
if (hotplug || vector == NMI_VECTOR) {
if (!cpumask_equal(cpu_online_mask, cpumask_of(cpu))) {
unsigned long mask = cpumask_bits(cpu_online_mask)[0];
@@ -105,16 +108,19 @@ static void flat_send_IPI_allbutself(int vector)
__default_send_IPI_shortcut(APIC_DEST_ALLBUT,
vector, apic->dest_logical);
}
+ put_online_cpus_atomic();
}
static void flat_send_IPI_all(int vector)
{
+ get_online_cpus_atomic();
if (vector == NMI_VECTOR) {
flat_send_IPI_mask(cpu_online_mask, vector);
} else {
__default_send_IPI_shortcut(APIC_DEST_ALLINC,
vector, apic->dest_logical);
}
+ put_online_cpus_atomic();
}
static unsigned int flat_get_apic_id(unsigned long x)
@@ -255,12 +261,16 @@ static void physflat_send_IPI_mask_allbutself(const struct cpumask *cpumask,
static void physflat_send_IPI_allbutself(int vector)
{
+ get_online_cpus_atomic();
default_send_IPI_mask_allbutself_phys(cpu_online_mask, vector);
+ put_online_cpus_atomic();
}
static void physflat_send_IPI_all(int vector)
{
+ get_online_cpus_atomic();
physflat_send_IPI_mask(cpu_online_mask, vector);
+ put_online_cpus_atomic();
}
static int physflat_probe(void)
diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c
index 9c2aa89..7d19c1d 100644
--- a/arch/x86/kernel/apic/apic_numachip.c
+++ b/arch/x86/kernel/apic/apic_numachip.c
@@ -14,6 +14,7 @@
#include <linux/errno.h>
#include <linux/threads.h>
#include <linux/cpumask.h>
+#include <linux/cpu.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <linux/module.h>
@@ -131,15 +132,19 @@ static void numachip_send_IPI_allbutself(int vector)
unsigned int this_cpu = smp_processor_id();
unsigned int cpu;
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
if (cpu != this_cpu)
numachip_send_IPI_one(cpu, vector);
}
+ put_online_cpus_atomic();
}
static void numachip_send_IPI_all(int vector)
{
+ get_online_cpus_atomic();
numachip_send_IPI_mask(cpu_online_mask, vector);
+ put_online_cpus_atomic();
}
static void numachip_send_IPI_self(int vector)
diff --git a/arch/x86/kernel/apic/es7000_32.c b/arch/x86/kernel/apic/es7000_32.c
index 0874799..ddf2995 100644
--- a/arch/x86/kernel/apic/es7000_32.c
+++ b/arch/x86/kernel/apic/es7000_32.c
@@ -45,6 +45,7 @@
#include <linux/gfp.h>
#include <linux/nmi.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/io.h>
#include <asm/apicdef.h>
@@ -412,12 +413,16 @@ static void es7000_send_IPI_mask(const struct cpumask *mask, int vector)
static void es7000_send_IPI_allbutself(int vector)
{
+ get_online_cpus_atomic();
default_send_IPI_mask_allbutself_phys(cpu_online_mask, vector);
+ put_online_cpus_atomic();
}
static void es7000_send_IPI_all(int vector)
{
+ get_online_cpus_atomic();
es7000_send_IPI_mask(cpu_online_mask, vector);
+ put_online_cpus_atomic();
}
static int es7000_apic_id_registered(void)
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index b739d39..ca1c2a5 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -25,6 +25,7 @@
#include <linux/init.h>
#include <linux/delay.h>
#include <linux/sched.h>
+#include <linux/cpu.h>
#include <linux/pci.h>
#include <linux/mc146818rtc.h>
#include <linux/compiler.h>
@@ -1788,13 +1789,13 @@ __apicdebuginit(void) print_local_APICs(int maxcpu)
if (!maxcpu)
return;
- preempt_disable();
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
if (cpu >= maxcpu)
break;
smp_call_function_single(cpu, print_local_APIC, NULL, 1);
}
- preempt_enable();
+ put_online_cpus_atomic();
}
__apicdebuginit(void) print_PIC(void)
@@ -2209,6 +2210,7 @@ void send_cleanup_vector(struct irq_cfg *cfg)
{
cpumask_var_t cleanup_mask;
+ get_online_cpus_atomic();
if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
unsigned int i;
for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
@@ -2219,6 +2221,7 @@ void send_cleanup_vector(struct irq_cfg *cfg)
free_cpumask_var(cleanup_mask);
}
cfg->move_in_progress = 0;
+ put_online_cpus_atomic();
}
asmlinkage void smp_irq_move_cleanup_interrupt(void)
diff --git a/arch/x86/kernel/apic/ipi.c b/arch/x86/kernel/apic/ipi.c
index cce91bf..c65aa77 100644
--- a/arch/x86/kernel/apic/ipi.c
+++ b/arch/x86/kernel/apic/ipi.c
@@ -29,12 +29,14 @@ void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector)
* to an arbitrary mask, so I do a unicast to each CPU instead.
* - mbligh
*/
+ get_online_cpus_atomic();
local_irq_save(flags);
for_each_cpu(query_cpu, mask) {
__default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
query_cpu), vector, APIC_DEST_PHYSICAL);
}
local_irq_restore(flags);
+ put_online_cpus_atomic();
}
void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
@@ -46,6 +48,7 @@ void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
/* See Hack comment above */
+ get_online_cpus_atomic();
local_irq_save(flags);
for_each_cpu(query_cpu, mask) {
if (query_cpu == this_cpu)
@@ -54,6 +57,7 @@ void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
query_cpu), vector, APIC_DEST_PHYSICAL);
}
local_irq_restore(flags);
+ put_online_cpus_atomic();
}
#ifdef CONFIG_X86_32
@@ -70,12 +74,14 @@ void default_send_IPI_mask_sequence_logical(const struct cpumask *mask,
* should be modified to do 1 message per cluster ID - mbligh
*/
+ get_online_cpus_atomic();
local_irq_save(flags);
for_each_cpu(query_cpu, mask)
__default_send_IPI_dest_field(
early_per_cpu(x86_cpu_to_logical_apicid, query_cpu),
vector, apic->dest_logical);
local_irq_restore(flags);
+ put_online_cpus_atomic();
}
void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
@@ -87,6 +93,7 @@ void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
/* See Hack comment above */
+ get_online_cpus_atomic();
local_irq_save(flags);
for_each_cpu(query_cpu, mask) {
if (query_cpu == this_cpu)
@@ -96,6 +103,7 @@ void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
vector, apic->dest_logical);
}
local_irq_restore(flags);
+ put_online_cpus_atomic();
}
/*
@@ -109,10 +117,12 @@ void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector)
if (WARN_ONCE(!mask, "empty IPI mask"))
return;
+ get_online_cpus_atomic();
local_irq_save(flags);
WARN_ON(mask & ~cpumask_bits(cpu_online_mask)[0]);
__default_send_IPI_dest_field(mask, vector, apic->dest_logical);
local_irq_restore(flags);
+ put_online_cpus_atomic();
}
void default_send_IPI_allbutself(int vector)
diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c
index c88baa4..cb08e6b 100644
--- a/arch/x86/kernel/apic/x2apic_cluster.c
+++ b/arch/x86/kernel/apic/x2apic_cluster.c
@@ -88,12 +88,16 @@ x2apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector)
static void x2apic_send_IPI_allbutself(int vector)
{
+ get_online_cpus_atomic();
__x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLBUT);
+ put_online_cpus_atomic();
}
static void x2apic_send_IPI_all(int vector)
{
+ get_online_cpus_atomic();
__x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLINC);
+ put_online_cpus_atomic();
}
static int
diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c
index 8cfade9..cc469a3 100644
--- a/arch/x86/kernel/apic/x2apic_uv_x.c
+++ b/arch/x86/kernel/apic/x2apic_uv_x.c
@@ -244,15 +244,19 @@ static void uv_send_IPI_allbutself(int vector)
unsigned int this_cpu = smp_processor_id();
unsigned int cpu;
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
if (cpu != this_cpu)
uv_send_IPI_one(cpu, vector);
}
+ put_online_cpus_atomic();
}
static void uv_send_IPI_all(int vector)
{
+ get_online_cpus_atomic();
uv_send_IPI_mask(cpu_online_mask, vector);
+ put_online_cpus_atomic();
}
static int uv_apic_id_valid(int apicid)
diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c
index 47a1870..d128ba4 100644
--- a/arch/x86/kernel/cpu/mcheck/therm_throt.c
+++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c
@@ -82,13 +82,13 @@ static ssize_t therm_throt_device_show_##event##_##name( \
unsigned int cpu = dev->id; \
ssize_t ret; \
\
- preempt_disable(); /* CPU hotplug */ \
+ get_online_cpus_atomic(); /* CPU hotplug */ \
if (cpu_online(cpu)) { \
ret = sprintf(buf, "%lu\n", \
per_cpu(thermal_state, cpu).event.name); \
} else \
ret = 0; \
- preempt_enable(); \
+ put_online_cpus_atomic(); \
\
return ret; \
}
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 13a6b29..2c3ec76 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -147,12 +147,12 @@ void flush_tlb_current_task(void)
{
struct mm_struct *mm = current->mm;
- preempt_disable();
+ get_online_cpus_atomic();
local_flush_tlb();
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
- preempt_enable();
+ put_online_cpus_atomic();
}
/*
@@ -187,7 +187,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
unsigned long addr;
unsigned act_entries, tlb_entries = 0;
- preempt_disable();
+ get_online_cpus_atomic();
if (current->active_mm != mm)
goto flush_all;
@@ -225,21 +225,21 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
if (cpumask_any_but(mm_cpumask(mm),
smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, start, end);
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
flush_all:
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
- preempt_enable();
+ put_online_cpus_atomic();
}
void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
{
struct mm_struct *mm = vma->vm_mm;
- preempt_disable();
+ get_online_cpus_atomic();
if (current->active_mm == mm) {
if (current->mm)
@@ -251,7 +251,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, start, 0UL);
- preempt_enable();
+ put_online_cpus_atomic();
}
static void do_flush_tlb_all(void *info)
^ permalink raw reply related
* [PATCH v5 26/45] perf/x86: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:39 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
The CPU_DYING notifier modifies the per-cpu pointer pmu->box, and this can
race with functions such as uncore_pmu_to_box() and uncore_pci_remove() when
we remove stop_machine() from the CPU offline path. So protect them using
get/put_online_cpus_atomic().
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86 at kernel.org
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/x86/kernel/cpu/perf_event_intel_uncore.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/arch/x86/kernel/cpu/perf_event_intel_uncore.c b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
index b43200d..6faae53 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_uncore.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
@@ -1,3 +1,4 @@
+#include <linux/cpu.h>
#include "perf_event_intel_uncore.h"
static struct intel_uncore_type *empty_uncore[] = { NULL, };
@@ -1965,6 +1966,7 @@ uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
if (box)
return box;
+ get_online_cpus_atomic();
raw_spin_lock(&uncore_box_lock);
list_for_each_entry(box, &pmu->box_list, list) {
if (box->phys_id == topology_physical_package_id(cpu)) {
@@ -1974,6 +1976,7 @@ uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
}
}
raw_spin_unlock(&uncore_box_lock);
+ put_online_cpus_atomic();
return *per_cpu_ptr(pmu->box, cpu);
}
@@ -2556,6 +2559,7 @@ static void uncore_pci_remove(struct pci_dev *pdev)
if (WARN_ON_ONCE(phys_id != box->phys_id))
return;
+ get_online_cpus_atomic();
raw_spin_lock(&uncore_box_lock);
list_del(&box->list);
raw_spin_unlock(&uncore_box_lock);
@@ -2569,6 +2573,7 @@ static void uncore_pci_remove(struct pci_dev *pdev)
WARN_ON_ONCE(atomic_read(&box->refcnt) != 1);
kfree(box);
+ put_online_cpus_atomic();
}
static int uncore_pci_probe(struct pci_dev *pdev,
^ permalink raw reply related
* [PATCH v5 27/45] KVM: Use get/put_online_cpus_atomic() to prevent CPU offline from atomic context
From: Srivatsa S. Bhat @ 2013-01-22 7:40 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Gleb Natapov <gleb@redhat.com>
Cc: kvm at vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
virt/kvm/kvm_main.c | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 1cd693a..47f9c30 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -174,7 +174,8 @@ static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
zalloc_cpumask_var(&cpus, GFP_ATOMIC);
- me = get_cpu();
+ get_online_cpus_atomic();
+ me = smp_processor_id();
kvm_for_each_vcpu(i, vcpu, kvm) {
kvm_make_request(req, vcpu);
cpu = vcpu->cpu;
@@ -192,7 +193,7 @@ static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
smp_call_function_many(cpus, ack_flush, NULL, 1);
else
called = false;
- put_cpu();
+ put_online_cpus_atomic();
free_cpumask_var(cpus);
return called;
}
@@ -1621,11 +1622,12 @@ void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
++vcpu->stat.halt_wakeup;
}
- me = get_cpu();
+ get_online_cpus_atomic();
+ me = smp_processor_id();
if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
if (kvm_arch_vcpu_should_kick(vcpu))
smp_send_reschedule(cpu);
- put_cpu();
+ put_online_cpus_atomic();
}
#endif /* !CONFIG_S390 */
^ permalink raw reply related
* [PATCH v5 28/45] kvm/vmx: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:40 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context (in vmx_vcpu_load() to prevent CPUs from
going offline while clearing vmcs).
Reported-by: Michael Wang <wangyun@linux.vnet.ibm.com>
Debugged-by: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Gleb Natapov <gleb@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86 at kernel.org
Cc: kvm at vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/x86/kvm/vmx.c | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 9120ae1..2886ff0 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -1557,10 +1557,14 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
struct vcpu_vmx *vmx = to_vmx(vcpu);
u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
- if (!vmm_exclusive)
+ if (!vmm_exclusive) {
kvm_cpu_vmxon(phys_addr);
- else if (vmx->loaded_vmcs->cpu != cpu)
+ } else if (vmx->loaded_vmcs->cpu != cpu) {
+ /* Prevent any CPU from going offline */
+ get_online_cpus_atomic();
loaded_vmcs_clear(vmx->loaded_vmcs);
+ put_online_cpus_atomic();
+ }
if (per_cpu(current_vmcs, cpu) != vmx->loaded_vmcs->vmcs) {
per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
^ permalink raw reply related
* [PATCH v5 29/45] x86/xen: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:40 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86 at kernel.org
Cc: xen-devel at lists.xensource.com
Cc: virtualization at lists.linux-foundation.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/x86/xen/mmu.c | 11 +++++++++--
arch/x86/xen/smp.c | 9 +++++++++
2 files changed, 18 insertions(+), 2 deletions(-)
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 01de35c..6a95a15 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -39,6 +39,7 @@
* Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
*/
#include <linux/sched.h>
+#include <linux/cpu.h>
#include <linux/highmem.h>
#include <linux/debugfs.h>
#include <linux/bug.h>
@@ -1163,9 +1164,13 @@ static void xen_drop_mm_ref(struct mm_struct *mm)
*/
static void xen_exit_mmap(struct mm_struct *mm)
{
- get_cpu(); /* make sure we don't move around */
+ /*
+ * Make sure we don't move around, and prevent CPUs from going
+ * offline.
+ */
+ get_online_cpus_atomic();
xen_drop_mm_ref(mm);
- put_cpu();
+ put_online_cpus_atomic();
spin_lock(&mm->page_table_lock);
@@ -1371,6 +1376,7 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
args->op.arg2.vcpumask = to_cpumask(args->mask);
/* Remove us, and any offline CPUS. */
+ get_online_cpus_atomic();
cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
@@ -1383,6 +1389,7 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
xen_mc_issue(PARAVIRT_LAZY_MMU);
+ put_online_cpus_atomic();
}
static unsigned long xen_read_cr3(void)
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 4f7d259..7d753ae 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -16,6 +16,7 @@
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/irq_work.h>
#include <asm/paravirt.h>
@@ -487,8 +488,10 @@ static void __xen_send_IPI_mask(const struct cpumask *mask,
{
unsigned cpu;
+ get_online_cpus_atomic();
for_each_cpu_and(cpu, mask, cpu_online_mask)
xen_send_IPI_one(cpu, vector);
+ put_online_cpus_atomic();
}
static void xen_smp_send_call_function_ipi(const struct cpumask *mask)
@@ -551,8 +554,10 @@ void xen_send_IPI_all(int vector)
{
int xen_vector = xen_map_vector(vector);
+ get_online_cpus_atomic();
if (xen_vector >= 0)
__xen_send_IPI_mask(cpu_online_mask, xen_vector);
+ put_online_cpus_atomic();
}
void xen_send_IPI_self(int vector)
@@ -572,20 +577,24 @@ void xen_send_IPI_mask_allbutself(const struct cpumask *mask,
if (!(num_online_cpus() > 1))
return;
+ get_online_cpus_atomic();
for_each_cpu_and(cpu, mask, cpu_online_mask) {
if (this_cpu == cpu)
continue;
xen_smp_send_call_function_single_ipi(cpu);
}
+ put_online_cpus_atomic();
}
void xen_send_IPI_allbutself(int vector)
{
int xen_vector = xen_map_vector(vector);
+ get_online_cpus_atomic();
if (xen_vector >= 0)
xen_send_IPI_mask_allbutself(cpu_online_mask, xen_vector);
+ put_online_cpus_atomic();
}
static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
^ permalink raw reply related
* [PATCH v5 30/45] alpha/smp: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:41 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Also, remove the non-ASCII character present in this file!
Cc: linux-alpha at vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/alpha/kernel/smp.c | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
diff --git a/arch/alpha/kernel/smp.c b/arch/alpha/kernel/smp.c
index 9603bc2..9213d5d 100644
--- a/arch/alpha/kernel/smp.c
+++ b/arch/alpha/kernel/smp.c
@@ -498,7 +498,6 @@ smp_cpus_done(unsigned int max_cpus)
((bogosum + 2500) / (5000/HZ)) % 100);
}
-
void
smp_percpu_timer_interrupt(struct pt_regs *regs)
{
@@ -682,7 +681,7 @@ ipi_flush_tlb_mm(void *x)
void
flush_tlb_mm(struct mm_struct *mm)
{
- preempt_disable();
+ get_online_cpus_atomic();
if (mm == current->active_mm) {
flush_tlb_current(mm);
@@ -694,7 +693,7 @@ flush_tlb_mm(struct mm_struct *mm)
if (mm->context[cpu])
mm->context[cpu] = 0;
}
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
}
@@ -703,7 +702,7 @@ flush_tlb_mm(struct mm_struct *mm)
printk(KERN_CRIT "flush_tlb_mm: timed out\n");
}
- preempt_enable();
+ put_online_cpus_atomic();
}
EXPORT_SYMBOL(flush_tlb_mm);
@@ -731,7 +730,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
struct flush_tlb_page_struct data;
struct mm_struct *mm = vma->vm_mm;
- preempt_disable();
+ get_online_cpus_atomic();
if (mm == current->active_mm) {
flush_tlb_current_page(mm, vma, addr);
@@ -743,7 +742,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
if (mm->context[cpu])
mm->context[cpu] = 0;
}
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
}
@@ -756,7 +755,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
printk(KERN_CRIT "flush_tlb_page: timed out\n");
}
- preempt_enable();
+ put_online_cpus_atomic();
}
EXPORT_SYMBOL(flush_tlb_page);
@@ -787,7 +786,7 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
if ((vma->vm_flags & VM_EXEC) == 0)
return;
- preempt_disable();
+ get_online_cpus_atomic();
if (mm == current->active_mm) {
__load_new_mm_context(mm);
@@ -799,7 +798,7 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
if (mm->context[cpu])
mm->context[cpu] = 0;
}
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
}
@@ -808,5 +807,5 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
printk(KERN_CRIT "flush_icache_page: timed out\n");
}
- preempt_enable();
+ put_online_cpus_atomic();
}
^ permalink raw reply related
* [PATCH v5 31/45] blackfin/smp: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:41 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Mike Frysinger <vapier@gentoo.org>
Cc: Bob Liu <lliubbo@gmail.com>
Cc: Steven Miao <realmz6@gmail.com>
Cc: uclinux-dist-devel at blackfin.uclinux.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/blackfin/mach-common/smp.c | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/arch/blackfin/mach-common/smp.c b/arch/blackfin/mach-common/smp.c
index bb61ae4..6cc6d7a 100644
--- a/arch/blackfin/mach-common/smp.c
+++ b/arch/blackfin/mach-common/smp.c
@@ -194,6 +194,7 @@ void send_ipi(const struct cpumask *cpumask, enum ipi_message_type msg)
struct ipi_data *bfin_ipi_data;
unsigned long flags;
+ get_online_cpus_atomic();
local_irq_save(flags);
smp_mb();
for_each_cpu(cpu, cpumask) {
@@ -205,6 +206,7 @@ void send_ipi(const struct cpumask *cpumask, enum ipi_message_type msg)
}
local_irq_restore(flags);
+ put_online_cpus_atomic();
}
void arch_send_call_function_single_ipi(int cpu)
@@ -238,13 +240,13 @@ void smp_send_stop(void)
{
cpumask_t callmap;
- preempt_disable();
+ get_online_cpus_atomic();
cpumask_copy(&callmap, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &callmap);
if (!cpumask_empty(&callmap))
send_ipi(&callmap, BFIN_IPI_CPU_STOP);
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
^ permalink raw reply related
* [PATCH v5 32/45] cris/smp: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:41 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Mikael Starvik <starvik@axis.com>
Cc: Jesper Nilsson <jesper.nilsson@axis.com>
Cc: linux-cris-kernel at axis.com
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/cris/arch-v32/kernel/smp.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/arch/cris/arch-v32/kernel/smp.c b/arch/cris/arch-v32/kernel/smp.c
index 04a16ed..644b358 100644
--- a/arch/cris/arch-v32/kernel/smp.c
+++ b/arch/cris/arch-v32/kernel/smp.c
@@ -15,6 +15,7 @@
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/cpumask.h>
+#include <linux/cpu.h>
#include <linux/interrupt.h>
#include <linux/module.h>
@@ -208,9 +209,12 @@ int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle)
void smp_send_reschedule(int cpu)
{
cpumask_t cpu_mask;
+
+ get_online_cpus_atomic();
cpumask_clear(&cpu_mask);
cpumask_set_cpu(cpu, &cpu_mask);
send_ipi(IPI_SCHEDULE, 0, cpu_mask);
+ put_online_cpus_atomic();
}
/* TLB flushing
@@ -224,6 +228,7 @@ void flush_tlb_common(struct mm_struct* mm, struct vm_area_struct* vma, unsigned
unsigned long flags;
cpumask_t cpu_mask;
+ get_online_cpus_atomic();
spin_lock_irqsave(&tlbstate_lock, flags);
cpu_mask = (mm == FLUSH_ALL ? cpu_all_mask : *mm_cpumask(mm));
cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
@@ -232,6 +237,7 @@ void flush_tlb_common(struct mm_struct* mm, struct vm_area_struct* vma, unsigned
flush_addr = addr;
send_ipi(IPI_FLUSH_TLB, 1, cpu_mask);
spin_unlock_irqrestore(&tlbstate_lock, flags);
+ put_online_cpus_atomic();
}
void flush_tlb_all(void)
@@ -312,6 +318,7 @@ int smp_call_function(void (*func)(void *info), void *info, int wait)
struct call_data_struct data;
int ret;
+ get_online_cpus_atomic();
cpumask_setall(&cpu_mask);
cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
@@ -325,6 +332,7 @@ int smp_call_function(void (*func)(void *info), void *info, int wait)
call_data = &data;
ret = send_ipi(IPI_CALL, wait, cpu_mask);
spin_unlock(&call_lock);
+ put_online_cpus_atomic();
return ret;
}
^ permalink raw reply related
* [PATCH v5 33/45] hexagon/smp: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:42 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Richard Kuo <rkuo@codeaurora.org>
Cc: linux-hexagon at vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/hexagon/kernel/smp.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/arch/hexagon/kernel/smp.c b/arch/hexagon/kernel/smp.c
index 8e095df..ec87de9 100644
--- a/arch/hexagon/kernel/smp.c
+++ b/arch/hexagon/kernel/smp.c
@@ -112,6 +112,7 @@ void send_ipi(const struct cpumask *cpumask, enum ipi_message_type msg)
unsigned long cpu;
unsigned long retval;
+ get_online_cpus_atomic();
local_irq_save(flags);
for_each_cpu(cpu, cpumask) {
@@ -128,6 +129,7 @@ void send_ipi(const struct cpumask *cpumask, enum ipi_message_type msg)
}
local_irq_restore(flags);
+ put_online_cpus_atomic();
}
static struct irqaction ipi_intdesc = {
@@ -241,9 +243,12 @@ void smp_send_reschedule(int cpu)
void smp_send_stop(void)
{
struct cpumask targets;
+
+ get_online_cpus_atomic();
cpumask_copy(&targets, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &targets);
send_ipi(&targets, IPI_CPU_STOP);
+ put_online_cpus_atomic();
}
void arch_send_call_function_single_ipi(int cpu)
^ permalink raw reply related
* [PATCH v5 34/45] ia64: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:42 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Tony Luck <tony.luck@intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: linux-ia64 at vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/ia64/kernel/irq_ia64.c | 13 +++++++++++++
arch/ia64/kernel/perfmon.c | 6 ++++++
arch/ia64/kernel/smp.c | 23 ++++++++++++++++-------
arch/ia64/mm/tlb.c | 6 ++++--
4 files changed, 39 insertions(+), 9 deletions(-)
diff --git a/arch/ia64/kernel/irq_ia64.c b/arch/ia64/kernel/irq_ia64.c
index 1034884..d0b4478 100644
--- a/arch/ia64/kernel/irq_ia64.c
+++ b/arch/ia64/kernel/irq_ia64.c
@@ -31,6 +31,7 @@
#include <linux/ratelimit.h>
#include <linux/acpi.h>
#include <linux/sched.h>
+#include <linux/cpu.h>
#include <asm/delay.h>
#include <asm/intrinsics.h>
@@ -190,9 +191,11 @@ static void clear_irq_vector(int irq)
{
unsigned long flags;
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
__clear_irq_vector(irq);
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
}
int
@@ -204,6 +207,7 @@ ia64_native_assign_irq_vector (int irq)
vector = -ENOSPC;
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
for_each_online_cpu(cpu) {
domain = vector_allocation_domain(cpu);
@@ -218,6 +222,7 @@ ia64_native_assign_irq_vector (int irq)
BUG_ON(__bind_irq_vector(irq, vector, domain));
out:
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
return vector;
}
@@ -302,9 +307,11 @@ int irq_prepare_move(int irq, int cpu)
unsigned long flags;
int ret;
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
ret = __irq_prepare_move(irq, cpu);
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
return ret;
}
@@ -320,11 +327,13 @@ void irq_complete_move(unsigned irq)
if (unlikely(cpu_isset(smp_processor_id(), cfg->old_domain)))
return;
+ get_online_cpus_atomic();
cpumask_and(&cleanup_mask, &cfg->old_domain, cpu_online_mask);
cfg->move_cleanup_count = cpus_weight(cleanup_mask);
for_each_cpu_mask(i, cleanup_mask)
platform_send_ipi(i, IA64_IRQ_MOVE_VECTOR, IA64_IPI_DM_INT, 0);
cfg->move_in_progress = 0;
+ put_online_cpus_atomic();
}
static irqreturn_t smp_irq_move_cleanup_interrupt(int irq, void *dev_id)
@@ -409,6 +418,8 @@ int create_irq(void)
cpumask_t domain = CPU_MASK_NONE;
irq = vector = -ENOSPC;
+
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
for_each_online_cpu(cpu) {
domain = vector_allocation_domain(cpu);
@@ -424,6 +435,8 @@ int create_irq(void)
BUG_ON(__bind_irq_vector(irq, vector, domain));
out:
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
+
if (irq >= 0)
dynamic_irq_init(irq);
return irq;
diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c
index ea39eba..6c6a029 100644
--- a/arch/ia64/kernel/perfmon.c
+++ b/arch/ia64/kernel/perfmon.c
@@ -34,6 +34,7 @@
#include <linux/poll.h>
#include <linux/vfs.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/pagemap.h>
#include <linux/mount.h>
#include <linux/bitops.h>
@@ -6485,6 +6486,7 @@ pfm_install_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
}
/* reserve our session */
+ get_online_cpus_atomic();
for_each_online_cpu(reserve_cpu) {
ret = pfm_reserve_session(NULL, 1, reserve_cpu);
if (ret) goto cleanup_reserve;
@@ -6500,6 +6502,7 @@ pfm_install_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
/* officially change to the alternate interrupt handler */
pfm_alt_intr_handler = hdl;
+ put_online_cpus_atomic();
spin_unlock(&pfm_alt_install_check);
return 0;
@@ -6512,6 +6515,7 @@ cleanup_reserve:
pfm_unreserve_session(NULL, 1, i);
}
+ put_online_cpus_atomic();
spin_unlock(&pfm_alt_install_check);
return ret;
@@ -6536,6 +6540,7 @@ pfm_remove_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
pfm_alt_intr_handler = NULL;
+ get_online_cpus_atomic();
ret = on_each_cpu(pfm_alt_restore_pmu_state, NULL, 1);
if (ret) {
DPRINT(("on_each_cpu() failed: %d\n", ret));
@@ -6545,6 +6550,7 @@ pfm_remove_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
pfm_unreserve_session(NULL, 1, i);
}
+ put_online_cpus_atomic();
spin_unlock(&pfm_alt_install_check);
return 0;
diff --git a/arch/ia64/kernel/smp.c b/arch/ia64/kernel/smp.c
index 9fcd4e6..d9a4636 100644
--- a/arch/ia64/kernel/smp.c
+++ b/arch/ia64/kernel/smp.c
@@ -24,6 +24,7 @@
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/kernel_stat.h>
#include <linux/mm.h>
#include <linux/cache.h>
@@ -154,12 +155,15 @@ send_IPI_single (int dest_cpu, int op)
static inline void
send_IPI_allbutself (int op)
{
- unsigned int i;
+ unsigned int i, cpu;
+ get_online_cpus_atomic();
+ cpu = smp_processor_id();
for_each_online_cpu(i) {
- if (i != smp_processor_id())
+ if (i != cpu)
send_IPI_single(i, op);
}
+ put_online_cpus_atomic();
}
/*
@@ -170,9 +174,11 @@ send_IPI_mask(const struct cpumask *mask, int op)
{
unsigned int cpu;
+ get_online_cpus_atomic();
for_each_cpu(cpu, mask) {
send_IPI_single(cpu, op);
}
+ put_online_cpus_atomic();
}
/*
@@ -183,9 +189,11 @@ send_IPI_all (int op)
{
int i;
+ get_online_cpus_atomic();
for_each_online_cpu(i) {
send_IPI_single(i, op);
}
+ put_online_cpus_atomic();
}
/*
@@ -259,7 +267,7 @@ smp_flush_tlb_cpumask(cpumask_t xcpumask)
cpumask_t cpumask = xcpumask;
int mycpu, cpu, flush_mycpu = 0;
- preempt_disable();
+ get_online_cpus_atomic();
mycpu = smp_processor_id();
for_each_cpu_mask(cpu, cpumask)
@@ -280,7 +288,7 @@ smp_flush_tlb_cpumask(cpumask_t xcpumask)
while(counts[cpu] == (local_tlb_flush_counts[cpu].count & 0xffff))
udelay(FLUSH_DELAY);
- preempt_enable();
+ put_online_cpus_atomic();
}
void
@@ -293,12 +301,13 @@ void
smp_flush_tlb_mm (struct mm_struct *mm)
{
cpumask_var_t cpus;
- preempt_disable();
+
+ get_online_cpus_atomic();
/* this happens for the common case of a single-threaded fork(): */
if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1))
{
local_finish_flush_tlb_mm(mm);
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
if (!alloc_cpumask_var(&cpus, GFP_ATOMIC)) {
@@ -313,7 +322,7 @@ smp_flush_tlb_mm (struct mm_struct *mm)
local_irq_disable();
local_finish_flush_tlb_mm(mm);
local_irq_enable();
- preempt_enable();
+ put_online_cpus_atomic();
}
void arch_send_call_function_single_ipi(int cpu)
diff --git a/arch/ia64/mm/tlb.c b/arch/ia64/mm/tlb.c
index ed61297..8f03b58 100644
--- a/arch/ia64/mm/tlb.c
+++ b/arch/ia64/mm/tlb.c
@@ -20,6 +20,7 @@
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/mm.h>
#include <linux/bootmem.h>
#include <linux/slab.h>
@@ -87,11 +88,12 @@ wrap_mmu_context (struct mm_struct *mm)
* can't call flush_tlb_all() here because of race condition
* with O(1) scheduler [EF]
*/
- cpu = get_cpu(); /* prevent preemption/migration */
+ get_online_cpus_atomic(); /* prevent preemption/migration */
+ cpu = smp_processor_id();
for_each_online_cpu(i)
if (i != cpu)
per_cpu(ia64_need_tlb_flush, i) = 1;
- put_cpu();
+ put_online_cpus_atomic();
local_flush_tlb_all();
}
^ permalink raw reply related
* [PATCH v5 35/45] m32r: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:42 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Hirokazu Takata <takata@linux-m32r.org>
Cc: linux-m32r at ml.linux-m32r.org
Cc: linux-m32r-ja at ml.linux-m32r.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/m32r/kernel/smp.c | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/arch/m32r/kernel/smp.c b/arch/m32r/kernel/smp.c
index ce7aea3..0dad4d7 100644
--- a/arch/m32r/kernel/smp.c
+++ b/arch/m32r/kernel/smp.c
@@ -151,7 +151,7 @@ void smp_flush_cache_all(void)
cpumask_t cpumask;
unsigned long *mask;
- preempt_disable();
+ get_online_cpus_atomic();
cpumask_copy(&cpumask, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &cpumask);
spin_lock(&flushcache_lock);
@@ -162,7 +162,7 @@ void smp_flush_cache_all(void)
while (flushcache_cpumask)
mb();
spin_unlock(&flushcache_lock);
- preempt_enable();
+ put_online_cpus_atomic();
}
void smp_flush_cache_all_interrupt(void)
@@ -250,7 +250,7 @@ void smp_flush_tlb_mm(struct mm_struct *mm)
unsigned long *mmc;
unsigned long flags;
- preempt_disable();
+ get_online_cpus_atomic();
cpu_id = smp_processor_id();
mmc = &mm->context[cpu_id];
cpumask_copy(&cpu_mask, mm_cpumask(mm));
@@ -268,7 +268,7 @@ void smp_flush_tlb_mm(struct mm_struct *mm)
if (!cpumask_empty(&cpu_mask))
flush_tlb_others(cpu_mask, mm, NULL, FLUSH_ALL);
- preempt_enable();
+ put_online_cpus_atomic();
}
/*==========================================================================*
@@ -715,10 +715,12 @@ static void send_IPI_allbutself(int ipi_num, int try)
{
cpumask_t cpumask;
+ get_online_cpus_atomic();
cpumask_copy(&cpumask, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &cpumask);
send_IPI_mask(&cpumask, ipi_num, try);
+ put_online_cpus_atomic();
}
/*==========================================================================*
@@ -750,6 +752,7 @@ static void send_IPI_mask(const struct cpumask *cpumask, int ipi_num, int try)
if (num_cpus <= 1) /* NO MP */
return;
+ get_online_cpus_atomic();
cpumask_and(&tmp, cpumask, cpu_online_mask);
BUG_ON(!cpumask_equal(cpumask, &tmp));
@@ -760,6 +763,7 @@ static void send_IPI_mask(const struct cpumask *cpumask, int ipi_num, int try)
}
send_IPI_mask_phys(&physid_mask, ipi_num, try);
+ put_online_cpus_atomic();
}
/*==========================================================================*
^ permalink raw reply related
* [PATCH v5 36/45] MIPS: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:42 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs fom
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: David Daney <david.daney@cavium.com>
Cc: linux-mips at linux-mips.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/mips/kernel/cevt-smtc.c | 8 ++++++++
arch/mips/kernel/smp.c | 16 ++++++++--------
arch/mips/kernel/smtc.c | 3 +++
arch/mips/mm/c-octeon.c | 4 ++--
4 files changed, 21 insertions(+), 10 deletions(-)
diff --git a/arch/mips/kernel/cevt-smtc.c b/arch/mips/kernel/cevt-smtc.c
index 2e72d30..6fb311b 100644
--- a/arch/mips/kernel/cevt-smtc.c
+++ b/arch/mips/kernel/cevt-smtc.c
@@ -11,6 +11,7 @@
#include <linux/interrupt.h>
#include <linux/percpu.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/irq.h>
#include <asm/smtc_ipi.h>
@@ -84,6 +85,8 @@ static int mips_next_event(unsigned long delta,
unsigned long nextcomp = 0L;
int vpe = current_cpu_data.vpe_id;
int cpu = smp_processor_id();
+
+ get_online_cpus_atomic();
local_irq_save(flags);
mtflags = dmt();
@@ -164,6 +167,7 @@ static int mips_next_event(unsigned long delta,
}
emt(mtflags);
local_irq_restore(flags);
+ put_online_cpus_atomic();
return 0;
}
@@ -180,6 +184,7 @@ void smtc_distribute_timer(int vpe)
repeat:
nextstamp = 0L;
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
/*
* Find virtual CPUs within the current VPE who have
@@ -221,6 +226,9 @@ repeat:
}
}
+
+ put_online_cpus_atomic()
+
/* Reprogram for interrupt at next soonest timestamp for VPE */
if (ISVALID(nextstamp)) {
write_c0_compare(nextstamp);
diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c
index 66bf4e2..3828afa 100644
--- a/arch/mips/kernel/smp.c
+++ b/arch/mips/kernel/smp.c
@@ -248,12 +248,12 @@ static inline void smp_on_other_tlbs(void (*func) (void *info), void *info)
static inline void smp_on_each_tlb(void (*func) (void *info), void *info)
{
- preempt_disable();
+ get_online_cpus_atomic();
smp_on_other_tlbs(func, info);
func(info);
- preempt_enable();
+ put_online_cpus_atomic();
}
/*
@@ -271,7 +271,7 @@ static inline void smp_on_each_tlb(void (*func) (void *info), void *info)
void flush_tlb_mm(struct mm_struct *mm)
{
- preempt_disable();
+ get_online_cpus_atomic();
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
smp_on_other_tlbs(flush_tlb_mm_ipi, mm);
@@ -285,7 +285,7 @@ void flush_tlb_mm(struct mm_struct *mm)
}
local_flush_tlb_mm(mm);
- preempt_enable();
+ put_online_cpus_atomic();
}
struct flush_tlb_data {
@@ -305,7 +305,7 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned l
{
struct mm_struct *mm = vma->vm_mm;
- preempt_disable();
+ get_online_cpus_atomic();
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
struct flush_tlb_data fd = {
.vma = vma,
@@ -323,7 +323,7 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned l
}
}
local_flush_tlb_range(vma, start, end);
- preempt_enable();
+ put_online_cpus_atomic();
}
static void flush_tlb_kernel_range_ipi(void *info)
@@ -352,7 +352,7 @@ static void flush_tlb_page_ipi(void *info)
void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
{
- preempt_disable();
+ get_online_cpus_atomic();
if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) {
struct flush_tlb_data fd = {
.vma = vma,
@@ -369,7 +369,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
}
}
local_flush_tlb_page(vma, page);
- preempt_enable();
+ put_online_cpus_atomic();
}
static void flush_tlb_one_ipi(void *info)
diff --git a/arch/mips/kernel/smtc.c b/arch/mips/kernel/smtc.c
index 1d47843..caf081e 100644
--- a/arch/mips/kernel/smtc.c
+++ b/arch/mips/kernel/smtc.c
@@ -22,6 +22,7 @@
#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/cpumask.h>
+#include <linux/cpu.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/module.h>
@@ -1143,6 +1144,7 @@ static irqreturn_t ipi_interrupt(int irq, void *dev_idm)
* for the current TC, so we ought not to have to do it explicitly here.
*/
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
if (cpu_data[cpu].vpe_id != my_vpe)
continue;
@@ -1179,6 +1181,7 @@ static irqreturn_t ipi_interrupt(int irq, void *dev_idm)
}
}
}
+ put_online_cpus_atomic();
return IRQ_HANDLED;
}
diff --git a/arch/mips/mm/c-octeon.c b/arch/mips/mm/c-octeon.c
index 6ec04da..cd2c1ce 100644
--- a/arch/mips/mm/c-octeon.c
+++ b/arch/mips/mm/c-octeon.c
@@ -73,7 +73,7 @@ static void octeon_flush_icache_all_cores(struct vm_area_struct *vma)
mb();
octeon_local_flush_icache();
#ifdef CONFIG_SMP
- preempt_disable();
+ get_online_cpus_atomic();
cpu = smp_processor_id();
/*
@@ -88,7 +88,7 @@ static void octeon_flush_icache_all_cores(struct vm_area_struct *vma)
for_each_cpu(cpu, &mask)
octeon_send_ipi_single(cpu, SMP_ICACHE_FLUSH);
- preempt_enable();
+ put_online_cpus_atomic();
#endif
}
^ permalink raw reply related
* [PATCH v5 37/45] mn10300: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-01-22 7:43 UTC (permalink / raw)
To: linux-arm-kernel
In-Reply-To: <20130122073210.13822.50434.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: David Howells <dhowells@redhat.com>
Cc: Koichi Yasutake <yasutake.koichi@jp.panasonic.com>
Cc: linux-am33-list at redhat.com
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/mn10300/kernel/smp.c | 2 ++
arch/mn10300/mm/cache-smp.c | 5 +++++
arch/mn10300/mm/tlb-smp.c | 15 +++++++++------
3 files changed, 16 insertions(+), 6 deletions(-)
diff --git a/arch/mn10300/kernel/smp.c b/arch/mn10300/kernel/smp.c
index 5d7e152..9dfa172 100644
--- a/arch/mn10300/kernel/smp.c
+++ b/arch/mn10300/kernel/smp.c
@@ -349,9 +349,11 @@ void send_IPI_allbutself(int irq)
{
cpumask_t cpumask;
+ get_online_cpus_atomic();
cpumask_copy(&cpumask, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &cpumask);
send_IPI_mask(&cpumask, irq);
+ put_online_cpus_atomic();
}
void arch_send_call_function_ipi_mask(const struct cpumask *mask)
diff --git a/arch/mn10300/mm/cache-smp.c b/arch/mn10300/mm/cache-smp.c
index 2d23b9e..47ca1c9 100644
--- a/arch/mn10300/mm/cache-smp.c
+++ b/arch/mn10300/mm/cache-smp.c
@@ -13,6 +13,7 @@
#include <linux/mman.h>
#include <linux/threads.h>
#include <linux/interrupt.h>
+#include <linux/cpu.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/processor.h>
@@ -94,6 +95,8 @@ void smp_cache_call(unsigned long opr_mask,
smp_cache_mask = opr_mask;
smp_cache_start = start;
smp_cache_end = end;
+
+ get_online_cpus_atomic();
cpumask_copy(&smp_cache_ipi_map, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &smp_cache_ipi_map);
@@ -102,4 +105,6 @@ void smp_cache_call(unsigned long opr_mask,
while (!cpumask_empty(&smp_cache_ipi_map))
/* nothing. lockup detection does not belong here */
mb();
+
+ put_online_cpus_atomic();
}
diff --git a/arch/mn10300/mm/tlb-smp.c b/arch/mn10300/mm/tlb-smp.c
index 3e57faf..d47304d 100644
--- a/arch/mn10300/mm/tlb-smp.c
+++ b/arch/mn10300/mm/tlb-smp.c
@@ -23,6 +23,7 @@
#include <linux/sched.h>
#include <linux/profile.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <asm/tlbflush.h>
#include <asm/bitops.h>
#include <asm/processor.h>
@@ -105,6 +106,7 @@ static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
BUG_ON(cpumask_empty(&cpumask));
BUG_ON(cpumask_test_cpu(smp_processor_id(), &cpumask));
+ get_online_cpus_atomic();
cpumask_and(&tmp, &cpumask, cpu_online_mask);
BUG_ON(!cpumask_equal(&cpumask, &tmp));
@@ -134,6 +136,7 @@ static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
flush_mm = NULL;
flush_va = 0;
spin_unlock(&tlbstate_lock);
+ put_online_cpus_atomic();
}
/**
@@ -144,7 +147,7 @@ void flush_tlb_mm(struct mm_struct *mm)
{
cpumask_t cpu_mask;
- preempt_disable();
+ get_online_cpus_atomic();
cpumask_copy(&cpu_mask, mm_cpumask(mm));
cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
@@ -152,7 +155,7 @@ void flush_tlb_mm(struct mm_struct *mm)
if (!cpumask_empty(&cpu_mask))
flush_tlb_others(cpu_mask, mm, FLUSH_ALL);
- preempt_enable();
+ put_online_cpus_atomic();
}
/**
@@ -163,7 +166,7 @@ void flush_tlb_current_task(void)
struct mm_struct *mm = current->mm;
cpumask_t cpu_mask;
- preempt_disable();
+ get_online_cpus_atomic();
cpumask_copy(&cpu_mask, mm_cpumask(mm));
cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
@@ -171,7 +174,7 @@ void flush_tlb_current_task(void)
if (!cpumask_empty(&cpu_mask))
flush_tlb_others(cpu_mask, mm, FLUSH_ALL);
- preempt_enable();
+ put_online_cpus_atomic();
}
/**
@@ -184,7 +187,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
struct mm_struct *mm = vma->vm_mm;
cpumask_t cpu_mask;
- preempt_disable();
+ get_online_cpus_atomic();
cpumask_copy(&cpu_mask, mm_cpumask(mm));
cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
@@ -192,7 +195,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
if (!cpumask_empty(&cpu_mask))
flush_tlb_others(cpu_mask, mm, va);
- preempt_enable();
+ put_online_cpus_atomic();
}
/**
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox