* [PATCH v2 19/45] irq: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:28 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, zhong, linux-pm, fweisbec, linux-kernel,
rostedt, xiaoguangrong, sbw, wangyun, Srivatsa S. Bhat, netdev,
Thomas Gleixner, linuxppc-dev
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
kernel/irq/manage.c | 7 +++++++
kernel/irq/proc.c | 3 +++
2 files changed, 10 insertions(+)
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index e16caa8..4d89f19 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -18,6 +18,7 @@
#include <linux/sched.h>
#include <linux/sched/rt.h>
#include <linux/task_work.h>
+#include <linux/cpu.h>
#include "internals.h"
@@ -202,9 +203,11 @@ int irq_set_affinity(unsigned int irq, const struct cpumask *mask)
if (!desc)
return -EINVAL;
+ get_online_cpus_atomic();
raw_spin_lock_irqsave(&desc->lock, flags);
ret = __irq_set_affinity_locked(irq_desc_get_irq_data(desc), mask);
raw_spin_unlock_irqrestore(&desc->lock, flags);
+ put_online_cpus_atomic();
return ret;
}
@@ -343,9 +346,11 @@ int irq_select_affinity_usr(unsigned int irq, struct cpumask *mask)
unsigned long flags;
int ret;
+ get_online_cpus_atomic();
raw_spin_lock_irqsave(&desc->lock, flags);
ret = setup_affinity(irq, desc, mask);
raw_spin_unlock_irqrestore(&desc->lock, flags);
+ put_online_cpus_atomic();
return ret;
}
@@ -1128,7 +1133,9 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
}
/* Set default affinity mask once everything is setup */
+ get_online_cpus_atomic();
setup_affinity(irq, desc, mask);
+ put_online_cpus_atomic();
} else if (new->flags & IRQF_TRIGGER_MASK) {
unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK;
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c
index 19ed5c4..47f9a74 100644
--- a/kernel/irq/proc.c
+++ b/kernel/irq/proc.c
@@ -7,6 +7,7 @@
*/
#include <linux/irq.h>
+#include <linux/cpu.h>
#include <linux/gfp.h>
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
@@ -441,6 +442,7 @@ int show_interrupts(struct seq_file *p, void *v)
if (!desc)
return 0;
+ get_online_cpus_atomic();
raw_spin_lock_irqsave(&desc->lock, flags);
for_each_online_cpu(j)
any_count |= kstat_irqs_cpu(i, j);
@@ -477,6 +479,7 @@ int show_interrupts(struct seq_file *p, void *v)
seq_putc(p, '\n');
out:
raw_spin_unlock_irqrestore(&desc->lock, flags);
+ put_online_cpus_atomic();
return 0;
}
#endif
^ permalink raw reply related
* [PATCH v2 20/45] net: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:29 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, Alexander Duyck, nikunj, zhong, linux-pm, fweisbec,
linux-kernel, rostedt, xiaoguangrong, sbw, Eric Dumazet, wangyun,
Srivatsa S. Bhat, netdev, Ben Hutchings, Cong Wang, linuxppc-dev,
David S. Miller
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Alexander Duyck <alexander.h.duyck@intel.com>
Cc: Cong Wang <xiyou.wangcong@gmail.com>
Cc: Ben Hutchings <bhutchings@solarflare.com>
Cc: netdev@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
net/core/dev.c | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/net/core/dev.c b/net/core/dev.c
index fc1e289..90519e9 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -3141,7 +3141,7 @@ int netif_rx(struct sk_buff *skb)
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu;
- preempt_disable();
+ get_online_cpus_atomic();
rcu_read_lock();
cpu = get_rps_cpu(skb->dev, skb, &rflow);
@@ -3151,7 +3151,7 @@ int netif_rx(struct sk_buff *skb)
ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
rcu_read_unlock();
- preempt_enable();
+ put_online_cpus_atomic();
} else
#endif
{
@@ -3570,6 +3570,7 @@ int netif_receive_skb(struct sk_buff *skb)
struct rps_dev_flow voidflow, *rflow = &voidflow;
int cpu, ret;
+ get_online_cpus_atomic();
rcu_read_lock();
cpu = get_rps_cpu(skb->dev, skb, &rflow);
@@ -3577,9 +3578,11 @@ int netif_receive_skb(struct sk_buff *skb)
if (cpu >= 0) {
ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
rcu_read_unlock();
+ put_online_cpus_atomic();
return ret;
}
rcu_read_unlock();
+ put_online_cpus_atomic();
}
#endif
return __netif_receive_skb(skb);
@@ -3957,6 +3960,7 @@ static void net_rps_action_and_irq_enable(struct softnet_data *sd)
local_irq_enable();
/* Send pending IPI's to kick RPS processing on remote cpus. */
+ get_online_cpus_atomic();
while (remsd) {
struct softnet_data *next = remsd->rps_ipi_next;
@@ -3965,6 +3969,7 @@ static void net_rps_action_and_irq_enable(struct softnet_data *sd)
&remsd->csd, 0);
remsd = next;
}
+ put_online_cpus_atomic();
} else
#endif
local_irq_enable();
^ permalink raw reply related
* [PATCH v2 21/45] block: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:29 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, Jens Axboe, nikunj, zhong, linux-pm, fweisbec,
linux-kernel, rostedt, xiaoguangrong, sbw, wangyun,
Srivatsa S. Bhat, netdev, linuxppc-dev
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
block/blk-softirq.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/block/blk-softirq.c b/block/blk-softirq.c
index 467c8de..bbab3d3 100644
--- a/block/blk-softirq.c
+++ b/block/blk-softirq.c
@@ -58,6 +58,7 @@ static void trigger_softirq(void *data)
*/
static int raise_blk_irq(int cpu, struct request *rq)
{
+ get_online_cpus_atomic();
if (cpu_online(cpu)) {
struct call_single_data *data = &rq->csd;
@@ -66,8 +67,10 @@ static int raise_blk_irq(int cpu, struct request *rq)
data->flags = 0;
__smp_call_function_single(cpu, data, 0);
+ put_online_cpus_atomic();
return 0;
}
+ put_online_cpus_atomic();
return 1;
}
^ permalink raw reply related
* [PATCH v2 22/45] percpu_counter: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:29 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, zhong, linux-pm, fweisbec, linux-kernel,
rostedt, xiaoguangrong, sbw, wangyun, Srivatsa S. Bhat, netdev,
Tejun Heo, linuxppc-dev, Al Viro
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Tejun Heo <tj@kernel.org>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
lib/percpu_counter.c | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c
index ba6085d..f5e718d 100644
--- a/lib/percpu_counter.c
+++ b/lib/percpu_counter.c
@@ -98,6 +98,15 @@ s64 __percpu_counter_sum(struct percpu_counter *fbc)
s64 ret;
int cpu;
+ /*
+ * The calls to get/put_online_cpus_atomic() is strictly not
+ * necessary, since CPU hotplug is explicitly handled via the
+ * hotplug callback which synchronizes through fbc->lock.
+ * But we add them here anyway to make it easier for the debug
+ * code under CONFIG_DEBUG_HOTPLUG_CPU to validate the correctness
+ * of hotplug synchronization.
+ */
+ get_online_cpus_atomic();
raw_spin_lock(&fbc->lock);
ret = fbc->count;
for_each_online_cpu(cpu) {
@@ -105,6 +114,7 @@ s64 __percpu_counter_sum(struct percpu_counter *fbc)
ret += *pcount;
}
raw_spin_unlock(&fbc->lock);
+ put_online_cpus_atomic();
return ret;
}
EXPORT_SYMBOL(__percpu_counter_sum);
^ permalink raw reply related
* [PATCH v2 23/45] infiniband: ehca: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:29 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, Roland Dreier, nikunj, zhong, linux-pm, linux-rdma,
fweisbec, linux-kernel, rostedt, xiaoguangrong, sbw,
Christoph Raisch, Hoang-Nam Nguyen, wangyun, Srivatsa S. Bhat,
netdev, Sean Hefty, linuxppc-dev, Hal Rosenstock
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Hoang-Nam Nguyen <hnguyen@de.ibm.com>
Cc: Christoph Raisch <raisch@de.ibm.com>
Cc: Roland Dreier <roland@kernel.org>
Cc: Sean Hefty <sean.hefty@intel.com>
Cc: Hal Rosenstock <hal.rosenstock@gmail.com>
Cc: linux-rdma@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
drivers/infiniband/hw/ehca/ehca_irq.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/drivers/infiniband/hw/ehca/ehca_irq.c b/drivers/infiniband/hw/ehca/ehca_irq.c
index 8615d7c..ace901e 100644
--- a/drivers/infiniband/hw/ehca/ehca_irq.c
+++ b/drivers/infiniband/hw/ehca/ehca_irq.c
@@ -43,6 +43,7 @@
#include <linux/slab.h>
#include <linux/smpboot.h>
+#include <linux/cpu.h>
#include "ehca_classes.h"
#include "ehca_irq.h"
@@ -703,6 +704,7 @@ static void queue_comp_task(struct ehca_cq *__cq)
int cq_jobs;
unsigned long flags;
+ get_online_cpus_atomic();
cpu_id = find_next_online_cpu(pool);
BUG_ON(!cpu_online(cpu_id));
@@ -720,6 +722,7 @@ static void queue_comp_task(struct ehca_cq *__cq)
BUG_ON(!cct || !thread);
}
__queue_comp_task(__cq, cct, thread);
+ put_online_cpus_atomic();
}
static void run_comp_task(struct ehca_cpu_comp_task *cct)
@@ -759,6 +762,7 @@ static void comp_task_park(unsigned int cpu)
list_splice_init(&cct->cq_list, &list);
spin_unlock_irq(&cct->task_lock);
+ get_online_cpus_atomic();
cpu = find_next_online_cpu(pool);
target = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
thread = *per_cpu_ptr(pool->cpu_comp_threads, cpu);
@@ -768,6 +772,7 @@ static void comp_task_park(unsigned int cpu)
__queue_comp_task(cq, target, thread);
}
spin_unlock_irq(&target->task_lock);
+ put_online_cpus_atomic();
}
static void comp_task_stop(unsigned int cpu, bool online)
^ permalink raw reply related
* [PATCH v2 24/45] [SCSI] fcoe: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:29 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, James E.J. Bottomley, nikunj, zhong, linux-pm,
fweisbec, linux-kernel, rostedt, xiaoguangrong, linux-scsi, sbw,
wangyun, Srivatsa S. Bhat, netdev, Robert Love, linuxppc-dev,
devel
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Robert Love <robert.w.love@intel.com>
Cc: "James E.J. Bottomley" <JBottomley@parallels.com>
Cc: devel@open-fcoe.org
Cc: linux-scsi@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
drivers/scsi/fcoe/fcoe.c | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/drivers/scsi/fcoe/fcoe.c b/drivers/scsi/fcoe/fcoe.c
index 292b24f..a107d3c 100644
--- a/drivers/scsi/fcoe/fcoe.c
+++ b/drivers/scsi/fcoe/fcoe.c
@@ -1484,6 +1484,7 @@ static int fcoe_rcv(struct sk_buff *skb, struct net_device *netdev,
* was originated, otherwise select cpu using rx exchange id
* or fcoe_select_cpu().
*/
+ get_online_cpus_atomic();
if (ntoh24(fh->fh_f_ctl) & FC_FC_EX_CTX)
cpu = ntohs(fh->fh_ox_id) & fc_cpu_mask;
else {
@@ -1493,8 +1494,10 @@ static int fcoe_rcv(struct sk_buff *skb, struct net_device *netdev,
cpu = ntohs(fh->fh_rx_id) & fc_cpu_mask;
}
- if (cpu >= nr_cpu_ids)
+ if (cpu >= nr_cpu_ids) {
+ put_online_cpus_atomic();
goto err;
+ }
fps = &per_cpu(fcoe_percpu, cpu);
spin_lock(&fps->fcoe_rx_list.lock);
@@ -1514,6 +1517,7 @@ static int fcoe_rcv(struct sk_buff *skb, struct net_device *netdev,
spin_lock(&fps->fcoe_rx_list.lock);
if (!fps->thread) {
spin_unlock(&fps->fcoe_rx_list.lock);
+ put_online_cpus_atomic();
goto err;
}
}
@@ -1535,6 +1539,7 @@ static int fcoe_rcv(struct sk_buff *skb, struct net_device *netdev,
if (fps->thread->state == TASK_INTERRUPTIBLE)
wake_up_process(fps->thread);
spin_unlock(&fps->fcoe_rx_list.lock);
+ put_online_cpus_atomic();
return 0;
err:
^ permalink raw reply related
* [PATCH v2 25/45] staging/octeon: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:30 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, devel, nikunj, zhong, linux-pm, fweisbec, David Daney,
linux-kernel, rostedt, xiaoguangrong, sbw, Greg Kroah-Hartman,
wangyun, Srivatsa S. Bhat, netdev, linuxppc-dev
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: devel@driverdev.osuosl.org
Acked-by: David Daney <david.daney@cavium.com>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
drivers/staging/octeon/ethernet-rx.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/drivers/staging/octeon/ethernet-rx.c b/drivers/staging/octeon/ethernet-rx.c
index 34afc16..8588b4d 100644
--- a/drivers/staging/octeon/ethernet-rx.c
+++ b/drivers/staging/octeon/ethernet-rx.c
@@ -36,6 +36,7 @@
#include <linux/prefetch.h>
#include <linux/ratelimit.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/interrupt.h>
#include <net/dst.h>
#ifdef CONFIG_XFRM
@@ -97,6 +98,7 @@ static void cvm_oct_enable_one_cpu(void)
return;
/* ... if a CPU is available, Turn on NAPI polling for that CPU. */
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
if (!cpu_test_and_set(cpu, core_state.cpu_state)) {
v = smp_call_function_single(cpu, cvm_oct_enable_napi,
@@ -106,6 +108,7 @@ static void cvm_oct_enable_one_cpu(void)
break;
}
}
+ put_online_cpus_atomic();
}
static void cvm_oct_no_more_work(void)
^ permalink raw reply related
* [PATCH v2 26/45] x86: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:30 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: fweisbec, Sebastian Andrzej Siewior, Jan Beulich, H. Peter Anvin,
linux-arch, Joonsoo Kim, Joerg Roedel, x86, xiaoguangrong,
Ingo Molnar, wangyun, nikunj, linux-pm, rostedt,
Konrad Rzeszutek Wilk, Borislav Petkov, Thomas Gleixner,
linux-edac, Tony Luck, zhong, netdev, linux-kernel, sbw,
Srivatsa S. Bhat, linuxppc-dev
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
Cc: Joerg Roedel <joro@8bytes.org>
Cc: Jan Beulich <jbeulich@suse.com>
Cc: Joonsoo Kim <js1304@gmail.com>
Cc: linux-edac@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/x86/kernel/apic/io_apic.c | 21 ++++++++++++++++++---
arch/x86/kernel/cpu/mcheck/therm_throt.c | 4 ++--
arch/x86/mm/tlb.c | 14 +++++++-------
3 files changed, 27 insertions(+), 12 deletions(-)
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 9ed796c..4c71c1e 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -25,6 +25,7 @@
#include <linux/init.h>
#include <linux/delay.h>
#include <linux/sched.h>
+#include <linux/cpu.h>
#include <linux/pci.h>
#include <linux/mc146818rtc.h>
#include <linux/compiler.h>
@@ -1169,9 +1170,11 @@ int assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
int err;
unsigned long flags;
+ get_online_cpus_atomic();
raw_spin_lock_irqsave(&vector_lock, flags);
err = __assign_irq_vector(irq, cfg, mask);
raw_spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
return err;
}
@@ -1757,13 +1760,13 @@ __apicdebuginit(void) print_local_APICs(int maxcpu)
if (!maxcpu)
return;
- preempt_disable();
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
if (cpu >= maxcpu)
break;
smp_call_function_single(cpu, print_local_APIC, NULL, 1);
}
- preempt_enable();
+ put_online_cpus_atomic();
}
__apicdebuginit(void) print_PIC(void)
@@ -2153,10 +2156,12 @@ static int ioapic_retrigger_irq(struct irq_data *data)
unsigned long flags;
int cpu;
+ get_online_cpus_atomic();
raw_spin_lock_irqsave(&vector_lock, flags);
cpu = cpumask_first_and(cfg->domain, cpu_online_mask);
apic->send_IPI_mask(cpumask_of(cpu), cfg->vector);
raw_spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
return 1;
}
@@ -2175,6 +2180,7 @@ void send_cleanup_vector(struct irq_cfg *cfg)
{
cpumask_var_t cleanup_mask;
+ get_online_cpus_atomic();
if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
unsigned int i;
for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
@@ -2185,6 +2191,7 @@ void send_cleanup_vector(struct irq_cfg *cfg)
free_cpumask_var(cleanup_mask);
}
cfg->move_in_progress = 0;
+ put_online_cpus_atomic();
}
asmlinkage void smp_irq_move_cleanup_interrupt(void)
@@ -2939,11 +2946,13 @@ unsigned int __create_irqs(unsigned int from, unsigned int count, int node)
goto out_irqs;
}
+ get_online_cpus_atomic();
raw_spin_lock_irqsave(&vector_lock, flags);
for (i = 0; i < count; i++)
if (__assign_irq_vector(irq + i, cfg[i], apic->target_cpus()))
goto out_vecs;
raw_spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
for (i = 0; i < count; i++) {
irq_set_chip_data(irq + i, cfg[i]);
@@ -2957,6 +2966,7 @@ out_vecs:
for (i--; i >= 0; i--)
__clear_irq_vector(irq + i, cfg[i]);
raw_spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
out_irqs:
for (i = 0; i < count; i++)
free_irq_at(irq + i, cfg[i]);
@@ -2994,9 +3004,11 @@ void destroy_irq(unsigned int irq)
free_remapped_irq(irq);
+ get_online_cpus_atomic();
raw_spin_lock_irqsave(&vector_lock, flags);
__clear_irq_vector(irq, cfg);
raw_spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
free_irq_at(irq, cfg);
}
@@ -3365,8 +3377,11 @@ io_apic_setup_irq_pin(unsigned int irq, int node, struct io_apic_irq_attr *attr)
if (!cfg)
return -EINVAL;
ret = __add_pin_to_irq_node(cfg, node, attr->ioapic, attr->ioapic_pin);
- if (!ret)
+ if (!ret) {
+ get_online_cpus_atomic();
setup_ioapic_irq(irq, cfg, attr);
+ put_online_cpus_atomic();
+ }
return ret;
}
diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c
index 2f3a799..3eea984 100644
--- a/arch/x86/kernel/cpu/mcheck/therm_throt.c
+++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c
@@ -83,13 +83,13 @@ static ssize_t therm_throt_device_show_##event##_##name( \
unsigned int cpu = dev->id; \
ssize_t ret; \
\
- preempt_disable(); /* CPU hotplug */ \
+ get_online_cpus_atomic(); /* CPU hotplug */ \
if (cpu_online(cpu)) { \
ret = sprintf(buf, "%lu\n", \
per_cpu(thermal_state, cpu).event.name); \
} else \
ret = 0; \
- preempt_enable(); \
+ put_online_cpus_atomic(); \
\
return ret; \
}
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 282375f..8126374 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -147,12 +147,12 @@ void flush_tlb_current_task(void)
{
struct mm_struct *mm = current->mm;
- preempt_disable();
+ get_online_cpus_atomic();
local_flush_tlb();
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
- preempt_enable();
+ put_online_cpus_atomic();
}
/*
@@ -187,7 +187,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
unsigned long addr;
unsigned act_entries, tlb_entries = 0;
- preempt_disable();
+ get_online_cpus_atomic();
if (current->active_mm != mm)
goto flush_all;
@@ -225,21 +225,21 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
if (cpumask_any_but(mm_cpumask(mm),
smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, start, end);
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
flush_all:
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
- preempt_enable();
+ put_online_cpus_atomic();
}
void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
{
struct mm_struct *mm = vma->vm_mm;
- preempt_disable();
+ get_online_cpus_atomic();
if (current->active_mm == mm) {
if (current->mm)
@@ -251,7 +251,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, start, 0UL);
- preempt_enable();
+ put_online_cpus_atomic();
}
static void do_flush_tlb_all(void *info)
^ permalink raw reply related
* [PATCH v2 27/45] perf/x86: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:30 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, x86, nikunj, zhong, linux-pm, fweisbec,
H. Peter Anvin, linux-kernel, rostedt, xiaoguangrong, sbw,
Ingo Molnar, Paul Mackerras, wangyun, Srivatsa S. Bhat, netdev,
Thomas Gleixner, linuxppc-dev, Arnaldo Carvalho de Melo,
Peter Zijlstra
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
The CPU_DYING notifier modifies the per-cpu pointer pmu->box, and this can
race with functions such as uncore_pmu_to_box() and uncore_pci_remove() when
we remove stop_machine() from the CPU offline path. So protect them using
get/put_online_cpus_atomic().
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Arnaldo Carvalho de Melo <acme@ghostprotocols.net>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/x86/kernel/cpu/perf_event_intel_uncore.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/arch/x86/kernel/cpu/perf_event_intel_uncore.c b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
index 9dd9975..7c2a064 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_uncore.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
@@ -1,3 +1,4 @@
+#include <linux/cpu.h>
#include "perf_event_intel_uncore.h"
static struct intel_uncore_type *empty_uncore[] = { NULL, };
@@ -2630,6 +2631,7 @@ uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
if (box)
return box;
+ get_online_cpus_atomic();
raw_spin_lock(&uncore_box_lock);
list_for_each_entry(box, &pmu->box_list, list) {
if (box->phys_id == topology_physical_package_id(cpu)) {
@@ -2639,6 +2641,7 @@ uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
}
}
raw_spin_unlock(&uncore_box_lock);
+ put_online_cpus_atomic();
return *per_cpu_ptr(pmu->box, cpu);
}
@@ -3229,6 +3232,7 @@ static void uncore_pci_remove(struct pci_dev *pdev)
list_del(&box->list);
raw_spin_unlock(&uncore_box_lock);
+ get_online_cpus_atomic();
for_each_possible_cpu(cpu) {
if (*per_cpu_ptr(pmu->box, cpu) == box) {
*per_cpu_ptr(pmu->box, cpu) = NULL;
@@ -3237,6 +3241,8 @@ static void uncore_pci_remove(struct pci_dev *pdev)
}
WARN_ON_ONCE(atomic_read(&box->refcnt) != 1);
+ put_online_cpus_atomic();
+
kfree(box);
}
^ permalink raw reply related
* [PATCH v2 28/45] KVM: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:30 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, kvm, Gleb Natapov, nikunj, zhong, linux-pm, fweisbec,
linux-kernel, rostedt, xiaoguangrong, sbw, wangyun,
Srivatsa S. Bhat, netdev, Paolo Bonzini, linuxppc-dev
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Gleb Natapov <gleb@redhat.com>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: kvm@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
virt/kvm/kvm_main.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 302681c..5bbfa30 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -174,7 +174,7 @@ static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
zalloc_cpumask_var(&cpus, GFP_ATOMIC);
- me = get_cpu();
+ me = get_online_cpus_atomic();
kvm_for_each_vcpu(i, vcpu, kvm) {
kvm_make_request(req, vcpu);
cpu = vcpu->cpu;
@@ -192,7 +192,7 @@ static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
smp_call_function_many(cpus, ack_flush, NULL, 1);
else
called = false;
- put_cpu();
+ put_online_cpus_atomic();
free_cpumask_var(cpus);
return called;
}
@@ -1707,11 +1707,11 @@ void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
++vcpu->stat.halt_wakeup;
}
- me = get_cpu();
+ me = get_online_cpus_atomic();
if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
if (kvm_arch_vcpu_should_kick(vcpu))
smp_send_reschedule(cpu);
- put_cpu();
+ put_online_cpus_atomic();
}
EXPORT_SYMBOL_GPL(kvm_vcpu_kick);
#endif /* !CONFIG_S390 */
^ permalink raw reply related
* [PATCH v2 29/45] kvm/vmx: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:30 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, kvm, Gleb Natapov, x86, nikunj, zhong, linux-pm,
fweisbec, H. Peter Anvin, linux-kernel, rostedt, xiaoguangrong,
sbw, Ingo Molnar, wangyun, Srivatsa S. Bhat, netdev,
Paolo Bonzini, Thomas Gleixner, linuxppc-dev
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Gleb Natapov <gleb@redhat.com>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Cc: kvm@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/x86/kvm/vmx.c | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 260a919..4e1e966 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -26,6 +26,7 @@
#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/sched.h>
+#include <linux/cpu.h>
#include <linux/moduleparam.h>
#include <linux/mod_devicetable.h>
#include <linux/ftrace_event.h>
@@ -7164,12 +7165,12 @@ static struct kvm_vcpu *vmx_create_vcpu(struct kvm *kvm, unsigned int id)
if (!vmm_exclusive)
kvm_cpu_vmxoff();
- cpu = get_cpu();
+ cpu = get_online_cpus_atomic();
vmx_vcpu_load(&vmx->vcpu, cpu);
vmx->vcpu.cpu = cpu;
err = vmx_vcpu_setup(vmx);
vmx_vcpu_put(&vmx->vcpu);
- put_cpu();
+ put_online_cpus_atomic();
if (err)
goto free_vmcs;
if (vm_need_virtualize_apic_accesses(kvm)) {
@@ -7706,12 +7707,12 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
vmx->nested.vmcs01_tsc_offset = vmcs_read64(TSC_OFFSET);
- cpu = get_cpu();
+ cpu = get_online_cpus_atomic();
vmx->loaded_vmcs = vmcs02;
vmx_vcpu_put(vcpu);
vmx_vcpu_load(vcpu, cpu);
vcpu->cpu = cpu;
- put_cpu();
+ put_online_cpus_atomic();
vmx_segment_cache_clear(vmx);
@@ -8023,12 +8024,12 @@ static void nested_vmx_vmexit(struct kvm_vcpu *vcpu)
leave_guest_mode(vcpu);
prepare_vmcs12(vcpu, vmcs12);
- cpu = get_cpu();
+ cpu = get_online_cpus_atomic();
vmx->loaded_vmcs = &vmx->vmcs01;
vmx_vcpu_put(vcpu);
vmx_vcpu_load(vcpu, cpu);
vcpu->cpu = cpu;
- put_cpu();
+ put_online_cpus_atomic();
vmx_segment_cache_clear(vmx);
^ permalink raw reply related
* [PATCH v2 30/45] x86/xen: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:30 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, Jeremy Fitzhardinge, x86, H. Peter Anvin, nikunj,
zhong, linux-pm, fweisbec, Konrad Rzeszutek Wilk, linux-kernel,
rostedt, xiaoguangrong, sbw, Ingo Molnar, wangyun,
Srivatsa S. Bhat, netdev, Thomas Gleixner, virtualization,
linuxppc-dev, xen-devel
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Cc: xen-devel@lists.xensource.com
Cc: virtualization@lists.linux-foundation.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/x86/xen/mmu.c | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index fdc3ba2..3229c4f 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -39,6 +39,7 @@
* Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
*/
#include <linux/sched.h>
+#include <linux/cpu.h>
#include <linux/highmem.h>
#include <linux/debugfs.h>
#include <linux/bug.h>
@@ -1163,9 +1164,13 @@ static void xen_drop_mm_ref(struct mm_struct *mm)
*/
static void xen_exit_mmap(struct mm_struct *mm)
{
- get_cpu(); /* make sure we don't move around */
+ /*
+ * Make sure we don't move around, and also prevent CPUs from
+ * going offline.
+ */
+ get_online_cpus_atomic();
xen_drop_mm_ref(mm);
- put_cpu();
+ put_online_cpus_atomic();
spin_lock(&mm->page_table_lock);
^ permalink raw reply related
* [PATCH v2 31/45] alpha/smp: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:31 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, zhong, linux-pm, fweisbec, linux-alpha,
linux-kernel, rostedt, xiaoguangrong, sbw, Ivan Kokshaysky,
wangyun, Srivatsa S. Bhat, netdev, Matt Turner, linuxppc-dev,
Thomas Gleixner, Richard Henderson
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Richard Henderson <rth@twiddle.net>
Cc: Ivan Kokshaysky <ink@jurassic.park.msu.ru>
Cc: Matt Turner <mattst88@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-alpha@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/alpha/kernel/smp.c | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
diff --git a/arch/alpha/kernel/smp.c b/arch/alpha/kernel/smp.c
index 7b60834..e147268 100644
--- a/arch/alpha/kernel/smp.c
+++ b/arch/alpha/kernel/smp.c
@@ -497,7 +497,6 @@ smp_cpus_done(unsigned int max_cpus)
((bogosum + 2500) / (5000/HZ)) % 100);
}
-\f
void
smp_percpu_timer_interrupt(struct pt_regs *regs)
{
@@ -681,7 +680,7 @@ ipi_flush_tlb_mm(void *x)
void
flush_tlb_mm(struct mm_struct *mm)
{
- preempt_disable();
+ get_online_cpus_atomic();
if (mm == current->active_mm) {
flush_tlb_current(mm);
@@ -693,7 +692,7 @@ flush_tlb_mm(struct mm_struct *mm)
if (mm->context[cpu])
mm->context[cpu] = 0;
}
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
}
@@ -702,7 +701,7 @@ flush_tlb_mm(struct mm_struct *mm)
printk(KERN_CRIT "flush_tlb_mm: timed out\n");
}
- preempt_enable();
+ put_online_cpus_atomic();
}
EXPORT_SYMBOL(flush_tlb_mm);
@@ -730,7 +729,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
struct flush_tlb_page_struct data;
struct mm_struct *mm = vma->vm_mm;
- preempt_disable();
+ get_online_cpus_atomic();
if (mm == current->active_mm) {
flush_tlb_current_page(mm, vma, addr);
@@ -742,7 +741,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
if (mm->context[cpu])
mm->context[cpu] = 0;
}
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
}
@@ -755,7 +754,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
printk(KERN_CRIT "flush_tlb_page: timed out\n");
}
- preempt_enable();
+ put_online_cpus_atomic();
}
EXPORT_SYMBOL(flush_tlb_page);
@@ -786,7 +785,7 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
if ((vma->vm_flags & VM_EXEC) == 0)
return;
- preempt_disable();
+ get_online_cpus_atomic();
if (mm == current->active_mm) {
__load_new_mm_context(mm);
@@ -798,7 +797,7 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
if (mm->context[cpu])
mm->context[cpu] = 0;
}
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
}
@@ -807,5 +806,5 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
printk(KERN_CRIT "flush_icache_page: timed out\n");
}
- preempt_enable();
+ put_online_cpus_atomic();
}
^ permalink raw reply related
* [PATCH v2 32/45] blackfin/smp: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:31 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, Bob Liu, Mike Frysinger, nikunj, zhong, linux-pm,
fweisbec, Steven Miao, linux-kernel, rostedt, xiaoguangrong, sbw,
wangyun, Srivatsa S. Bhat, netdev, uclinux-dist-devel,
Thomas Gleixner, linuxppc-dev
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Mike Frysinger <vapier@gentoo.org>
Cc: Bob Liu <lliubbo@gmail.com>
Cc: Steven Miao <realmz6@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: uclinux-dist-devel@blackfin.uclinux.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/blackfin/mach-common/smp.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/arch/blackfin/mach-common/smp.c b/arch/blackfin/mach-common/smp.c
index 1bc2ce6..11496cd 100644
--- a/arch/blackfin/mach-common/smp.c
+++ b/arch/blackfin/mach-common/smp.c
@@ -238,13 +238,13 @@ void smp_send_stop(void)
{
cpumask_t callmap;
- preempt_disable();
+ get_online_cpus_atomic();
cpumask_copy(&callmap, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &callmap);
if (!cpumask_empty(&callmap))
send_ipi(&callmap, BFIN_IPI_CPU_STOP);
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
^ permalink raw reply related
* [PATCH v2 33/45] cris/smp: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:31 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, Jesper Nilsson, nikunj, zhong, linux-pm,
linux-cris-kernel, fweisbec, linux-kernel, rostedt, xiaoguangrong,
Mikael Starvik, sbw, wangyun, Srivatsa S. Bhat, netdev,
Thomas Gleixner, linuxppc-dev
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Acked-by: Jesper Nilsson <jesper.nilsson@axis.com>
Cc: Mikael Starvik <starvik@axis.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-cris-kernel@axis.com
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/cris/arch-v32/kernel/smp.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/arch/cris/arch-v32/kernel/smp.c b/arch/cris/arch-v32/kernel/smp.c
index cdd1202..b2d4612 100644
--- a/arch/cris/arch-v32/kernel/smp.c
+++ b/arch/cris/arch-v32/kernel/smp.c
@@ -13,6 +13,7 @@
#include <linux/init.h>
#include <linux/timex.h>
#include <linux/sched.h>
+#include <linux/cpu.h>
#include <linux/kernel.h>
#include <linux/cpumask.h>
#include <linux/interrupt.h>
@@ -222,6 +223,7 @@ void flush_tlb_common(struct mm_struct* mm, struct vm_area_struct* vma, unsigned
unsigned long flags;
cpumask_t cpu_mask;
+ get_online_cpus_atomic();
spin_lock_irqsave(&tlbstate_lock, flags);
cpu_mask = (mm == FLUSH_ALL ? cpu_all_mask : *mm_cpumask(mm));
cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
@@ -230,6 +232,7 @@ void flush_tlb_common(struct mm_struct* mm, struct vm_area_struct* vma, unsigned
flush_addr = addr;
send_ipi(IPI_FLUSH_TLB, 1, cpu_mask);
spin_unlock_irqrestore(&tlbstate_lock, flags);
+ put_online_cpus_atomic();
}
void flush_tlb_all(void)
@@ -319,10 +322,12 @@ int smp_call_function(void (*func)(void *info), void *info, int wait)
data.info = info;
data.wait = wait;
+ get_online_cpus_atomic();
spin_lock(&call_lock);
call_data = &data;
ret = send_ipi(IPI_CALL, wait, cpu_mask);
spin_unlock(&call_lock);
+ put_online_cpus_atomic();
return ret;
}
^ permalink raw reply related
* [PATCH v2 34/45] hexagon/smp: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:32 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, zhong, linux-pm, fweisbec, linux-kernel,
rostedt, xiaoguangrong, sbw, wangyun, Srivatsa S. Bhat, netdev,
linux-hexagon, Thomas Gleixner, Richard Kuo, linuxppc-dev
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Richard Kuo <rkuo@codeaurora.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-hexagon@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/hexagon/kernel/smp.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/arch/hexagon/kernel/smp.c b/arch/hexagon/kernel/smp.c
index 0e364ca..30d4318 100644
--- a/arch/hexagon/kernel/smp.c
+++ b/arch/hexagon/kernel/smp.c
@@ -241,9 +241,12 @@ void smp_send_reschedule(int cpu)
void smp_send_stop(void)
{
struct cpumask targets;
+
+ get_online_cpus_atomic();
cpumask_copy(&targets, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &targets);
send_ipi(&targets, IPI_CPU_STOP);
+ put_online_cpus_atomic();
}
void arch_send_call_function_single_ipi(int cpu)
^ permalink raw reply related
* [PATCH v2 35/45] ia64: irq, perfmon: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:32 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, Fenghua Yu, Tony Luck, linux-ia64, nikunj, zhong,
linux-pm, fweisbec, linux-kernel, rostedt, xiaoguangrong, sbw,
wangyun, Srivatsa S. Bhat, netdev, Andrew Morton, linuxppc-dev,
Thomas Gleixner, Eric W. Biederman
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Tony Luck <tony.luck@intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-ia64@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/ia64/kernel/irq_ia64.c | 15 +++++++++++++++
arch/ia64/kernel/perfmon.c | 8 +++++++-
2 files changed, 22 insertions(+), 1 deletion(-)
diff --git a/arch/ia64/kernel/irq_ia64.c b/arch/ia64/kernel/irq_ia64.c
index 1034884..f58b162 100644
--- a/arch/ia64/kernel/irq_ia64.c
+++ b/arch/ia64/kernel/irq_ia64.c
@@ -25,6 +25,7 @@
#include <linux/ptrace.h>
#include <linux/signal.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/threads.h>
#include <linux/bitops.h>
#include <linux/irq.h>
@@ -160,9 +161,11 @@ int bind_irq_vector(int irq, int vector, cpumask_t domain)
unsigned long flags;
int ret;
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
ret = __bind_irq_vector(irq, vector, domain);
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
return ret;
}
@@ -190,9 +193,11 @@ static void clear_irq_vector(int irq)
{
unsigned long flags;
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
__clear_irq_vector(irq);
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
}
int
@@ -204,6 +209,7 @@ ia64_native_assign_irq_vector (int irq)
vector = -ENOSPC;
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
for_each_online_cpu(cpu) {
domain = vector_allocation_domain(cpu);
@@ -218,6 +224,7 @@ ia64_native_assign_irq_vector (int irq)
BUG_ON(__bind_irq_vector(irq, vector, domain));
out:
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
return vector;
}
@@ -302,9 +309,11 @@ int irq_prepare_move(int irq, int cpu)
unsigned long flags;
int ret;
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
ret = __irq_prepare_move(irq, cpu);
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
return ret;
}
@@ -320,11 +329,13 @@ void irq_complete_move(unsigned irq)
if (unlikely(cpu_isset(smp_processor_id(), cfg->old_domain)))
return;
+ get_online_cpus_atomic();
cpumask_and(&cleanup_mask, &cfg->old_domain, cpu_online_mask);
cfg->move_cleanup_count = cpus_weight(cleanup_mask);
for_each_cpu_mask(i, cleanup_mask)
platform_send_ipi(i, IA64_IRQ_MOVE_VECTOR, IA64_IPI_DM_INT, 0);
cfg->move_in_progress = 0;
+ put_online_cpus_atomic();
}
static irqreturn_t smp_irq_move_cleanup_interrupt(int irq, void *dev_id)
@@ -393,10 +404,12 @@ void destroy_and_reserve_irq(unsigned int irq)
dynamic_irq_cleanup(irq);
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
__clear_irq_vector(irq);
irq_status[irq] = IRQ_RSVD;
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
}
/*
@@ -409,6 +422,7 @@ int create_irq(void)
cpumask_t domain = CPU_MASK_NONE;
irq = vector = -ENOSPC;
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
for_each_online_cpu(cpu) {
domain = vector_allocation_domain(cpu);
@@ -424,6 +438,7 @@ int create_irq(void)
BUG_ON(__bind_irq_vector(irq, vector, domain));
out:
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
if (irq >= 0)
dynamic_irq_init(irq);
return irq;
diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c
index 9ea25fc..16c8303 100644
--- a/arch/ia64/kernel/perfmon.c
+++ b/arch/ia64/kernel/perfmon.c
@@ -6476,9 +6476,12 @@ pfm_install_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
/* do the easy test first */
if (pfm_alt_intr_handler) return -EBUSY;
+ get_online_cpus_atomic();
+
/* one at a time in the install or remove, just fail the others */
if (!spin_trylock(&pfm_alt_install_check)) {
- return -EBUSY;
+ ret = -EBUSY;
+ goto out;
}
/* reserve our session */
@@ -6498,6 +6501,7 @@ pfm_install_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
pfm_alt_intr_handler = hdl;
spin_unlock(&pfm_alt_install_check);
+ put_online_cpus_atomic();
return 0;
@@ -6510,6 +6514,8 @@ cleanup_reserve:
}
spin_unlock(&pfm_alt_install_check);
+out:
+ put_online_cpus_atomic();
return ret;
}
^ permalink raw reply related
* [PATCH v2 36/45] ia64: smp, tlb: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:32 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, Fenghua Yu, Tony Luck, linux-ia64, nikunj, zhong,
linux-pm, fweisbec, linux-kernel, rostedt, xiaoguangrong, sbw,
wangyun, Srivatsa S. Bhat, netdev, linuxppc-dev
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Tony Luck <tony.luck@intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: linux-ia64@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/ia64/kernel/smp.c | 12 ++++++------
arch/ia64/mm/tlb.c | 4 ++--
2 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/arch/ia64/kernel/smp.c b/arch/ia64/kernel/smp.c
index 9fcd4e6..25991ba 100644
--- a/arch/ia64/kernel/smp.c
+++ b/arch/ia64/kernel/smp.c
@@ -24,6 +24,7 @@
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/kernel_stat.h>
#include <linux/mm.h>
#include <linux/cache.h>
@@ -259,8 +260,7 @@ smp_flush_tlb_cpumask(cpumask_t xcpumask)
cpumask_t cpumask = xcpumask;
int mycpu, cpu, flush_mycpu = 0;
- preempt_disable();
- mycpu = smp_processor_id();
+ mycpu = get_online_cpus_atomic();
for_each_cpu_mask(cpu, cpumask)
counts[cpu] = local_tlb_flush_counts[cpu].count & 0xffff;
@@ -280,7 +280,7 @@ smp_flush_tlb_cpumask(cpumask_t xcpumask)
while(counts[cpu] == (local_tlb_flush_counts[cpu].count & 0xffff))
udelay(FLUSH_DELAY);
- preempt_enable();
+ put_online_cpus_atomic();
}
void
@@ -293,12 +293,12 @@ void
smp_flush_tlb_mm (struct mm_struct *mm)
{
cpumask_var_t cpus;
- preempt_disable();
+ get_online_cpus_atomic();
/* this happens for the common case of a single-threaded fork(): */
if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1))
{
local_finish_flush_tlb_mm(mm);
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
if (!alloc_cpumask_var(&cpus, GFP_ATOMIC)) {
@@ -313,7 +313,7 @@ smp_flush_tlb_mm (struct mm_struct *mm)
local_irq_disable();
local_finish_flush_tlb_mm(mm);
local_irq_enable();
- preempt_enable();
+ put_online_cpus_atomic();
}
void arch_send_call_function_single_ipi(int cpu)
diff --git a/arch/ia64/mm/tlb.c b/arch/ia64/mm/tlb.c
index ed61297..8c55ef5 100644
--- a/arch/ia64/mm/tlb.c
+++ b/arch/ia64/mm/tlb.c
@@ -87,11 +87,11 @@ wrap_mmu_context (struct mm_struct *mm)
* can't call flush_tlb_all() here because of race condition
* with O(1) scheduler [EF]
*/
- cpu = get_cpu(); /* prevent preemption/migration */
+ cpu = get_online_cpus_atomic(); /* prevent preemption/migration */
for_each_online_cpu(i)
if (i != cpu)
per_cpu(ia64_need_tlb_flush, i) = 1;
- put_cpu();
+ put_online_cpus_atomic();
local_flush_tlb_all();
}
^ permalink raw reply related
* [PATCH v2 37/45] m32r: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:32 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, linux-m32r-ja, linux-m32r, nikunj, zhong, linux-pm,
fweisbec, Hirokazu Takata, linux-kernel, rostedt, xiaoguangrong,
sbw, wangyun, Srivatsa S. Bhat, netdev, linuxppc-dev
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Hirokazu Takata <takata@linux-m32r.org>
Cc: linux-m32r@ml.linux-m32r.org
Cc: linux-m32r-ja@ml.linux-m32r.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/m32r/kernel/smp.c | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/arch/m32r/kernel/smp.c b/arch/m32r/kernel/smp.c
index ce7aea3..ffafdba 100644
--- a/arch/m32r/kernel/smp.c
+++ b/arch/m32r/kernel/smp.c
@@ -151,7 +151,7 @@ void smp_flush_cache_all(void)
cpumask_t cpumask;
unsigned long *mask;
- preempt_disable();
+ get_online_cpus_atomic();
cpumask_copy(&cpumask, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &cpumask);
spin_lock(&flushcache_lock);
@@ -162,7 +162,7 @@ void smp_flush_cache_all(void)
while (flushcache_cpumask)
mb();
spin_unlock(&flushcache_lock);
- preempt_enable();
+ put_online_cpus_atomic();
}
void smp_flush_cache_all_interrupt(void)
@@ -197,12 +197,12 @@ void smp_flush_tlb_all(void)
{
unsigned long flags;
- preempt_disable();
+ get_online_cpus_atomic();
local_irq_save(flags);
__flush_tlb_all();
local_irq_restore(flags);
smp_call_function(flush_tlb_all_ipi, NULL, 1);
- preempt_enable();
+ put_online_cpus_atomic();
}
/*==========================================================================*
@@ -250,7 +250,7 @@ void smp_flush_tlb_mm(struct mm_struct *mm)
unsigned long *mmc;
unsigned long flags;
- preempt_disable();
+ get_online_cpus_atomic();
cpu_id = smp_processor_id();
mmc = &mm->context[cpu_id];
cpumask_copy(&cpu_mask, mm_cpumask(mm));
@@ -268,7 +268,7 @@ void smp_flush_tlb_mm(struct mm_struct *mm)
if (!cpumask_empty(&cpu_mask))
flush_tlb_others(cpu_mask, mm, NULL, FLUSH_ALL);
- preempt_enable();
+ put_online_cpus_atomic();
}
/*==========================================================================*
@@ -320,7 +320,7 @@ void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
unsigned long *mmc;
unsigned long flags;
- preempt_disable();
+ get_online_cpus_atomic();
cpu_id = smp_processor_id();
mmc = &mm->context[cpu_id];
cpumask_copy(&cpu_mask, mm_cpumask(mm));
@@ -341,7 +341,7 @@ void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
if (!cpumask_empty(&cpu_mask))
flush_tlb_others(cpu_mask, mm, vma, va);
- preempt_enable();
+ put_online_cpus_atomic();
}
/*==========================================================================*
^ permalink raw reply related
* [PATCH v2 38/45] MIPS: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:32 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-mips, David Daney, fweisbec, linux-arch, xiaoguangrong,
Steven J. Hill, wangyun, nikunj, linux-pm, rostedt, Sanjay Lal,
Thomas Gleixner, Florian Fainelli, John Crispin, zhong,
Yong Zhang, netdev, linux-kernel, Ralf Baechle, sbw,
Srivatsa S. Bhat, linuxppc-dev
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: David Daney <david.daney@cavium.com>
Cc: Yong Zhang <yong.zhang0@gmail.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Sanjay Lal <sanjayl@kymasys.com>
Cc: "Steven J. Hill" <sjhill@mips.com>
Cc: John Crispin <blogic@openwrt.org>
Cc: Florian Fainelli <florian@openwrt.org>
Cc: linux-mips@linux-mips.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/mips/kernel/cevt-smtc.c | 7 +++++++
arch/mips/kernel/smp.c | 16 ++++++++--------
arch/mips/kernel/smtc.c | 12 ++++++++++++
arch/mips/mm/c-octeon.c | 4 ++--
4 files changed, 29 insertions(+), 10 deletions(-)
diff --git a/arch/mips/kernel/cevt-smtc.c b/arch/mips/kernel/cevt-smtc.c
index 9de5ed7..2e6c0cd 100644
--- a/arch/mips/kernel/cevt-smtc.c
+++ b/arch/mips/kernel/cevt-smtc.c
@@ -11,6 +11,7 @@
#include <linux/interrupt.h>
#include <linux/percpu.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/irq.h>
#include <asm/smtc_ipi.h>
@@ -84,6 +85,8 @@ static int mips_next_event(unsigned long delta,
unsigned long nextcomp = 0L;
int vpe = current_cpu_data.vpe_id;
int cpu = smp_processor_id();
+
+ get_online_cpus_atomic();
local_irq_save(flags);
mtflags = dmt();
@@ -164,6 +167,7 @@ static int mips_next_event(unsigned long delta,
}
emt(mtflags);
local_irq_restore(flags);
+ put_online_cpus_atomic();
return 0;
}
@@ -177,6 +181,7 @@ void smtc_distribute_timer(int vpe)
unsigned long nextstamp;
unsigned long reference;
+ get_online_cpus_atomic();
repeat:
nextstamp = 0L;
@@ -229,6 +234,8 @@ repeat:
> (unsigned long)LONG_MAX)
goto repeat;
}
+
+ put_online_cpus_atomic();
}
diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c
index 6e7862a..be152b6 100644
--- a/arch/mips/kernel/smp.c
+++ b/arch/mips/kernel/smp.c
@@ -250,12 +250,12 @@ static inline void smp_on_other_tlbs(void (*func) (void *info), void *info)
static inline void smp_on_each_tlb(void (*func) (void *info), void *info)
{
- preempt_disable();
+ get_online_cpus_atomic();
smp_on_other_tlbs(func, info);
func(info);
- preempt_enable();
+ put_online_cpus_atomic();
}
/*
@@ -273,7 +273,7 @@ static inline void smp_on_each_tlb(void (*func) (void *info), void *info)
void flush_tlb_mm(struct mm_struct *mm)
{
- preempt_disable();
+ get_online_cpus_atomic();
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
smp_on_other_tlbs(flush_tlb_mm_ipi, mm);
@@ -287,7 +287,7 @@ void flush_tlb_mm(struct mm_struct *mm)
}
local_flush_tlb_mm(mm);
- preempt_enable();
+ put_online_cpus_atomic();
}
struct flush_tlb_data {
@@ -307,7 +307,7 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned l
{
struct mm_struct *mm = vma->vm_mm;
- preempt_disable();
+ get_online_cpus_atomic();
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
struct flush_tlb_data fd = {
.vma = vma,
@@ -325,7 +325,7 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned l
}
}
local_flush_tlb_range(vma, start, end);
- preempt_enable();
+ put_online_cpus_atomic();
}
static void flush_tlb_kernel_range_ipi(void *info)
@@ -354,7 +354,7 @@ static void flush_tlb_page_ipi(void *info)
void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
{
- preempt_disable();
+ get_online_cpus_atomic();
if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) {
struct flush_tlb_data fd = {
.vma = vma,
@@ -371,7 +371,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
}
}
local_flush_tlb_page(vma, page);
- preempt_enable();
+ put_online_cpus_atomic();
}
static void flush_tlb_one_ipi(void *info)
diff --git a/arch/mips/kernel/smtc.c b/arch/mips/kernel/smtc.c
index 75a4fd7..3cda8eb 100644
--- a/arch/mips/kernel/smtc.c
+++ b/arch/mips/kernel/smtc.c
@@ -21,6 +21,7 @@
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/cpumask.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
@@ -1143,6 +1144,8 @@ static irqreturn_t ipi_interrupt(int irq, void *dev_idm)
* for the current TC, so we ought not to have to do it explicitly here.
*/
+ get_online_cpus_atomic();
+
for_each_online_cpu(cpu) {
if (cpu_data[cpu].vpe_id != my_vpe)
continue;
@@ -1180,6 +1183,8 @@ static irqreturn_t ipi_interrupt(int irq, void *dev_idm)
}
}
+ put_online_cpus_atomic();
+
return IRQ_HANDLED;
}
@@ -1383,6 +1388,7 @@ void smtc_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu)
* them, let's be really careful...
*/
+ get_online_cpus_atomic();
local_irq_save(flags);
if (smtc_status & SMTC_TLB_SHARED) {
mtflags = dvpe();
@@ -1438,6 +1444,7 @@ void smtc_get_new_mmu_context(struct mm_struct *mm, unsigned long cpu)
else
emt(mtflags);
local_irq_restore(flags);
+ put_online_cpus_atomic();
}
/*
@@ -1496,6 +1503,7 @@ void smtc_cflush_lockdown(void)
{
int cpu;
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
if (cpu != smp_processor_id()) {
settc(cpu_data[cpu].tc_id);
@@ -1504,6 +1512,7 @@ void smtc_cflush_lockdown(void)
}
}
mips_ihb();
+ put_online_cpus_atomic();
}
/* It would be cheating to change the cpu_online states during a flush! */
@@ -1512,6 +1521,8 @@ void smtc_cflush_release(void)
{
int cpu;
+ get_online_cpus_atomic();
+
/*
* Start with a hazard barrier to ensure
* that all CACHE ops have played through.
@@ -1525,4 +1536,5 @@ void smtc_cflush_release(void)
}
}
mips_ihb();
+ put_online_cpus_atomic();
}
diff --git a/arch/mips/mm/c-octeon.c b/arch/mips/mm/c-octeon.c
index 8557fb5..8e1bcf6 100644
--- a/arch/mips/mm/c-octeon.c
+++ b/arch/mips/mm/c-octeon.c
@@ -73,7 +73,7 @@ static void octeon_flush_icache_all_cores(struct vm_area_struct *vma)
mb();
octeon_local_flush_icache();
#ifdef CONFIG_SMP
- preempt_disable();
+ get_online_cpus_atomic();
cpu = smp_processor_id();
/*
@@ -88,7 +88,7 @@ static void octeon_flush_icache_all_cores(struct vm_area_struct *vma)
for_each_cpu(cpu, &mask)
octeon_send_ipi_single(cpu, SMP_ICACHE_FLUSH);
- preempt_enable();
+ put_online_cpus_atomic();
#endif
}
^ permalink raw reply related
* [PATCH v2 39/45] mn10300: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:33 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, linux-am33-list, nikunj, zhong, linux-pm, fweisbec,
linux-kernel, rostedt, xiaoguangrong, David Howells, sbw, wangyun,
Srivatsa S. Bhat, netdev, Koichi Yasutake, linuxppc-dev
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: David Howells <dhowells@redhat.com>
Cc: Koichi Yasutake <yasutake.koichi@jp.panasonic.com>
Cc: linux-am33-list@redhat.com
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/mn10300/mm/cache-smp.c | 3 +++
arch/mn10300/mm/tlb-smp.c | 17 +++++++++--------
2 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/arch/mn10300/mm/cache-smp.c b/arch/mn10300/mm/cache-smp.c
index 2d23b9e..406357d 100644
--- a/arch/mn10300/mm/cache-smp.c
+++ b/arch/mn10300/mm/cache-smp.c
@@ -13,6 +13,7 @@
#include <linux/mman.h>
#include <linux/threads.h>
#include <linux/interrupt.h>
+#include <linux/cpu.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/processor.h>
@@ -91,6 +92,7 @@ void smp_cache_interrupt(void)
void smp_cache_call(unsigned long opr_mask,
unsigned long start, unsigned long end)
{
+ get_online_cpus_atomic();
smp_cache_mask = opr_mask;
smp_cache_start = start;
smp_cache_end = end;
@@ -102,4 +104,5 @@ void smp_cache_call(unsigned long opr_mask,
while (!cpumask_empty(&smp_cache_ipi_map))
/* nothing. lockup detection does not belong here */
mb();
+ put_online_cpus_atomic();
}
diff --git a/arch/mn10300/mm/tlb-smp.c b/arch/mn10300/mm/tlb-smp.c
index 3e57faf..8856fd3 100644
--- a/arch/mn10300/mm/tlb-smp.c
+++ b/arch/mn10300/mm/tlb-smp.c
@@ -23,6 +23,7 @@
#include <linux/sched.h>
#include <linux/profile.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <asm/tlbflush.h>
#include <asm/bitops.h>
#include <asm/processor.h>
@@ -61,7 +62,7 @@ void smp_flush_tlb(void *unused)
{
unsigned long cpu_id;
- cpu_id = get_cpu();
+ cpu_id = get_online_cpus_atomic();
if (!cpumask_test_cpu(cpu_id, &flush_cpumask))
/* This was a BUG() but until someone can quote me the line
@@ -82,7 +83,7 @@ void smp_flush_tlb(void *unused)
cpumask_clear_cpu(cpu_id, &flush_cpumask);
smp_mb__after_clear_bit();
out:
- put_cpu();
+ put_online_cpus_atomic();
}
/**
@@ -144,7 +145,7 @@ void flush_tlb_mm(struct mm_struct *mm)
{
cpumask_t cpu_mask;
- preempt_disable();
+ get_online_cpus_atomic();
cpumask_copy(&cpu_mask, mm_cpumask(mm));
cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
@@ -152,7 +153,7 @@ void flush_tlb_mm(struct mm_struct *mm)
if (!cpumask_empty(&cpu_mask))
flush_tlb_others(cpu_mask, mm, FLUSH_ALL);
- preempt_enable();
+ put_online_cpus_atomic();
}
/**
@@ -163,7 +164,7 @@ void flush_tlb_current_task(void)
struct mm_struct *mm = current->mm;
cpumask_t cpu_mask;
- preempt_disable();
+ get_online_cpus_atomic();
cpumask_copy(&cpu_mask, mm_cpumask(mm));
cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
@@ -171,7 +172,7 @@ void flush_tlb_current_task(void)
if (!cpumask_empty(&cpu_mask))
flush_tlb_others(cpu_mask, mm, FLUSH_ALL);
- preempt_enable();
+ put_online_cpus_atomic();
}
/**
@@ -184,7 +185,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
struct mm_struct *mm = vma->vm_mm;
cpumask_t cpu_mask;
- preempt_disable();
+ get_online_cpus_atomic();
cpumask_copy(&cpu_mask, mm_cpumask(mm));
cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
@@ -192,7 +193,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
if (!cpumask_empty(&cpu_mask))
flush_tlb_others(cpu_mask, mm, va);
- preempt_enable();
+ put_online_cpus_atomic();
}
/**
^ permalink raw reply related
* [PATCH v2 40/45] powerpc, irq: Use GFP_ATOMIC allocations in atomic context
From: Srivatsa S. Bhat @ 2013-06-25 20:33 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, Li Zhong, linux-pm, fweisbec, linux-kernel,
Steven Rostedt, xiaoguangrong, sbw, Paul Mackerras, wangyun,
Srivatsa S. Bhat, netdev, linuxppc-dev, Ian Munsie
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
The function migrate_irqs() is called with interrupts disabled
and hence its not safe to do GFP_KERNEL allocations inside it,
because they can sleep. So change the gfp mask to GFP_ATOMIC.
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Michael Ellerman <michael@ellerman.id.au>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Ian Munsie <imunsie@au1.ibm.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Michael Ellerman <michael@ellerman.id.au>
Cc: Li Zhong <zhong@linux.vnet.ibm.com>
Cc: linuxppc-dev@lists.ozlabs.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/powerpc/kernel/irq.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index ea185e0..ca39bac 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -412,7 +412,7 @@ void migrate_irqs(void)
cpumask_var_t mask;
const struct cpumask *map = cpu_online_mask;
- alloc_cpumask_var(&mask, GFP_KERNEL);
+ alloc_cpumask_var(&mask, GFP_ATOMIC);
for_each_irq_desc(irq, desc) {
struct irq_data *data;
^ permalink raw reply related
* [PATCH v2 41/45] powerpc: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:33 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: Gleb Natapov, linux-arch, Zhao Chenhui, Alexander Graf,
xiaoguangrong, oprofile-list, wangyun, fweisbec, Rob Herring,
cbe-oss-dev, nikunj, linux-pm, rostedt, kvm-ppc, kvm, zhong,
netdev, linux-kernel, sbw, Srivatsa S. Bhat, linuxppc-dev
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Gleb Natapov <gleb@redhat.com>
Cc: Alexander Graf <agraf@suse.de>
Cc: Rob Herring <rob.herring@calxeda.com>
Cc: Grant Likely <grant.likely@secretlab.ca>
Cc: Kumar Gala <galak@kernel.crashing.org>
Cc: Zhao Chenhui <chenhui.zhao@freescale.com>
Cc: linuxppc-dev@lists.ozlabs.org
Cc: kvm@vger.kernel.org
Cc: kvm-ppc@vger.kernel.org
Cc: oprofile-list@lists.sf.net
Cc: cbe-oss-dev@lists.ozlabs.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/powerpc/kernel/irq.c | 7 ++++++-
arch/powerpc/kernel/machine_kexec_64.c | 4 ++--
arch/powerpc/kernel/smp.c | 2 ++
arch/powerpc/kvm/book3s_hv.c | 5 +++--
arch/powerpc/mm/mmu_context_nohash.c | 3 +++
arch/powerpc/oprofile/cell/spu_profiler.c | 3 +++
arch/powerpc/oprofile/cell/spu_task_sync.c | 4 ++++
arch/powerpc/oprofile/op_model_cell.c | 6 ++++++
8 files changed, 29 insertions(+), 5 deletions(-)
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index ca39bac..41e9961 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -45,6 +45,7 @@
#include <linux/irq.h>
#include <linux/seq_file.h>
#include <linux/cpumask.h>
+#include <linux/cpu.h>
#include <linux/profile.h>
#include <linux/bitops.h>
#include <linux/list.h>
@@ -410,7 +411,10 @@ void migrate_irqs(void)
unsigned int irq;
static int warned;
cpumask_var_t mask;
- const struct cpumask *map = cpu_online_mask;
+ const struct cpumask *map;
+
+ get_online_cpus_atomic();
+ map = cpu_online_mask;
alloc_cpumask_var(&mask, GFP_ATOMIC);
@@ -436,6 +440,7 @@ void migrate_irqs(void)
}
free_cpumask_var(mask);
+ put_online_cpus_atomic();
local_irq_enable();
mdelay(1);
diff --git a/arch/powerpc/kernel/machine_kexec_64.c b/arch/powerpc/kernel/machine_kexec_64.c
index 611acdf..38f6d75 100644
--- a/arch/powerpc/kernel/machine_kexec_64.c
+++ b/arch/powerpc/kernel/machine_kexec_64.c
@@ -187,7 +187,7 @@ static void kexec_prepare_cpus_wait(int wait_state)
int my_cpu, i, notified=-1;
hw_breakpoint_disable();
- my_cpu = get_cpu();
+ my_cpu = get_online_cpus_atomic();
/* Make sure each CPU has at least made it to the state we need.
*
* FIXME: There is a (slim) chance of a problem if not all of the CPUs
@@ -266,7 +266,7 @@ static void kexec_prepare_cpus(void)
*/
kexec_prepare_cpus_wait(KEXEC_STATE_REAL_MODE);
- put_cpu();
+ put_online_cpus_atomic();
}
#else /* ! SMP */
diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index ee7ac5e..2123bec 100644
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -277,9 +277,11 @@ void smp_send_debugger_break(void)
if (unlikely(!smp_ops))
return;
+ get_online_cpus_atomic();
for_each_online_cpu(cpu)
if (cpu != me)
do_message_pass(cpu, PPC_MSG_DEBUGGER_BREAK);
+ put_online_cpus_atomic();
}
#endif
diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
index 2efa9dd..9d8a973 100644
--- a/arch/powerpc/kvm/book3s_hv.c
+++ b/arch/powerpc/kvm/book3s_hv.c
@@ -28,6 +28,7 @@
#include <linux/fs.h>
#include <linux/anon_inodes.h>
#include <linux/cpumask.h>
+#include <linux/cpu.h>
#include <linux/spinlock.h>
#include <linux/page-flags.h>
#include <linux/srcu.h>
@@ -78,7 +79,7 @@ void kvmppc_fast_vcpu_kick(struct kvm_vcpu *vcpu)
++vcpu->stat.halt_wakeup;
}
- me = get_cpu();
+ me = get_online_cpus_atomic();
/* CPU points to the first thread of the core */
if (cpu != me && cpu >= 0 && cpu < nr_cpu_ids) {
@@ -88,7 +89,7 @@ void kvmppc_fast_vcpu_kick(struct kvm_vcpu *vcpu)
else if (cpu_online(cpu))
smp_send_reschedule(cpu);
}
- put_cpu();
+ put_online_cpus_atomic();
}
/*
diff --git a/arch/powerpc/mm/mmu_context_nohash.c b/arch/powerpc/mm/mmu_context_nohash.c
index e779642..c7bdcb4 100644
--- a/arch/powerpc/mm/mmu_context_nohash.c
+++ b/arch/powerpc/mm/mmu_context_nohash.c
@@ -194,6 +194,8 @@ void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
unsigned int i, id, cpu = smp_processor_id();
unsigned long *map;
+ get_online_cpus_atomic();
+
/* No lockless fast path .. yet */
raw_spin_lock(&context_lock);
@@ -280,6 +282,7 @@ void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
pr_hardcont(" -> %d\n", id);
set_context(id, next->pgd);
raw_spin_unlock(&context_lock);
+ put_online_cpus_atomic();
}
/*
diff --git a/arch/powerpc/oprofile/cell/spu_profiler.c b/arch/powerpc/oprofile/cell/spu_profiler.c
index b129d00..ab6e6c1 100644
--- a/arch/powerpc/oprofile/cell/spu_profiler.c
+++ b/arch/powerpc/oprofile/cell/spu_profiler.c
@@ -14,6 +14,7 @@
#include <linux/hrtimer.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/slab.h>
#include <asm/cell-pmu.h>
#include <asm/time.h>
@@ -142,6 +143,7 @@ static enum hrtimer_restart profile_spus(struct hrtimer *timer)
if (!spu_prof_running)
goto stop;
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
if (cbe_get_hw_thread_id(cpu))
continue;
@@ -177,6 +179,7 @@ static enum hrtimer_restart profile_spus(struct hrtimer *timer)
oprof_spu_smpl_arry_lck_flags);
}
+ put_online_cpus_atomic();
smp_wmb(); /* insure spu event buffer updates are written */
/* don't want events intermingled... */
diff --git a/arch/powerpc/oprofile/cell/spu_task_sync.c b/arch/powerpc/oprofile/cell/spu_task_sync.c
index 28f1af2..8464ef6 100644
--- a/arch/powerpc/oprofile/cell/spu_task_sync.c
+++ b/arch/powerpc/oprofile/cell/spu_task_sync.c
@@ -28,6 +28,7 @@
#include <linux/oprofile.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
+#include <linux/cpu.h>
#include "pr_util.h"
#define RELEASE_ALL 9999
@@ -448,11 +449,14 @@ static int number_of_online_nodes(void)
{
u32 cpu; u32 tmp;
int nodes = 0;
+
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
tmp = cbe_cpu_to_node(cpu) + 1;
if (tmp > nodes)
nodes++;
}
+ put_online_cpus_atomic();
return nodes;
}
diff --git a/arch/powerpc/oprofile/op_model_cell.c b/arch/powerpc/oprofile/op_model_cell.c
index b9589c1..c9bb028 100644
--- a/arch/powerpc/oprofile/op_model_cell.c
+++ b/arch/powerpc/oprofile/op_model_cell.c
@@ -22,6 +22,7 @@
#include <linux/oprofile.h>
#include <linux/percpu.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/spinlock.h>
#include <linux/timer.h>
#include <asm/cell-pmu.h>
@@ -463,6 +464,7 @@ static void cell_virtual_cntr(unsigned long data)
* not both playing with the counters on the same node.
*/
+ get_online_cpus_atomic();
spin_lock_irqsave(&cntr_lock, flags);
prev_hdw_thread = hdw_thread;
@@ -550,6 +552,7 @@ static void cell_virtual_cntr(unsigned long data)
}
spin_unlock_irqrestore(&cntr_lock, flags);
+ put_online_cpus_atomic();
mod_timer(&timer_virt_cntr, jiffies + HZ / 10);
}
@@ -608,6 +611,8 @@ static void spu_evnt_swap(unsigned long data)
/* Make sure spu event interrupt handler and spu event swap
* don't access the counters simultaneously.
*/
+
+ get_online_cpus_atomic();
spin_lock_irqsave(&cntr_lock, flags);
cur_spu_evnt_phys_spu_indx = spu_evnt_phys_spu_indx;
@@ -673,6 +678,7 @@ static void spu_evnt_swap(unsigned long data)
}
spin_unlock_irqrestore(&cntr_lock, flags);
+ put_online_cpus_atomic();
/* swap approximately every 0.1 seconds */
mod_timer(&timer_spu_event_swap, jiffies + HZ / 25);
^ permalink raw reply related
* [PATCH v2 42/45] powerpc: Use get/put_online_cpus_atomic() to avoid false-positive warning
From: Srivatsa S. Bhat @ 2013-06-25 20:33 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, zhong, linux-pm, fweisbec, linux-kernel,
rostedt, xiaoguangrong, sbw, Paul Mackerras, wangyun,
Srivatsa S. Bhat, netdev, Thomas Gleixner, linuxppc-dev,
Zhao Chenhui
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Bringing a secondary CPU online is a special case in which, accessing
the cpu_online_mask is safe, even though that task (which running on the
CPU coming online) is not the hotplug writer.
It is a little hard to teach this to the debugging checks under
CONFIG_DEBUG_HOTPLUG_CPU. But luckily powerpc is one of the few places
where the CPU coming online traverses the cpu_online_mask before fully
coming online. So wrap that part under get/put_online_cpus_atomic(), to
avoid false-positive warnings from the CPU hotplug debug code.
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Kumar Gala <galak@kernel.crashing.org>
Cc: Zhao Chenhui <chenhui.zhao@freescale.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linuxppc-dev@lists.ozlabs.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/powerpc/kernel/smp.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index 2123bec..59c9a09 100644
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -657,6 +657,7 @@ __cpuinit void start_secondary(void *unused)
cpumask_set_cpu(base + i, cpu_core_mask(cpu));
}
l2_cache = cpu_to_l2cache(cpu);
+ get_online_cpus_atomic();
for_each_online_cpu(i) {
struct device_node *np = cpu_to_l2cache(i);
if (!np)
@@ -667,6 +668,7 @@ __cpuinit void start_secondary(void *unused)
}
of_node_put(np);
}
+ put_online_cpus_atomic();
of_node_put(l2_cache);
local_irq_enable();
^ permalink raw reply related
* [PATCH v2 43/45] sh: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-06-25 20:33 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung,
walken, vincent.guittot, laijs
Cc: linux-arch, nikunj, zhong, linux-pm, fweisbec, linux-sh,
linux-kernel, rostedt, xiaoguangrong, sbw, Paul Mundt, wangyun,
Srivatsa S. Bhat, netdev, Thomas Gleixner, linuxppc-dev
In-Reply-To: <20130625202452.16593.22810.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able
to depend on disabling preemption to prevent CPUs from going offline
from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going
offline, while invoking from atomic context.
Cc: Paul Mundt <lethal@linux-sh.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-sh@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/sh/kernel/smp.c | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/arch/sh/kernel/smp.c b/arch/sh/kernel/smp.c
index 4569645..42ec182 100644
--- a/arch/sh/kernel/smp.c
+++ b/arch/sh/kernel/smp.c
@@ -357,7 +357,7 @@ static void flush_tlb_mm_ipi(void *mm)
*/
void flush_tlb_mm(struct mm_struct *mm)
{
- preempt_disable();
+ get_online_cpus_atomic();
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
smp_call_function(flush_tlb_mm_ipi, (void *)mm, 1);
@@ -369,7 +369,7 @@ void flush_tlb_mm(struct mm_struct *mm)
}
local_flush_tlb_mm(mm);
- preempt_enable();
+ put_online_cpus_atomic();
}
struct flush_tlb_data {
@@ -390,7 +390,7 @@ void flush_tlb_range(struct vm_area_struct *vma,
{
struct mm_struct *mm = vma->vm_mm;
- preempt_disable();
+ get_online_cpus_atomic();
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
struct flush_tlb_data fd;
@@ -405,7 +405,7 @@ void flush_tlb_range(struct vm_area_struct *vma,
cpu_context(i, mm) = 0;
}
local_flush_tlb_range(vma, start, end);
- preempt_enable();
+ put_online_cpus_atomic();
}
static void flush_tlb_kernel_range_ipi(void *info)
@@ -433,7 +433,7 @@ static void flush_tlb_page_ipi(void *info)
void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
{
- preempt_disable();
+ get_online_cpus_atomic();
if ((atomic_read(&vma->vm_mm->mm_users) != 1) ||
(current->mm != vma->vm_mm)) {
struct flush_tlb_data fd;
@@ -448,7 +448,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
cpu_context(i, vma->vm_mm) = 0;
}
local_flush_tlb_page(vma, page);
- preempt_enable();
+ put_online_cpus_atomic();
}
static void flush_tlb_one_ipi(void *info)
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox