* [PATCH 4/6] iommu/fsl: Add window permission flags for iommu_domain_window_enable API.
From: Varun Sethi @ 2013-02-18 12:52 UTC (permalink / raw)
To: iommu, linuxppc-dev, linux-kernel, scottwood, joro, stuart.yoder
Cc: Varun Sethi
In-Reply-To: <1361191939-21260-1-git-send-email-Varun.Sethi@freescale.com>
Each iommu window can have access permissions associated with it. Extended the
window_enable API to incorporate window access permissions.
Signed-off-by: Varun Sethi <Varun.Sethi@freescale.com>
---
drivers/iommu/iommu.c | 5 +++--
include/linux/iommu.h | 7 ++++---
2 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index b972d43..1a2564d 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -854,12 +854,13 @@ EXPORT_SYMBOL_GPL(iommu_unmap);
int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr,
- phys_addr_t paddr, u64 size)
+ phys_addr_t paddr, u64 size, int iommu_prot)
{
if (unlikely(domain->ops->domain_window_enable == NULL))
return -ENODEV;
- return domain->ops->domain_window_enable(domain, wnd_nr, paddr, size);
+ return domain->ops->domain_window_enable(domain, wnd_nr, paddr, size,
+ iommu_prot);
}
EXPORT_SYMBOL_GPL(iommu_domain_window_enable);
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index ba3b8a9..529987c 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -105,7 +105,7 @@ struct iommu_ops {
/* Window handling functions */
int (*domain_window_enable)(struct iommu_domain *domain, u32 wnd_nr,
- phys_addr_t paddr, u64 size);
+ phys_addr_t paddr, u64 size, int iommu_prot);
void (*domain_window_disable)(struct iommu_domain *domain, u32 wnd_nr);
/* Set the numer of window per domain */
int (*domain_set_windows)(struct iommu_domain *domain, u32 w_count);
@@ -171,7 +171,8 @@ extern int iommu_domain_set_attr(struct iommu_domain *domain, enum iommu_attr,
/* Window handling function prototypes */
extern int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr,
- phys_addr_t offset, u64 size);
+ phys_addr_t offset, u64 size,
+ int iommu_prot);
extern void iommu_domain_window_disable(struct iommu_domain *domain, u32 wnd_nr);
/**
* report_iommu_fault() - report about an IOMMU fault to the IOMMU framework
@@ -257,7 +258,7 @@ static inline int iommu_unmap(struct iommu_domain *domain, unsigned long iova,
static inline int iommu_domain_window_enable(struct iommu_domain *domain,
u32 wnd_nr, phys_addr_t paddr,
- u64 size)
+ u64 size, int iommu_prot)
{
return -ENODEV;
}
--
1.7.4.1
^ permalink raw reply related
* [PATCH 2/6] powerpc/fsl_pci: Store the platform device information corresponding to the pci controller.
From: Varun Sethi @ 2013-02-18 12:52 UTC (permalink / raw)
To: iommu, linuxppc-dev, linux-kernel, scottwood, joro, stuart.yoder
Cc: Varun Sethi
In-Reply-To: <1361191939-21260-1-git-send-email-Varun.Sethi@freescale.com>
The pci controller structure has a provision to store the device strcuture
pointer of the corresponding platform device. Currently this information is
not stored during fsl pci controller initialization. This information is
required while dealing with iommu groups for pci devices connected to the fsl
pci controller. For the case where the pci devices can't be paritioned, they
would fall under the same device group as the pci controller.
This patch stores the platform device information in the pci controller
structure during initialization.
Signed-off-by: Varun Sethi <Varun.Sethi@freescale.com>
---
arch/powerpc/sysdev/fsl_pci.c | 9 +++++++--
arch/powerpc/sysdev/fsl_pci.h | 2 +-
2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/arch/powerpc/sysdev/fsl_pci.c b/arch/powerpc/sysdev/fsl_pci.c
index 92a5915..b393ae7 100644
--- a/arch/powerpc/sysdev/fsl_pci.c
+++ b/arch/powerpc/sysdev/fsl_pci.c
@@ -421,13 +421,16 @@ void fsl_pcibios_fixup_bus(struct pci_bus *bus)
}
}
-int __init fsl_add_bridge(struct device_node *dev, int is_primary)
+int __init fsl_add_bridge(struct platform_device *pdev, int is_primary)
{
int len;
struct pci_controller *hose;
struct resource rsrc;
const int *bus_range;
u8 hdr_type, progif;
+ struct device_node *dev;
+
+ dev = pdev->dev.of_node;
if (!of_device_is_available(dev)) {
pr_warning("%s: disabled\n", dev->full_name);
@@ -453,6 +456,8 @@ int __init fsl_add_bridge(struct device_node *dev, int is_primary)
if (!hose)
return -ENOMEM;
+ /* set platform device as the parent */
+ hose->parent = &pdev->dev;
hose->first_busno = bus_range ? bus_range[0] : 0x0;
hose->last_busno = bus_range ? bus_range[1] : 0xff;
@@ -880,7 +885,7 @@ static int fsl_pci_probe(struct platform_device *pdev)
#endif
node = pdev->dev.of_node;
- ret = fsl_add_bridge(node, fsl_pci_primary == node);
+ ret = fsl_add_bridge(pdev, fsl_pci_primary == node);
#ifdef CONFIG_SWIOTLB
if (ret == 0) {
diff --git a/arch/powerpc/sysdev/fsl_pci.h b/arch/powerpc/sysdev/fsl_pci.h
index d078537..c495c00 100644
--- a/arch/powerpc/sysdev/fsl_pci.h
+++ b/arch/powerpc/sysdev/fsl_pci.h
@@ -91,7 +91,7 @@ struct ccsr_pci {
__be32 pex_err_cap_r3; /* 0x.e34 - PCIE error capture register 0 */
};
-extern int fsl_add_bridge(struct device_node *dev, int is_primary);
+extern int fsl_add_bridge(struct platform_device *pdev, int is_primary);
extern void fsl_pcibios_fixup_bus(struct pci_bus *bus);
extern int mpc83xx_add_bridge(struct device_node *dev);
u64 fsl_pci_immrbar_base(struct pci_controller *hose);
--
1.7.4.1
^ permalink raw reply related
* [PATCH 1/6 v8] iommu/fsl: Store iommu domain information pointer in archdata.
From: Varun Sethi @ 2013-02-18 12:52 UTC (permalink / raw)
To: iommu, linuxppc-dev, linux-kernel, scottwood, joro, stuart.yoder
Cc: Varun Sethi
In-Reply-To: <1361191939-21260-1-git-send-email-Varun.Sethi@freescale.com>
Add a new field in the device (powerpc) archdata structure for storing iommu domain
information pointer. This pointer is stored when the device is attached to a particular
domain.
Signed-off-by: Varun Sethi <Varun.Sethi@freescale.com>
---
- no change.
arch/powerpc/include/asm/device.h | 4 ++++
1 files changed, 4 insertions(+), 0 deletions(-)
diff --git a/arch/powerpc/include/asm/device.h b/arch/powerpc/include/asm/device.h
index 77e97dd..6dc79fe 100644
--- a/arch/powerpc/include/asm/device.h
+++ b/arch/powerpc/include/asm/device.h
@@ -28,6 +28,10 @@ struct dev_archdata {
void *iommu_table_base;
} dma_data;
+ /* IOMMU domain information pointer. This would be set
+ * when this device is attached to an iommu_domain.
+ */
+ void *iommu_domain;
#ifdef CONFIG_SWIOTLB
dma_addr_t max_direct_dma_addr;
#endif
--
1.7.4.1
^ permalink raw reply related
* [PATCH 0/6 v8] iommu/fsl: Freescale PAMU driver and IOMMU API implementation.
From: Varun Sethi @ 2013-02-18 12:52 UTC (permalink / raw)
To: iommu, linuxppc-dev, linux-kernel, scottwood, joro, stuart.yoder
Cc: Varun Sethi
This patchset provides the Freescale PAMU (Peripheral Access Management Unit) driver
and the corresponding IOMMU API implementation. PAMU is the IOMMU present on Freescale
QorIQ platforms. PAMU can authorize memory access, remap the memory address, and remap
the I/O transaction type.
This set consists of the following patches:
1. Addition of new field in the device (powerpc) archdata structure for storing iommu domain information
pointer. This pointer is stored when the device is attached to a particular iommu domain.
2. Store PCI controller platform device information in the PCI controller structure.
3. Add defines for FSL PCI controller BRR1 register.
4. Add window permission flags in the iommu_domain_window_enable API.
5. Add domain attributes for FSL PAMU driver.
6. PAMU driver and IOMMU API implementation.
This patch set is based on the next branch of the iommu git tree maintained by Joerg.
Varun Sethi (6):
Store iommu domain information in the device structure.
Store the platform device information corresponding to the pci
controller.
Added defines for the FSL PCI controller BRR1 register.
Add window permission flags for iommu_domain_window_enable API.
Add addtional attributes specific to the PAMU driver.
FSL PAMU driver and IOMMU API implementation.
arch/powerpc/include/asm/device.h | 4 +
arch/powerpc/include/asm/pci-bridge.h | 4 +
arch/powerpc/sysdev/fsl_pci.c | 9 +-
arch/powerpc/sysdev/fsl_pci.h | 2 +-
drivers/iommu/Kconfig | 8 +
drivers/iommu/Makefile | 1 +
drivers/iommu/fsl_pamu.c | 1260 +++++++++++++++++++++++++++++++++
drivers/iommu/fsl_pamu.h | 398 +++++++++++
drivers/iommu/fsl_pamu_domain.c | 1135 +++++++++++++++++++++++++++++
drivers/iommu/fsl_pamu_domain.h | 89 +++
drivers/iommu/iommu.c | 5 +-
include/linux/iommu.h | 40 +-
12 files changed, 2947 insertions(+), 8 deletions(-)
create mode 100644 drivers/iommu/fsl_pamu.c
create mode 100644 drivers/iommu/fsl_pamu.h
create mode 100644 drivers/iommu/fsl_pamu_domain.c
create mode 100644 drivers/iommu/fsl_pamu_domain.h
--
1.7.4.1
^ permalink raw reply
* [PATCH v6 46/46] Documentation/cpu-hotplug: Remove references to stop_machine()
From: Srivatsa S. Bhat @ 2013-02-18 12:44 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Since stop_machine() is no longer used in the CPU offline path, we cannot
disable CPU hotplug using preempt_disable()/local_irq_disable() etc. We
need to use the newly introduced get/put_online_cpus_atomic() APIs.
Reflect this in the documentation.
Cc: Rob Landley <rob@landley.net>
Cc: linux-doc@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Reviewed-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
Documentation/cpu-hotplug.txt | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
diff --git a/Documentation/cpu-hotplug.txt b/Documentation/cpu-hotplug.txt
index 9f40135..7f907ec 100644
--- a/Documentation/cpu-hotplug.txt
+++ b/Documentation/cpu-hotplug.txt
@@ -113,13 +113,15 @@ Never use anything other than cpumask_t to represent bitmap of CPUs.
#include <linux/cpu.h>
get_online_cpus() and put_online_cpus():
-The above calls are used to inhibit cpu hotplug operations. While the
+The above calls are used to inhibit cpu hotplug operations, when invoked from
+non-atomic context (because the above functions can sleep). While the
cpu_hotplug.refcount is non zero, the cpu_online_mask will not change.
-If you merely need to avoid cpus going away, you could also use
-preempt_disable() and preempt_enable() for those sections.
-Just remember the critical section cannot call any
-function that can sleep or schedule this process away. The preempt_disable()
-will work as long as stop_machine_run() is used to take a cpu down.
+
+However, if you are executing in atomic context (ie., you can't afford to
+sleep), and you merely need to avoid cpus going offline, you can use
+get_online_cpus_atomic() and put_online_cpus_atomic() for those sections.
+Just remember the critical section cannot call any function that can sleep or
+schedule this process away.
CPU Hotplug - Frequently Asked Questions.
@@ -360,6 +362,9 @@ A: There are two ways. If your code can be run in interrupt context, use
return err;
}
+ If my_func_on_cpu() itself cannot block, use get/put_online_cpus_atomic()
+ instead of get/put_online_cpus() to prevent CPUs from going offline.
+
Q: How do we determine how many CPUs are available for hotplug.
A: There is no clear spec defined way from ACPI that can give us that
information today. Based on some input from Natalie of Unisys,
^ permalink raw reply related
* [PATCH v6 45/46] CPU hotplug, stop_machine: Decouple CPU hotplug from stop_machine() in Kconfig
From: Srivatsa S. Bhat @ 2013-02-18 12:44 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Simply dropping HOTPLUG_CPU from the dependency list of STOP_MACHINE will
lead to the following kconfig issue, reported by Fengguang Wu:
"warning: (HAVE_TEXT_POKE_SMP) selects STOP_MACHINE which has unmet direct
dependencies (SMP && MODULE_UNLOAD)"
So drop HOTPLUG_CPU and add HAVE_TEXT_POKE_SMP to the dependency list of
STOP_MACHINE.
And while at it, also cleanup a comment that refers to CPU hotplug being
dependent on stop_machine().
Cc: David Howells <dhowells@redhat.com>
Reported-by: Fengguang Wu <fengguang.wu@intel.com>
Suggested-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
Reviewed-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
include/linux/stop_machine.h | 2 +-
init/Kconfig | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/include/linux/stop_machine.h b/include/linux/stop_machine.h
index 3b5e910..ce2d3c4 100644
--- a/include/linux/stop_machine.h
+++ b/include/linux/stop_machine.h
@@ -120,7 +120,7 @@ int stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus);
* @cpus: the cpus to run the @fn() on (NULL = any online cpu)
*
* Description: This is a special version of the above, which assumes cpus
- * won't come or go while it's being called. Used by hotplug cpu.
+ * won't come or go while it's being called.
*/
int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus);
diff --git a/init/Kconfig b/init/Kconfig
index be8b7f5..f4f79af 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -1711,7 +1711,7 @@ config INIT_ALL_POSSIBLE
config STOP_MACHINE
bool
default y
- depends on (SMP && MODULE_UNLOAD) || HOTPLUG_CPU
+ depends on (SMP && (MODULE_UNLOAD || HAVE_TEXT_POKE_SMP))
help
Need stop_machine() primitive.
^ permalink raw reply related
* [PATCH v6 44/46] cpu: No more __stop_machine() in _cpu_down()
From: Srivatsa S. Bhat @ 2013-02-18 12:44 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
From: Paul E. McKenney <paul.mckenney@linaro.org>
The _cpu_down() function invoked as part of the CPU-hotplug offlining
process currently invokes __stop_machine(), which is slow and inflicts
substantial real-time latencies on the entire system. This patch
substitutes stop_one_cpu() for __stop_machine() in order to improve
both performance and real-time latency.
There were a number of uses of preempt_disable() or local_irq_disable()
that were intended to block CPU-hotplug offlining. These were fixed by
using get/put_online_cpus_atomic(), which is the new synchronization
primitive to prevent CPU offline, while invoking from atomic context.
Signed-off-by: Paul E. McKenney <paul.mckenney@linaro.org>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
[ srivatsa.bhat@linux.vnet.ibm.com: Refer to the new sync primitives for
readers (in the changelog); s/stop_cpus/stop_one_cpu and fix comment
referring to stop_machine in the code]
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
kernel/cpu.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 58dd1df..652fb53 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -337,7 +337,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
}
smpboot_park_threads(cpu);
- err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu));
+ err = stop_one_cpu(cpu, take_cpu_down, &tcd_param);
if (err) {
/* CPU didn't die: tell everyone. Can't complain. */
smpboot_unpark_threads(cpu);
@@ -349,7 +349,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
/*
* The migration_call() CPU_DYING callback will have removed all
* runnable tasks from the cpu, there's only the idle task left now
- * that the migration thread is done doing the stop_machine thing.
+ * that the migration thread is done doing the stop_one_cpu() thing.
*
* Wait for the stop thread to go away.
*/
^ permalink raw reply related
* [PATCH v6 43/46] tile: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-02-18 12:44 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Chris Metcalf <cmetcalf@tilera.com>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/tile/kernel/smp.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/arch/tile/kernel/smp.c b/arch/tile/kernel/smp.c
index cbc73a8..fb30624 100644
--- a/arch/tile/kernel/smp.c
+++ b/arch/tile/kernel/smp.c
@@ -15,6 +15,7 @@
*/
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/irq.h>
@@ -82,9 +83,12 @@ void send_IPI_many(const struct cpumask *mask, int tag)
void send_IPI_allbutself(int tag)
{
struct cpumask mask;
+
+ get_online_cpus_atomic();
cpumask_copy(&mask, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &mask);
send_IPI_many(&mask, tag);
+ put_online_cpus_atomic();
}
/*
^ permalink raw reply related
* [PATCH v6 42/46] sparc: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-02-18 12:44 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Sam Ravnborg <sam@ravnborg.org>
Cc: sparclinux@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/sparc/kernel/leon_smp.c | 2 ++
arch/sparc/kernel/smp_64.c | 9 +++++----
arch/sparc/kernel/sun4d_smp.c | 2 ++
arch/sparc/kernel/sun4m_smp.c | 3 +++
4 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/arch/sparc/kernel/leon_smp.c b/arch/sparc/kernel/leon_smp.c
index 0f3fb6d..441d3ac 100644
--- a/arch/sparc/kernel/leon_smp.c
+++ b/arch/sparc/kernel/leon_smp.c
@@ -420,6 +420,7 @@ static void leon_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1,
unsigned long flags;
spin_lock_irqsave(&cross_call_lock, flags);
+ get_online_cpus_atomic();
{
/* If you make changes here, make sure gcc generates proper code... */
@@ -476,6 +477,7 @@ static void leon_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1,
} while (++i <= high);
}
+ put_online_cpus_atomic();
spin_unlock_irqrestore(&cross_call_lock, flags);
}
}
diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c
index 537eb66..e1d7300 100644
--- a/arch/sparc/kernel/smp_64.c
+++ b/arch/sparc/kernel/smp_64.c
@@ -894,7 +894,8 @@ void smp_flush_dcache_page_impl(struct page *page, int cpu)
atomic_inc(&dcpage_flushes);
#endif
- this_cpu = get_cpu();
+ get_online_cpus_atomic();
+ this_cpu = smp_processor_id();
if (cpu == this_cpu) {
__local_flush_dcache_page(page);
@@ -920,7 +921,7 @@ void smp_flush_dcache_page_impl(struct page *page, int cpu)
}
}
- put_cpu();
+ put_online_cpus_atomic();
}
void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
@@ -931,7 +932,7 @@ void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
if (tlb_type == hypervisor)
return;
- preempt_disable();
+ get_online_cpus_atomic();
#ifdef CONFIG_DEBUG_DCFLUSH
atomic_inc(&dcpage_flushes);
@@ -956,7 +957,7 @@ void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
}
__local_flush_dcache_page(page);
- preempt_enable();
+ put_online_cpus_atomic();
}
void __irq_entry smp_new_mmu_context_version_client(int irq, struct pt_regs *regs)
diff --git a/arch/sparc/kernel/sun4d_smp.c b/arch/sparc/kernel/sun4d_smp.c
index ddaea31..1fa7ff2 100644
--- a/arch/sparc/kernel/sun4d_smp.c
+++ b/arch/sparc/kernel/sun4d_smp.c
@@ -300,6 +300,7 @@ static void sun4d_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1,
unsigned long flags;
spin_lock_irqsave(&cross_call_lock, flags);
+ get_online_cpus_atomic();
{
/*
@@ -356,6 +357,7 @@ static void sun4d_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1,
} while (++i <= high);
}
+ put_online_cpus_atomic();
spin_unlock_irqrestore(&cross_call_lock, flags);
}
}
diff --git a/arch/sparc/kernel/sun4m_smp.c b/arch/sparc/kernel/sun4m_smp.c
index 128af73..5599548 100644
--- a/arch/sparc/kernel/sun4m_smp.c
+++ b/arch/sparc/kernel/sun4m_smp.c
@@ -192,6 +192,7 @@ static void sun4m_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1,
unsigned long flags;
spin_lock_irqsave(&cross_call_lock, flags);
+ get_online_cpus_atomic();
/* Init function glue. */
ccall_info.func = func;
@@ -238,6 +239,8 @@ static void sun4m_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1,
barrier();
} while (++i < ncpus);
}
+
+ put_online_cpus_atomic();
spin_unlock_irqrestore(&cross_call_lock, flags);
}
^ permalink raw reply related
* [PATCH v6 41/46] sh: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-02-18 12:44 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Paul Mundt <lethal@linux-sh.org>
Cc: linux-sh@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/sh/kernel/smp.c | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/arch/sh/kernel/smp.c b/arch/sh/kernel/smp.c
index 2062aa8..232fabe 100644
--- a/arch/sh/kernel/smp.c
+++ b/arch/sh/kernel/smp.c
@@ -357,7 +357,7 @@ static void flush_tlb_mm_ipi(void *mm)
*/
void flush_tlb_mm(struct mm_struct *mm)
{
- preempt_disable();
+ get_online_cpus_atomic();
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
smp_call_function(flush_tlb_mm_ipi, (void *)mm, 1);
@@ -369,7 +369,7 @@ void flush_tlb_mm(struct mm_struct *mm)
}
local_flush_tlb_mm(mm);
- preempt_enable();
+ put_online_cpus_atomic();
}
struct flush_tlb_data {
@@ -390,7 +390,7 @@ void flush_tlb_range(struct vm_area_struct *vma,
{
struct mm_struct *mm = vma->vm_mm;
- preempt_disable();
+ get_online_cpus_atomic();
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
struct flush_tlb_data fd;
@@ -405,7 +405,7 @@ void flush_tlb_range(struct vm_area_struct *vma,
cpu_context(i, mm) = 0;
}
local_flush_tlb_range(vma, start, end);
- preempt_enable();
+ put_online_cpus_atomic();
}
static void flush_tlb_kernel_range_ipi(void *info)
@@ -433,7 +433,7 @@ static void flush_tlb_page_ipi(void *info)
void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
{
- preempt_disable();
+ get_online_cpus_atomic();
if ((atomic_read(&vma->vm_mm->mm_users) != 1) ||
(current->mm != vma->vm_mm)) {
struct flush_tlb_data fd;
@@ -448,7 +448,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
cpu_context(i, vma->vm_mm) = 0;
}
local_flush_tlb_page(vma, page);
- preempt_enable();
+ put_online_cpus_atomic();
}
static void flush_tlb_one_ipi(void *info)
^ permalink raw reply related
* [PATCH v6 40/46] powerpc: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-02-18 12:43 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: linuxppc-dev@lists.ozlabs.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/powerpc/mm/mmu_context_nohash.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/arch/powerpc/mm/mmu_context_nohash.c b/arch/powerpc/mm/mmu_context_nohash.c
index e779642..29f58cf 100644
--- a/arch/powerpc/mm/mmu_context_nohash.c
+++ b/arch/powerpc/mm/mmu_context_nohash.c
@@ -196,6 +196,7 @@ void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
/* No lockless fast path .. yet */
raw_spin_lock(&context_lock);
+ get_online_cpus_atomic();
pr_hard("[%d] activating context for mm @%p, active=%d, id=%d",
cpu, next, next->context.active, next->context.id);
@@ -279,6 +280,7 @@ void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
/* Flick the MMU and release lock */
pr_hardcont(" -> %d\n", id);
set_context(id, next->pgd);
+ put_online_cpus_atomic();
raw_spin_unlock(&context_lock);
}
^ permalink raw reply related
* [PATCH v6 39/46] parisc: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-02-18 12:43 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: "James E.J. Bottomley" <jejb@parisc-linux.org>
Cc: linux-parisc@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/parisc/kernel/smp.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/arch/parisc/kernel/smp.c b/arch/parisc/kernel/smp.c
index 6266730..f7851b7 100644
--- a/arch/parisc/kernel/smp.c
+++ b/arch/parisc/kernel/smp.c
@@ -229,11 +229,13 @@ static inline void
send_IPI_allbutself(enum ipi_message_type op)
{
int i;
-
+
+ get_online_cpus_atomic();
for_each_online_cpu(i) {
if (i != smp_processor_id())
send_IPI_single(i, op);
}
+ put_online_cpus_atomic();
}
^ permalink raw reply related
* [PATCH v6 38/46] mn10300: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-02-18 12:43 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: David Howells <dhowells@redhat.com>
Cc: Koichi Yasutake <yasutake.koichi@jp.panasonic.com>
Cc: linux-am33-list@redhat.com
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/mn10300/kernel/smp.c | 2 ++
arch/mn10300/mm/cache-smp.c | 5 +++++
arch/mn10300/mm/tlb-smp.c | 15 +++++++++------
3 files changed, 16 insertions(+), 6 deletions(-)
diff --git a/arch/mn10300/kernel/smp.c b/arch/mn10300/kernel/smp.c
index 5d7e152..9dfa172 100644
--- a/arch/mn10300/kernel/smp.c
+++ b/arch/mn10300/kernel/smp.c
@@ -349,9 +349,11 @@ void send_IPI_allbutself(int irq)
{
cpumask_t cpumask;
+ get_online_cpus_atomic();
cpumask_copy(&cpumask, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &cpumask);
send_IPI_mask(&cpumask, irq);
+ put_online_cpus_atomic();
}
void arch_send_call_function_ipi_mask(const struct cpumask *mask)
diff --git a/arch/mn10300/mm/cache-smp.c b/arch/mn10300/mm/cache-smp.c
index 2d23b9e..47ca1c9 100644
--- a/arch/mn10300/mm/cache-smp.c
+++ b/arch/mn10300/mm/cache-smp.c
@@ -13,6 +13,7 @@
#include <linux/mman.h>
#include <linux/threads.h>
#include <linux/interrupt.h>
+#include <linux/cpu.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/processor.h>
@@ -94,6 +95,8 @@ void smp_cache_call(unsigned long opr_mask,
smp_cache_mask = opr_mask;
smp_cache_start = start;
smp_cache_end = end;
+
+ get_online_cpus_atomic();
cpumask_copy(&smp_cache_ipi_map, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &smp_cache_ipi_map);
@@ -102,4 +105,6 @@ void smp_cache_call(unsigned long opr_mask,
while (!cpumask_empty(&smp_cache_ipi_map))
/* nothing. lockup detection does not belong here */
mb();
+
+ put_online_cpus_atomic();
}
diff --git a/arch/mn10300/mm/tlb-smp.c b/arch/mn10300/mm/tlb-smp.c
index 3e57faf..d47304d 100644
--- a/arch/mn10300/mm/tlb-smp.c
+++ b/arch/mn10300/mm/tlb-smp.c
@@ -23,6 +23,7 @@
#include <linux/sched.h>
#include <linux/profile.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <asm/tlbflush.h>
#include <asm/bitops.h>
#include <asm/processor.h>
@@ -105,6 +106,7 @@ static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
BUG_ON(cpumask_empty(&cpumask));
BUG_ON(cpumask_test_cpu(smp_processor_id(), &cpumask));
+ get_online_cpus_atomic();
cpumask_and(&tmp, &cpumask, cpu_online_mask);
BUG_ON(!cpumask_equal(&cpumask, &tmp));
@@ -134,6 +136,7 @@ static void flush_tlb_others(cpumask_t cpumask, struct mm_struct *mm,
flush_mm = NULL;
flush_va = 0;
spin_unlock(&tlbstate_lock);
+ put_online_cpus_atomic();
}
/**
@@ -144,7 +147,7 @@ void flush_tlb_mm(struct mm_struct *mm)
{
cpumask_t cpu_mask;
- preempt_disable();
+ get_online_cpus_atomic();
cpumask_copy(&cpu_mask, mm_cpumask(mm));
cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
@@ -152,7 +155,7 @@ void flush_tlb_mm(struct mm_struct *mm)
if (!cpumask_empty(&cpu_mask))
flush_tlb_others(cpu_mask, mm, FLUSH_ALL);
- preempt_enable();
+ put_online_cpus_atomic();
}
/**
@@ -163,7 +166,7 @@ void flush_tlb_current_task(void)
struct mm_struct *mm = current->mm;
cpumask_t cpu_mask;
- preempt_disable();
+ get_online_cpus_atomic();
cpumask_copy(&cpu_mask, mm_cpumask(mm));
cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
@@ -171,7 +174,7 @@ void flush_tlb_current_task(void)
if (!cpumask_empty(&cpu_mask))
flush_tlb_others(cpu_mask, mm, FLUSH_ALL);
- preempt_enable();
+ put_online_cpus_atomic();
}
/**
@@ -184,7 +187,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
struct mm_struct *mm = vma->vm_mm;
cpumask_t cpu_mask;
- preempt_disable();
+ get_online_cpus_atomic();
cpumask_copy(&cpu_mask, mm_cpumask(mm));
cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
@@ -192,7 +195,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
if (!cpumask_empty(&cpu_mask))
flush_tlb_others(cpu_mask, mm, va);
- preempt_enable();
+ put_online_cpus_atomic();
}
/**
^ permalink raw reply related
* [PATCH v6 37/46] MIPS: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-02-18 12:43 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs fom
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: David Daney <david.daney@cavium.com>
Cc: linux-mips@linux-mips.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/mips/kernel/cevt-smtc.c | 8 ++++++++
arch/mips/kernel/smp.c | 16 ++++++++--------
arch/mips/kernel/smtc.c | 3 +++
arch/mips/mm/c-octeon.c | 4 ++--
4 files changed, 21 insertions(+), 10 deletions(-)
diff --git a/arch/mips/kernel/cevt-smtc.c b/arch/mips/kernel/cevt-smtc.c
index 2e72d30..6fb311b 100644
--- a/arch/mips/kernel/cevt-smtc.c
+++ b/arch/mips/kernel/cevt-smtc.c
@@ -11,6 +11,7 @@
#include <linux/interrupt.h>
#include <linux/percpu.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/irq.h>
#include <asm/smtc_ipi.h>
@@ -84,6 +85,8 @@ static int mips_next_event(unsigned long delta,
unsigned long nextcomp = 0L;
int vpe = current_cpu_data.vpe_id;
int cpu = smp_processor_id();
+
+ get_online_cpus_atomic();
local_irq_save(flags);
mtflags = dmt();
@@ -164,6 +167,7 @@ static int mips_next_event(unsigned long delta,
}
emt(mtflags);
local_irq_restore(flags);
+ put_online_cpus_atomic();
return 0;
}
@@ -180,6 +184,7 @@ void smtc_distribute_timer(int vpe)
repeat:
nextstamp = 0L;
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
/*
* Find virtual CPUs within the current VPE who have
@@ -221,6 +226,9 @@ repeat:
}
}
+
+ put_online_cpus_atomic()
+
/* Reprogram for interrupt at next soonest timestamp for VPE */
if (ISVALID(nextstamp)) {
write_c0_compare(nextstamp);
diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c
index 66bf4e2..3828afa 100644
--- a/arch/mips/kernel/smp.c
+++ b/arch/mips/kernel/smp.c
@@ -248,12 +248,12 @@ static inline void smp_on_other_tlbs(void (*func) (void *info), void *info)
static inline void smp_on_each_tlb(void (*func) (void *info), void *info)
{
- preempt_disable();
+ get_online_cpus_atomic();
smp_on_other_tlbs(func, info);
func(info);
- preempt_enable();
+ put_online_cpus_atomic();
}
/*
@@ -271,7 +271,7 @@ static inline void smp_on_each_tlb(void (*func) (void *info), void *info)
void flush_tlb_mm(struct mm_struct *mm)
{
- preempt_disable();
+ get_online_cpus_atomic();
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
smp_on_other_tlbs(flush_tlb_mm_ipi, mm);
@@ -285,7 +285,7 @@ void flush_tlb_mm(struct mm_struct *mm)
}
local_flush_tlb_mm(mm);
- preempt_enable();
+ put_online_cpus_atomic();
}
struct flush_tlb_data {
@@ -305,7 +305,7 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned l
{
struct mm_struct *mm = vma->vm_mm;
- preempt_disable();
+ get_online_cpus_atomic();
if ((atomic_read(&mm->mm_users) != 1) || (current->mm != mm)) {
struct flush_tlb_data fd = {
.vma = vma,
@@ -323,7 +323,7 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned l
}
}
local_flush_tlb_range(vma, start, end);
- preempt_enable();
+ put_online_cpus_atomic();
}
static void flush_tlb_kernel_range_ipi(void *info)
@@ -352,7 +352,7 @@ static void flush_tlb_page_ipi(void *info)
void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
{
- preempt_disable();
+ get_online_cpus_atomic();
if ((atomic_read(&vma->vm_mm->mm_users) != 1) || (current->mm != vma->vm_mm)) {
struct flush_tlb_data fd = {
.vma = vma,
@@ -369,7 +369,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
}
}
local_flush_tlb_page(vma, page);
- preempt_enable();
+ put_online_cpus_atomic();
}
static void flush_tlb_one_ipi(void *info)
diff --git a/arch/mips/kernel/smtc.c b/arch/mips/kernel/smtc.c
index 1d47843..caf081e 100644
--- a/arch/mips/kernel/smtc.c
+++ b/arch/mips/kernel/smtc.c
@@ -22,6 +22,7 @@
#include <linux/sched.h>
#include <linux/smp.h>
#include <linux/cpumask.h>
+#include <linux/cpu.h>
#include <linux/interrupt.h>
#include <linux/kernel_stat.h>
#include <linux/module.h>
@@ -1143,6 +1144,7 @@ static irqreturn_t ipi_interrupt(int irq, void *dev_idm)
* for the current TC, so we ought not to have to do it explicitly here.
*/
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
if (cpu_data[cpu].vpe_id != my_vpe)
continue;
@@ -1179,6 +1181,7 @@ static irqreturn_t ipi_interrupt(int irq, void *dev_idm)
}
}
}
+ put_online_cpus_atomic();
return IRQ_HANDLED;
}
diff --git a/arch/mips/mm/c-octeon.c b/arch/mips/mm/c-octeon.c
index 6ec04da..cd2c1ce 100644
--- a/arch/mips/mm/c-octeon.c
+++ b/arch/mips/mm/c-octeon.c
@@ -73,7 +73,7 @@ static void octeon_flush_icache_all_cores(struct vm_area_struct *vma)
mb();
octeon_local_flush_icache();
#ifdef CONFIG_SMP
- preempt_disable();
+ get_online_cpus_atomic();
cpu = smp_processor_id();
/*
@@ -88,7 +88,7 @@ static void octeon_flush_icache_all_cores(struct vm_area_struct *vma)
for_each_cpu(cpu, &mask)
octeon_send_ipi_single(cpu, SMP_ICACHE_FLUSH);
- preempt_enable();
+ put_online_cpus_atomic();
#endif
}
^ permalink raw reply related
* [PATCH v6 36/46] m32r: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-02-18 12:43 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Hirokazu Takata <takata@linux-m32r.org>
Cc: linux-m32r@ml.linux-m32r.org
Cc: linux-m32r-ja@ml.linux-m32r.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/m32r/kernel/smp.c | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/arch/m32r/kernel/smp.c b/arch/m32r/kernel/smp.c
index ce7aea3..0dad4d7 100644
--- a/arch/m32r/kernel/smp.c
+++ b/arch/m32r/kernel/smp.c
@@ -151,7 +151,7 @@ void smp_flush_cache_all(void)
cpumask_t cpumask;
unsigned long *mask;
- preempt_disable();
+ get_online_cpus_atomic();
cpumask_copy(&cpumask, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &cpumask);
spin_lock(&flushcache_lock);
@@ -162,7 +162,7 @@ void smp_flush_cache_all(void)
while (flushcache_cpumask)
mb();
spin_unlock(&flushcache_lock);
- preempt_enable();
+ put_online_cpus_atomic();
}
void smp_flush_cache_all_interrupt(void)
@@ -250,7 +250,7 @@ void smp_flush_tlb_mm(struct mm_struct *mm)
unsigned long *mmc;
unsigned long flags;
- preempt_disable();
+ get_online_cpus_atomic();
cpu_id = smp_processor_id();
mmc = &mm->context[cpu_id];
cpumask_copy(&cpu_mask, mm_cpumask(mm));
@@ -268,7 +268,7 @@ void smp_flush_tlb_mm(struct mm_struct *mm)
if (!cpumask_empty(&cpu_mask))
flush_tlb_others(cpu_mask, mm, NULL, FLUSH_ALL);
- preempt_enable();
+ put_online_cpus_atomic();
}
/*==========================================================================*
@@ -715,10 +715,12 @@ static void send_IPI_allbutself(int ipi_num, int try)
{
cpumask_t cpumask;
+ get_online_cpus_atomic();
cpumask_copy(&cpumask, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &cpumask);
send_IPI_mask(&cpumask, ipi_num, try);
+ put_online_cpus_atomic();
}
/*==========================================================================*
@@ -750,6 +752,7 @@ static void send_IPI_mask(const struct cpumask *cpumask, int ipi_num, int try)
if (num_cpus <= 1) /* NO MP */
return;
+ get_online_cpus_atomic();
cpumask_and(&tmp, cpumask, cpu_online_mask);
BUG_ON(!cpumask_equal(cpumask, &tmp));
@@ -760,6 +763,7 @@ static void send_IPI_mask(const struct cpumask *cpumask, int ipi_num, int try)
}
send_IPI_mask_phys(&physid_mask, ipi_num, try);
+ put_online_cpus_atomic();
}
/*==========================================================================*
^ permalink raw reply related
* [PATCH v6 35/46] ia64: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-02-18 12:43 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Tony Luck <tony.luck@intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: linux-ia64@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/ia64/kernel/irq_ia64.c | 13 +++++++++++++
arch/ia64/kernel/perfmon.c | 6 ++++++
arch/ia64/kernel/smp.c | 23 ++++++++++++++++-------
arch/ia64/mm/tlb.c | 6 ++++--
4 files changed, 39 insertions(+), 9 deletions(-)
diff --git a/arch/ia64/kernel/irq_ia64.c b/arch/ia64/kernel/irq_ia64.c
index 1034884..d0b4478 100644
--- a/arch/ia64/kernel/irq_ia64.c
+++ b/arch/ia64/kernel/irq_ia64.c
@@ -31,6 +31,7 @@
#include <linux/ratelimit.h>
#include <linux/acpi.h>
#include <linux/sched.h>
+#include <linux/cpu.h>
#include <asm/delay.h>
#include <asm/intrinsics.h>
@@ -190,9 +191,11 @@ static void clear_irq_vector(int irq)
{
unsigned long flags;
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
__clear_irq_vector(irq);
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
}
int
@@ -204,6 +207,7 @@ ia64_native_assign_irq_vector (int irq)
vector = -ENOSPC;
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
for_each_online_cpu(cpu) {
domain = vector_allocation_domain(cpu);
@@ -218,6 +222,7 @@ ia64_native_assign_irq_vector (int irq)
BUG_ON(__bind_irq_vector(irq, vector, domain));
out:
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
return vector;
}
@@ -302,9 +307,11 @@ int irq_prepare_move(int irq, int cpu)
unsigned long flags;
int ret;
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
ret = __irq_prepare_move(irq, cpu);
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
return ret;
}
@@ -320,11 +327,13 @@ void irq_complete_move(unsigned irq)
if (unlikely(cpu_isset(smp_processor_id(), cfg->old_domain)))
return;
+ get_online_cpus_atomic();
cpumask_and(&cleanup_mask, &cfg->old_domain, cpu_online_mask);
cfg->move_cleanup_count = cpus_weight(cleanup_mask);
for_each_cpu_mask(i, cleanup_mask)
platform_send_ipi(i, IA64_IRQ_MOVE_VECTOR, IA64_IPI_DM_INT, 0);
cfg->move_in_progress = 0;
+ put_online_cpus_atomic();
}
static irqreturn_t smp_irq_move_cleanup_interrupt(int irq, void *dev_id)
@@ -409,6 +418,8 @@ int create_irq(void)
cpumask_t domain = CPU_MASK_NONE;
irq = vector = -ENOSPC;
+
+ get_online_cpus_atomic();
spin_lock_irqsave(&vector_lock, flags);
for_each_online_cpu(cpu) {
domain = vector_allocation_domain(cpu);
@@ -424,6 +435,8 @@ int create_irq(void)
BUG_ON(__bind_irq_vector(irq, vector, domain));
out:
spin_unlock_irqrestore(&vector_lock, flags);
+ put_online_cpus_atomic();
+
if (irq >= 0)
dynamic_irq_init(irq);
return irq;
diff --git a/arch/ia64/kernel/perfmon.c b/arch/ia64/kernel/perfmon.c
index ea39eba..6c6a029 100644
--- a/arch/ia64/kernel/perfmon.c
+++ b/arch/ia64/kernel/perfmon.c
@@ -34,6 +34,7 @@
#include <linux/poll.h>
#include <linux/vfs.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/pagemap.h>
#include <linux/mount.h>
#include <linux/bitops.h>
@@ -6485,6 +6486,7 @@ pfm_install_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
}
/* reserve our session */
+ get_online_cpus_atomic();
for_each_online_cpu(reserve_cpu) {
ret = pfm_reserve_session(NULL, 1, reserve_cpu);
if (ret) goto cleanup_reserve;
@@ -6500,6 +6502,7 @@ pfm_install_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
/* officially change to the alternate interrupt handler */
pfm_alt_intr_handler = hdl;
+ put_online_cpus_atomic();
spin_unlock(&pfm_alt_install_check);
return 0;
@@ -6512,6 +6515,7 @@ cleanup_reserve:
pfm_unreserve_session(NULL, 1, i);
}
+ put_online_cpus_atomic();
spin_unlock(&pfm_alt_install_check);
return ret;
@@ -6536,6 +6540,7 @@ pfm_remove_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
pfm_alt_intr_handler = NULL;
+ get_online_cpus_atomic();
ret = on_each_cpu(pfm_alt_restore_pmu_state, NULL, 1);
if (ret) {
DPRINT(("on_each_cpu() failed: %d\n", ret));
@@ -6545,6 +6550,7 @@ pfm_remove_alt_pmu_interrupt(pfm_intr_handler_desc_t *hdl)
pfm_unreserve_session(NULL, 1, i);
}
+ put_online_cpus_atomic();
spin_unlock(&pfm_alt_install_check);
return 0;
diff --git a/arch/ia64/kernel/smp.c b/arch/ia64/kernel/smp.c
index 9fcd4e6..d9a4636 100644
--- a/arch/ia64/kernel/smp.c
+++ b/arch/ia64/kernel/smp.c
@@ -24,6 +24,7 @@
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/kernel_stat.h>
#include <linux/mm.h>
#include <linux/cache.h>
@@ -154,12 +155,15 @@ send_IPI_single (int dest_cpu, int op)
static inline void
send_IPI_allbutself (int op)
{
- unsigned int i;
+ unsigned int i, cpu;
+ get_online_cpus_atomic();
+ cpu = smp_processor_id();
for_each_online_cpu(i) {
- if (i != smp_processor_id())
+ if (i != cpu)
send_IPI_single(i, op);
}
+ put_online_cpus_atomic();
}
/*
@@ -170,9 +174,11 @@ send_IPI_mask(const struct cpumask *mask, int op)
{
unsigned int cpu;
+ get_online_cpus_atomic();
for_each_cpu(cpu, mask) {
send_IPI_single(cpu, op);
}
+ put_online_cpus_atomic();
}
/*
@@ -183,9 +189,11 @@ send_IPI_all (int op)
{
int i;
+ get_online_cpus_atomic();
for_each_online_cpu(i) {
send_IPI_single(i, op);
}
+ put_online_cpus_atomic();
}
/*
@@ -259,7 +267,7 @@ smp_flush_tlb_cpumask(cpumask_t xcpumask)
cpumask_t cpumask = xcpumask;
int mycpu, cpu, flush_mycpu = 0;
- preempt_disable();
+ get_online_cpus_atomic();
mycpu = smp_processor_id();
for_each_cpu_mask(cpu, cpumask)
@@ -280,7 +288,7 @@ smp_flush_tlb_cpumask(cpumask_t xcpumask)
while(counts[cpu] == (local_tlb_flush_counts[cpu].count & 0xffff))
udelay(FLUSH_DELAY);
- preempt_enable();
+ put_online_cpus_atomic();
}
void
@@ -293,12 +301,13 @@ void
smp_flush_tlb_mm (struct mm_struct *mm)
{
cpumask_var_t cpus;
- preempt_disable();
+
+ get_online_cpus_atomic();
/* this happens for the common case of a single-threaded fork(): */
if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1))
{
local_finish_flush_tlb_mm(mm);
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
if (!alloc_cpumask_var(&cpus, GFP_ATOMIC)) {
@@ -313,7 +322,7 @@ smp_flush_tlb_mm (struct mm_struct *mm)
local_irq_disable();
local_finish_flush_tlb_mm(mm);
local_irq_enable();
- preempt_enable();
+ put_online_cpus_atomic();
}
void arch_send_call_function_single_ipi(int cpu)
diff --git a/arch/ia64/mm/tlb.c b/arch/ia64/mm/tlb.c
index ed61297..8f03b58 100644
--- a/arch/ia64/mm/tlb.c
+++ b/arch/ia64/mm/tlb.c
@@ -20,6 +20,7 @@
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/mm.h>
#include <linux/bootmem.h>
#include <linux/slab.h>
@@ -87,11 +88,12 @@ wrap_mmu_context (struct mm_struct *mm)
* can't call flush_tlb_all() here because of race condition
* with O(1) scheduler [EF]
*/
- cpu = get_cpu(); /* prevent preemption/migration */
+ get_online_cpus_atomic(); /* prevent preemption/migration */
+ cpu = smp_processor_id();
for_each_online_cpu(i)
if (i != cpu)
per_cpu(ia64_need_tlb_flush, i) = 1;
- put_cpu();
+ put_online_cpus_atomic();
local_flush_tlb_all();
}
^ permalink raw reply related
* [PATCH v6 34/46] hexagon/smp: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-02-18 12:43 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Richard Kuo <rkuo@codeaurora.org>
Cc: linux-hexagon@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/hexagon/kernel/smp.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/arch/hexagon/kernel/smp.c b/arch/hexagon/kernel/smp.c
index 8e095df..ec87de9 100644
--- a/arch/hexagon/kernel/smp.c
+++ b/arch/hexagon/kernel/smp.c
@@ -112,6 +112,7 @@ void send_ipi(const struct cpumask *cpumask, enum ipi_message_type msg)
unsigned long cpu;
unsigned long retval;
+ get_online_cpus_atomic();
local_irq_save(flags);
for_each_cpu(cpu, cpumask) {
@@ -128,6 +129,7 @@ void send_ipi(const struct cpumask *cpumask, enum ipi_message_type msg)
}
local_irq_restore(flags);
+ put_online_cpus_atomic();
}
static struct irqaction ipi_intdesc = {
@@ -241,9 +243,12 @@ void smp_send_reschedule(int cpu)
void smp_send_stop(void)
{
struct cpumask targets;
+
+ get_online_cpus_atomic();
cpumask_copy(&targets, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &targets);
send_ipi(&targets, IPI_CPU_STOP);
+ put_online_cpus_atomic();
}
void arch_send_call_function_single_ipi(int cpu)
^ permalink raw reply related
* [PATCH v6 33/46] cris/smp: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-02-18 12:42 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Mikael Starvik <starvik@axis.com>
Cc: Jesper Nilsson <jesper.nilsson@axis.com>
Cc: linux-cris-kernel@axis.com
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/cris/arch-v32/kernel/smp.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/arch/cris/arch-v32/kernel/smp.c b/arch/cris/arch-v32/kernel/smp.c
index 04a16ed..644b358 100644
--- a/arch/cris/arch-v32/kernel/smp.c
+++ b/arch/cris/arch-v32/kernel/smp.c
@@ -15,6 +15,7 @@
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/cpumask.h>
+#include <linux/cpu.h>
#include <linux/interrupt.h>
#include <linux/module.h>
@@ -208,9 +209,12 @@ int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *tidle)
void smp_send_reschedule(int cpu)
{
cpumask_t cpu_mask;
+
+ get_online_cpus_atomic();
cpumask_clear(&cpu_mask);
cpumask_set_cpu(cpu, &cpu_mask);
send_ipi(IPI_SCHEDULE, 0, cpu_mask);
+ put_online_cpus_atomic();
}
/* TLB flushing
@@ -224,6 +228,7 @@ void flush_tlb_common(struct mm_struct* mm, struct vm_area_struct* vma, unsigned
unsigned long flags;
cpumask_t cpu_mask;
+ get_online_cpus_atomic();
spin_lock_irqsave(&tlbstate_lock, flags);
cpu_mask = (mm == FLUSH_ALL ? cpu_all_mask : *mm_cpumask(mm));
cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
@@ -232,6 +237,7 @@ void flush_tlb_common(struct mm_struct* mm, struct vm_area_struct* vma, unsigned
flush_addr = addr;
send_ipi(IPI_FLUSH_TLB, 1, cpu_mask);
spin_unlock_irqrestore(&tlbstate_lock, flags);
+ put_online_cpus_atomic();
}
void flush_tlb_all(void)
@@ -312,6 +318,7 @@ int smp_call_function(void (*func)(void *info), void *info, int wait)
struct call_data_struct data;
int ret;
+ get_online_cpus_atomic();
cpumask_setall(&cpu_mask);
cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
@@ -325,6 +332,7 @@ int smp_call_function(void (*func)(void *info), void *info, int wait)
call_data = &data;
ret = send_ipi(IPI_CALL, wait, cpu_mask);
spin_unlock(&call_lock);
+ put_online_cpus_atomic();
return ret;
}
^ permalink raw reply related
* [PATCH v6 32/46] blackfin/smp: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-02-18 12:42 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Mike Frysinger <vapier@gentoo.org>
Cc: Bob Liu <lliubbo@gmail.com>
Cc: Steven Miao <realmz6@gmail.com>
Cc: uclinux-dist-devel@blackfin.uclinux.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/blackfin/mach-common/smp.c | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/arch/blackfin/mach-common/smp.c b/arch/blackfin/mach-common/smp.c
index bb61ae4..6cc6d7a 100644
--- a/arch/blackfin/mach-common/smp.c
+++ b/arch/blackfin/mach-common/smp.c
@@ -194,6 +194,7 @@ void send_ipi(const struct cpumask *cpumask, enum ipi_message_type msg)
struct ipi_data *bfin_ipi_data;
unsigned long flags;
+ get_online_cpus_atomic();
local_irq_save(flags);
smp_mb();
for_each_cpu(cpu, cpumask) {
@@ -205,6 +206,7 @@ void send_ipi(const struct cpumask *cpumask, enum ipi_message_type msg)
}
local_irq_restore(flags);
+ put_online_cpus_atomic();
}
void arch_send_call_function_single_ipi(int cpu)
@@ -238,13 +240,13 @@ void smp_send_stop(void)
{
cpumask_t callmap;
- preempt_disable();
+ get_online_cpus_atomic();
cpumask_copy(&callmap, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), &callmap);
if (!cpumask_empty(&callmap))
send_ipi(&callmap, BFIN_IPI_CPU_STOP);
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
^ permalink raw reply related
* [PATCH v6 31/46] alpha/smp: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-02-18 12:42 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Also, remove the non-ASCII character present in this file!
Cc: linux-alpha@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/alpha/kernel/smp.c | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
diff --git a/arch/alpha/kernel/smp.c b/arch/alpha/kernel/smp.c
index 9603bc2..9213d5d 100644
--- a/arch/alpha/kernel/smp.c
+++ b/arch/alpha/kernel/smp.c
@@ -498,7 +498,6 @@ smp_cpus_done(unsigned int max_cpus)
((bogosum + 2500) / (5000/HZ)) % 100);
}
-\f
void
smp_percpu_timer_interrupt(struct pt_regs *regs)
{
@@ -682,7 +681,7 @@ ipi_flush_tlb_mm(void *x)
void
flush_tlb_mm(struct mm_struct *mm)
{
- preempt_disable();
+ get_online_cpus_atomic();
if (mm == current->active_mm) {
flush_tlb_current(mm);
@@ -694,7 +693,7 @@ flush_tlb_mm(struct mm_struct *mm)
if (mm->context[cpu])
mm->context[cpu] = 0;
}
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
}
@@ -703,7 +702,7 @@ flush_tlb_mm(struct mm_struct *mm)
printk(KERN_CRIT "flush_tlb_mm: timed out\n");
}
- preempt_enable();
+ put_online_cpus_atomic();
}
EXPORT_SYMBOL(flush_tlb_mm);
@@ -731,7 +730,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
struct flush_tlb_page_struct data;
struct mm_struct *mm = vma->vm_mm;
- preempt_disable();
+ get_online_cpus_atomic();
if (mm == current->active_mm) {
flush_tlb_current_page(mm, vma, addr);
@@ -743,7 +742,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
if (mm->context[cpu])
mm->context[cpu] = 0;
}
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
}
@@ -756,7 +755,7 @@ flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
printk(KERN_CRIT "flush_tlb_page: timed out\n");
}
- preempt_enable();
+ put_online_cpus_atomic();
}
EXPORT_SYMBOL(flush_tlb_page);
@@ -787,7 +786,7 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
if ((vma->vm_flags & VM_EXEC) == 0)
return;
- preempt_disable();
+ get_online_cpus_atomic();
if (mm == current->active_mm) {
__load_new_mm_context(mm);
@@ -799,7 +798,7 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
if (mm->context[cpu])
mm->context[cpu] = 0;
}
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
}
@@ -808,5 +807,5 @@ flush_icache_user_range(struct vm_area_struct *vma, struct page *page,
printk(KERN_CRIT "flush_icache_page: timed out\n");
}
- preempt_enable();
+ put_online_cpus_atomic();
}
^ permalink raw reply related
* [PATCH v6 30/46] x86/xen: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-02-18 12:42 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Cc: Jeremy Fitzhardinge <jeremy@goop.org>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Cc: xen-devel@lists.xensource.com
Cc: virtualization@lists.linux-foundation.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/x86/xen/mmu.c | 11 +++++++++--
arch/x86/xen/smp.c | 9 +++++++++
2 files changed, 18 insertions(+), 2 deletions(-)
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 01de35c..6a95a15 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -39,6 +39,7 @@
* Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
*/
#include <linux/sched.h>
+#include <linux/cpu.h>
#include <linux/highmem.h>
#include <linux/debugfs.h>
#include <linux/bug.h>
@@ -1163,9 +1164,13 @@ static void xen_drop_mm_ref(struct mm_struct *mm)
*/
static void xen_exit_mmap(struct mm_struct *mm)
{
- get_cpu(); /* make sure we don't move around */
+ /*
+ * Make sure we don't move around, and prevent CPUs from going
+ * offline.
+ */
+ get_online_cpus_atomic();
xen_drop_mm_ref(mm);
- put_cpu();
+ put_online_cpus_atomic();
spin_lock(&mm->page_table_lock);
@@ -1371,6 +1376,7 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
args->op.arg2.vcpumask = to_cpumask(args->mask);
/* Remove us, and any offline CPUS. */
+ get_online_cpus_atomic();
cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
@@ -1383,6 +1389,7 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
xen_mc_issue(PARAVIRT_LAZY_MMU);
+ put_online_cpus_atomic();
}
static unsigned long xen_read_cr3(void)
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 34bc4ce..cb82e71 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -16,6 +16,7 @@
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/irq_work.h>
#include <asm/paravirt.h>
@@ -480,8 +481,10 @@ static void __xen_send_IPI_mask(const struct cpumask *mask,
{
unsigned cpu;
+ get_online_cpus_atomic();
for_each_cpu_and(cpu, mask, cpu_online_mask)
xen_send_IPI_one(cpu, vector);
+ put_online_cpus_atomic();
}
static void xen_smp_send_call_function_ipi(const struct cpumask *mask)
@@ -544,8 +547,10 @@ void xen_send_IPI_all(int vector)
{
int xen_vector = xen_map_vector(vector);
+ get_online_cpus_atomic();
if (xen_vector >= 0)
__xen_send_IPI_mask(cpu_online_mask, xen_vector);
+ put_online_cpus_atomic();
}
void xen_send_IPI_self(int vector)
@@ -565,20 +570,24 @@ void xen_send_IPI_mask_allbutself(const struct cpumask *mask,
if (!(num_online_cpus() > 1))
return;
+ get_online_cpus_atomic();
for_each_cpu_and(cpu, mask, cpu_online_mask) {
if (this_cpu == cpu)
continue;
xen_smp_send_call_function_single_ipi(cpu);
}
+ put_online_cpus_atomic();
}
void xen_send_IPI_allbutself(int vector)
{
int xen_vector = xen_map_vector(vector);
+ get_online_cpus_atomic();
if (xen_vector >= 0)
xen_send_IPI_mask_allbutself(cpu_online_mask, xen_vector);
+ put_online_cpus_atomic();
}
static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id)
^ permalink raw reply related
* [PATCH v6 29/46] kvm/vmx: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-02-18 12:42 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context (in vmx_vcpu_load() to prevent CPUs from
going offline while clearing vmcs).
Reported-by: Michael Wang <wangyun@linux.vnet.ibm.com>
Debugged-by: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Gleb Natapov <gleb@redhat.com>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Cc: kvm@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/x86/kvm/vmx.c | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 9120ae1..2886ff0 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -1557,10 +1557,14 @@ static void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
struct vcpu_vmx *vmx = to_vmx(vcpu);
u64 phys_addr = __pa(per_cpu(vmxarea, cpu));
- if (!vmm_exclusive)
+ if (!vmm_exclusive) {
kvm_cpu_vmxon(phys_addr);
- else if (vmx->loaded_vmcs->cpu != cpu)
+ } else if (vmx->loaded_vmcs->cpu != cpu) {
+ /* Prevent any CPU from going offline */
+ get_online_cpus_atomic();
loaded_vmcs_clear(vmx->loaded_vmcs);
+ put_online_cpus_atomic();
+ }
if (per_cpu(current_vmcs, cpu) != vmx->loaded_vmcs->vmcs) {
per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
^ permalink raw reply related
* [PATCH v6 28/46] KVM: Use get/put_online_cpus_atomic() to prevent CPU offline from atomic context
From: Srivatsa S. Bhat @ 2013-02-18 12:42 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Gleb Natapov <gleb@redhat.com>
Cc: kvm@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
virt/kvm/kvm_main.c | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 1cd693a..47f9c30 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -174,7 +174,8 @@ static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
zalloc_cpumask_var(&cpus, GFP_ATOMIC);
- me = get_cpu();
+ get_online_cpus_atomic();
+ me = smp_processor_id();
kvm_for_each_vcpu(i, vcpu, kvm) {
kvm_make_request(req, vcpu);
cpu = vcpu->cpu;
@@ -192,7 +193,7 @@ static bool make_all_cpus_request(struct kvm *kvm, unsigned int req)
smp_call_function_many(cpus, ack_flush, NULL, 1);
else
called = false;
- put_cpu();
+ put_online_cpus_atomic();
free_cpumask_var(cpus);
return called;
}
@@ -1621,11 +1622,12 @@ void kvm_vcpu_kick(struct kvm_vcpu *vcpu)
++vcpu->stat.halt_wakeup;
}
- me = get_cpu();
+ get_online_cpus_atomic();
+ me = smp_processor_id();
if (cpu != me && (unsigned)cpu < nr_cpu_ids && cpu_online(cpu))
if (kvm_arch_vcpu_should_kick(vcpu))
smp_send_reschedule(cpu);
- put_cpu();
+ put_online_cpus_atomic();
}
#endif /* !CONFIG_S390 */
^ permalink raw reply related
* [PATCH v6 27/46] perf/x86: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-02-18 12:42 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
The CPU_DYING notifier modifies the per-cpu pointer pmu->box, and this can
race with functions such as uncore_pmu_to_box() and uncore_pci_remove() when
we remove stop_machine() from the CPU offline path. So protect them using
get/put_online_cpus_atomic().
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/x86/kernel/cpu/perf_event_intel_uncore.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/arch/x86/kernel/cpu/perf_event_intel_uncore.c b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
index b43200d..6faae53 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_uncore.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
@@ -1,3 +1,4 @@
+#include <linux/cpu.h>
#include "perf_event_intel_uncore.h"
static struct intel_uncore_type *empty_uncore[] = { NULL, };
@@ -1965,6 +1966,7 @@ uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
if (box)
return box;
+ get_online_cpus_atomic();
raw_spin_lock(&uncore_box_lock);
list_for_each_entry(box, &pmu->box_list, list) {
if (box->phys_id == topology_physical_package_id(cpu)) {
@@ -1974,6 +1976,7 @@ uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
}
}
raw_spin_unlock(&uncore_box_lock);
+ put_online_cpus_atomic();
return *per_cpu_ptr(pmu->box, cpu);
}
@@ -2556,6 +2559,7 @@ static void uncore_pci_remove(struct pci_dev *pdev)
if (WARN_ON_ONCE(phys_id != box->phys_id))
return;
+ get_online_cpus_atomic();
raw_spin_lock(&uncore_box_lock);
list_del(&box->list);
raw_spin_unlock(&uncore_box_lock);
@@ -2569,6 +2573,7 @@ static void uncore_pci_remove(struct pci_dev *pdev)
WARN_ON_ONCE(atomic_read(&box->refcnt) != 1);
kfree(box);
+ put_online_cpus_atomic();
}
static int uncore_pci_probe(struct pci_dev *pdev,
^ permalink raw reply related
* [PATCH v6 26/46] x86: Use get/put_online_cpus_atomic() to prevent CPU offline
From: Srivatsa S. Bhat @ 2013-02-18 12:41 UTC (permalink / raw)
To: tglx, peterz, tj, oleg, paulmck, rusty, mingo, akpm, namhyung
Cc: linux-arch, linux, nikunj, linux-pm, fweisbec, linux-doc,
linux-kernel, rostedt, xiaoguangrong, rjw, sbw, wangyun,
srivatsa.bhat, netdev, vincent.guittot, walken, linuxppc-dev,
linux-arm-kernel
In-Reply-To: <20130218123714.26245.61816.stgit@srivatsabhat.in.ibm.com>
Once stop_machine() is gone from the CPU offline path, we won't be able to
depend on preempt_disable() or local_irq_disable() to prevent CPUs from
going offline from under us.
Use the get/put_online_cpus_atomic() APIs to prevent CPUs from going offline,
while invoking from atomic context.
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: x86@kernel.org
Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Yinghai Lu <yinghai@kernel.org>
Cc: Daniel J Blueman <daniel@numascale-asia.com>
Cc: Steffen Persvold <sp@numascale.com>
Cc: Joerg Roedel <joerg.roedel@amd.com>
Cc: linux-edac@vger.kernel.org
Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>
---
arch/x86/include/asm/ipi.h | 5 +++++
arch/x86/kernel/apic/apic_flat_64.c | 10 ++++++++++
arch/x86/kernel/apic/apic_numachip.c | 5 +++++
arch/x86/kernel/apic/es7000_32.c | 5 +++++
arch/x86/kernel/apic/io_apic.c | 7 +++++--
arch/x86/kernel/apic/ipi.c | 10 ++++++++++
arch/x86/kernel/apic/x2apic_cluster.c | 4 ++++
arch/x86/kernel/apic/x2apic_uv_x.c | 4 ++++
arch/x86/kernel/cpu/mcheck/therm_throt.c | 4 ++--
arch/x86/mm/tlb.c | 14 +++++++-------
10 files changed, 57 insertions(+), 11 deletions(-)
diff --git a/arch/x86/include/asm/ipi.h b/arch/x86/include/asm/ipi.h
index 615fa90..112249c 100644
--- a/arch/x86/include/asm/ipi.h
+++ b/arch/x86/include/asm/ipi.h
@@ -20,6 +20,7 @@
* Subject to the GNU Public License, v.2
*/
+#include <linux/cpu.h>
#include <asm/hw_irq.h>
#include <asm/apic.h>
#include <asm/smp.h>
@@ -131,18 +132,22 @@ extern int no_broadcast;
static inline void __default_local_send_IPI_allbutself(int vector)
{
+ get_online_cpus_atomic();
if (no_broadcast || vector == NMI_VECTOR)
apic->send_IPI_mask_allbutself(cpu_online_mask, vector);
else
__default_send_IPI_shortcut(APIC_DEST_ALLBUT, vector, apic->dest_logical);
+ put_online_cpus_atomic();
}
static inline void __default_local_send_IPI_all(int vector)
{
+ get_online_cpus_atomic();
if (no_broadcast || vector == NMI_VECTOR)
apic->send_IPI_mask(cpu_online_mask, vector);
else
__default_send_IPI_shortcut(APIC_DEST_ALLINC, vector, apic->dest_logical);
+ put_online_cpus_atomic();
}
#ifdef CONFIG_X86_32
diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c
index 00c77cf..8207ade 100644
--- a/arch/x86/kernel/apic/apic_flat_64.c
+++ b/arch/x86/kernel/apic/apic_flat_64.c
@@ -11,6 +11,7 @@
#include <linux/errno.h>
#include <linux/threads.h>
#include <linux/cpumask.h>
+#include <linux/cpu.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <linux/ctype.h>
@@ -92,6 +93,8 @@ static void flat_send_IPI_allbutself(int vector)
#else
int hotplug = 0;
#endif
+
+ get_online_cpus_atomic();
if (hotplug || vector == NMI_VECTOR) {
if (!cpumask_equal(cpu_online_mask, cpumask_of(cpu))) {
unsigned long mask = cpumask_bits(cpu_online_mask)[0];
@@ -105,16 +108,19 @@ static void flat_send_IPI_allbutself(int vector)
__default_send_IPI_shortcut(APIC_DEST_ALLBUT,
vector, apic->dest_logical);
}
+ put_online_cpus_atomic();
}
static void flat_send_IPI_all(int vector)
{
+ get_online_cpus_atomic();
if (vector == NMI_VECTOR) {
flat_send_IPI_mask(cpu_online_mask, vector);
} else {
__default_send_IPI_shortcut(APIC_DEST_ALLINC,
vector, apic->dest_logical);
}
+ put_online_cpus_atomic();
}
static unsigned int flat_get_apic_id(unsigned long x)
@@ -255,12 +261,16 @@ static void physflat_send_IPI_mask_allbutself(const struct cpumask *cpumask,
static void physflat_send_IPI_allbutself(int vector)
{
+ get_online_cpus_atomic();
default_send_IPI_mask_allbutself_phys(cpu_online_mask, vector);
+ put_online_cpus_atomic();
}
static void physflat_send_IPI_all(int vector)
{
+ get_online_cpus_atomic();
physflat_send_IPI_mask(cpu_online_mask, vector);
+ put_online_cpus_atomic();
}
static int physflat_probe(void)
diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c
index 9c2aa89..7d19c1d 100644
--- a/arch/x86/kernel/apic/apic_numachip.c
+++ b/arch/x86/kernel/apic/apic_numachip.c
@@ -14,6 +14,7 @@
#include <linux/errno.h>
#include <linux/threads.h>
#include <linux/cpumask.h>
+#include <linux/cpu.h>
#include <linux/string.h>
#include <linux/kernel.h>
#include <linux/module.h>
@@ -131,15 +132,19 @@ static void numachip_send_IPI_allbutself(int vector)
unsigned int this_cpu = smp_processor_id();
unsigned int cpu;
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
if (cpu != this_cpu)
numachip_send_IPI_one(cpu, vector);
}
+ put_online_cpus_atomic();
}
static void numachip_send_IPI_all(int vector)
{
+ get_online_cpus_atomic();
numachip_send_IPI_mask(cpu_online_mask, vector);
+ put_online_cpus_atomic();
}
static void numachip_send_IPI_self(int vector)
diff --git a/arch/x86/kernel/apic/es7000_32.c b/arch/x86/kernel/apic/es7000_32.c
index 0874799..ddf2995 100644
--- a/arch/x86/kernel/apic/es7000_32.c
+++ b/arch/x86/kernel/apic/es7000_32.c
@@ -45,6 +45,7 @@
#include <linux/gfp.h>
#include <linux/nmi.h>
#include <linux/smp.h>
+#include <linux/cpu.h>
#include <linux/io.h>
#include <asm/apicdef.h>
@@ -412,12 +413,16 @@ static void es7000_send_IPI_mask(const struct cpumask *mask, int vector)
static void es7000_send_IPI_allbutself(int vector)
{
+ get_online_cpus_atomic();
default_send_IPI_mask_allbutself_phys(cpu_online_mask, vector);
+ put_online_cpus_atomic();
}
static void es7000_send_IPI_all(int vector)
{
+ get_online_cpus_atomic();
es7000_send_IPI_mask(cpu_online_mask, vector);
+ put_online_cpus_atomic();
}
static int es7000_apic_id_registered(void)
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index b739d39..ca1c2a5 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -25,6 +25,7 @@
#include <linux/init.h>
#include <linux/delay.h>
#include <linux/sched.h>
+#include <linux/cpu.h>
#include <linux/pci.h>
#include <linux/mc146818rtc.h>
#include <linux/compiler.h>
@@ -1788,13 +1789,13 @@ __apicdebuginit(void) print_local_APICs(int maxcpu)
if (!maxcpu)
return;
- preempt_disable();
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
if (cpu >= maxcpu)
break;
smp_call_function_single(cpu, print_local_APIC, NULL, 1);
}
- preempt_enable();
+ put_online_cpus_atomic();
}
__apicdebuginit(void) print_PIC(void)
@@ -2209,6 +2210,7 @@ void send_cleanup_vector(struct irq_cfg *cfg)
{
cpumask_var_t cleanup_mask;
+ get_online_cpus_atomic();
if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
unsigned int i;
for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
@@ -2219,6 +2221,7 @@ void send_cleanup_vector(struct irq_cfg *cfg)
free_cpumask_var(cleanup_mask);
}
cfg->move_in_progress = 0;
+ put_online_cpus_atomic();
}
asmlinkage void smp_irq_move_cleanup_interrupt(void)
diff --git a/arch/x86/kernel/apic/ipi.c b/arch/x86/kernel/apic/ipi.c
index cce91bf..c65aa77 100644
--- a/arch/x86/kernel/apic/ipi.c
+++ b/arch/x86/kernel/apic/ipi.c
@@ -29,12 +29,14 @@ void default_send_IPI_mask_sequence_phys(const struct cpumask *mask, int vector)
* to an arbitrary mask, so I do a unicast to each CPU instead.
* - mbligh
*/
+ get_online_cpus_atomic();
local_irq_save(flags);
for_each_cpu(query_cpu, mask) {
__default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
query_cpu), vector, APIC_DEST_PHYSICAL);
}
local_irq_restore(flags);
+ put_online_cpus_atomic();
}
void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
@@ -46,6 +48,7 @@ void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
/* See Hack comment above */
+ get_online_cpus_atomic();
local_irq_save(flags);
for_each_cpu(query_cpu, mask) {
if (query_cpu == this_cpu)
@@ -54,6 +57,7 @@ void default_send_IPI_mask_allbutself_phys(const struct cpumask *mask,
query_cpu), vector, APIC_DEST_PHYSICAL);
}
local_irq_restore(flags);
+ put_online_cpus_atomic();
}
#ifdef CONFIG_X86_32
@@ -70,12 +74,14 @@ void default_send_IPI_mask_sequence_logical(const struct cpumask *mask,
* should be modified to do 1 message per cluster ID - mbligh
*/
+ get_online_cpus_atomic();
local_irq_save(flags);
for_each_cpu(query_cpu, mask)
__default_send_IPI_dest_field(
early_per_cpu(x86_cpu_to_logical_apicid, query_cpu),
vector, apic->dest_logical);
local_irq_restore(flags);
+ put_online_cpus_atomic();
}
void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
@@ -87,6 +93,7 @@ void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
/* See Hack comment above */
+ get_online_cpus_atomic();
local_irq_save(flags);
for_each_cpu(query_cpu, mask) {
if (query_cpu == this_cpu)
@@ -96,6 +103,7 @@ void default_send_IPI_mask_allbutself_logical(const struct cpumask *mask,
vector, apic->dest_logical);
}
local_irq_restore(flags);
+ put_online_cpus_atomic();
}
/*
@@ -109,10 +117,12 @@ void default_send_IPI_mask_logical(const struct cpumask *cpumask, int vector)
if (WARN_ONCE(!mask, "empty IPI mask"))
return;
+ get_online_cpus_atomic();
local_irq_save(flags);
WARN_ON(mask & ~cpumask_bits(cpu_online_mask)[0]);
__default_send_IPI_dest_field(mask, vector, apic->dest_logical);
local_irq_restore(flags);
+ put_online_cpus_atomic();
}
void default_send_IPI_allbutself(int vector)
diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c
index c88baa4..cb08e6b 100644
--- a/arch/x86/kernel/apic/x2apic_cluster.c
+++ b/arch/x86/kernel/apic/x2apic_cluster.c
@@ -88,12 +88,16 @@ x2apic_send_IPI_mask_allbutself(const struct cpumask *mask, int vector)
static void x2apic_send_IPI_allbutself(int vector)
{
+ get_online_cpus_atomic();
__x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLBUT);
+ put_online_cpus_atomic();
}
static void x2apic_send_IPI_all(int vector)
{
+ get_online_cpus_atomic();
__x2apic_send_IPI_mask(cpu_online_mask, vector, APIC_DEST_ALLINC);
+ put_online_cpus_atomic();
}
static int
diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c
index 8cfade9..cc469a3 100644
--- a/arch/x86/kernel/apic/x2apic_uv_x.c
+++ b/arch/x86/kernel/apic/x2apic_uv_x.c
@@ -244,15 +244,19 @@ static void uv_send_IPI_allbutself(int vector)
unsigned int this_cpu = smp_processor_id();
unsigned int cpu;
+ get_online_cpus_atomic();
for_each_online_cpu(cpu) {
if (cpu != this_cpu)
uv_send_IPI_one(cpu, vector);
}
+ put_online_cpus_atomic();
}
static void uv_send_IPI_all(int vector)
{
+ get_online_cpus_atomic();
uv_send_IPI_mask(cpu_online_mask, vector);
+ put_online_cpus_atomic();
}
static int uv_apic_id_valid(int apicid)
diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c
index 47a1870..d128ba4 100644
--- a/arch/x86/kernel/cpu/mcheck/therm_throt.c
+++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c
@@ -82,13 +82,13 @@ static ssize_t therm_throt_device_show_##event##_##name( \
unsigned int cpu = dev->id; \
ssize_t ret; \
\
- preempt_disable(); /* CPU hotplug */ \
+ get_online_cpus_atomic(); /* CPU hotplug */ \
if (cpu_online(cpu)) { \
ret = sprintf(buf, "%lu\n", \
per_cpu(thermal_state, cpu).event.name); \
} else \
ret = 0; \
- preempt_enable(); \
+ put_online_cpus_atomic(); \
\
return ret; \
}
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 13a6b29..2c3ec76 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -147,12 +147,12 @@ void flush_tlb_current_task(void)
{
struct mm_struct *mm = current->mm;
- preempt_disable();
+ get_online_cpus_atomic();
local_flush_tlb();
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
- preempt_enable();
+ put_online_cpus_atomic();
}
/*
@@ -187,7 +187,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
unsigned long addr;
unsigned act_entries, tlb_entries = 0;
- preempt_disable();
+ get_online_cpus_atomic();
if (current->active_mm != mm)
goto flush_all;
@@ -225,21 +225,21 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
if (cpumask_any_but(mm_cpumask(mm),
smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, start, end);
- preempt_enable();
+ put_online_cpus_atomic();
return;
}
flush_all:
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
- preempt_enable();
+ put_online_cpus_atomic();
}
void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
{
struct mm_struct *mm = vma->vm_mm;
- preempt_disable();
+ get_online_cpus_atomic();
if (current->active_mm == mm) {
if (current->mm)
@@ -251,7 +251,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, start, 0UL);
- preempt_enable();
+ put_online_cpus_atomic();
}
static void do_flush_tlb_all(void *info)
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox