* [PATCH v2 18/18] of: unittest: initialize args before calling of_*parse_*()
From: frowand.list @ 2018-10-13 4:53 UTC (permalink / raw)
To: Rob Herring, Pantelis Antoniou, Michael Ellerman,
Benjamin Herrenschmidt, Paul Mackerras, Alan Tull, Moritz Fischer,
Guenter Roeck
Cc: devicetree, linux-fpga, linuxppc-dev, linux-kernel
In-Reply-To: <1539406418-18162-1-git-send-email-frowand.list@gmail.com>
From: Frank Rowand <frank.rowand@sony.com>
Callers of of_irq_parse_one() blindly use the pointer args.np
without checking whether of_irq_parse_one() had an error and
thus did not set the value of args.np. Initialize args to
zero so that using the format "%pOF" to show the value of
args.np will show "(null)" when of_irq_parse_one() has an
error. This prevents the dereference of a random value.
Make the same fix for callers of of_parse_phandle_with_args()
and of_parse_phandle_with_args_map().
Reported-by: Guenter Roeck <linux@roeck-us.net>
Signed-off-by: Frank Rowand <frank.rowand@sony.com>
---
changes since v1:
- add same fix for of_parse_phandle_with_args()
- add same fix for of_parse_phandle_with_args_map()
drivers/of/unittest.c | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c
index 785985bdbfa6..5f4db23e4752 100644
--- a/drivers/of/unittest.c
+++ b/drivers/of/unittest.c
@@ -375,6 +375,7 @@ static void __init of_unittest_parse_phandle_with_args(void)
for (i = 0; i < 8; i++) {
bool passed = true;
+ memset(&args, 0, sizeof(args));
rc = of_parse_phandle_with_args(np, "phandle-list",
"#phandle-cells", i, &args);
@@ -428,6 +429,7 @@ static void __init of_unittest_parse_phandle_with_args(void)
}
/* Check for missing list property */
+ memset(&args, 0, sizeof(args));
rc = of_parse_phandle_with_args(np, "phandle-list-missing",
"#phandle-cells", 0, &args);
unittest(rc == -ENOENT, "expected:%i got:%i\n", -ENOENT, rc);
@@ -436,6 +438,7 @@ static void __init of_unittest_parse_phandle_with_args(void)
unittest(rc == -ENOENT, "expected:%i got:%i\n", -ENOENT, rc);
/* Check for missing cells property */
+ memset(&args, 0, sizeof(args));
rc = of_parse_phandle_with_args(np, "phandle-list",
"#phandle-cells-missing", 0, &args);
unittest(rc == -EINVAL, "expected:%i got:%i\n", -EINVAL, rc);
@@ -444,6 +447,7 @@ static void __init of_unittest_parse_phandle_with_args(void)
unittest(rc == -EINVAL, "expected:%i got:%i\n", -EINVAL, rc);
/* Check for bad phandle in list */
+ memset(&args, 0, sizeof(args));
rc = of_parse_phandle_with_args(np, "phandle-list-bad-phandle",
"#phandle-cells", 0, &args);
unittest(rc == -EINVAL, "expected:%i got:%i\n", -EINVAL, rc);
@@ -452,6 +456,7 @@ static void __init of_unittest_parse_phandle_with_args(void)
unittest(rc == -EINVAL, "expected:%i got:%i\n", -EINVAL, rc);
/* Check for incorrectly formed argument list */
+ memset(&args, 0, sizeof(args));
rc = of_parse_phandle_with_args(np, "phandle-list-bad-args",
"#phandle-cells", 1, &args);
unittest(rc == -EINVAL, "expected:%i got:%i\n", -EINVAL, rc);
@@ -502,6 +507,7 @@ static void __init of_unittest_parse_phandle_with_args_map(void)
for (i = 0; i < 8; i++) {
bool passed = true;
+ memset(&args, 0, sizeof(args));
rc = of_parse_phandle_with_args_map(np, "phandle-list",
"phandle", i, &args);
@@ -559,21 +565,25 @@ static void __init of_unittest_parse_phandle_with_args_map(void)
}
/* Check for missing list property */
+ memset(&args, 0, sizeof(args));
rc = of_parse_phandle_with_args_map(np, "phandle-list-missing",
"phandle", 0, &args);
unittest(rc == -ENOENT, "expected:%i got:%i\n", -ENOENT, rc);
/* Check for missing cells,map,mask property */
+ memset(&args, 0, sizeof(args));
rc = of_parse_phandle_with_args_map(np, "phandle-list",
"phandle-missing", 0, &args);
unittest(rc == -EINVAL, "expected:%i got:%i\n", -EINVAL, rc);
/* Check for bad phandle in list */
+ memset(&args, 0, sizeof(args));
rc = of_parse_phandle_with_args_map(np, "phandle-list-bad-phandle",
"phandle", 0, &args);
unittest(rc == -EINVAL, "expected:%i got:%i\n", -EINVAL, rc);
/* Check for incorrectly formed argument list */
+ memset(&args, 0, sizeof(args));
rc = of_parse_phandle_with_args_map(np, "phandle-list-bad-args",
"phandle", 1, &args);
unittest(rc == -EINVAL, "expected:%i got:%i\n", -EINVAL, rc);
@@ -780,7 +790,7 @@ static void __init of_unittest_parse_interrupts(void)
for (i = 0; i < 4; i++) {
bool passed = true;
- args.args_count = 0;
+ memset(&args, 0, sizeof(args));
rc = of_irq_parse_one(np, i, &args);
passed &= !rc;
@@ -801,7 +811,7 @@ static void __init of_unittest_parse_interrupts(void)
for (i = 0; i < 4; i++) {
bool passed = true;
- args.args_count = 0;
+ memset(&args, 0, sizeof(args));
rc = of_irq_parse_one(np, i, &args);
/* Test the values from tests-phandle.dtsi */
@@ -854,6 +864,7 @@ static void __init of_unittest_parse_interrupts_extended(void)
for (i = 0; i < 7; i++) {
bool passed = true;
+ memset(&args, 0, sizeof(args));
rc = of_irq_parse_one(np, i, &args);
/* Test the values from tests-phandle.dtsi */
--
Frank Rowand <frank.rowand@sony.com>
^ permalink raw reply related
* Re: [PATCH v2 2/2] mm: speed up mremap by 500x on large regions
From: Anton Ivanov @ 2018-10-13 6:10 UTC (permalink / raw)
To: Kirill A. Shutemov
Cc: linux-mips, Rich Felker, linux-ia64, linux-sh, Peter Zijlstra,
Catalin Marinas, Dave Hansen, Will Deacon, mhocko, linux-mm,
lokeshgidra, Joel Fernandes (Google), linux-riscv, elfring,
Jonas Bonn, linux-s390, dancol, Yoshinori Sato, Max Filippov,
linux-hexagon, Helge Deller,
maintainer:X86 ARCHITECTURE 32-BIT AND 64-BIT, hughd,
James E.J. Bottomley, kasan-dev, kvmarm, Ingo Molnar,
Geert Uytterhoeven, Andrey Ryabinin, linux-snps-arc, kernel-team,
Sam Creasey, linux-xtensa, Jeff Dike, linux-alpha, linux-um,
Stefan Kristiansson, Julia Lawall, linux-m68k, openrisc,
Borislav Petkov, Andy Lutomirski, nios2-dev, Stafford Horne,
Guan Xuetao, linux-arm-kernel, Chris Zankel, Tony Luck,
linux-parisc, pantin, linux-kernel, Fenghua Yu, minchan,
Thomas Gleixner, Richard Weinberger, sparclinux, akpm,
linuxppc-dev, David S. Miller, Ley Foon Tan
In-Reply-To: <20181012214008.jj7vhh76vte7xj7i@kshutemo-mobl1>
On 12/10/2018 22:40, Kirill A. Shutemov wrote:
> On Fri, Oct 12, 2018 at 05:42:24PM +0100, Anton Ivanov wrote:
>> On 10/12/18 3:48 PM, Anton Ivanov wrote:
>>> On 12/10/2018 15:37, Kirill A. Shutemov wrote:
>>>> On Fri, Oct 12, 2018 at 03:09:49PM +0100, Anton Ivanov wrote:
>>>>> On 10/12/18 2:37 AM, Joel Fernandes (Google) wrote:
>>>>>> Android needs to mremap large regions of memory during
>>>>>> memory management
>>>>>> related operations. The mremap system call can be really
>>>>>> slow if THP is
>>>>>> not enabled. The bottleneck is move_page_tables, which is copying each
>>>>>> pte at a time, and can be really slow across a large map.
>>>>>> Turning on THP
>>>>>> may not be a viable option, and is not for us. This patch
>>>>>> speeds up the
>>>>>> performance for non-THP system by copying at the PMD level
>>>>>> when possible.
>>>>>>
>>>>>> The speed up is three orders of magnitude. On a 1GB mremap, the mremap
>>>>>> completion times drops from 160-250 millesconds to 380-400
>>>>>> microseconds.
>>>>>>
>>>>>> Before:
>>>>>> Total mremap time for 1GB data: 242321014 nanoseconds.
>>>>>> Total mremap time for 1GB data: 196842467 nanoseconds.
>>>>>> Total mremap time for 1GB data: 167051162 nanoseconds.
>>>>>>
>>>>>> After:
>>>>>> Total mremap time for 1GB data: 385781 nanoseconds.
>>>>>> Total mremap time for 1GB data: 388959 nanoseconds.
>>>>>> Total mremap time for 1GB data: 402813 nanoseconds.
>>>>>>
>>>>>> Incase THP is enabled, the optimization is skipped. I also flush the
>>>>>> tlb every time we do this optimization since I couldn't find a way to
>>>>>> determine if the low-level PTEs are dirty. It is seen that the cost of
>>>>>> doing so is not much compared the improvement, on both
>>>>>> x86-64 and arm64.
>>>>>>
>>>>>> Cc: minchan@kernel.org
>>>>>> Cc: pantin@google.com
>>>>>> Cc: hughd@google.com
>>>>>> Cc: lokeshgidra@google.com
>>>>>> Cc: dancol@google.com
>>>>>> Cc: mhocko@kernel.org
>>>>>> Cc: kirill@shutemov.name
>>>>>> Cc: akpm@linux-foundation.org
>>>>>> Signed-off-by: Joel Fernandes (Google) <joel@joelfernandes.org>
>>>>>> ---
>>>>>> mm/mremap.c | 62
>>>>>> +++++++++++++++++++++++++++++++++++++++++++++++++++++
>>>>>> 1 file changed, 62 insertions(+)
>>>>>>
>>>>>> diff --git a/mm/mremap.c b/mm/mremap.c
>>>>>> index 9e68a02a52b1..d82c485822ef 100644
>>>>>> --- a/mm/mremap.c
>>>>>> +++ b/mm/mremap.c
>>>>>> @@ -191,6 +191,54 @@ static void move_ptes(struct
>>>>>> vm_area_struct *vma, pmd_t *old_pmd,
>>>>>> drop_rmap_locks(vma);
>>>>>> }
>>>>>> +static bool move_normal_pmd(struct vm_area_struct *vma,
>>>>>> unsigned long old_addr,
>>>>>> + unsigned long new_addr, unsigned long old_end,
>>>>>> + pmd_t *old_pmd, pmd_t *new_pmd, bool *need_flush)
>>>>>> +{
>>>>>> + spinlock_t *old_ptl, *new_ptl;
>>>>>> + struct mm_struct *mm = vma->vm_mm;
>>>>>> +
>>>>>> + if ((old_addr & ~PMD_MASK) || (new_addr & ~PMD_MASK)
>>>>>> + || old_end - old_addr < PMD_SIZE)
>>>>>> + return false;
>>>>>> +
>>>>>> + /*
>>>>>> + * The destination pmd shouldn't be established, free_pgtables()
>>>>>> + * should have release it.
>>>>>> + */
>>>>>> + if (WARN_ON(!pmd_none(*new_pmd)))
>>>>>> + return false;
>>>>>> +
>>>>>> + /*
>>>>>> + * We don't have to worry about the ordering of src and dst
>>>>>> + * ptlocks because exclusive mmap_sem prevents deadlock.
>>>>>> + */
>>>>>> + old_ptl = pmd_lock(vma->vm_mm, old_pmd);
>>>>>> + if (old_ptl) {
>>>>>> + pmd_t pmd;
>>>>>> +
>>>>>> + new_ptl = pmd_lockptr(mm, new_pmd);
>>>>>> + if (new_ptl != old_ptl)
>>>>>> + spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
>>>>>> +
>>>>>> + /* Clear the pmd */
>>>>>> + pmd = *old_pmd;
>>>>>> + pmd_clear(old_pmd);
>>>>>> +
>>>>>> + VM_BUG_ON(!pmd_none(*new_pmd));
>>>>>> +
>>>>>> + /* Set the new pmd */
>>>>>> + set_pmd_at(mm, new_addr, new_pmd, pmd);
>>>>> UML does not have set_pmd_at at all
>>>> Every architecture does. :)
>>> I tried to build it patching vs 4.19-rc before I made this statement and
>>> ran into that.
>>>
>>> Presently it does not.
>>>
>>> https://elixir.bootlin.com/linux/v4.19-rc7/ident/set_pmd_at - UML is not
>>> on the list.
>> Once this problem as well as the omissions in the include changes for UML in
>> patch one have been fixed it appears to be working.
>>
>> What it needs is attached.
> Well, the optization is only suitable for arch that has 3 or more levels
> of page tables. Otherwise it will not have [non-folded] pmd.
>
> And in this case arch/um already should have set_pmd_at(), see
> 3_LEVEL_PGTABLES.
>
> To port on 2-level paging, it has to be handled on pgd level. It
> complicates the code and will not bring much value.
>
UML has 3 level page tables on 64 bit.
A.
^ permalink raw reply
* Re: [PATCH] powerpc: Fix stackprotector detection for non-glibc toolchains
From: Christophe LEROY @ 2018-10-13 7:32 UTC (permalink / raw)
To: Michael Ellerman, linuxppc-dev
In-Reply-To: <20181012225832.13827-1-mpe@ellerman.id.au>
Le 13/10/2018 à 00:58, Michael Ellerman a écrit :
> If GCC is not built with glibc support then we must explicitly tell it
> which register to use for TLS mode stack protector, otherwise it will
> error out and the cc-option check will fail.
Oh ? I didn't encounter such a problem with the nolibc GCC from
https://mirrors.edge.kernel.org/pub/tools/crosstool/
I did all my tests with powerpc64-linux-gcc 8.1 on x86_64
Christophe
>
> Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Reviewed-by: Christophe Leroy <christophe.leroy@c-s.fr>
> ---
> arch/powerpc/Kconfig | 3 ++-
> 1 file changed, 2 insertions(+), 1 deletion(-)
>
> diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
> index 1888636c9eb6..3d008115fe18 100644
> --- a/arch/powerpc/Kconfig
> +++ b/arch/powerpc/Kconfig
> @@ -180,7 +180,8 @@ config PPC
> select HAVE_ARCH_SECCOMP_FILTER
> select HAVE_ARCH_TRACEHOOK
> select HAVE_CBPF_JIT if !PPC64
> - select HAVE_STACKPROTECTOR if $(cc-option,-mstack-protector-guard=tls)
> + select HAVE_STACKPROTECTOR if PPC64 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r13)
> + select HAVE_STACKPROTECTOR if PPC32 && $(cc-option,-mstack-protector-guard=tls -mstack-protector-guard-reg=r2)
> select HAVE_CONTEXT_TRACKING if PPC64
> select HAVE_DEBUG_KMEMLEAK
> select HAVE_DEBUG_STACKOVERFLOW
>
^ permalink raw reply
* Re: [PATCH 4/4] powerpc: Add -Wimplicit-fallthrough to arch CFLAGS
From: Gustavo A. R. Silva @ 2018-10-13 7:39 UTC (permalink / raw)
To: Kees Cook; +Cc: maddy, linuxppc-dev
In-Reply-To: <CAGXu5j+-iRoDpxQMwNfEb-qnxN2VXyHz0JX2=WH51v7g+avwFQ@mail.gmail.com>
On 10/13/18 3:23 AM, Kees Cook wrote:
>>
>> $ scripts/get_maintainer.pl --nokeywords --nogit --nogit-fallback include/linux/compat.h
>> linux-kernel@vger.kernel.org (open list)
>
> Normally things like that go through akpm, but I'm happy to carry them
> if needed.
>
Oh okay. Let me try through apkm first. :)
Thanks
--
Gustavo
^ permalink raw reply
* Re: [PATCH v2 3/3] powerpc: machine check interrupt is a non-maskable interrupt
From: Christophe Leroy @ 2018-10-13 8:29 UTC (permalink / raw)
To: Nicholas Piggin; +Cc: Mahesh Jagannath Salgaonkar, linuxppc-dev
In-Reply-To: <9f0cbf48-d278-08bf-cb32-8b9608768025@c-s.fr>
On 10/11/2018 02:31 PM, Christophe LEROY wrote:
>
>
> Le 09/10/2018 à 13:16, Nicholas Piggin a écrit :
>> On Tue, 9 Oct 2018 09:36:18 +0000
>> Christophe Leroy <christophe.leroy@c-s.fr> wrote:
>>
>>> On 10/09/2018 05:30 AM, Nicholas Piggin wrote:
>>>> On Tue, 9 Oct 2018 06:46:30 +0200
>>>> Christophe LEROY <christophe.leroy@c-s.fr> wrote:
>>>>> Le 09/10/2018 à 06:32, Nicholas Piggin a écrit :
>>>>>> On Mon, 8 Oct 2018 17:39:11 +0200
>>>>>> Christophe LEROY <christophe.leroy@c-s.fr> wrote:
>>>>>>> Hi Nick,
>>>>>>>
>>>>>>> Le 19/07/2017 à 08:59, Nicholas Piggin a écrit :
>>>>>>>> Use nmi_enter similarly to system reset interrupts. This uses NMI
>>>>>>>> printk NMI buffers and turns off various debugging facilities that
>>>>>>>> helps avoid tripping on ourselves or other CPUs.
>>>>>>>>
>>>>>>>> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
>>>>>>>> ---
>>>>>>>> arch/powerpc/kernel/traps.c | 9 ++++++---
>>>>>>>> 1 file changed, 6 insertions(+), 3 deletions(-)
>>>>>>>>
>>>>>>>> diff --git a/arch/powerpc/kernel/traps.c
>>>>>>>> b/arch/powerpc/kernel/traps.c
>>>>>>>> index 2849c4f50324..6d31f9d7c333 100644
>>>>>>>> --- a/arch/powerpc/kernel/traps.c
>>>>>>>> +++ b/arch/powerpc/kernel/traps.c
>>>>>>>> @@ -789,8 +789,10 @@ int machine_check_generic(struct pt_regs
>>>>>>>> *regs)
>>>>>>>> void machine_check_exception(struct pt_regs *regs)
>>>>>>>> {
>>>>>>>> - enum ctx_state prev_state = exception_enter();
>>>>>>>> int recover = 0;
>>>>>>>> + bool nested = in_nmi();
>>>>>>>> + if (!nested)
>>>>>>>> + nmi_enter();
>>>>>>>
>>>>>>> This alters preempt_count, then when die() is called
>>>>>>> in_interrupt() returns true allthough the trap didn't happen in
>>>>>>> interrupt, so oops_end() panics for "fatal exception in interrupt"
>>>>>>> instead of gently sending SIGBUS the faulting app.
>>>>>>
>>>>>> Thanks for tracking that down.
>>>>>>> Any idea on how to fix this ?
>>>>>>
>>>>>> I would say we have to deliver the sigbus by hand.
>>>>>>
>>>>>> if ((user_mode(regs)))
>>>>>> _exception(SIGBUS, regs, BUS_MCEERR_AR, regs->nip);
>>>>>> else
>>>>>> die("Machine check", regs, SIGBUS);
>>>>>
>>>>> And what about all the other things done by 'die()' ?
>>>>>
>>>>> And what if it is a kernel thread ?
>>>>>
>>>>> In one of my boards, I have a kernel thread regularly checking the HW,
>>>>> and if it gets a machine check I expect it to gently stop and the die
>>>>> notification to be delivered to all registered notifiers.
>>>>>
>>>>> Until before this patch, it was working well.
>>>>
>>>> I guess the alternative is we could check regs->trap for machine
>>>> check in the die test. Complication is having to account for MCE
>>>> in an interrupt handler.
>>>>
>>>> if (in_interrupt()) {
>>>> if (!IS_MCHECK_EXC(regs) || (irq_count() -
>>>> (NMI_OFFSET + HARDIRQ_OFFSET)))
>>>> panic("Fatal exception in interrupt");
>>>> }
>>>>
>>>> Something like that might work for you? We needs a ppc64 macro for the
>>>> MCE, and can probably add something like in_nmi_from_interrupt() for
>>>> the second part of the test.
>>>
>>> Don't know, I'm away from home on business trip so I won't be able to
>>> test anything before next week. However it looks more or less like a
>>> hack, doesn't it ?
>>
>> I thought it seemed okay (with the right functions added). Actually it
>> could be a bit nicer to do this, then it works generally :
>>
>> if (in_interrupt()) {
>> if (!in_nmi() || in_nmi_from_interrupt())
>> panic("Fatal exception in interrupt");
>> }
>>
>>>
>>> What about the following ?
>>
>> Hmm, in some ways maybe it's nicer. One complication is I would like the
>> same thing to be available for platform specific machine check
>> handlers, so then you need to pass is_in_interrupt to them. Which you
>> can do without any problem... But is it cleaner than the above?
>
> For me it looks cleaner than twiddle the preempt_count depending on
> whether we were or not already in nmi() .
>
> Let's draft something and see what it looks like.
Ok, finaly I went to your solution, see below, as it avoids having to
modify all subarch and platform specific machine check handlers.
Unfortunately it doesn't solves the issue, it only delays it:
oops_end() calls do_exit(), which has the following test:
if (unlikely(in_interrupt()))
panic("Aiee, killing interrupt handler!");
So at the time being I still have no idea how to fix that, have you ?
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index fd58749b4d6b..3569e826f0c2 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -132,6 +132,21 @@ static void pmac_backlight_unblank(void)
static inline void pmac_backlight_unblank(void) { }
#endif
+static bool from_interrupt(void)
+{
+ if (!in_nmi())
+ return in_interrupt();
+ /*
+ * if we are in NMI, we need to determine if we were already in
+ * interrupt before entering NMI. To do that, we recalculate irq_count()
+ * from before the call to nmi_enter().
+ * If we were already in NMI and reentered in a new one, we have
+ * increased the preempt count by HARDIRQ_OFFSET, so the calculated
+ * value will be not null
+ */
+ return irq_count() - NMI_OFFSET - HARDIRQ_OFFSET;
+}
+
/*
* If oops/die is expected to crash the machine, return true here.
*
@@ -147,8 +162,7 @@ bool die_will_crash(void)
return true;
if (kexec_should_crash(current))
return true;
- if (in_interrupt() || panic_on_oops ||
- !current->pid || is_global_init(current))
+ if (from_interrupt() || panic_on_oops || !current->pid ||
is_global_init(current))
return true;
return false;
@@ -242,12 +256,12 @@ static void oops_end(unsigned long flags, struct
pt_regs *regs,
* know we are going to panic, delay for 1 second so we have a
* chance to get clean backtraces from all CPUs that are oopsing.
*/
- if (in_interrupt() || panic_on_oops || !current->pid ||
+ if (from_interrupt() || panic_on_oops || !current->pid ||
is_global_init(current)) {
mdelay(MSEC_PER_SEC);
}
- if (in_interrupt())
+ if (from_interrupt())
panic("Fatal exception in interrupt");
if (panic_on_oops)
panic("Fatal exception");
@@ -378,15 +392,37 @@ void _exception(int signr, struct pt_regs *regs,
int code, unsigned long addr)
_exception_pkey(signr, regs, code, addr, 0);
}
+static bool exception_nmi_enter(void)
+{
+ bool nested = in_nmi();
+
+ /*
+ * In case we are already in an NMI, increase preempt_count by
+ * HARDIRQ_OFFSET in order to get from_interrupt() return true
+ */
+ if (nested)
+ preempt_count_add(HARDIRQ_OFFSET);
+ else
+ nmi_enter();
+
+ return nested;
+}
+
+static void exception_nmi_exit(bool nested)
+{
+ if (nested)
+ preempt_count_sub(HARDIRQ_OFFSET);
+ else
+ nmi_exit();
+}
+
void system_reset_exception(struct pt_regs *regs)
{
/*
* Avoid crashes in case of nested NMI exceptions. Recoverability
* is determined by RI and in_nmi
*/
- bool nested = in_nmi();
- if (!nested)
- nmi_enter();
+ bool nested = exception_nmi_enter();
__this_cpu_inc(irq_stat.sreset_irqs);
@@ -435,8 +471,7 @@ void system_reset_exception(struct pt_regs *regs)
if (!(regs->msr & MSR_RI))
nmi_panic(regs, "Unrecoverable System Reset");
- if (!nested)
- nmi_exit();
+ exception_nmi_exit(nested);
/* What should we do here? We could issue a shutdown or hard reset. */
}
@@ -737,9 +772,7 @@ int machine_check_generic(struct pt_regs *regs)
void machine_check_exception(struct pt_regs *regs)
{
int recover = 0;
- bool nested = in_nmi();
- if (!nested)
- nmi_enter();
+ bool nested = exception_nmi_enter();
__this_cpu_inc(irq_stat.mce_exceptions);
@@ -772,8 +805,7 @@ void machine_check_exception(struct pt_regs *regs)
nmi_panic(regs, "Unrecoverable Machine check");
bail:
- if (!nested)
- nmi_exit();
+ exception_nmi_exit(nested);
}
void SMIException(struct pt_regs *regs)
>
>
>>
>> I guess one advantage of yours is that a BUG somewhere in the NMI path
>> will panic the system. Or is that a disadvantage?
>
> Why would it panic the system more than now ? And is it an issue at all
> ? Doesn't BUG() panic in any case ?
>
Christophe
^ permalink raw reply related
* Re: [PATCH v2 3/3] powerpc: machine check interrupt is a non-maskable interrupt
From: Nicholas Piggin @ 2018-10-13 8:48 UTC (permalink / raw)
To: Christophe Leroy; +Cc: Mahesh Jagannath Salgaonkar, linuxppc-dev
In-Reply-To: <bb79aa53-8924-b334-93d6-fc907c8880e1@c-s.fr>
On Sat, 13 Oct 2018 08:29:48 +0000
Christophe Leroy <christophe.leroy@c-s.fr> wrote:
> On 10/11/2018 02:31 PM, Christophe LEROY wrote:
> >
> >
> > Le 09/10/2018 à 13:16, Nicholas Piggin a écrit :
> >> On Tue, 9 Oct 2018 09:36:18 +0000
> >> Christophe Leroy <christophe.leroy@c-s.fr> wrote:
> >>
> >>> On 10/09/2018 05:30 AM, Nicholas Piggin wrote:
> >>>> On Tue, 9 Oct 2018 06:46:30 +0200
> >>>> Christophe LEROY <christophe.leroy@c-s.fr> wrote:
> >>>>> Le 09/10/2018 à 06:32, Nicholas Piggin a écrit :
> >>>>>> On Mon, 8 Oct 2018 17:39:11 +0200
> >>>>>> Christophe LEROY <christophe.leroy@c-s.fr> wrote:
> >>>>>>> Hi Nick,
> >>>>>>>
> >>>>>>> Le 19/07/2017 à 08:59, Nicholas Piggin a écrit :
> >>>>>>>> Use nmi_enter similarly to system reset interrupts. This uses NMI
> >>>>>>>> printk NMI buffers and turns off various debugging facilities that
> >>>>>>>> helps avoid tripping on ourselves or other CPUs.
> >>>>>>>>
> >>>>>>>> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
> >>>>>>>> ---
> >>>>>>>> arch/powerpc/kernel/traps.c | 9 ++++++---
> >>>>>>>> 1 file changed, 6 insertions(+), 3 deletions(-)
> >>>>>>>>
> >>>>>>>> diff --git a/arch/powerpc/kernel/traps.c
> >>>>>>>> b/arch/powerpc/kernel/traps.c
> >>>>>>>> index 2849c4f50324..6d31f9d7c333 100644
> >>>>>>>> --- a/arch/powerpc/kernel/traps.c
> >>>>>>>> +++ b/arch/powerpc/kernel/traps.c
> >>>>>>>> @@ -789,8 +789,10 @@ int machine_check_generic(struct pt_regs
> >>>>>>>> *regs)
> >>>>>>>> void machine_check_exception(struct pt_regs *regs)
> >>>>>>>> {
> >>>>>>>> - enum ctx_state prev_state = exception_enter();
> >>>>>>>> int recover = 0;
> >>>>>>>> + bool nested = in_nmi();
> >>>>>>>> + if (!nested)
> >>>>>>>> + nmi_enter();
> >>>>>>>
> >>>>>>> This alters preempt_count, then when die() is called
> >>>>>>> in_interrupt() returns true allthough the trap didn't happen in
> >>>>>>> interrupt, so oops_end() panics for "fatal exception in interrupt"
> >>>>>>> instead of gently sending SIGBUS the faulting app.
> >>>>>>
> >>>>>> Thanks for tracking that down.
> >>>>>>> Any idea on how to fix this ?
> >>>>>>
> >>>>>> I would say we have to deliver the sigbus by hand.
> >>>>>>
> >>>>>> if ((user_mode(regs)))
> >>>>>> _exception(SIGBUS, regs, BUS_MCEERR_AR, regs->nip);
> >>>>>> else
> >>>>>> die("Machine check", regs, SIGBUS);
> >>>>>
> >>>>> And what about all the other things done by 'die()' ?
> >>>>>
> >>>>> And what if it is a kernel thread ?
> >>>>>
> >>>>> In one of my boards, I have a kernel thread regularly checking the HW,
> >>>>> and if it gets a machine check I expect it to gently stop and the die
> >>>>> notification to be delivered to all registered notifiers.
> >>>>>
> >>>>> Until before this patch, it was working well.
> >>>>
> >>>> I guess the alternative is we could check regs->trap for machine
> >>>> check in the die test. Complication is having to account for MCE
> >>>> in an interrupt handler.
> >>>>
> >>>> if (in_interrupt()) {
> >>>> if (!IS_MCHECK_EXC(regs) || (irq_count() -
> >>>> (NMI_OFFSET + HARDIRQ_OFFSET)))
> >>>> panic("Fatal exception in interrupt");
> >>>> }
> >>>>
> >>>> Something like that might work for you? We needs a ppc64 macro for the
> >>>> MCE, and can probably add something like in_nmi_from_interrupt() for
> >>>> the second part of the test.
> >>>
> >>> Don't know, I'm away from home on business trip so I won't be able to
> >>> test anything before next week. However it looks more or less like a
> >>> hack, doesn't it ?
> >>
> >> I thought it seemed okay (with the right functions added). Actually it
> >> could be a bit nicer to do this, then it works generally :
> >>
> >> if (in_interrupt()) {
> >> if (!in_nmi() || in_nmi_from_interrupt())
> >> panic("Fatal exception in interrupt");
> >> }
> >>
> >>>
> >>> What about the following ?
> >>
> >> Hmm, in some ways maybe it's nicer. One complication is I would like the
> >> same thing to be available for platform specific machine check
> >> handlers, so then you need to pass is_in_interrupt to them. Which you
> >> can do without any problem... But is it cleaner than the above?
> >
> > For me it looks cleaner than twiddle the preempt_count depending on
> > whether we were or not already in nmi() .
> >
> > Let's draft something and see what it looks like.
>
> Ok, finaly I went to your solution, see below, as it avoids having to
> modify all subarch and platform specific machine check handlers.
>
> Unfortunately it doesn't solves the issue, it only delays it:
>
> oops_end() calls do_exit(), which has the following test:
>
> if (unlikely(in_interrupt()))
> panic("Aiee, killing interrupt handler!");
>
>
> So at the time being I still have no idea how to fix that, have you ?
Huh, I'm not sure. x86's MCE handling looks like it does this:
/*
* We might have interrupted pretty much anything. In
* fact, if we're a machine check, we can even interrupt
* NMI processing. We don't want in_nmi() to return true,
* but we need to notify RCU.
*/
rcu_nmi_enter();
But I don't see why they don't want the full NMI treatment there. I
thought the whole point was to do everything so you would get e.g.,
the NMI-safe printk and so on.
The reason the in_interrupt checks work below is because the synchronous
trap handlers e.g., for BUG do not enter interrupt context so the
question is about they context they interrupted. Maybe the right way to
go is nmi_exit just before deciding to oops.
Perhaps we could ask lkml.
Thanks,
Nick
^ permalink raw reply
* Re: [PATCH v2 3/3] powerpc: machine check interrupt is a non-maskable interrupt
From: Christophe LEROY @ 2018-10-13 8:56 UTC (permalink / raw)
To: Nicholas Piggin; +Cc: Mahesh Jagannath Salgaonkar, linuxppc-dev
In-Reply-To: <20181013184815.6a80d196@roar.ozlabs.ibm.com>
Le 13/10/2018 à 10:48, Nicholas Piggin a écrit :
> On Sat, 13 Oct 2018 08:29:48 +0000
> Christophe Leroy <christophe.leroy@c-s.fr> wrote:
>
>> On 10/11/2018 02:31 PM, Christophe LEROY wrote:
>>>
>>>
>>> Le 09/10/2018 à 13:16, Nicholas Piggin a écrit :
>>>> On Tue, 9 Oct 2018 09:36:18 +0000
>>>> Christophe Leroy <christophe.leroy@c-s.fr> wrote:
>>>>
>>>>> On 10/09/2018 05:30 AM, Nicholas Piggin wrote:
>>>>>> On Tue, 9 Oct 2018 06:46:30 +0200
>>>>>> Christophe LEROY <christophe.leroy@c-s.fr> wrote:
>>>>>>> Le 09/10/2018 à 06:32, Nicholas Piggin a écrit :
>>>>>>>> On Mon, 8 Oct 2018 17:39:11 +0200
>>>>>>>> Christophe LEROY <christophe.leroy@c-s.fr> wrote:
>>>>>>>>> Hi Nick,
>>>>>>>>>
>>>>>>>>> Le 19/07/2017 à 08:59, Nicholas Piggin a écrit :
>>>>>>>>>> Use nmi_enter similarly to system reset interrupts. This uses NMI
>>>>>>>>>> printk NMI buffers and turns off various debugging facilities that
>>>>>>>>>> helps avoid tripping on ourselves or other CPUs.
>>>>>>>>>>
>>>>>>>>>> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
>>>>>>>>>> ---
>>>>>>>>>> arch/powerpc/kernel/traps.c | 9 ++++++---
>>>>>>>>>> 1 file changed, 6 insertions(+), 3 deletions(-)
>>>>>>>>>>
>>>>>>>>>> diff --git a/arch/powerpc/kernel/traps.c
>>>>>>>>>> b/arch/powerpc/kernel/traps.c
>>>>>>>>>> index 2849c4f50324..6d31f9d7c333 100644
>>>>>>>>>> --- a/arch/powerpc/kernel/traps.c
>>>>>>>>>> +++ b/arch/powerpc/kernel/traps.c
>>>>>>>>>> @@ -789,8 +789,10 @@ int machine_check_generic(struct pt_regs
>>>>>>>>>> *regs)
>>>>>>>>>> void machine_check_exception(struct pt_regs *regs)
>>>>>>>>>> {
>>>>>>>>>> - enum ctx_state prev_state = exception_enter();
>>>>>>>>>> int recover = 0;
>>>>>>>>>> + bool nested = in_nmi();
>>>>>>>>>> + if (!nested)
>>>>>>>>>> + nmi_enter();
>>>>>>>>>
>>>>>>>>> This alters preempt_count, then when die() is called
>>>>>>>>> in_interrupt() returns true allthough the trap didn't happen in
>>>>>>>>> interrupt, so oops_end() panics for "fatal exception in interrupt"
>>>>>>>>> instead of gently sending SIGBUS the faulting app.
>>>>>>>>
>>>>>>>> Thanks for tracking that down.
>>>>>>>>> Any idea on how to fix this ?
>>>>>>>>
>>>>>>>> I would say we have to deliver the sigbus by hand.
>>>>>>>>
>>>>>>>> if ((user_mode(regs)))
>>>>>>>> _exception(SIGBUS, regs, BUS_MCEERR_AR, regs->nip);
>>>>>>>> else
>>>>>>>> die("Machine check", regs, SIGBUS);
>>>>>>>
>>>>>>> And what about all the other things done by 'die()' ?
>>>>>>>
>>>>>>> And what if it is a kernel thread ?
>>>>>>>
>>>>>>> In one of my boards, I have a kernel thread regularly checking the HW,
>>>>>>> and if it gets a machine check I expect it to gently stop and the die
>>>>>>> notification to be delivered to all registered notifiers.
>>>>>>>
>>>>>>> Until before this patch, it was working well.
>>>>>>
>>>>>> I guess the alternative is we could check regs->trap for machine
>>>>>> check in the die test. Complication is having to account for MCE
>>>>>> in an interrupt handler.
>>>>>>
>>>>>> if (in_interrupt()) {
>>>>>> if (!IS_MCHECK_EXC(regs) || (irq_count() -
>>>>>> (NMI_OFFSET + HARDIRQ_OFFSET)))
>>>>>> panic("Fatal exception in interrupt");
>>>>>> }
>>>>>>
>>>>>> Something like that might work for you? We needs a ppc64 macro for the
>>>>>> MCE, and can probably add something like in_nmi_from_interrupt() for
>>>>>> the second part of the test.
>>>>>
>>>>> Don't know, I'm away from home on business trip so I won't be able to
>>>>> test anything before next week. However it looks more or less like a
>>>>> hack, doesn't it ?
>>>>
>>>> I thought it seemed okay (with the right functions added). Actually it
>>>> could be a bit nicer to do this, then it works generally :
>>>>
>>>> if (in_interrupt()) {
>>>> if (!in_nmi() || in_nmi_from_interrupt())
>>>> panic("Fatal exception in interrupt");
>>>> }
>>>>
>>>>>
>>>>> What about the following ?
>>>>
>>>> Hmm, in some ways maybe it's nicer. One complication is I would like the
>>>> same thing to be available for platform specific machine check
>>>> handlers, so then you need to pass is_in_interrupt to them. Which you
>>>> can do without any problem... But is it cleaner than the above?
>>>
>>> For me it looks cleaner than twiddle the preempt_count depending on
>>> whether we were or not already in nmi() .
>>>
>>> Let's draft something and see what it looks like.
>>
>> Ok, finaly I went to your solution, see below, as it avoids having to
>> modify all subarch and platform specific machine check handlers.
>>
>> Unfortunately it doesn't solves the issue, it only delays it:
>>
>> oops_end() calls do_exit(), which has the following test:
>>
>> if (unlikely(in_interrupt()))
>> panic("Aiee, killing interrupt handler!");
>>
>>
>> So at the time being I still have no idea how to fix that, have you ?
>
> Huh, I'm not sure. x86's MCE handling looks like it does this:
>
> /*
> * We might have interrupted pretty much anything. In
> * fact, if we're a machine check, we can even interrupt
> * NMI processing. We don't want in_nmi() to return true,
> * but we need to notify RCU.
> */
> rcu_nmi_enter();
>
> But I don't see why they don't want the full NMI treatment there. I
> thought the whole point was to do everything so you would get e.g.,
> the NMI-safe printk and so on.
>
> The reason the in_interrupt checks work below is because the synchronous
> trap handlers e.g., for BUG do not enter interrupt context so the
> question is about they context they interrupted. Maybe the right way to
> go is nmi_exit just before deciding to oops.
Yes I arrived at the same conclusion. I tested it just now and it works
for me. Thanks.
Christophe
>
> Perhaps we could ask lkml.
>
> Thanks,
> Nick
>
^ permalink raw reply
* [PATCH] powerpc/traps: restore recoverability of machine_check interrupts
From: Christophe Leroy @ 2018-10-13 9:16 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman, npiggin
Cc: linuxppc-dev, linux-kernel
commit b96672dd840f ("powerpc: Machine check interrupt is a non-
maskable interrupt") added a call to nmi_enter() at the beginning of
machine check restart exception handler. Due to that, in_interrupt()
always returns true regardless of the state before entering the
exception, and die() panics even when the system was not already in
interrupt.
This patch calls nmi_exit() before calling die() in order to restore
the interrupt state we had before calling nmi_enter()
Fixes: b96672dd840f ("powerpc: Machine check interrupt is a non-maskable interrupt")
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
arch/powerpc/kernel/traps.c | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c
index fd58749b4d6b..4f880c2a6e4c 100644
--- a/arch/powerpc/kernel/traps.c
+++ b/arch/powerpc/kernel/traps.c
@@ -765,12 +765,17 @@ void machine_check_exception(struct pt_regs *regs)
if (check_io_access(regs))
goto bail;
- die("Machine check", regs, SIGBUS);
-
/* Must die if the interrupt is not recoverable */
if (!(regs->msr & MSR_RI))
nmi_panic(regs, "Unrecoverable Machine check");
+ if (!nested)
+ nmi_exit();
+
+ die("Machine check", regs, SIGBUS);
+
+ return;
+
bail:
if (!nested)
nmi_exit();
--
2.13.3
^ permalink raw reply related
* [PATCH] powerpc/64: properly initialise the stackprotector canary on SMP.
From: Christophe Leroy @ 2018-10-13 9:45 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
abdhalee
Cc: linuxppc-dev, linux-kernel
commit 06ec27aea9fc ("powerpc/64: add stack protector support")
doesn't initialise the stack canary on SMP secondary CPU's paca,
leading to the following false positive report from the
stack protector.
smp: Bringing up secondary CPUs ...
Kernel panic - not syncing: stack-protector: Kernel stack is corrupted in: __schedule+0x978/0xa80
CPU: 1 PID: 0 Comm: swapper/1 Not tainted 4.19.0-rc7-next-20181010-autotest-autotest #1
Call Trace:
[c000001fed5b3bf0] [c000000000a0ef3c] dump_stack+0xb0/0xf4 (unreliable)
[c000001fed5b3c30] [c0000000000f9d68] panic+0x140/0x308
[c000001fed5b3cc0] [c0000000000f9844] __stack_chk_fail+0x24/0x30
[c000001fed5b3d20] [c000000000a2c3a8] __schedule+0x978/0xa80
[c000001fed5b3e00] [c000000000a2c9b4] schedule_idle+0x34/0x60
[c000001fed5b3e30] [c00000000013d344] do_idle+0x224/0x3d0
[c000001fed5b3ec0] [c00000000013d6e0] cpu_startup_entry+0x30/0x50
[c000001fed5b3ef0] [c000000000047f34] start_secondary+0x4d4/0x520
[c000001fed5b3f90] [c00000000000b370] start_secondary_prolog+0x10/0x14
This patch properly initialises the stack_canary of the secondary
idle tasks.
Reported-by: Abdul Haleem <abdhalee@linux.vnet.ibm.com>
Fixes: 06ec27aea9fc ("powerpc/64: add stack protector support")
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
arch/powerpc/kernel/smp.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index 61c1fadbc644..e774d3bf3a03 100644
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -34,6 +34,7 @@
#include <linux/topology.h>
#include <linux/profile.h>
#include <linux/processor.h>
+#include <linux/random.h>
#include <asm/ptrace.h>
#include <linux/atomic.h>
@@ -810,9 +811,16 @@ static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle)
{
struct thread_info *ti = task_thread_info(idle);
+#ifdef CONFIG_STACKPROTECTOR
+ idle->stack_canary = get_random_canary();
+#endif
+
#ifdef CONFIG_PPC64
paca_ptrs[cpu]->__current = idle;
paca_ptrs[cpu]->kstack = (unsigned long)ti + THREAD_SIZE - STACK_FRAME_OVERHEAD;
+#ifdef CONFIG_STACKPROTECTOR
+ paca_ptrs[cpu]->canary = idle->stack_canary;
+#endif
#endif
ti->cpu = cpu;
secondary_ti = current_set[cpu] = ti;
--
2.13.3
^ permalink raw reply related
* Re: [BUG][linux-next][ppc] kernel panic - not syncing: stack-protector: Kernel stack is corrupted in: __schedule+0x978/0xa80
From: Christophe LEROY @ 2018-10-13 9:47 UTC (permalink / raw)
To: Abdul Haleem
Cc: sachinp, Stephen Rothwell, manvanth, linux-kernel, npiggin,
linux-next, linuxppc-dev
In-Reply-To: <1539350620.17569.39.camel@abdul.in.ibm.com>
Le 12/10/2018 à 15:23, Abdul Haleem a écrit :
> On Fri, 2018-10-12 at 12:36 +0200, Christophe LEROY wrote:
>>
>> Le 12/10/2018 à 12:31, Abdul Haleem a écrit :
>>> On Fri, 2018-10-12 at 10:08 +0200, Christophe LEROY wrote:
>>>>
>>>> Le 12/10/2018 à 09:48, Abdul Haleem a écrit :
>>>>> On Fri, 2018-10-12 at 13:15 +0530, Abdul Haleem wrote:
>>>>>> Greeting's
>>>>>>
>>>>>> Today's linux-next fails to boot on powerpc bare-metal with this error
>>>>>>
>>>>>> POWER8 performance monitor hardware support registered
>>>>>> rcu: Hierarchical SRCU implementation.
>>>>>> smp: Bringing up secondary CPUs ...
>>>>>> Kernel panic - not syncing: stack-protector: Kernel stack is corrupted in: __schedule+0x978/0xa80
>>>>>> CPU: 1 PID: 0 Comm: swapper/1 Not tainted 4.19.0-rc7-next-20181010-autotest-autotest #1
>>>>>> Call Trace:
>>>>>> [c000001fed5b3bf0] [c000000000a0ef3c] dump_stack+0xb0/0xf4 (unreliable)
>>>>>> [c000001fed5b3c30] [c0000000000f9d68] panic+0x140/0x308
>>>>>> [c000001fed5b3cc0] [c0000000000f9844] __stack_chk_fail+0x24/0x30
>>>>>> [c000001fed5b3d20] [c000000000a2c3a8] __schedule+0x978/0xa80
>>>>>> [c000001fed5b3e00] [c000000000a2c9b4] schedule_idle+0x34/0x60
>>>>>> [c000001fed5b3e30] [c00000000013d344] do_idle+0x224/0x3d0
>>>>>> [c000001fed5b3ec0] [c00000000013d6e0] cpu_startup_entry+0x30/0x50
>>>>>> [c000001fed5b3ef0] [c000000000047f34] start_secondary+0x4d4/0x520
>>>>>> [c000001fed5b3f90] [c00000000000b370] start_secondary_prolog+0x10/0x14
>>>>>> Rebooting in 10 seconds..
>>>>>>
>>>>>> Machine: Power 8 bare-metal
>>>>>> kernel version: 4.19.0-rc7-next-20181010
>>>>>> gcc version: 4.8.5 20150623
>>>>>> config attach
>>>>>
>>>>> Attaching the kernel config file
>>>>>
>>>>
>>>> # Linux/powerpc 4.11.0-rc4 Kernel Configuration
>>>>
>>>> This is not the correct config file. Can you send the .config ?
>>>
>>> CONFIG_HAVE_STACKPROTECTOR=y
>>> CONFIG_CC_HAS_STACKPROTECTOR_NONE=y
>>> CONFIG_STACKPROTECTOR=y
>>> CONFIG_STACKPROTECTOR_STRONG=y
>>>
>>> ah yes, I have attached the correct config now.
>>
>> Ok,
>>
>> Could you please try and add a call to function boot_init_stack_canary()
>> in function start_secondary() in arch/powerpc/kernel/smp.c just before
>> the call to cpu_startup_entry() ?
>
> Yes, it works. kernel boots fine with above changes to arch/powerpc/kernel/smp.c
>
Thanks,
Could you please test the patch I just sent ?
I that patch, I initialises the stack canary when preparing the idle
tasks rather than doing it at its startup.
Christophe
^ permalink raw reply
* [PATCH 1/3] powerpc: Split user/kernel definitions of struct pt_regs
From: Michael Ellerman @ 2018-10-13 10:56 UTC (permalink / raw)
To: linuxppc-dev; +Cc: npiggin
We use a shared definition for struct pt_regs in uapi/asm/ptrace.h.
That means the layout of the structure is ABI, ie. we can't change it.
That would be fine if it was only used to describe the user-visible
register state of a process, but it's also the struct we use in the
kernel to describe the registers saved in an interrupt frame.
We'd like more flexibility in the content (and possibly layout) of the
kernel version of the struct, but currently that's not possible.
So split the definition into a user-visible definition which remains
unchanged, and a kernel internal one.
At the moment they're still identical, and we check that at build
time. That's because we have code (in ptrace etc.) that assumes that
they are the same. We will fix that code in future patches, and then
we can break the strict symmetry between the two structs.
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
---
arch/powerpc/include/asm/ptrace.h | 27 ++++++++++++++++++
arch/powerpc/include/uapi/asm/ptrace.h | 7 ++++-
arch/powerpc/kernel/ptrace.c | 39 ++++++++++++++++++++++++++
3 files changed, 72 insertions(+), 1 deletion(-)
diff --git a/arch/powerpc/include/asm/ptrace.h b/arch/powerpc/include/asm/ptrace.h
index 447cbd1bee99..3dd15024db93 100644
--- a/arch/powerpc/include/asm/ptrace.h
+++ b/arch/powerpc/include/asm/ptrace.h
@@ -26,6 +26,33 @@
#include <uapi/asm/ptrace.h>
#include <asm/asm-const.h>
+#ifndef __ASSEMBLY__
+struct pt_regs
+{
+ union {
+ struct user_pt_regs user_regs;
+ struct {
+ unsigned long gpr[32];
+ unsigned long nip;
+ unsigned long msr;
+ unsigned long orig_gpr3;
+ unsigned long ctr;
+ unsigned long link;
+ unsigned long xer;
+ unsigned long ccr;
+#ifdef CONFIG_PPC64
+ unsigned long softe;
+#else
+ unsigned long mq;
+#endif
+ unsigned long trap;
+ unsigned long dar;
+ unsigned long dsisr;
+ unsigned long result;
+ };
+ };
+};
+#endif
#ifdef __powerpc64__
diff --git a/arch/powerpc/include/uapi/asm/ptrace.h b/arch/powerpc/include/uapi/asm/ptrace.h
index 55c7a131d2ab..f5f1ccc740fc 100644
--- a/arch/powerpc/include/uapi/asm/ptrace.h
+++ b/arch/powerpc/include/uapi/asm/ptrace.h
@@ -29,7 +29,12 @@
#ifndef __ASSEMBLY__
-struct pt_regs {
+#ifdef __KERNEL__
+struct user_pt_regs
+#else
+struct pt_regs
+#endif
+{
unsigned long gpr[32];
unsigned long nip;
unsigned long msr;
diff --git a/arch/powerpc/kernel/ptrace.c b/arch/powerpc/kernel/ptrace.c
index 4e372f54088f..939d7f81bbbe 100644
--- a/arch/powerpc/kernel/ptrace.c
+++ b/arch/powerpc/kernel/ptrace.c
@@ -3335,3 +3335,42 @@ void do_syscall_trace_leave(struct pt_regs *regs)
user_enter();
}
+
+void __init pt_regs_check(void)
+{
+ BUILD_BUG_ON(offsetof(struct pt_regs, gpr) !=
+ offsetof(struct user_pt_regs, gpr));
+ BUILD_BUG_ON(offsetof(struct pt_regs, nip) !=
+ offsetof(struct user_pt_regs, nip));
+ BUILD_BUG_ON(offsetof(struct pt_regs, msr) !=
+ offsetof(struct user_pt_regs, msr));
+ BUILD_BUG_ON(offsetof(struct pt_regs, msr) !=
+ offsetof(struct user_pt_regs, msr));
+ BUILD_BUG_ON(offsetof(struct pt_regs, orig_gpr3) !=
+ offsetof(struct user_pt_regs, orig_gpr3));
+ BUILD_BUG_ON(offsetof(struct pt_regs, ctr) !=
+ offsetof(struct user_pt_regs, ctr));
+ BUILD_BUG_ON(offsetof(struct pt_regs, link) !=
+ offsetof(struct user_pt_regs, link));
+ BUILD_BUG_ON(offsetof(struct pt_regs, xer) !=
+ offsetof(struct user_pt_regs, xer));
+ BUILD_BUG_ON(offsetof(struct pt_regs, ccr) !=
+ offsetof(struct user_pt_regs, ccr));
+#ifdef __powerpc64__
+ BUILD_BUG_ON(offsetof(struct pt_regs, softe) !=
+ offsetof(struct user_pt_regs, softe));
+#else
+ BUILD_BUG_ON(offsetof(struct pt_regs, mq) !=
+ offsetof(struct user_pt_regs, mq));
+#endif
+ BUILD_BUG_ON(offsetof(struct pt_regs, trap) !=
+ offsetof(struct user_pt_regs, trap));
+ BUILD_BUG_ON(offsetof(struct pt_regs, dar) !=
+ offsetof(struct user_pt_regs, dar));
+ BUILD_BUG_ON(offsetof(struct pt_regs, dsisr) !=
+ offsetof(struct user_pt_regs, dsisr));
+ BUILD_BUG_ON(offsetof(struct pt_regs, result) !=
+ offsetof(struct user_pt_regs, result));
+
+ BUILD_BUG_ON(sizeof(struct user_pt_regs) > sizeof(struct pt_regs));
+}
--
2.17.1
^ permalink raw reply related
* [PATCH 2/3] powerpc/ptrace: Don't use sizeof(struct pt_regs) in ptrace code
From: Michael Ellerman @ 2018-10-13 10:56 UTC (permalink / raw)
To: linuxppc-dev; +Cc: npiggin
In-Reply-To: <20181013105646.5147-1-mpe@ellerman.id.au>
Now that we've split the user & kernel versions of pt_regs we need to
be more careful in the ptrace code.
For now we've ensured the location of the fields in both structs is
the same, so most of the ptrace code doesn't need updating.
But there are a few places where we use sizeof(pt_regs), and these
will be wrong as soon as we increase the size of the kernel structure.
So flip them all to use sizeof(user_pt_regs).
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
---
arch/powerpc/kernel/ptrace.c | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/arch/powerpc/kernel/ptrace.c b/arch/powerpc/kernel/ptrace.c
index 939d7f81bbbe..c7d0d0c1e34d 100644
--- a/arch/powerpc/kernel/ptrace.c
+++ b/arch/powerpc/kernel/ptrace.c
@@ -297,7 +297,7 @@ int ptrace_get_reg(struct task_struct *task, int regno, unsigned long *data)
}
#endif
- if (regno < (sizeof(struct pt_regs) / sizeof(unsigned long))) {
+ if (regno < (sizeof(struct user_pt_regs) / sizeof(unsigned long))) {
*data = ((unsigned long *)task->thread.regs)[regno];
return 0;
}
@@ -360,10 +360,10 @@ static int gpr_get(struct task_struct *target, const struct user_regset *regset,
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
&target->thread.regs->orig_gpr3,
offsetof(struct pt_regs, orig_gpr3),
- sizeof(struct pt_regs));
+ sizeof(struct user_pt_regs));
if (!ret)
ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- sizeof(struct pt_regs), -1);
+ sizeof(struct user_pt_regs), -1);
return ret;
}
@@ -853,10 +853,10 @@ static int tm_cgpr_get(struct task_struct *target,
ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
&target->thread.ckpt_regs.orig_gpr3,
offsetof(struct pt_regs, orig_gpr3),
- sizeof(struct pt_regs));
+ sizeof(struct user_pt_regs));
if (!ret)
ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
- sizeof(struct pt_regs), -1);
+ sizeof(struct user_pt_regs), -1);
return ret;
}
@@ -3131,7 +3131,7 @@ long arch_ptrace(struct task_struct *child, long request,
case PTRACE_GETREGS: /* Get all pt_regs from the child. */
return copy_regset_to_user(child, &user_ppc_native_view,
REGSET_GPR,
- 0, sizeof(struct pt_regs),
+ 0, sizeof(struct user_pt_regs),
datavp);
#ifdef CONFIG_PPC64
@@ -3140,7 +3140,7 @@ long arch_ptrace(struct task_struct *child, long request,
case PTRACE_SETREGS: /* Set all gp regs in the child. */
return copy_regset_from_user(child, &user_ppc_native_view,
REGSET_GPR,
- 0, sizeof(struct pt_regs),
+ 0, sizeof(struct user_pt_regs),
datavp);
case PTRACE_GETFPREGS: /* Get the child FPU state (FPR0...31 + FPSCR) */
--
2.17.1
^ permalink raw reply related
* [PATCH 3/3] powerpc/64: Interrupts save PPR on stack rather than thread_struct
From: Michael Ellerman @ 2018-10-13 10:56 UTC (permalink / raw)
To: linuxppc-dev; +Cc: npiggin
In-Reply-To: <20181013105646.5147-1-mpe@ellerman.id.au>
From: Nicholas Piggin <npiggin@gmail.com>
PPR is the odd register out when it comes to interrupt handling, it is
saved in current->thread.ppr while all others are saved on the stack.
The difficulty with this is that accessing thread.ppr can cause a SLB
fault, but the SLB fault handler implementation in C change had
assumed the normal exception entry handlers would not cause an SLB
fault.
Fix this by allocating room in the interrupt stack to save PPR.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
---
arch/powerpc/include/asm/exception-64s.h | 9 ++++-----
arch/powerpc/include/asm/processor.h | 6 ++----
arch/powerpc/include/asm/ptrace.h | 4 ++++
arch/powerpc/kernel/asm-offsets.c | 2 +-
arch/powerpc/kernel/entry_64.S | 15 +++++----------
arch/powerpc/kernel/process.c | 2 +-
arch/powerpc/kernel/ptrace.c | 4 ++--
7 files changed, 19 insertions(+), 23 deletions(-)
diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h
index a86feddddad0..403d73898a9a 100644
--- a/arch/powerpc/include/asm/exception-64s.h
+++ b/arch/powerpc/include/asm/exception-64s.h
@@ -236,11 +236,10 @@
* PPR save/restore macros used in exceptions_64s.S
* Used for P7 or later processors
*/
-#define SAVE_PPR(area, ra, rb) \
+#define SAVE_PPR(area, ra) \
BEGIN_FTR_SECTION_NESTED(940) \
- ld ra,PACACURRENT(r13); \
- ld rb,area+EX_PPR(r13); /* Read PPR from paca */ \
- std rb,TASKTHREADPPR(ra); \
+ ld ra,area+EX_PPR(r13); /* Read PPR from paca */ \
+ std ra,_PPR(r1); \
END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,940)
#define RESTORE_PPR_PACA(area, ra) \
@@ -508,7 +507,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
3: EXCEPTION_PROLOG_COMMON_1(); \
beq 4f; /* if from kernel mode */ \
ACCOUNT_CPU_USER_ENTRY(r13, r9, r10); \
- SAVE_PPR(area, r9, r10); \
+ SAVE_PPR(area, r9); \
4: EXCEPTION_PROLOG_COMMON_2(area) \
EXCEPTION_PROLOG_COMMON_3(n) \
ACCOUNT_STOLEN_TIME
diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index 52fadded5c1e..3fefb8a65b17 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -32,9 +32,9 @@
/* Default SMT priority is set to 3. Use 11- 13bits to save priority. */
#define PPR_PRIORITY 3
#ifdef __ASSEMBLY__
-#define INIT_PPR (PPR_PRIORITY << 50)
+#define DEFAULT_PPR (PPR_PRIORITY << 50)
#else
-#define INIT_PPR ((u64)PPR_PRIORITY << 50)
+#define DEFAULT_PPR ((u64)PPR_PRIORITY << 50)
#endif /* __ASSEMBLY__ */
#endif /* CONFIG_PPC64 */
@@ -341,7 +341,6 @@ struct thread_struct {
* onwards.
*/
int dscr_inherit;
- unsigned long ppr; /* used to save/restore SMT priority */
unsigned long tidr;
#endif
#ifdef CONFIG_PPC_BOOK3S_64
@@ -389,7 +388,6 @@ struct thread_struct {
.regs = (struct pt_regs *)INIT_SP - 1, /* XXX bogus, I think */ \
.addr_limit = KERNEL_DS, \
.fpexc_mode = 0, \
- .ppr = INIT_PPR, \
.fscr = FSCR_TAR | FSCR_EBB \
}
#endif
diff --git a/arch/powerpc/include/asm/ptrace.h b/arch/powerpc/include/asm/ptrace.h
index 3dd15024db93..2ba2a1e52291 100644
--- a/arch/powerpc/include/asm/ptrace.h
+++ b/arch/powerpc/include/asm/ptrace.h
@@ -51,6 +51,10 @@ struct pt_regs
unsigned long result;
};
};
+
+#ifdef CONFIG_PPC64
+ unsigned long ppr;
+#endif
};
#endif
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 2eb4923f8468..92156c61d21c 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -89,7 +89,6 @@ int main(void)
#ifdef CONFIG_PPC64
DEFINE(SIGSEGV, SIGSEGV);
DEFINE(NMI_MASK, NMI_MASK);
- OFFSET(TASKTHREADPPR, task_struct, thread.ppr);
#else
OFFSET(THREAD_INFO, task_struct, stack);
DEFINE(THREAD_INFO_GAP, _ALIGN_UP(sizeof(struct thread_info), 16));
@@ -323,6 +322,7 @@ int main(void)
STACK_PT_REGS_OFFSET(_ESR, dsisr);
#else /* CONFIG_PPC64 */
STACK_PT_REGS_OFFSET(SOFTE, softe);
+ STACK_PT_REGS_OFFSET(_PPR, ppr);
#endif /* CONFIG_PPC64 */
#if defined(CONFIG_PPC32)
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index 7db00ee6be48..7b1693adff2a 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -386,10 +386,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
4: /* Anything else left to do? */
BEGIN_FTR_SECTION
- lis r3,INIT_PPR@highest /* Set thread.ppr = 3 */
- ld r10,PACACURRENT(r13)
+ lis r3,DEFAULT_PPR@highest /* Set default PPR */
sldi r3,r3,32 /* bits 11-13 are used for ppr */
- std r3,TASKTHREADPPR(r10)
+ std r3,_PPR(r1)
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
@@ -942,12 +941,6 @@ fast_exception_return:
andi. r0,r3,MSR_RI
beq- .Lunrecov_restore
- /* Load PPR from thread struct before we clear MSR:RI */
-BEGIN_FTR_SECTION
- ld r2,PACACURRENT(r13)
- ld r2,TASKTHREADPPR(r2)
-END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
-
/*
* Clear RI before restoring r13. If we are returning to
* userspace and we take an exception after restoring r13,
@@ -968,7 +961,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
andi. r0,r3,MSR_PR
beq 1f
BEGIN_FTR_SECTION
- mtspr SPRN_PPR,r2 /* Restore PPR */
+ /* Restore PPR */
+ ld r2,_PPR(r1)
+ mtspr SPRN_PPR,r2
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
ACCOUNT_CPU_USER_EXIT(r13, r2, r4)
REST_GPR(13, r1)
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 0ed8d0968515..f9d1cca28cce 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -1710,7 +1710,7 @@ int copy_thread(unsigned long clone_flags, unsigned long usp,
p->thread.dscr = mfspr(SPRN_DSCR);
}
if (cpu_has_feature(CPU_FTR_HAS_PPR))
- p->thread.ppr = INIT_PPR;
+ childregs->ppr = DEFAULT_PPR;
p->thread.tidr = 0;
#endif
diff --git a/arch/powerpc/kernel/ptrace.c b/arch/powerpc/kernel/ptrace.c
index c7d0d0c1e34d..afb819f4ca68 100644
--- a/arch/powerpc/kernel/ptrace.c
+++ b/arch/powerpc/kernel/ptrace.c
@@ -1609,7 +1609,7 @@ static int ppr_get(struct task_struct *target,
void *kbuf, void __user *ubuf)
{
return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
- &target->thread.ppr, 0, sizeof(u64));
+ &target->thread.regs->ppr, 0, sizeof(u64));
}
static int ppr_set(struct task_struct *target,
@@ -1618,7 +1618,7 @@ static int ppr_set(struct task_struct *target,
const void *kbuf, const void __user *ubuf)
{
return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
- &target->thread.ppr, 0, sizeof(u64));
+ &target->thread.regs->ppr, 0, sizeof(u64));
}
static int dscr_get(struct task_struct *target,
--
2.17.1
^ permalink raw reply related
* Re: [PATCH] powerpc: Fix stackprotector detection for non-glibc toolchains
From: Michael Ellerman @ 2018-10-13 11:55 UTC (permalink / raw)
To: Christophe LEROY, linuxppc-dev, Segher Boessenkool
In-Reply-To: <7e1acec1-c840-321b-0a93-99679fce6cb3@c-s.fr>
Christophe LEROY <christophe.leroy@c-s.fr> writes:
> Le 13/10/2018 à 00:58, Michael Ellerman a écrit :
>> If GCC is not built with glibc support then we must explicitly tell it
>> which register to use for TLS mode stack protector, otherwise it will
>> error out and the cc-option check will fail.
>
> Oh ? I didn't encounter such a problem with the nolibc GCC from
> https://mirrors.edge.kernel.org/pub/tools/crosstool/
Yes, you're right.
$ /opt/cross/kisskb/korg/gcc-8.1.0-nolibc/powerpc64-linux/bin/powerpc64-linux-gcc -o empty.o -Wall -c empty.c -mstack-protector-guard=tls
$ echo $?
0
But with mine:
$ /home/kerkins/toolchains/ppc/gcc-8-branch/powerpc-linux/bin/powerpc-linux-gcc -o empty.o -Wall -c empty.c -mstack-protector-guard=tls
cc1: error: ‘-mstack-protector-guard=tls’ needs a valid base register
So it's only my cross compilers that don't work.
The kernel.org ones are:
Configured with: /home/arnd/git/gcc/configure --target=powerpc64-linux
--enable-targets=all
--prefix=/home/arnd/cross/x86_64/gcc-8.1.0-nolibc/powerpc64-linux
--enable-languages=c --without-headers --disable-bootstrap
--disable-nls --disable-threads --disable-shared --disable-libmudflap
--disable-libssp --disable-libgomp --disable-decimal-float
--disable-libquadmath --disable-libatomic --disable-libcc1
--disable-libmpx --enable-checking=release
Whereas mine is:
Configured with: ../../src/gcc/configure
--prefix=/home/kerkins/workspace/gcc-build/gcc/gcc-8-branch/target/ppc/build/install/powerpc-linux
--disable-multilib --disable-bootstrap --enable-languages=c
--with-pkgversion='Custom 2c79ff811dfcee1c' --target=powerpc-linux
--enable-targets=all
So I wonder if something in there is making the difference?
I guess I'll just rewrite the change log to say "some toolchains".
cheers
^ permalink raw reply
* [PATCH v6] powerpc/64s: reimplement book3s idle code in C
From: Nicholas Piggin @ 2018-10-13 12:04 UTC (permalink / raw)
To: linuxppc-dev
Cc: Gautham R . Shenoy, Mahesh Jagannath Salgaonkar, Nicholas Piggin,
kvm-ppc, Aneesh Kumar K.V, Akshay Adiga
Reimplement Book3S idle code in C, moving POWER7/8/9 implementation
speific HV idle code to the powernv platform code.
Book3S assembly stubs are kept in common code and used only to save
the stack frame and non-volatile GPRs before executing architected
idle instructions, and restoring the stack and reloading GPRs then
returning to C after waking from idle.
The complex logic dealing with threads and subcores, locking, SPRs,
HMIs, timebase resync, etc., is all done in C which makes it more
maintainable.
This is not a strict translation to C code, there are some
significant differences:
- Idle wakeup no longer uses the ->cpu_restore call to reinit SPRs,
but saves and restores them itself.
- The optimisation where EC=ESL=0 idle modes did not have to save GPRs
or change MSR is restored, because it's now simple to do. ESL=1
sleeps that do not lose GPRs can use this optimization too.
- KVM secondary entry and cede is now more of a call/return style
rather than branchy. nap_state_lost is not required because KVM
always returns via NVGPR restoring path.
Reviewed-by: Gautham R. Shenoy <ego@linux.vnet.ibm.com>
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
Notes:
- P9 restores some of the PMU SPRs, but not others, and P8 only zeroes
them. There are improvmets to be made to SPR save restore policies and
documentation, but this first pass tries to keep things as they were.
Left to do:
- Test actual POWER7 hardware.
- KVM could use more review, it's pretty tricky. Not sure if what
I'm doing with the emergency stack is kosher. But it runs pretty fine
here with a POWER8 SMP+SMT guest. Possible to streamline
KVM cede code now that idle function saves nv gprs for us?
Since RFC v1:
- Now tested and working with POWER9 hash and radix.
- KVM support added. This took a bit of work to untangle and might
still have some issues, but POWER9 seems to work including hash on
radix with dependent threads mode.
- This snowballed a bit because of KVM and other details making it
not feasible to leave POWER7/8 code alone. That's only half done
at the moment.
- So far this trades about 800 lines of asm for 500 of C. With POWER7/8
support done it might be another hundred or so lines of C.
Since RFC v2:
- Fixed deep state SLB reloading
- Now tested and working with POWER8.
- Accounted for most feedback.
Since RFC v3:
- Rebased to powerpc merge + idle state bugfix
- Split SLB flush/restore code out and shared with MCE code (pseries
MCE patches can also use).
- More testing on POWER8 including KVM with secondaries.
- Performance testing looks good. EC=ESL=0 is about 5% faster, other
stop states look a little faster too.
- Adjusted SPR saving to handler POWER7, haven't tested it.
Since v1:
- More review comments from Gautham.
- Rename isa3_ to isa300_ prefix.
- Tinkered with some comments, copyright notice, changelog.
- Cede and regular idle do not go via KVM secondary wakeup code path,
so hwthread_state stores and barriers can be simplified, and some
KVM code paths simplified a little.
Since v2:
- Rebase, SLB reload patch has been merged.
- More testing. Tested machine check idle wakeup path with mambo stepping
through instructions.
Since v3:
- Build fixes caught by CI
Since v4:
- PSSCR test PLS rather than RL (Akshay)
Since v5:
- Fix TB loss test to use PLS instead of RL as well
- Rename hv_loss variable to spr_loss to better describe its usage
- Clamp the SPR loss level to shallower of SPR loss or TB loss in case
future CPU has that behaviour (P8 type behaviour).
- Added a few more comments.
- Re-tested on P8, P9 and KVM.
---
arch/powerpc/include/asm/cpuidle.h | 19 +-
arch/powerpc/include/asm/paca.h | 40 +-
arch/powerpc/include/asm/processor.h | 9 +-
arch/powerpc/include/asm/reg.h | 8 +-
arch/powerpc/kernel/asm-offsets.c | 18 -
arch/powerpc/kernel/dt_cpu_ftrs.c | 21 +-
arch/powerpc/kernel/exceptions-64s.S | 17 +-
arch/powerpc/kernel/idle_book3s.S | 1012 +++-------------------
arch/powerpc/kernel/setup-common.c | 4 +-
arch/powerpc/kvm/book3s_hv_rmhandlers.S | 86 +-
arch/powerpc/platforms/powernv/idle.c | 850 ++++++++++++++----
arch/powerpc/platforms/powernv/subcore.c | 2 +-
arch/powerpc/xmon/xmon.c | 24 +-
13 files changed, 904 insertions(+), 1206 deletions(-)
diff --git a/arch/powerpc/include/asm/cpuidle.h b/arch/powerpc/include/asm/cpuidle.h
index 43e5f31fe64d..9844b3ded187 100644
--- a/arch/powerpc/include/asm/cpuidle.h
+++ b/arch/powerpc/include/asm/cpuidle.h
@@ -27,10 +27,11 @@
* the THREAD_WINKLE_BITS are set, which indicate which threads have not
* yet woken from the winkle state.
*/
-#define PNV_CORE_IDLE_LOCK_BIT 0x10000000
+#define NR_PNV_CORE_IDLE_LOCK_BIT 28
+#define PNV_CORE_IDLE_LOCK_BIT (1ULL << NR_PNV_CORE_IDLE_LOCK_BIT)
+#define PNV_CORE_IDLE_WINKLE_COUNT_SHIFT 16
#define PNV_CORE_IDLE_WINKLE_COUNT 0x00010000
-#define PNV_CORE_IDLE_WINKLE_COUNT_ALL_BIT 0x00080000
#define PNV_CORE_IDLE_WINKLE_COUNT_BITS 0x000F0000
#define PNV_CORE_IDLE_THREAD_WINKLE_BITS_SHIFT 8
#define PNV_CORE_IDLE_THREAD_WINKLE_BITS 0x0000FF00
@@ -68,16 +69,6 @@
#define ERR_DEEP_STATE_ESL_MISMATCH -2
#ifndef __ASSEMBLY__
-/* Additional SPRs that need to be saved/restored during stop */
-struct stop_sprs {
- u64 pid;
- u64 ldbar;
- u64 fscr;
- u64 hfscr;
- u64 mmcr1;
- u64 mmcr2;
- u64 mmcra;
-};
#define PNV_IDLE_NAME_LEN 16
struct pnv_idle_states_t {
@@ -92,10 +83,6 @@ struct pnv_idle_states_t {
extern struct pnv_idle_states_t *pnv_idle_states;
extern int nr_pnv_idle_states;
-extern u32 pnv_fastsleep_workaround_at_entry[];
-extern u32 pnv_fastsleep_workaround_at_exit[];
-
-extern u64 pnv_first_deep_stop_state;
unsigned long pnv_cpu_offline(unsigned int cpu);
int validate_psscr_val_mask(u64 *psscr_val, u64 *psscr_mask, u32 flags);
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index c6d01f0aa898..cad1e440ca58 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -168,7 +168,6 @@ struct paca_struct {
u8 irq_happened; /* irq happened while soft-disabled */
u8 io_sync; /* writel() needs spin_unlock sync */
u8 irq_work_pending; /* IRQ_WORK interrupt while soft-disable */
- u8 nap_state_lost; /* NV GPR values lost in power7_idle */
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
u8 pmcregs_in_use; /* pseries puts this in lppaca */
#endif
@@ -178,23 +177,28 @@ struct paca_struct {
#endif
#ifdef CONFIG_PPC_POWERNV
- /* Per-core mask tracking idle threads and a lock bit-[L][TTTTTTTT] */
- u32 *core_idle_state_ptr;
- u8 thread_idle_state; /* PNV_THREAD_RUNNING/NAP/SLEEP */
- /* Mask to indicate thread id in core */
- u8 thread_mask;
- /* Mask to denote subcore sibling threads */
- u8 subcore_sibling_mask;
- /* Flag to request this thread not to stop */
- atomic_t dont_stop;
- /* The PSSCR value that the kernel requested before going to stop */
- u64 requested_psscr;
-
- /*
- * Save area for additional SPRs that need to be
- * saved/restored during cpuidle stop.
- */
- struct stop_sprs stop_sprs;
+ /* PowerNV idle fields */
+ /* PNV_CORE_IDLE_* bits, all siblings work on thread 0 paca */
+ unsigned long idle_state;
+ union {
+ /* P7/P8 specific fields */
+ struct {
+ /* PNV_THREAD_RUNNING/NAP/SLEEP */
+ u8 thread_idle_state;
+ /* Mask to denote subcore sibling threads */
+ u8 subcore_sibling_mask;
+ };
+
+ /* P9 specific fields */
+ struct {
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+ /* The PSSCR value that the kernel requested before going to stop */
+ u64 requested_psscr;
+ /* Flag to request this thread not to stop */
+ atomic_t dont_stop;
+#endif
+ };
+ };
#endif
#ifdef CONFIG_PPC_BOOK3S_64
diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index 52fadded5c1e..12b76ecdc57d 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -506,14 +506,17 @@ static inline unsigned long get_clean_sp(unsigned long sp, int is_32)
}
#endif
+/* asm stubs */
+extern unsigned long isa300_idle_stop_noloss(unsigned long psscr_val);
+extern unsigned long isa300_idle_stop_mayloss(unsigned long psscr_val);
+extern unsigned long isa206_idle_insn_mayloss(unsigned long type);
+
extern unsigned long cpuidle_disable;
enum idle_boot_override {IDLE_NO_OVERRIDE = 0, IDLE_POWERSAVE_OFF};
extern int powersave_nap; /* set if nap mode can be used in idle loop */
-extern unsigned long power7_idle_insn(unsigned long type); /* PNV_THREAD_NAP/etc*/
+
extern void power7_idle_type(unsigned long type);
-extern unsigned long power9_idle_stop(unsigned long psscr_val);
-extern unsigned long power9_offline_stop(unsigned long psscr_val);
extern void power9_idle_type(unsigned long stop_psscr_val,
unsigned long stop_psscr_mask);
diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h
index 640a4d818772..7f71897db246 100644
--- a/arch/powerpc/include/asm/reg.h
+++ b/arch/powerpc/include/asm/reg.h
@@ -168,6 +168,7 @@
#define PSSCR_ESL 0x00200000 /* Enable State Loss */
#define PSSCR_SD 0x00400000 /* Status Disable */
#define PSSCR_PLS 0xf000000000000000 /* Power-saving Level Status */
+#define PSSCR_PLS_SHIFT 60
#define PSSCR_GUEST_VIS 0xf0000000000003ffUL /* Guest-visible PSSCR fields */
#define PSSCR_FAKE_SUSPEND 0x00000400 /* Fake-suspend bit (P9 DD2.2) */
#define PSSCR_FAKE_SUSPEND_LG 10 /* Fake-suspend bit position */
@@ -757,10 +758,9 @@
#define SRR1_WAKERESET 0x00100000 /* System reset */
#define SRR1_WAKEHDBELL 0x000c0000 /* Hypervisor doorbell on P8 */
#define SRR1_WAKESTATE 0x00030000 /* Powersave exit mask [46:47] */
-#define SRR1_WS_DEEPEST 0x00030000 /* Some resources not maintained,
- * may not be recoverable */
-#define SRR1_WS_DEEPER 0x00020000 /* Some resources not maintained */
-#define SRR1_WS_DEEP 0x00010000 /* All resources maintained */
+#define SRR1_WS_HVLOSS 0x00030000 /* HV resources not maintained */
+#define SRR1_WS_GPRLOSS 0x00020000 /* GPRs not maintained */
+#define SRR1_WS_NOLOSS 0x00010000 /* All resources maintained */
#define SRR1_PROGTM 0x00200000 /* TM Bad Thing */
#define SRR1_PROGFPE 0x00100000 /* Floating Point Enabled */
#define SRR1_PROGILL 0x00080000 /* Illegal instruction */
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index a6d70fd2e499..fba5805bdb77 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -262,7 +262,6 @@ int main(void)
OFFSET(ACCOUNT_USER_TIME, paca_struct, accounting.utime);
OFFSET(ACCOUNT_SYSTEM_TIME, paca_struct, accounting.stime);
OFFSET(PACA_TRAP_SAVE, paca_struct, trap_save);
- OFFSET(PACA_NAPSTATELOST, paca_struct, nap_state_lost);
OFFSET(PACA_SPRG_VDSO, paca_struct, sprg_vdso);
#else /* CONFIG_PPC64 */
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
@@ -767,23 +766,6 @@ int main(void)
OFFSET(VCPU_TIMING_LAST_ENTER_TBL, kvm_vcpu, arch.timing_last_enter.tv32.tbl);
#endif
-#ifdef CONFIG_PPC_POWERNV
- OFFSET(PACA_CORE_IDLE_STATE_PTR, paca_struct, core_idle_state_ptr);
- OFFSET(PACA_THREAD_IDLE_STATE, paca_struct, thread_idle_state);
- OFFSET(PACA_THREAD_MASK, paca_struct, thread_mask);
- OFFSET(PACA_SUBCORE_SIBLING_MASK, paca_struct, subcore_sibling_mask);
- OFFSET(PACA_REQ_PSSCR, paca_struct, requested_psscr);
- OFFSET(PACA_DONT_STOP, paca_struct, dont_stop);
-#define STOP_SPR(x, f) OFFSET(x, paca_struct, stop_sprs.f)
- STOP_SPR(STOP_PID, pid);
- STOP_SPR(STOP_LDBAR, ldbar);
- STOP_SPR(STOP_FSCR, fscr);
- STOP_SPR(STOP_HFSCR, hfscr);
- STOP_SPR(STOP_MMCR1, mmcr1);
- STOP_SPR(STOP_MMCR2, mmcr2);
- STOP_SPR(STOP_MMCRA, mmcra);
-#endif
-
DEFINE(PPC_DBELL_SERVER, PPC_DBELL_SERVER);
DEFINE(PPC_DBELL_MSGTYPE, PPC_DBELL_MSGTYPE);
diff --git a/arch/powerpc/kernel/dt_cpu_ftrs.c b/arch/powerpc/kernel/dt_cpu_ftrs.c
index f432054234a4..d635d78facdc 100644
--- a/arch/powerpc/kernel/dt_cpu_ftrs.c
+++ b/arch/powerpc/kernel/dt_cpu_ftrs.c
@@ -71,7 +71,6 @@ static int hv_mode;
static struct {
u64 lpcr;
- u64 lpcr_clear;
u64 hfscr;
u64 fscr;
} system_registers;
@@ -80,24 +79,7 @@ static void (*init_pmu_registers)(void);
static void __restore_cpu_cpufeatures(void)
{
- u64 lpcr;
-
- /*
- * LPCR is restored by the power on engine already. It can be changed
- * after early init e.g., by radix enable, and we have no unified API
- * for saving and restoring such SPRs.
- *
- * This ->restore hook should really be removed from idle and register
- * restore moved directly into the idle restore code, because this code
- * doesn't know how idle is implemented or what it needs restored here.
- *
- * The best we can do to accommodate secondary boot and idle restore
- * for now is "or" LPCR with existing.
- */
- lpcr = mfspr(SPRN_LPCR);
- lpcr |= system_registers.lpcr;
- lpcr &= ~system_registers.lpcr_clear;
- mtspr(SPRN_LPCR, lpcr);
+ mtspr(SPRN_LPCR, system_registers.lpcr);
if (hv_mode) {
mtspr(SPRN_LPID, 0);
mtspr(SPRN_HFSCR, system_registers.hfscr);
@@ -318,7 +300,6 @@ static int __init feat_enable_mmu_hash_v3(struct dt_cpu_feature *f)
{
u64 lpcr;
- system_registers.lpcr_clear |= (LPCR_ISL | LPCR_UPRT | LPCR_HR);
lpcr = mfspr(SPRN_LPCR);
lpcr &= ~(LPCR_ISL | LPCR_UPRT | LPCR_HR);
mtspr(SPRN_LPCR, lpcr);
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index b9239dbf6d59..c88650109f29 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -136,8 +136,9 @@ TRAMP_KVM(PACA_EXNMI, 0x100)
#ifdef CONFIG_PPC_P7_NAP
EXC_COMMON_BEGIN(system_reset_idle_common)
- mfspr r12,SPRN_SRR1
- b pnv_powersave_wakeup
+ mfspr r3,SPRN_SRR1
+ bltlr cr3 /* no state loss, return to idle caller */
+ b idle_return_gpr_loss
#endif
/*
@@ -419,17 +420,17 @@ EXC_COMMON_BEGIN(machine_check_idle_common)
* Then decrement MCE nesting after finishing with the stack.
*/
ld r3,_MSR(r1)
+ ld r4,_LINK(r1)
lhz r11,PACA_IN_MCE(r13)
subi r11,r11,1
sth r11,PACA_IN_MCE(r13)
- /* Turn off the RI bit because SRR1 is used by idle wakeup code. */
- /* Recoverability could be improved by reducing the use of SRR1. */
- li r11,0
- mtmsrd r11,1
-
- b pnv_powersave_wakeup_mce
+ mtlr r4
+ rlwinm r10,r3,47-31,30,31
+ cmpwi cr3,r10,2
+ bltlr cr3 /* no state loss, return to idle caller */
+ b idle_return_gpr_loss
#endif
/*
* Handle machine check early in real mode. We come here with
diff --git a/arch/powerpc/kernel/idle_book3s.S b/arch/powerpc/kernel/idle_book3s.S
index 7f5ac2e8581b..ffdee1ab4388 100644
--- a/arch/powerpc/kernel/idle_book3s.S
+++ b/arch/powerpc/kernel/idle_book3s.S
@@ -1,227 +1,87 @@
/*
- * This file contains idle entry/exit functions for POWER7,
- * POWER8 and POWER9 CPUs.
+ * Copyright 2018, IBM Corporation.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version
* 2 of the License, or (at your option) any later version.
+ *
+ * This file contains general idle entry/exit functions to save
+ * and restore stack and NVGPRs which allows C code to call idle
+ * states that lose GPRs, and it will return transparently with
+ * SRR1 wakeup reason return value.
+ *
+ * The platform / CPU caller must ensure SPRs and any other non-GPR
+ * state is saved and restored correctly, handle KVM, interrupts, etc.
*/
-#include <linux/threads.h>
-#include <asm/processor.h>
-#include <asm/page.h>
-#include <asm/cputable.h>
-#include <asm/thread_info.h>
#include <asm/ppc_asm.h>
#include <asm/asm-offsets.h>
#include <asm/ppc-opcode.h>
-#include <asm/hw_irq.h>
-#include <asm/kvm_book3s_asm.h>
-#include <asm/opal.h>
#include <asm/cpuidle.h>
-#include <asm/exception-64s.h>
-#include <asm/book3s/64/mmu-hash.h>
-#include <asm/mmu.h>
-#include <asm/asm-compat.h>
-#include <asm/feature-fixups.h>
-
-#undef DEBUG
/*
- * Use unused space in the interrupt stack to save and restore
- * registers for winkle support.
- */
-#define _MMCR0 GPR0
-#define _SDR1 GPR3
-#define _PTCR GPR3
-#define _RPR GPR4
-#define _SPURR GPR5
-#define _PURR GPR6
-#define _TSCR GPR7
-#define _DSCR GPR8
-#define _AMOR GPR9
-#define _WORT GPR10
-#define _WORC GPR11
-#define _LPCR GPR12
-
-#define PSSCR_EC_ESL_MASK_SHIFTED (PSSCR_EC | PSSCR_ESL) >> 16
-
- .text
-
-/*
- * Used by threads before entering deep idle states. Saves SPRs
- * in interrupt stack frame
- */
-save_sprs_to_stack:
- /*
- * Note all register i.e per-core, per-subcore or per-thread is saved
- * here since any thread in the core might wake up first
- */
-BEGIN_FTR_SECTION
- /*
- * Note - SDR1 is dropped in Power ISA v3. Hence not restoring
- * SDR1 here
- */
- mfspr r3,SPRN_PTCR
- std r3,_PTCR(r1)
- mfspr r3,SPRN_LPCR
- std r3,_LPCR(r1)
-FTR_SECTION_ELSE
- mfspr r3,SPRN_SDR1
- std r3,_SDR1(r1)
-ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
- mfspr r3,SPRN_RPR
- std r3,_RPR(r1)
- mfspr r3,SPRN_SPURR
- std r3,_SPURR(r1)
- mfspr r3,SPRN_PURR
- std r3,_PURR(r1)
- mfspr r3,SPRN_TSCR
- std r3,_TSCR(r1)
- mfspr r3,SPRN_DSCR
- std r3,_DSCR(r1)
- mfspr r3,SPRN_AMOR
- std r3,_AMOR(r1)
- mfspr r3,SPRN_WORT
- std r3,_WORT(r1)
- mfspr r3,SPRN_WORC
- std r3,_WORC(r1)
-/*
- * On POWER9, there are idle states such as stop4, invoked via cpuidle,
- * that lose hypervisor resources. In such cases, we need to save
- * additional SPRs before entering those idle states so that they can
- * be restored to their older values on wakeup from the idle state.
+ * Desired PSSCR in r3
+ *
+ * No state will be lost regardless of wakeup mechanism (interrupt or NIA).
*
- * On POWER8, the only such deep idle state is winkle which is used
- * only in the context of CPU-Hotplug, where these additional SPRs are
- * reinitiazed to a sane value. Hence there is no need to save/restore
- * these SPRs.
+ * An EC=0 type wakeup will return with a value of 0. SRESET wakeup (which can
+ * happen with xscom SRESET and possibly MCE) may clobber volatiles except LR,
+ * and must blr, to return to caller with r3 set according to caller's expected
+ * return code (for Book3S/64 that is SRR1).
*/
-BEGIN_FTR_SECTION
- blr
-END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
-
-power9_save_additional_sprs:
- mfspr r3, SPRN_PID
- mfspr r4, SPRN_LDBAR
- std r3, STOP_PID(r13)
- std r4, STOP_LDBAR(r13)
-
- mfspr r3, SPRN_FSCR
- mfspr r4, SPRN_HFSCR
- std r3, STOP_FSCR(r13)
- std r4, STOP_HFSCR(r13)
-
- mfspr r3, SPRN_MMCRA
- mfspr r4, SPRN_MMCR0
- std r3, STOP_MMCRA(r13)
- std r4, _MMCR0(r1)
-
- mfspr r3, SPRN_MMCR1
- mfspr r4, SPRN_MMCR2
- std r3, STOP_MMCR1(r13)
- std r4, STOP_MMCR2(r13)
- blr
-
-power9_restore_additional_sprs:
- ld r3,_LPCR(r1)
- ld r4, STOP_PID(r13)
- mtspr SPRN_LPCR,r3
- mtspr SPRN_PID, r4
-
- ld r3, STOP_LDBAR(r13)
- ld r4, STOP_FSCR(r13)
- mtspr SPRN_LDBAR, r3
- mtspr SPRN_FSCR, r4
-
- ld r3, STOP_HFSCR(r13)
- ld r4, STOP_MMCRA(r13)
- mtspr SPRN_HFSCR, r3
- mtspr SPRN_MMCRA, r4
-
- ld r3, _MMCR0(r1)
- ld r4, STOP_MMCR1(r13)
- mtspr SPRN_MMCR0, r3
- mtspr SPRN_MMCR1, r4
-
- ld r3, STOP_MMCR2(r13)
- ld r4, PACA_SPRG_VDSO(r13)
- mtspr SPRN_MMCR2, r3
- mtspr SPRN_SPRG3, r4
+_GLOBAL(isa300_idle_stop_noloss)
+ mtspr SPRN_PSSCR,r3
+ PPC_STOP
+ li r3,0
blr
/*
- * Used by threads when the lock bit of core_idle_state is set.
- * Threads will spin in HMT_LOW until the lock bit is cleared.
- * r14 - pointer to core_idle_state
- * r15 - used to load contents of core_idle_state
- * r9 - used as a temporary variable
+ * Desired PSSCR in r3
+ *
+ * GPRs may be lost, so they are saved here. Wakeup is by interrupt only.
+ * The SRESET wakeup returns to this function's caller by calling
+ * idle_return_gpr_loss with r3 set to desired return value.
+ *
+ * A wakeup without GPR loss may alteratively be handled as in
+ * isa300_idle_stop_noloss and blr directly, as an optimisation.
+ *
+ * The caller is responsible for saving/restoring SPRs, MSR, timebase,
+ * etc.
*/
-
-core_idle_lock_held:
- HMT_LOW
-3: lwz r15,0(r14)
- andis. r15,r15,PNV_CORE_IDLE_LOCK_BIT@h
- bne 3b
- HMT_MEDIUM
- lwarx r15,0,r14
- andis. r9,r15,PNV_CORE_IDLE_LOCK_BIT@h
- bne- core_idle_lock_held
- blr
+_GLOBAL(isa300_idle_stop_mayloss)
+ mtspr SPRN_PSSCR,r3
+ std r1,PACAR1(r13)
+ mflr r4
+ mfcr r5
+ /* use stack red zone rather than a new frame */
+ addi r6,r1,-INT_FRAME_SIZE
+ SAVE_GPR(2, r6)
+ SAVE_NVGPRS(r6)
+ std r4,_LINK(r6)
+ std r5,_CCR(r6)
+ PPC_STOP
+ b . /* catch bugs */
/*
- * Pass requested state in r3:
- * r3 - PNV_THREAD_NAP/SLEEP/WINKLE in POWER8
- * - Requested PSSCR value in POWER9
+ * Desired return value in r3
*
- * Address of idle handler to branch to in realmode in r4
+ * The idle wakeup SRESET interrupt can call this after calling
+ * to return to the idle sleep function caller with r3 as the return code.
+ *
+ * This must not be used if idle was entered via a _noloss function (use
+ * a simple blr instead).
*/
-pnv_powersave_common:
- /* Use r3 to pass state nap/sleep/winkle */
- /* NAP is a state loss, we create a regs frame on the
- * stack, fill it up with the state we care about and
- * stick a pointer to it in PACAR1. We really only
- * need to save PC, some CR bits and the NV GPRs,
- * but for now an interrupt frame will do.
- */
- mtctr r4
-
- mflr r0
- std r0,16(r1)
- stdu r1,-INT_FRAME_SIZE(r1)
- std r0,_LINK(r1)
- std r0,_NIP(r1)
-
- /* We haven't lost state ... yet */
- li r0,0
- stb r0,PACA_NAPSTATELOST(r13)
-
- /* Continue saving state */
- SAVE_GPR(2, r1)
- SAVE_NVGPRS(r1)
- mfcr r5
- std r5,_CCR(r1)
- std r1,PACAR1(r13)
-
-BEGIN_FTR_SECTION
- /*
- * POWER9 does not require real mode to stop, and presently does not
- * set hwthread_state for KVM (threads don't share MMU context), so
- * we can remain in virtual mode for this.
- */
- bctr
-END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
- /*
- * POWER8
- * Go to real mode to do the nap, as required by the architecture.
- * Also, we need to be in real mode before setting hwthread_state,
- * because as soon as we do that, another thread can switch
- * the MMU context to the guest.
- */
- LOAD_REG_IMMEDIATE(r7, MSR_IDLE)
- mtmsrd r7,0
- bctr
+_GLOBAL(idle_return_gpr_loss)
+ ld r1,PACAR1(r13)
+ addi r6,r1,-INT_FRAME_SIZE
+ ld r4,_LINK(r6)
+ ld r5,_CCR(r6)
+ REST_NVGPRS(r6)
+ REST_GPR(2, r6)
+ mtlr r4
+ mtcr r5
+ blr
/*
* This is the sequence required to execute idle instructions, as
@@ -234,723 +94,57 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
ld r0,0(r1); \
236: cmpd cr0,r0,r0; \
bne 236b; \
- IDLE_INST;
-
-
- .globl pnv_enter_arch207_idle_mode
-pnv_enter_arch207_idle_mode:
-#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
- /* Tell KVM we're entering idle */
- li r4,KVM_HWTHREAD_IN_IDLE
- /******************************************************/
- /* N O T E W E L L ! ! ! N O T E W E L L */
- /* The following store to HSTATE_HWTHREAD_STATE(r13) */
- /* MUST occur in real mode, i.e. with the MMU off, */
- /* and the MMU must stay off until we clear this flag */
- /* and test HSTATE_HWTHREAD_REQ(r13) in */
- /* pnv_powersave_wakeup in this file. */
- /* The reason is that another thread can switch the */
- /* MMU to a guest context whenever this flag is set */
- /* to KVM_HWTHREAD_IN_IDLE, and if the MMU was on, */
- /* that would potentially cause this thread to start */
- /* executing instructions from guest memory in */
- /* hypervisor mode, leading to a host crash or data */
- /* corruption, or worse. */
- /******************************************************/
- stb r4,HSTATE_HWTHREAD_STATE(r13)
-#endif
- stb r3,PACA_THREAD_IDLE_STATE(r13)
- cmpwi cr3,r3,PNV_THREAD_SLEEP
- bge cr3,2f
- IDLE_STATE_ENTER_SEQ_NORET(PPC_NAP)
- /* No return */
-2:
- /* Sleep or winkle */
- lbz r7,PACA_THREAD_MASK(r13)
- ld r14,PACA_CORE_IDLE_STATE_PTR(r13)
- li r5,0
- beq cr3,3f
- lis r5,PNV_CORE_IDLE_WINKLE_COUNT@h
-3:
-lwarx_loop1:
- lwarx r15,0,r14
-
- andis. r9,r15,PNV_CORE_IDLE_LOCK_BIT@h
- bnel- core_idle_lock_held
-
- add r15,r15,r5 /* Add if winkle */
- andc r15,r15,r7 /* Clear thread bit */
-
- andi. r9,r15,PNV_CORE_IDLE_THREAD_BITS
-
-/*
- * If cr0 = 0, then current thread is the last thread of the core entering
- * sleep. Last thread needs to execute the hardware bug workaround code if
- * required by the platform.
- * Make the workaround call unconditionally here. The below branch call is
- * patched out when the idle states are discovered if the platform does not
- * require it.
- */
-.global pnv_fastsleep_workaround_at_entry
-pnv_fastsleep_workaround_at_entry:
- beq fastsleep_workaround_at_entry
-
- stwcx. r15,0,r14
- bne- lwarx_loop1
- isync
-
-common_enter: /* common code for all the threads entering sleep or winkle */
- bgt cr3,enter_winkle
- IDLE_STATE_ENTER_SEQ_NORET(PPC_SLEEP)
-
-fastsleep_workaround_at_entry:
- oris r15,r15,PNV_CORE_IDLE_LOCK_BIT@h
- stwcx. r15,0,r14
- bne- lwarx_loop1
- isync
-
- /* Fast sleep workaround */
- li r3,1
- li r4,1
- bl opal_config_cpu_idle_state
-
- /* Unlock */
- xoris r15,r15,PNV_CORE_IDLE_LOCK_BIT@h
- lwsync
- stw r15,0(r14)
- b common_enter
-
-enter_winkle:
- bl save_sprs_to_stack
-
- IDLE_STATE_ENTER_SEQ_NORET(PPC_WINKLE)
-
-/*
- * r3 - PSSCR value corresponding to the requested stop state.
- */
-power_enter_stop:
-/*
- * Check if we are executing the lite variant with ESL=EC=0
- */
- andis. r4,r3,PSSCR_EC_ESL_MASK_SHIFTED
- clrldi r3,r3,60 /* r3 = Bits[60:63] = Requested Level (RL) */
- bne .Lhandle_esl_ec_set
- PPC_STOP
- li r3,0 /* Since we didn't lose state, return 0 */
- std r3, PACA_REQ_PSSCR(r13)
-
- /*
- * pnv_wakeup_noloss() expects r12 to contain the SRR1 value so
- * it can determine if the wakeup reason is an HMI in
- * CHECK_HMI_INTERRUPT.
- *
- * However, when we wakeup with ESL=0, SRR1 will not contain the wakeup
- * reason, so there is no point setting r12 to SRR1.
- *
- * Further, we clear r12 here, so that we don't accidentally enter the
- * HMI in pnv_wakeup_noloss() if the value of r12[42:45] == WAKE_HMI.
- */
- li r12, 0
- b pnv_wakeup_noloss
-
-.Lhandle_esl_ec_set:
-BEGIN_FTR_SECTION
- /*
- * POWER9 DD2.0 or earlier can incorrectly set PMAO when waking up after
- * a state-loss idle. Saving and restoring MMCR0 over idle is a
- * workaround.
- */
- mfspr r4,SPRN_MMCR0
- std r4,_MMCR0(r1)
-END_FTR_SECTION_IFCLR(CPU_FTR_POWER9_DD2_1)
-
-/*
- * Check if the requested state is a deep idle state.
- */
- LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state)
- ld r4,ADDROFF(pnv_first_deep_stop_state)(r5)
- cmpd r3,r4
- bge .Lhandle_deep_stop
- PPC_STOP /* Does not return (system reset interrupt) */
-
-.Lhandle_deep_stop:
-/*
- * Entering deep idle state.
- * Clear thread bit in PACA_CORE_IDLE_STATE, save SPRs to
- * stack and enter stop
- */
- lbz r7,PACA_THREAD_MASK(r13)
- ld r14,PACA_CORE_IDLE_STATE_PTR(r13)
-
-lwarx_loop_stop:
- lwarx r15,0,r14
- andis. r9,r15,PNV_CORE_IDLE_LOCK_BIT@h
- bnel- core_idle_lock_held
- andc r15,r15,r7 /* Clear thread bit */
-
- stwcx. r15,0,r14
- bne- lwarx_loop_stop
- isync
-
- bl save_sprs_to_stack
-
- PPC_STOP /* Does not return (system reset interrupt) */
-
-/*
- * Entered with MSR[EE]=0 and no soft-masked interrupts pending.
- * r3 contains desired idle state (PNV_THREAD_NAP/SLEEP/WINKLE).
- */
-_GLOBAL(power7_idle_insn)
- /* Now check if user or arch enabled NAP mode */
- LOAD_REG_ADDR(r4, pnv_enter_arch207_idle_mode)
- b pnv_powersave_common
-
-#define CHECK_HMI_INTERRUPT \
-BEGIN_FTR_SECTION_NESTED(66); \
- rlwinm r0,r12,45-31,0xf; /* extract wake reason field (P8) */ \
-FTR_SECTION_ELSE_NESTED(66); \
- rlwinm r0,r12,45-31,0xe; /* P7 wake reason field is 3 bits */ \
-ALT_FTR_SECTION_END_NESTED_IFSET(CPU_FTR_ARCH_207S, 66); \
- cmpwi r0,0xa; /* Hypervisor maintenance ? */ \
- bne+ 20f; \
- /* Invoke opal call to handle hmi */ \
- ld r2,PACATOC(r13); \
- ld r1,PACAR1(r13); \
- std r3,ORIG_GPR3(r1); /* Save original r3 */ \
- li r3,0; /* NULL argument */ \
- bl hmi_exception_realmode; \
- nop; \
- ld r3,ORIG_GPR3(r1); /* Restore original r3 */ \
-20: nop;
-
-/*
- * Entered with MSR[EE]=0 and no soft-masked interrupts pending.
- * r3 contains desired PSSCR register value.
- *
- * Offline (CPU unplug) case also must notify KVM that the CPU is
- * idle.
- */
-_GLOBAL(power9_offline_stop)
-#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
- /*
- * Tell KVM we're entering idle.
- * This does not have to be done in real mode because the P9 MMU
- * is independent per-thread. Some steppings share radix/hash mode
- * between threads, but in that case KVM has a barrier sync in real
- * mode before and after switching between radix and hash.
- */
- li r4,KVM_HWTHREAD_IN_IDLE
- stb r4,HSTATE_HWTHREAD_STATE(r13)
-#endif
- /* fall through */
-
-_GLOBAL(power9_idle_stop)
- std r3, PACA_REQ_PSSCR(r13)
-#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
-BEGIN_FTR_SECTION
- sync
- lwz r5, PACA_DONT_STOP(r13)
- cmpwi r5, 0
- bne 1f
-END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG)
-#endif
- mtspr SPRN_PSSCR,r3
- LOAD_REG_ADDR(r4,power_enter_stop)
- b pnv_powersave_common
- /* No return */
-#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
-1:
- /*
- * We get here when TM / thread reconfiguration bug workaround
- * code wants to get the CPU into SMT4 mode, and therefore
- * we are being asked not to stop.
- */
- li r3, 0
- std r3, PACA_REQ_PSSCR(r13)
- blr /* return 0 for wakeup cause / SRR1 value */
-#endif
-
-/*
- * Called from machine check handler for powersave wakeups.
- * Low level machine check processing has already been done. Now just
- * go through the wake up path to get everything in order.
- *
- * r3 - The original SRR1 value.
- * Original SRR[01] have been clobbered.
- * MSR_RI is clear.
- */
-.global pnv_powersave_wakeup_mce
-pnv_powersave_wakeup_mce:
- /* Set cr3 for pnv_powersave_wakeup */
- rlwinm r11,r3,47-31,30,31
- cmpwi cr3,r11,2
-
- /*
- * Now put the original SRR1 with SRR1_WAKEMCE_RESVD as the wake
- * reason into r12, which allows reuse of the system reset wakeup
- * code without being mistaken for another type of wakeup.
- */
- oris r12,r3,SRR1_WAKEMCE_RESVD@h
-
- b pnv_powersave_wakeup
-
-/*
- * Called from reset vector for powersave wakeups.
- * cr3 - set to gt if waking up with partial/complete hypervisor state loss
- * r12 - SRR1
- */
-.global pnv_powersave_wakeup
-pnv_powersave_wakeup:
- ld r2, PACATOC(r13)
-
-BEGIN_FTR_SECTION
- bl pnv_restore_hyp_resource_arch300
-FTR_SECTION_ELSE
- bl pnv_restore_hyp_resource_arch207
-ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
-
- li r0,PNV_THREAD_RUNNING
- stb r0,PACA_THREAD_IDLE_STATE(r13) /* Clear thread state */
-
- mr r3,r12
-
-#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
- lbz r0,HSTATE_HWTHREAD_STATE(r13)
- cmpwi r0,KVM_HWTHREAD_IN_KERNEL
- beq 0f
- li r0,KVM_HWTHREAD_IN_KERNEL
- stb r0,HSTATE_HWTHREAD_STATE(r13)
- /* Order setting hwthread_state vs. testing hwthread_req */
- sync
-0: lbz r0,HSTATE_HWTHREAD_REQ(r13)
- cmpwi r0,0
- beq 1f
- b kvm_start_guest
-1:
-#endif
-
- /* Return SRR1 from power7_nap() */
- blt cr3,pnv_wakeup_noloss
- b pnv_wakeup_loss
+ IDLE_INST; \
+ b . /* catch bugs */
/*
- * Check whether we have woken up with hypervisor state loss.
- * If yes, restore hypervisor state and return back to link.
+ * Desired instruction type in r3
*
- * cr3 - set to gt if waking up with partial/complete hypervisor state loss
- */
-pnv_restore_hyp_resource_arch300:
- /*
- * Workaround for POWER9, if we lost resources, the ERAT
- * might have been mixed up and needs flushing. We also need
- * to reload MMCR0 (see comment above). We also need to set
- * then clear bit 60 in MMCRA to ensure the PMU starts running.
- */
- blt cr3,1f
-BEGIN_FTR_SECTION
- PPC_INVALIDATE_ERAT
- ld r1,PACAR1(r13)
- ld r4,_MMCR0(r1)
- mtspr SPRN_MMCR0,r4
-END_FTR_SECTION_IFCLR(CPU_FTR_POWER9_DD2_1)
- mfspr r4,SPRN_MMCRA
- ori r4,r4,(1 << (63-60))
- mtspr SPRN_MMCRA,r4
- xori r4,r4,(1 << (63-60))
- mtspr SPRN_MMCRA,r4
-1:
- /*
- * POWER ISA 3. Use PSSCR to determine if we
- * are waking up from deep idle state
- */
- LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state)
- ld r4,ADDROFF(pnv_first_deep_stop_state)(r5)
-
- /*
- * 0-3 bits correspond to Power-Saving Level Status
- * which indicates the idle state we are waking up from
- */
- mfspr r5, SPRN_PSSCR
- rldicl r5,r5,4,60
- li r0, 0 /* clear requested_psscr to say we're awake */
- std r0, PACA_REQ_PSSCR(r13)
- cmpd cr4,r5,r4
- bge cr4,pnv_wakeup_tb_loss /* returns to caller */
-
- blr /* Waking up without hypervisor state loss. */
-
-/* Same calling convention as arch300 */
-pnv_restore_hyp_resource_arch207:
- /*
- * POWER ISA 2.07 or less.
- * Check if we slept with sleep or winkle.
- */
- lbz r4,PACA_THREAD_IDLE_STATE(r13)
- cmpwi cr2,r4,PNV_THREAD_NAP
- bgt cr2,pnv_wakeup_tb_loss /* Either sleep or Winkle */
-
- /*
- * We fall through here if PACA_THREAD_IDLE_STATE shows we are waking
- * up from nap. At this stage CR3 shouldn't contains 'gt' since that
- * indicates we are waking with hypervisor state loss from nap.
- */
- bgt cr3,.
-
- blr /* Waking up without hypervisor state loss */
-
-/*
- * Called if waking up from idle state which can cause either partial or
- * complete hyp state loss.
- * In POWER8, called if waking up from fastsleep or winkle
- * In POWER9, called if waking up from stop state >= pnv_first_deep_stop_state
+ * GPRs may be lost, so they are saved here. Wakeup is by interrupt only.
+ * The SRESET wakeup returns to this function's caller by calling
+ * idle_return_gpr_loss with r3 set to desired return value.
*
- * r13 - PACA
- * cr3 - gt if waking up with partial/complete hypervisor state loss
+ * A wakeup without GPR loss may alteratively be handled as in
+ * isa300_idle_stop_noloss and blr directly, as an optimisation.
*
- * If ISA300:
- * cr4 - gt or eq if waking up from complete hypervisor state loss.
+ * The caller is responsible for saving/restoring SPRs, MSR, timebase,
+ * etc.
*
- * If ISA207:
- * r4 - PACA_THREAD_IDLE_STATE
+ * This must be called in real-mode.
*/
-pnv_wakeup_tb_loss:
- ld r1,PACAR1(r13)
- /*
- * Before entering any idle state, the NVGPRs are saved in the stack.
- * If there was a state loss, or PACA_NAPSTATELOST was set, then the
- * NVGPRs are restored. If we are here, it is likely that state is lost,
- * but not guaranteed -- neither ISA207 nor ISA300 tests to reach
- * here are the same as the test to restore NVGPRS:
- * PACA_THREAD_IDLE_STATE test for ISA207, PSSCR test for ISA300,
- * and SRR1 test for restoring NVGPRs.
- *
- * We are about to clobber NVGPRs now, so set NAPSTATELOST to
- * guarantee they will always be restored. This might be tightened
- * with careful reading of specs (particularly for ISA300) but this
- * is already a slow wakeup path and it's simpler to be safe.
- */
- li r0,1
- stb r0,PACA_NAPSTATELOST(r13)
-
- /*
- *
- * Save SRR1 and LR in NVGPRs as they might be clobbered in
- * opal_call() (called in CHECK_HMI_INTERRUPT). SRR1 is required
- * to determine the wakeup reason if we branch to kvm_start_guest. LR
- * is required to return back to reset vector after hypervisor state
- * restore is complete.
- */
- mr r19,r12
- mr r18,r4
- mflr r17
-BEGIN_FTR_SECTION
- CHECK_HMI_INTERRUPT
-END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
-
- ld r14,PACA_CORE_IDLE_STATE_PTR(r13)
- lbz r7,PACA_THREAD_MASK(r13)
-
- /*
- * Take the core lock to synchronize against other threads.
- *
- * Lock bit is set in one of the 2 cases-
- * a. In the sleep/winkle enter path, the last thread is executing
- * fastsleep workaround code.
- * b. In the wake up path, another thread is executing fastsleep
- * workaround undo code or resyncing timebase or restoring context
- * In either case loop until the lock bit is cleared.
- */
-1:
- lwarx r15,0,r14
- andis. r9,r15,PNV_CORE_IDLE_LOCK_BIT@h
- bnel- core_idle_lock_held
- oris r15,r15,PNV_CORE_IDLE_LOCK_BIT@h
- stwcx. r15,0,r14
- bne- 1b
- isync
-
- andi. r9,r15,PNV_CORE_IDLE_THREAD_BITS
- cmpwi cr2,r9,0
-
- /*
- * At this stage
- * cr2 - eq if first thread to wakeup in core
- * cr3- gt if waking up with partial/complete hypervisor state loss
- * ISA300:
- * cr4 - gt or eq if waking up from complete hypervisor state loss.
- */
-
-BEGIN_FTR_SECTION
- /*
- * Were we in winkle?
- * If yes, check if all threads were in winkle, decrement our
- * winkle count, set all thread winkle bits if all were in winkle.
- * Check if our thread has a winkle bit set, and set cr4 accordingly
- * (to match ISA300, above). Pseudo-code for core idle state
- * transitions for ISA207 is as follows (everything happens atomically
- * due to store conditional and/or lock bit):
- *
- * nap_idle() { }
- * nap_wake() { }
- *
- * sleep_idle()
- * {
- * core_idle_state &= ~thread_in_core
- * }
- *
- * sleep_wake()
- * {
- * bool first_in_core, first_in_subcore;
- *
- * first_in_core = (core_idle_state & IDLE_THREAD_BITS) == 0;
- * first_in_subcore = (core_idle_state & SUBCORE_SIBLING_MASK) == 0;
- *
- * core_idle_state |= thread_in_core;
- * }
- *
- * winkle_idle()
- * {
- * core_idle_state &= ~thread_in_core;
- * core_idle_state += 1 << WINKLE_COUNT_SHIFT;
- * }
- *
- * winkle_wake()
- * {
- * bool first_in_core, first_in_subcore, winkle_state_lost;
- *
- * first_in_core = (core_idle_state & IDLE_THREAD_BITS) == 0;
- * first_in_subcore = (core_idle_state & SUBCORE_SIBLING_MASK) == 0;
- *
- * core_idle_state |= thread_in_core;
- *
- * if ((core_idle_state & WINKLE_MASK) == (8 << WINKLE_COUNT_SIHFT))
- * core_idle_state |= THREAD_WINKLE_BITS;
- * core_idle_state -= 1 << WINKLE_COUNT_SHIFT;
- *
- * winkle_state_lost = core_idle_state &
- * (thread_in_core << WINKLE_THREAD_SHIFT);
- * core_idle_state &= ~(thread_in_core << WINKLE_THREAD_SHIFT);
- * }
- *
- */
- cmpwi r18,PNV_THREAD_WINKLE
+_GLOBAL(isa206_idle_insn_mayloss)
+ std r1,PACAR1(r13)
+ mflr r4
+ mfcr r5
+ /* use stack red zone rather than a new frame */
+ addi r6,r1,-INT_FRAME_SIZE
+ SAVE_GPR(2, r6)
+ SAVE_NVGPRS(r6)
+ std r4,_LINK(r6)
+ std r5,_CCR(r6)
+ cmpwi r3,PNV_THREAD_NAP
+ bne 1f
+ IDLE_STATE_ENTER_SEQ_NORET(PPC_NAP)
+1: cmpwi r3,PNV_THREAD_SLEEP
bne 2f
- andis. r9,r15,PNV_CORE_IDLE_WINKLE_COUNT_ALL_BIT@h
- subis r15,r15,PNV_CORE_IDLE_WINKLE_COUNT@h
- beq 2f
- ori r15,r15,PNV_CORE_IDLE_THREAD_WINKLE_BITS /* all were winkle */
-2:
- /* Shift thread bit to winkle mask, then test if this thread is set,
- * and remove it from the winkle bits */
- slwi r8,r7,8
- and r8,r8,r15
- andc r15,r15,r8
- cmpwi cr4,r8,1 /* cr4 will be gt if our bit is set, lt if not */
-
- lbz r4,PACA_SUBCORE_SIBLING_MASK(r13)
- and r4,r4,r15
- cmpwi r4,0 /* Check if first in subcore */
-
- or r15,r15,r7 /* Set thread bit */
- beq first_thread_in_subcore
-END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300)
-
- or r15,r15,r7 /* Set thread bit */
- beq cr2,first_thread_in_core
-
- /* Not first thread in core or subcore to wake up */
- b clear_lock
-
-first_thread_in_subcore:
- /*
- * If waking up from sleep, subcore state is not lost. Hence
- * skip subcore state restore
- */
- blt cr4,subcore_state_restored
-
- /* Restore per-subcore state */
- ld r4,_SDR1(r1)
- mtspr SPRN_SDR1,r4
-
- ld r4,_RPR(r1)
- mtspr SPRN_RPR,r4
- ld r4,_AMOR(r1)
- mtspr SPRN_AMOR,r4
-
-subcore_state_restored:
- /*
- * Check if the thread is also the first thread in the core. If not,
- * skip to clear_lock.
- */
- bne cr2,clear_lock
-
-first_thread_in_core:
-
- /*
- * First thread in the core waking up from any state which can cause
- * partial or complete hypervisor state loss. It needs to
- * call the fastsleep workaround code if the platform requires it.
- * Call it unconditionally here. The below branch instruction will
- * be patched out if the platform does not have fastsleep or does not
- * require the workaround. Patching will be performed during the
- * discovery of idle-states.
- */
-.global pnv_fastsleep_workaround_at_exit
-pnv_fastsleep_workaround_at_exit:
- b fastsleep_workaround_at_exit
-
-timebase_resync:
- /*
- * Use cr3 which indicates that we are waking up with atleast partial
- * hypervisor state loss to determine if TIMEBASE RESYNC is needed.
- */
- ble cr3,.Ltb_resynced
- /* Time base re-sync */
- bl opal_resync_timebase;
- /*
- * If waking up from sleep (POWER8), per core state
- * is not lost, skip to clear_lock.
- */
-.Ltb_resynced:
- blt cr4,clear_lock
-
- /*
- * First thread in the core to wake up and its waking up with
- * complete hypervisor state loss. Restore per core hypervisor
- * state.
- */
-BEGIN_FTR_SECTION
- ld r4,_PTCR(r1)
- mtspr SPRN_PTCR,r4
- ld r4,_RPR(r1)
- mtspr SPRN_RPR,r4
- ld r4,_AMOR(r1)
- mtspr SPRN_AMOR,r4
-END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
-
- ld r4,_TSCR(r1)
- mtspr SPRN_TSCR,r4
- ld r4,_WORC(r1)
- mtspr SPRN_WORC,r4
-
-clear_lock:
- xoris r15,r15,PNV_CORE_IDLE_LOCK_BIT@h
- lwsync
- stw r15,0(r14)
-
-common_exit:
- /*
- * Common to all threads.
- *
- * If waking up from sleep, hypervisor state is not lost. Hence
- * skip hypervisor state restore.
- */
- blt cr4,hypervisor_state_restored
-
- /* Waking up from winkle */
-
-BEGIN_MMU_FTR_SECTION
- b no_segments
-END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX)
- /* Restore SLB from PACA */
- ld r8,PACA_SLBSHADOWPTR(r13)
-
- .rept SLB_NUM_BOLTED
- li r3, SLBSHADOW_SAVEAREA
- LDX_BE r5, r8, r3
- addi r3, r3, 8
- LDX_BE r6, r8, r3
- andis. r7,r5,SLB_ESID_V@h
- beq 1f
- slbmte r6,r5
-1: addi r8,r8,16
- .endr
-no_segments:
-
- /* Restore per thread state */
-
- ld r4,_SPURR(r1)
- mtspr SPRN_SPURR,r4
- ld r4,_PURR(r1)
- mtspr SPRN_PURR,r4
- ld r4,_DSCR(r1)
- mtspr SPRN_DSCR,r4
- ld r4,_WORT(r1)
- mtspr SPRN_WORT,r4
-
- /* Call cur_cpu_spec->cpu_restore() */
- LOAD_REG_ADDR(r4, cur_cpu_spec)
- ld r4,0(r4)
- ld r12,CPU_SPEC_RESTORE(r4)
-#ifdef PPC64_ELF_ABI_v1
- ld r12,0(r12)
-#endif
- mtctr r12
- bctrl
-
-/*
- * On POWER9, we can come here on wakeup from a cpuidle stop state.
- * Hence restore the additional SPRs to the saved value.
- *
- * On POWER8, we come here only on winkle. Since winkle is used
- * only in the case of CPU-Hotplug, we don't need to restore
- * the additional SPRs.
- */
-BEGIN_FTR_SECTION
- bl power9_restore_additional_sprs
-END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
-hypervisor_state_restored:
-
- mr r12,r19
- mtlr r17
- blr /* return to pnv_powersave_wakeup */
-
-fastsleep_workaround_at_exit:
- li r3,1
- li r4,0
- bl opal_config_cpu_idle_state
- b timebase_resync
-
-/*
- * R3 here contains the value that will be returned to the caller
- * of power7_nap.
- * R12 contains SRR1 for CHECK_HMI_INTERRUPT.
- */
-.global pnv_wakeup_loss
-pnv_wakeup_loss:
- ld r1,PACAR1(r13)
-BEGIN_FTR_SECTION
- CHECK_HMI_INTERRUPT
-END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
- REST_NVGPRS(r1)
- REST_GPR(2, r1)
- ld r4,PACAKMSR(r13)
- ld r5,_LINK(r1)
- ld r6,_CCR(r1)
- addi r1,r1,INT_FRAME_SIZE
- mtlr r5
- mtcr r6
- mtmsrd r4
- blr
+ IDLE_STATE_ENTER_SEQ_NORET(PPC_SLEEP)
+2: IDLE_STATE_ENTER_SEQ_NORET(PPC_WINKLE)
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
/*
- * R3 here contains the value that will be returned to the caller
- * of power7_nap.
- * R12 contains SRR1 for CHECK_HMI_INTERRUPT.
+ * This is used to branch into kvm_start_guest with non-volatiles saved.
+ * KVM should branch to idle_return_gpr_loss to return back out to the
+ * caller, with r3 as return code.
*/
-pnv_wakeup_noloss:
- lbz r0,PACA_NAPSTATELOST(r13)
- cmpwi r0,0
- bne pnv_wakeup_loss
- ld r1,PACAR1(r13)
-BEGIN_FTR_SECTION
- CHECK_HMI_INTERRUPT
-END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
- ld r4,PACAKMSR(r13)
- ld r5,_NIP(r1)
- ld r6,_CCR(r1)
- addi r1,r1,INT_FRAME_SIZE
- mtlr r5
- mtcr r6
- mtmsrd r4
- blr
+_GLOBAL(idle_kvm_start_guest)
+ std r1,PACAR1(r13)
+ mflr r4
+ mfcr r5
+ /* use stack red zone rather than a new frame */
+ addi r6,r1,-INT_FRAME_SIZE
+ SAVE_GPR(2, r6)
+ SAVE_NVGPRS(r6)
+ std r4,_LINK(r6)
+ std r5,_CCR(r6)
+ b kvm_start_guest
+#endif
diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c
index 9ca9db707bcb..8b89abbfa295 100644
--- a/arch/powerpc/kernel/setup-common.c
+++ b/arch/powerpc/kernel/setup-common.c
@@ -402,8 +402,8 @@ void __init check_for_initrd(void)
#ifdef CONFIG_SMP
-int threads_per_core, threads_per_subcore, threads_shift;
-cpumask_t threads_core_mask;
+int threads_per_core, threads_per_subcore, threads_shift __read_mostly;
+cpumask_t threads_core_mask __read_mostly;
EXPORT_SYMBOL_GPL(threads_per_core);
EXPORT_SYMBOL_GPL(threads_per_subcore);
EXPORT_SYMBOL_GPL(threads_shift);
diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
index 1d14046124a0..3da500e5f14f 100644
--- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
+++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
@@ -34,6 +34,7 @@
#include <asm/thread_info.h>
#include <asm/asm-compat.h>
#include <asm/feature-fixups.h>
+#include <asm/cpuidle.h>
/* Sign-extend HDEC if not on POWER9 */
#define EXTEND_HDEC(reg) \
@@ -322,42 +323,27 @@ kvm_novcpu_exit:
b kvmhv_switch_to_host
/*
- * We come in here when wakened from nap mode.
- * Relocation is off and most register values are lost.
- * r13 points to the PACA.
+ * We come in here when wakened from Linux offline idle code.
+ * Relocation is off
* r3 contains the SRR1 wakeup value, SRR1 is trashed.
*/
.globl kvm_start_guest
kvm_start_guest:
- /* Set runlatch bit the minute you wake up from nap */
- mfspr r0, SPRN_CTRLF
- ori r0, r0, 1
- mtspr SPRN_CTRLT, r0
-
/*
* Could avoid this and pass it through in r3. For now,
* code expects it to be in SRR1.
*/
mtspr SPRN_SRR1,r3
- ld r2,PACATOC(r13)
-
li r0,0
stb r0,PACA_FTRACE_ENABLED(r13)
li r0,KVM_HWTHREAD_IN_KVM
stb r0,HSTATE_HWTHREAD_STATE(r13)
- /* NV GPR values from power7_idle() will no longer be valid */
- li r0,1
- stb r0,PACA_NAPSTATELOST(r13)
-
- /* were we napping due to cede? */
+ /* kvm cede / napping does not come through here */
lbz r0,HSTATE_NAPPING(r13)
- cmpwi r0,NAPPING_CEDE
- beq kvm_end_cede
- cmpwi r0,NAPPING_NOVCPU
- beq kvm_novcpu_wakeup
+ twnei r0,0
ld r1,PACAEMERGSP(r13)
subi r1,r1,STACK_FRAME_OVERHEAD
@@ -469,19 +455,17 @@ kvm_no_guest:
lbz r3, HSTATE_HWTHREAD_REQ(r13)
cmpwi r3, 0
bne 54f
-/*
- * We jump to pnv_wakeup_loss, which will return to the caller
- * of power7_nap in the powernv cpu offline loop. The value we
- * put in r3 becomes the return value for power7_nap. pnv_wakeup_loss
- * requires SRR1 in r12.
- */
+
+ /*
+ * Jump to idle_return_gpr_loss, which returns to the
+ * idle_kvm_start_guest caller.
+ */
li r3, LPCR_PECE0
mfspr r4, SPRN_LPCR
rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
mtspr SPRN_LPCR, r4
li r3, 0
- mfspr r12,SPRN_SRR1
- b pnv_wakeup_loss
+ b idle_return_gpr_loss
53: HMT_LOW
ld r5, HSTATE_KVM_VCORE(r13)
@@ -2760,21 +2744,47 @@ BEGIN_FTR_SECTION
li r4, LPCR_PECE_HVEE@higher
sldi r4, r4, 32
or r5, r5, r4
-END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
+FTR_SECTION_ELSE
+ li r3, PNV_THREAD_NAP
+ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
mtspr SPRN_LPCR,r5
isync
- li r0, 0
- std r0, HSTATE_SCRATCH0(r13)
- ptesync
- ld r0, HSTATE_SCRATCH0(r13)
-1: cmpd r0, r0
- bne 1b
+
+ mr r0, r1
+ ld r1, PACAEMERGSP(r13)
+ subi r1, r1, STACK_FRAME_OVERHEAD
+ std r0, 0(r1)
+ ld r0, PACAR1(r13)
+ std r0, 8(r1)
+
BEGIN_FTR_SECTION
- nap
+ bl isa300_idle_stop_mayloss
FTR_SECTION_ELSE
- PPC_STOP
-ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300)
- b .
+ bl isa206_idle_insn_mayloss
+ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
+
+ mfspr r0, SPRN_CTRLF
+ ori r0, r0, 1
+ mtspr SPRN_CTRLT, r0
+
+ ld r0, 8(r1)
+ std r0, PACAR1(r13)
+ ld r1, 0(r1)
+
+ mtspr SPRN_SRR1, r3
+
+ li r0, 0
+ stb r0, PACA_FTRACE_ENABLED(r13)
+
+ li r0, KVM_HWTHREAD_IN_KVM
+ stb r0, HSTATE_HWTHREAD_STATE(r13)
+
+ lbz r0, HSTATE_NAPPING(r13)
+ cmpwi r0, NAPPING_CEDE
+ beq kvm_end_cede
+ cmpwi r0, NAPPING_NOVCPU
+ beq kvm_novcpu_wakeup
+ twi 31,0,0 /* Nap state must not be zero */
33: mr r4, r3
li r3, 0
diff --git a/arch/powerpc/platforms/powernv/idle.c b/arch/powerpc/platforms/powernv/idle.c
index 35f699ebb662..d5fec02ea026 100644
--- a/arch/powerpc/platforms/powernv/idle.c
+++ b/arch/powerpc/platforms/powernv/idle.c
@@ -16,6 +16,7 @@
#include <linux/device.h>
#include <linux/cpu.h>
+#include <asm/asm-prototypes.h>
#include <asm/firmware.h>
#include <asm/machdep.h>
#include <asm/opal.h>
@@ -48,10 +49,10 @@ static u64 pnv_default_stop_mask;
static bool default_stop_found;
/*
- * First deep stop state. Used to figure out when to save/restore
- * hypervisor context.
+ * First stop state levels when SPR and TB loss can occur.
*/
-u64 pnv_first_deep_stop_state = MAX_STOP_STATE;
+static u64 pnv_first_tb_loss_level = MAX_STOP_STATE + 1;
+static u64 pnv_first_spr_loss_level = MAX_STOP_STATE + 1;
/*
* psscr value and mask of the deepest stop idle state.
@@ -62,6 +63,8 @@ static u64 pnv_deepest_stop_psscr_mask;
static u64 pnv_deepest_stop_flag;
static bool deepest_stop_found;
+static unsigned long power7_offline_type;
+
static int pnv_save_sprs_for_deep_states(void)
{
int cpu;
@@ -72,12 +75,12 @@ static int pnv_save_sprs_for_deep_states(void)
* all cpus at boot. Get these reg values of current cpu and use the
* same across all cpus.
*/
- uint64_t lpcr_val = mfspr(SPRN_LPCR);
- uint64_t hid0_val = mfspr(SPRN_HID0);
- uint64_t hid1_val = mfspr(SPRN_HID1);
- uint64_t hid4_val = mfspr(SPRN_HID4);
- uint64_t hid5_val = mfspr(SPRN_HID5);
- uint64_t hmeer_val = mfspr(SPRN_HMEER);
+ uint64_t lpcr_val = mfspr(SPRN_LPCR);
+ uint64_t hid0_val = mfspr(SPRN_HID0);
+ uint64_t hid1_val = mfspr(SPRN_HID1);
+ uint64_t hid4_val = mfspr(SPRN_HID4);
+ uint64_t hid5_val = mfspr(SPRN_HID5);
+ uint64_t hmeer_val = mfspr(SPRN_HMEER);
uint64_t msr_val = MSR_IDLE;
uint64_t psscr_val = pnv_deepest_stop_psscr_val;
@@ -137,89 +140,6 @@ static int pnv_save_sprs_for_deep_states(void)
return 0;
}
-static void pnv_alloc_idle_core_states(void)
-{
- int i, j;
- int nr_cores = cpu_nr_cores();
- u32 *core_idle_state;
-
- /*
- * core_idle_state - The lower 8 bits track the idle state of
- * each thread of the core.
- *
- * The most significant bit is the lock bit.
- *
- * Initially all the bits corresponding to threads_per_core
- * are set. They are cleared when the thread enters deep idle
- * state like sleep and winkle/stop.
- *
- * Initially the lock bit is cleared. The lock bit has 2
- * purposes:
- * a. While the first thread in the core waking up from
- * idle is restoring core state, it prevents other
- * threads in the core from switching to process
- * context.
- * b. While the last thread in the core is saving the
- * core state, it prevents a different thread from
- * waking up.
- */
- for (i = 0; i < nr_cores; i++) {
- int first_cpu = i * threads_per_core;
- int node = cpu_to_node(first_cpu);
- size_t paca_ptr_array_size;
-
- core_idle_state = kmalloc_node(sizeof(u32), GFP_KERNEL, node);
- *core_idle_state = (1 << threads_per_core) - 1;
- paca_ptr_array_size = (threads_per_core *
- sizeof(struct paca_struct *));
-
- for (j = 0; j < threads_per_core; j++) {
- int cpu = first_cpu + j;
-
- paca_ptrs[cpu]->core_idle_state_ptr = core_idle_state;
- paca_ptrs[cpu]->thread_idle_state = PNV_THREAD_RUNNING;
- paca_ptrs[cpu]->thread_mask = 1 << j;
- }
- }
-
- update_subcore_sibling_mask();
-
- if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) {
- int rc = pnv_save_sprs_for_deep_states();
-
- if (likely(!rc))
- return;
-
- /*
- * The stop-api is unable to restore hypervisor
- * resources on wakeup from platform idle states which
- * lose full context. So disable such states.
- */
- supported_cpuidle_states &= ~OPAL_PM_LOSE_FULL_CONTEXT;
- pr_warn("cpuidle-powernv: Disabling idle states that lose full context\n");
- pr_warn("cpuidle-powernv: Idle power-savings, CPU-Hotplug affected\n");
-
- if (cpu_has_feature(CPU_FTR_ARCH_300) &&
- (pnv_deepest_stop_flag & OPAL_PM_LOSE_FULL_CONTEXT)) {
- /*
- * Use the default stop state for CPU-Hotplug
- * if available.
- */
- if (default_stop_found) {
- pnv_deepest_stop_psscr_val =
- pnv_default_stop_val;
- pnv_deepest_stop_psscr_mask =
- pnv_default_stop_mask;
- pr_warn("cpuidle-powernv: Offlined CPUs will stop with psscr = 0x%016llx\n",
- pnv_deepest_stop_psscr_val);
- } else { /* Fallback to snooze loop for CPU-Hotplug */
- deepest_stop_found = false;
- pr_warn("cpuidle-powernv: Offlined CPUs will busy wait\n");
- }
- }
- }
-}
-
u32 pnv_get_supported_cpuidle_states(void)
{
return supported_cpuidle_states;
@@ -238,6 +158,9 @@ static void pnv_fastsleep_workaround_apply(void *info)
*err = 1;
}
+static bool power7_fastsleep_workaround_entry = true;
+static bool power7_fastsleep_workaround_exit = true;
+
/*
* Used to store fastsleep workaround state
* 0 - Workaround applied/undone at fastsleep entry/exit path (Default)
@@ -277,13 +200,7 @@ static ssize_t store_fastsleep_workaround_applyonce(struct device *dev,
* offlined, as last thread of the core entering fastsleep or deeper
* state would have applied workaround.
*/
- err = patch_instruction(
- (unsigned int *)pnv_fastsleep_workaround_at_exit,
- PPC_INST_NOP);
- if (err) {
- pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_exit");
- goto fail;
- }
+ power7_fastsleep_workaround_exit = false;
get_online_cpus();
primary_thread_mask = cpu_online_cores_map();
@@ -296,13 +213,7 @@ static ssize_t store_fastsleep_workaround_applyonce(struct device *dev,
goto fail;
}
- err = patch_instruction(
- (unsigned int *)pnv_fastsleep_workaround_at_entry,
- PPC_INST_NOP);
- if (err) {
- pr_err("fastsleep_workaround_applyonce change failed while patching pnv_fastsleep_workaround_at_entry");
- goto fail;
- }
+ power7_fastsleep_workaround_entry = false;
fastsleep_workaround_applyonce = 1;
@@ -315,6 +226,311 @@ static DEVICE_ATTR(fastsleep_workaround_applyonce, 0600,
show_fastsleep_workaround_applyonce,
store_fastsleep_workaround_applyonce);
+static inline void atomic_start_thread_idle(void)
+{
+ int cpu = raw_smp_processor_id();
+ int first = cpu_first_thread_sibling(cpu);
+ int thread_nr = cpu_thread_in_core(cpu);
+ unsigned long *state = &paca_ptrs[first]->idle_state;
+
+ clear_bit(thread_nr, state);
+}
+
+static inline void atomic_stop_thread_idle(void)
+{
+ int cpu = raw_smp_processor_id();
+ int first = cpu_first_thread_sibling(cpu);
+ int thread_nr = cpu_thread_in_core(cpu);
+ unsigned long *state = &paca_ptrs[first]->idle_state;
+
+ set_bit(thread_nr, state);
+}
+
+static inline void atomic_lock_thread_idle(void)
+{
+ int cpu = raw_smp_processor_id();
+ int first = cpu_first_thread_sibling(cpu);
+ unsigned long *state = &paca_ptrs[first]->idle_state;
+
+ while (unlikely(test_and_set_bit_lock(NR_PNV_CORE_IDLE_LOCK_BIT, state)))
+ barrier();
+}
+
+static inline void atomic_unlock_and_stop_thread_idle(void)
+{
+ int cpu = raw_smp_processor_id();
+ int first = cpu_first_thread_sibling(cpu);
+ unsigned long thread = 1UL << cpu_thread_in_core(cpu);
+ unsigned long *state = &paca_ptrs[first]->idle_state;
+ u64 s = READ_ONCE(*state);
+ u64 new, tmp;
+
+ BUG_ON(!(s & PNV_CORE_IDLE_LOCK_BIT));
+ BUG_ON(s & thread);
+
+again:
+ new = (s | thread) & ~PNV_CORE_IDLE_LOCK_BIT;
+ tmp = cmpxchg(state, s, new);
+ if (unlikely(tmp != s)) {
+ s = tmp;
+ goto again;
+ }
+}
+
+static inline void atomic_unlock_thread_idle(void)
+{
+ int cpu = raw_smp_processor_id();
+ int first = cpu_first_thread_sibling(cpu);
+ unsigned long *state = &paca_ptrs[first]->idle_state;
+
+ BUG_ON(!test_bit(NR_PNV_CORE_IDLE_LOCK_BIT, state));
+ clear_bit_unlock(NR_PNV_CORE_IDLE_LOCK_BIT, state);
+}
+
+/* P7 and P8 */
+struct p7_sprs {
+ /* per core */
+ u64 tscr;
+ u64 worc;
+
+ /* per subcore */
+ u64 sdr1;
+ u64 rpr;
+ u64 amor;
+
+ /* per thread */
+ u64 lpcr;
+ u64 hfscr;
+ u64 fscr;
+ u64 purr;
+ u64 spurr;
+ u64 dscr;
+ u64 wort;
+};
+
+static unsigned long power7_idle_insn(unsigned long type)
+{
+ int cpu = raw_smp_processor_id();
+ int first = cpu_first_thread_sibling(cpu);
+ unsigned long *state = &paca_ptrs[first]->idle_state;
+ unsigned long thread = 1UL << cpu_thread_in_core(cpu);
+ unsigned long core_thread_mask = (1UL << threads_per_core) - 1;
+ unsigned long srr1;
+ bool full_winkle;
+ struct p7_sprs sprs;
+ bool sprs_saved = false;
+ int rc;
+
+ memset(&sprs, 0, sizeof(sprs));
+
+ if (unlikely(type != PNV_THREAD_NAP)) {
+ atomic_lock_thread_idle();
+
+ BUG_ON(!(*state & thread));
+ *state &= ~thread;
+
+ if (power7_fastsleep_workaround_entry) {
+ if ((*state & core_thread_mask) == 0) {
+ rc = opal_config_cpu_idle_state(
+ OPAL_CONFIG_IDLE_FASTSLEEP,
+ OPAL_CONFIG_IDLE_APPLY);
+ BUG_ON(rc);
+ }
+ }
+
+ if (type == PNV_THREAD_WINKLE) {
+ sprs.tscr = mfspr(SPRN_TSCR);
+ sprs.worc = mfspr(SPRN_WORC);
+
+ sprs.sdr1 = mfspr(SPRN_SDR1);
+ sprs.rpr = mfspr(SPRN_RPR);
+ sprs.amor = mfspr(SPRN_AMOR);
+
+ sprs.lpcr = mfspr(SPRN_LPCR);
+ if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
+ sprs.hfscr = mfspr(SPRN_HFSCR);
+ sprs.fscr = mfspr(SPRN_FSCR);
+ }
+ sprs.purr = mfspr(SPRN_PURR);
+ sprs.spurr = mfspr(SPRN_SPURR);
+ sprs.dscr = mfspr(SPRN_DSCR);
+ sprs.wort = mfspr(SPRN_WORT);
+
+ sprs_saved = true;
+
+ /*
+ * Increment winkle counter and set all winkle bits if
+ * all threads are winkling. This allows wakeup side to
+ * distinguish between fast sleep and winkle state
+ * loss. Fast sleep still has to resync the timebase so
+ * this may not be a really big win.
+ */
+ *state += 1 << PNV_CORE_IDLE_WINKLE_COUNT_SHIFT;
+ if ((*state & PNV_CORE_IDLE_WINKLE_COUNT_BITS)
+ >> PNV_CORE_IDLE_WINKLE_COUNT_SHIFT
+ == threads_per_core)
+ *state |= PNV_CORE_IDLE_THREAD_WINKLE_BITS;
+ WARN_ON((*state & PNV_CORE_IDLE_WINKLE_COUNT_BITS) == 0);
+ }
+
+ atomic_unlock_thread_idle();
+ }
+
+ local_paca->thread_idle_state = type;
+ srr1 = isa206_idle_insn_mayloss(type); /* go idle */
+ local_paca->thread_idle_state = PNV_THREAD_RUNNING;
+
+ WARN_ON_ONCE(!srr1);
+ WARN_ON_ONCE(mfmsr() & (MSR_IR|MSR_DR));
+
+ if (unlikely((srr1 & SRR1_WAKEMASK_P8) == SRR1_WAKEHMI))
+ hmi_exception_realmode(NULL);
+
+ if (likely((srr1 & SRR1_WAKESTATE) != SRR1_WS_HVLOSS)) {
+ if (unlikely(type != PNV_THREAD_NAP)) {
+ atomic_lock_thread_idle();
+ if (type == PNV_THREAD_WINKLE) {
+ WARN_ON((*state & PNV_CORE_IDLE_WINKLE_COUNT_BITS) == 0);
+ *state -= 1 << PNV_CORE_IDLE_WINKLE_COUNT_SHIFT;
+ *state &= ~(thread << PNV_CORE_IDLE_THREAD_WINKLE_BITS_SHIFT);
+ }
+ atomic_unlock_and_stop_thread_idle();
+ }
+ return srr1;
+ }
+
+ /* HV state loss */
+ BUG_ON(type == PNV_THREAD_NAP);
+
+ atomic_lock_thread_idle();
+
+ full_winkle = false;
+ if (type == PNV_THREAD_WINKLE) {
+ WARN_ON((*state & PNV_CORE_IDLE_WINKLE_COUNT_BITS) == 0);
+ *state -= 1 << PNV_CORE_IDLE_WINKLE_COUNT_SHIFT;
+ if (*state & (thread << PNV_CORE_IDLE_THREAD_WINKLE_BITS_SHIFT)) {
+ *state &= ~(thread << PNV_CORE_IDLE_THREAD_WINKLE_BITS_SHIFT);
+ full_winkle = true;
+ BUG_ON(!sprs_saved);
+ }
+ }
+
+ WARN_ON(*state & thread);
+
+ if ((*state & core_thread_mask) != 0)
+ goto core_woken;
+
+ /* Per-core SPRs */
+ if (full_winkle) {
+ mtspr(SPRN_TSCR, sprs.tscr);
+ mtspr(SPRN_WORC, sprs.worc);
+ }
+
+ if (power7_fastsleep_workaround_exit) {
+ rc = opal_config_cpu_idle_state(OPAL_CONFIG_IDLE_FASTSLEEP,
+ OPAL_CONFIG_IDLE_UNDO);
+ BUG_ON(rc);
+ }
+
+ /* TB */
+ if (opal_resync_timebase() != OPAL_SUCCESS)
+ BUG();
+
+core_woken:
+ if (!full_winkle)
+ goto subcore_woken;
+
+ if ((*state & local_paca->subcore_sibling_mask) != 0)
+ goto subcore_woken;
+
+ /* Per-subcore SPRs */
+ mtspr(SPRN_SDR1, sprs.sdr1);
+ mtspr(SPRN_RPR, sprs.rpr);
+ mtspr(SPRN_AMOR, sprs.amor);
+
+subcore_woken:
+ /*
+ * isync after restoring shared SPRs and before unlocking. Unlock
+ * only contains hwsync which does not necessarily do the right
+ * thing for SPRs.
+ */
+ isync();
+ atomic_unlock_and_stop_thread_idle();
+
+ /* Fast sleep does not lose SPRs */
+ if (!full_winkle)
+ return srr1;
+
+ /* Per-thread SPRs */
+ mtspr(SPRN_LPCR, sprs.lpcr);
+ if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
+ mtspr(SPRN_HFSCR, sprs.hfscr);
+ mtspr(SPRN_FSCR, sprs.fscr);
+ }
+ mtspr(SPRN_PURR, sprs.purr);
+ mtspr(SPRN_SPURR, sprs.spurr);
+ mtspr(SPRN_DSCR, sprs.dscr);
+ mtspr(SPRN_WORT, sprs.wort);
+
+ mtspr(SPRN_SPRG3, local_paca->sprg_vdso);
+
+ /*
+ * The SLB has to be restored here, but it sometimes still
+ * contains entries, so the __ variant must be used to prevent
+ * multi hits.
+ */
+ __slb_restore_bolted_realmode();
+
+ return srr1;
+}
+
+extern unsigned long idle_kvm_start_guest(unsigned long srr1);
+
+#ifdef CONFIG_HOTPLUG_CPU
+static unsigned long power7_offline(void)
+{
+ unsigned long srr1;
+
+ mtmsr(MSR_IDLE);
+
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+ /* Tell KVM we're entering idle. */
+ /******************************************************/
+ /* N O T E W E L L ! ! ! N O T E W E L L */
+ /* The following store to HSTATE_HWTHREAD_STATE(r13) */
+ /* MUST occur in real mode, i.e. with the MMU off, */
+ /* and the MMU must stay off until we clear this flag */
+ /* and test HSTATE_HWTHREAD_REQ(r13) in */
+ /* pnv_powersave_wakeup in this file. */
+ /* The reason is that another thread can switch the */
+ /* MMU to a guest context whenever this flag is set */
+ /* to KVM_HWTHREAD_IN_IDLE, and if the MMU was on, */
+ /* that would potentially cause this thread to start */
+ /* executing instructions from guest memory in */
+ /* hypervisor mode, leading to a host crash or data */
+ /* corruption, or worse. */
+ /******************************************************/
+ local_paca->kvm_hstate.hwthread_state = KVM_HWTHREAD_IN_IDLE;
+#endif
+
+ __ppc64_runlatch_off();
+ srr1 = power7_idle_insn(power7_offline_type);
+ __ppc64_runlatch_on();
+
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+ local_paca->kvm_hstate.hwthread_state = KVM_HWTHREAD_IN_KERNEL;
+ /* Order setting hwthread_state vs. testing hwthread_req */
+ smp_mb();
+ if (local_paca->kvm_hstate.hwthread_req)
+ srr1 = idle_kvm_start_guest(srr1);
+#endif
+
+ mtmsr(MSR_KERNEL);
+
+ return srr1;
+}
+#endif
+
static unsigned long __power7_idle_type(unsigned long type)
{
unsigned long srr1;
@@ -322,9 +538,11 @@ static unsigned long __power7_idle_type(unsigned long type)
if (!prep_irq_for_idle_irqsoff())
return 0;
+ mtmsr(MSR_IDLE);
__ppc64_runlatch_off();
srr1 = power7_idle_insn(type);
__ppc64_runlatch_on();
+ mtmsr(MSR_KERNEL);
fini_irq_for_idle_irqsoff();
@@ -347,6 +565,261 @@ void power7_idle(void)
power7_idle_type(PNV_THREAD_NAP);
}
+struct p9_sprs {
+ /* per core */
+ u64 ptcr;
+ u64 rpr;
+ u64 tscr;
+ u64 ldbar;
+ u64 amor;
+
+ /* per thread */
+ u64 lpcr;
+ u64 hfscr;
+ u64 fscr;
+ u64 pid;
+ u64 purr;
+ u64 spurr;
+ u64 dscr;
+ u64 wort;
+
+ u64 mmcra;
+ u32 mmcr0;
+ u32 mmcr1;
+ u64 mmcr2;
+};
+
+static unsigned long power9_idle_stop(unsigned long psscr, bool mmu_on)
+{
+ int cpu = raw_smp_processor_id();
+ int first = cpu_first_thread_sibling(cpu);
+ unsigned long *state = &paca_ptrs[first]->idle_state;
+ unsigned long core_thread_mask = (1UL << threads_per_core) - 1;
+ unsigned long srr1;
+ unsigned long pls;
+ unsigned long mmcr0 = 0;
+ struct p9_sprs sprs;
+ bool sprs_saved = false;
+
+ /* This should not be required but GCC warns about used uninitialized */
+ memset(&sprs, 0, sizeof(sprs));
+
+ if (!(psscr & (PSSCR_EC|PSSCR_ESL))) {
+ /* EC=ESL=0 case */
+
+ BUG_ON(!mmu_on);
+
+ /*
+ * Wake synchronously. SRESET via xscom may still cause
+ * a 0x100 powersave wakeup with SRR1 reason!
+ */
+ srr1 = isa300_idle_stop_noloss(psscr); /* go idle */
+ if (likely(!srr1))
+ return 0;
+
+ /*
+ * Registers not saved, can't recover!
+ * This would be a hardware bug
+ */
+ BUG_ON((srr1 & SRR1_WAKESTATE) != SRR1_WS_NOLOSS);
+
+ goto out;
+ }
+
+ /* EC=ESL=1 case */
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+ if (cpu_has_feature(CPU_FTR_P9_TM_XER_SO_BUG)) {
+ local_paca->requested_psscr = psscr;
+ /* order setting requested_psscr vs testing dont_stop */
+ smp_mb();
+ if (atomic_read(&local_paca->dont_stop)) {
+ local_paca->requested_psscr = 0;
+ return 0;
+ }
+ }
+#endif
+
+ if (!cpu_has_feature(CPU_FTR_POWER9_DD2_1)) {
+ /*
+ * POWER9 DD2 can incorrectly set PMAO when waking up
+ * after a state-loss idle. Saving and restoring MMCR0
+ * over idle is a workaround.
+ */
+ mmcr0 = mfspr(SPRN_MMCR0);
+ }
+ if ((psscr & PSSCR_RL_MASK) >= pnv_first_spr_loss_level) {
+ sprs.lpcr = mfspr(SPRN_LPCR);
+ sprs.hfscr = mfspr(SPRN_HFSCR);
+ sprs.fscr = mfspr(SPRN_FSCR);
+ sprs.pid = mfspr(SPRN_PID);
+ sprs.purr = mfspr(SPRN_PURR);
+ sprs.spurr = mfspr(SPRN_SPURR);
+ sprs.dscr = mfspr(SPRN_DSCR);
+ sprs.wort = mfspr(SPRN_WORT);
+
+ sprs.mmcra = mfspr(SPRN_MMCRA);
+ sprs.mmcr0 = mfspr(SPRN_MMCR0);
+ sprs.mmcr1 = mfspr(SPRN_MMCR1);
+ sprs.mmcr2 = mfspr(SPRN_MMCR2);
+
+ sprs.ptcr = mfspr(SPRN_PTCR);
+ sprs.rpr = mfspr(SPRN_RPR);
+ sprs.tscr = mfspr(SPRN_TSCR);
+ sprs.ldbar = mfspr(SPRN_LDBAR);
+ sprs.amor = mfspr(SPRN_AMOR);
+
+ sprs_saved = true;
+
+ atomic_start_thread_idle();
+ }
+
+ srr1 = isa300_idle_stop_mayloss(psscr); /* go idle */
+
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+ local_paca->requested_psscr = 0;
+#endif
+
+ psscr = mfspr(SPRN_PSSCR);
+
+ WARN_ON_ONCE(!srr1);
+ WARN_ON_ONCE(mfmsr() & (MSR_IR|MSR_DR));
+
+ /* Workarounds for SMT thread switch problems */
+ if ((srr1 & SRR1_WAKESTATE) != SRR1_WS_NOLOSS) {
+ unsigned long mmcra;
+
+ /*
+ * Workaround for POWER9 DD2.0, if we lost resources, the ERAT
+ * might have been corrupted and needs flushing. We also need
+ * to reload MMCR0 (see mmcr0 comment above).
+ */
+ if (!cpu_has_feature(CPU_FTR_POWER9_DD2_1)) {
+ asm volatile(PPC_INVALIDATE_ERAT);
+ mtspr(SPRN_MMCR0, mmcr0);
+ }
+
+ /*
+ * DD2.2 and earlier need to set then clear bit 60 in MMCRA
+ * to ensure the PMU starts running.
+ */
+ mmcra = mfspr(SPRN_MMCRA);
+ mmcra |= PPC_BIT(60);
+ mtspr(SPRN_MMCRA, mmcra);
+ mmcra &= ~PPC_BIT(60);
+ mtspr(SPRN_MMCRA, mmcra);
+ }
+
+ if (unlikely((srr1 & SRR1_WAKEMASK_P8) == SRR1_WAKEHMI))
+ hmi_exception_realmode(NULL);
+
+ /*
+ * On POWER9, SRR1 bits do not match exactly as expected.
+ * SRR1_WS_GPRLOSS (10b) can also result in SPR loss, so
+ * just always test PSSCR for SPR/TB state loss.
+ */
+ pls = (psscr & PSSCR_PLS) >> PSSCR_PLS_SHIFT;
+ if (likely(pls < pnv_first_spr_loss_level)) {
+ if (sprs_saved)
+ atomic_stop_thread_idle();
+ goto out;
+ }
+
+ /* HV state loss */
+ BUG_ON(!sprs_saved);
+
+ atomic_lock_thread_idle();
+
+ if ((*state & core_thread_mask) != 0)
+ goto core_woken;
+
+ /* Per-core SPRs */
+ mtspr(SPRN_PTCR, sprs.ptcr);
+ mtspr(SPRN_RPR, sprs.rpr);
+ mtspr(SPRN_TSCR, sprs.tscr);
+ mtspr(SPRN_LDBAR, sprs.ldbar);
+ mtspr(SPRN_AMOR, sprs.amor);
+
+ if (pls >= pnv_first_tb_loss_level) {
+ /* TB loss */
+ if (opal_resync_timebase() != OPAL_SUCCESS)
+ BUG();
+ }
+
+ /*
+ * isync after restoring shared SPRs and before unlocking. Unlock
+ * only contains hwsync which does not necessarily do the right
+ * thing for SPRs.
+ */
+ isync();
+
+core_woken:
+ atomic_unlock_and_stop_thread_idle();
+
+ /* Per-thread SPRs */
+ mtspr(SPRN_LPCR, sprs.lpcr);
+ mtspr(SPRN_HFSCR, sprs.hfscr);
+ mtspr(SPRN_FSCR, sprs.fscr);
+ mtspr(SPRN_PID, sprs.pid);
+ mtspr(SPRN_PURR, sprs.purr);
+ mtspr(SPRN_SPURR, sprs.spurr);
+ mtspr(SPRN_DSCR, sprs.dscr);
+ mtspr(SPRN_WORT, sprs.wort);
+
+ mtspr(SPRN_MMCRA, sprs.mmcra);
+ mtspr(SPRN_MMCR0, sprs.mmcr0);
+ mtspr(SPRN_MMCR1, sprs.mmcr1);
+ mtspr(SPRN_MMCR2, sprs.mmcr2);
+
+ mtspr(SPRN_SPRG3, local_paca->sprg_vdso);
+
+ if (!radix_enabled())
+ __slb_restore_bolted_realmode();
+
+out:
+ if (mmu_on)
+ mtmsr(MSR_KERNEL);
+
+ return srr1;
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+static unsigned long power9_offline_stop(unsigned long psscr)
+{
+ unsigned long srr1;
+
+#ifndef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+ __ppc64_runlatch_off();
+ srr1 = power9_idle_stop(psscr, true);
+ __ppc64_runlatch_on();
+#else
+ /*
+ * Tell KVM we're entering idle.
+ * This does not have to be done in real mode because the P9 MMU
+ * is independent per-thread. Some steppings share radix/hash mode
+ * between threads, but in that case KVM has a barrier sync in real
+ * mode before and after switching between radix and hash.
+ *
+ * kvm_start_guest must still be called in real mode though, hence
+ * the false argument.
+ */
+ local_paca->kvm_hstate.hwthread_state = KVM_HWTHREAD_IN_IDLE;
+
+ __ppc64_runlatch_off();
+ srr1 = power9_idle_stop(psscr, false);
+ __ppc64_runlatch_on();
+
+ local_paca->kvm_hstate.hwthread_state = KVM_HWTHREAD_IN_KERNEL;
+ /* Order setting hwthread_state vs. testing hwthread_req */
+ smp_mb();
+ if (local_paca->kvm_hstate.hwthread_req)
+ srr1 = idle_kvm_start_guest(srr1);
+ mtmsr(MSR_KERNEL);
+#endif
+
+ return srr1;
+}
+#endif
+
static unsigned long __power9_idle_type(unsigned long stop_psscr_val,
unsigned long stop_psscr_mask)
{
@@ -360,7 +833,7 @@ static unsigned long __power9_idle_type(unsigned long stop_psscr_val,
psscr = (psscr & ~stop_psscr_mask) | stop_psscr_val;
__ppc64_runlatch_off();
- srr1 = power9_idle_stop(psscr);
+ srr1 = power9_idle_stop(psscr, true);
__ppc64_runlatch_on();
fini_irq_for_idle_irqsoff();
@@ -409,7 +882,7 @@ void pnv_power9_force_smt4_catch(void)
atomic_inc(&paca_ptrs[cpu0+thr]->dont_stop);
}
/* order setting dont_stop vs testing requested_psscr */
- mb();
+ smp_mb();
for (thr = 0; thr < threads_per_core; ++thr) {
if (!paca_ptrs[cpu0+thr]->requested_psscr)
++awake_threads;
@@ -480,7 +953,6 @@ static void pnv_program_cpu_hotplug_lpcr(unsigned int cpu, u64 lpcr_val)
unsigned long pnv_cpu_offline(unsigned int cpu)
{
unsigned long srr1;
- u32 idle_states = pnv_get_supported_cpuidle_states();
u64 lpcr_val;
/*
@@ -505,15 +977,8 @@ unsigned long pnv_cpu_offline(unsigned int cpu)
psscr = (psscr & ~pnv_deepest_stop_psscr_mask) |
pnv_deepest_stop_psscr_val;
srr1 = power9_offline_stop(psscr);
-
- } else if ((idle_states & OPAL_PM_WINKLE_ENABLED) &&
- (idle_states & OPAL_PM_LOSE_FULL_CONTEXT)) {
- srr1 = power7_idle_insn(PNV_THREAD_WINKLE);
- } else if ((idle_states & OPAL_PM_SLEEP_ENABLED) ||
- (idle_states & OPAL_PM_SLEEP_ENABLED_ER1)) {
- srr1 = power7_idle_insn(PNV_THREAD_SLEEP);
- } else if (idle_states & OPAL_PM_NAP_ENABLED) {
- srr1 = power7_idle_insn(PNV_THREAD_NAP);
+ } else if (cpu_has_feature(CPU_FTR_ARCH_206) && power7_offline_type) {
+ srr1 = power7_offline();
} else {
/* This is the fallback method. We emulate snooze */
while (!generic_check_cpu_restart(cpu)) {
@@ -619,33 +1084,44 @@ int validate_psscr_val_mask(u64 *psscr_val, u64 *psscr_mask, u32 flags)
* @dt_idle_states: Number of idle state entries
* Returns 0 on success
*/
-static int __init pnv_power9_idle_init(void)
+static void __init pnv_power9_idle_init(void)
{
u64 max_residency_ns = 0;
int i;
/*
- * Set pnv_first_deep_stop_state, pnv_deepest_stop_psscr_{val,mask},
- * and the pnv_default_stop_{val,mask}.
- *
- * pnv_first_deep_stop_state should be set to the first stop
- * level to cause hypervisor state loss.
- *
* pnv_deepest_stop_{val,mask} should be set to values corresponding to
* the deepest stop state.
*
* pnv_default_stop_{val,mask} should be set to values corresponding to
- * the shallowest (OPAL_PM_STOP_INST_FAST) loss-less stop state.
+ * the deepest loss-less (OPAL_PM_STOP_INST_FAST) stop state.
*/
- pnv_first_deep_stop_state = MAX_STOP_STATE;
+ pnv_first_tb_loss_level = MAX_STOP_STATE + 1;
+ pnv_first_spr_loss_level = MAX_STOP_STATE + 1;
for (i = 0; i < nr_pnv_idle_states; i++) {
int err;
struct pnv_idle_states_t *state = &pnv_idle_states[i];
u64 psscr_rl = state->psscr_val & PSSCR_RL_MASK;
+ if ((state->flags & OPAL_PM_TIMEBASE_STOP) &&
+ (pnv_first_tb_loss_level > psscr_rl))
+ pnv_first_tb_loss_level = psscr_rl;
+
if ((state->flags & OPAL_PM_LOSE_FULL_CONTEXT) &&
- pnv_first_deep_stop_state > psscr_rl)
- pnv_first_deep_stop_state = psscr_rl;
+ (pnv_first_spr_loss_level > psscr_rl))
+ pnv_first_spr_loss_level = psscr_rl;
+
+ /*
+ * The idle code does not deal with TB loss occurring
+ * in a shallower state than SPR loss, so force it to
+ * behave like SPRs are lost if TB is lost. POWER9 would
+ * never encouter this, but a POWER8 core would if it
+ * implemented the stop instruction. So this is for forward
+ * compatibility.
+ */
+ if ((state->flags & OPAL_PM_TIMEBASE_STOP) &&
+ (pnv_first_spr_loss_level > psscr_rl))
+ pnv_first_spr_loss_level = psscr_rl;
err = validate_psscr_val_mask(&state->psscr_val,
&state->psscr_mask,
@@ -670,6 +1146,7 @@ static int __init pnv_power9_idle_init(void)
pnv_default_stop_val = state->psscr_val;
pnv_default_stop_mask = state->psscr_mask;
default_stop_found = true;
+ WARN_ON(state->flags & OPAL_PM_LOSE_FULL_CONTEXT);
}
}
@@ -689,10 +1166,40 @@ static int __init pnv_power9_idle_init(void)
pnv_deepest_stop_psscr_mask);
}
- pr_info("cpuidle-powernv: Requested Level (RL) value of first deep stop = 0x%llx\n",
- pnv_first_deep_stop_state);
+ pr_info("cpuidle-powernv: First stop level that may lose SPRs = 0x%lld\n",
+ pnv_first_spr_loss_level);
- return 0;
+ pr_info("cpuidle-powernv: First stop level that may lose timebase = 0x%lld\n",
+ pnv_first_tb_loss_level);
+}
+
+static void __init pnv_disable_deep_states(void)
+{
+ /*
+ * The stop-api is unable to restore hypervisor
+ * resources on wakeup from platform idle states which
+ * lose full context. So disable such states.
+ */
+ supported_cpuidle_states &= ~OPAL_PM_LOSE_FULL_CONTEXT;
+ pr_warn("cpuidle-powernv: Disabling idle states that lose full context\n");
+ pr_warn("cpuidle-powernv: Idle power-savings, CPU-Hotplug affected\n");
+
+ if (cpu_has_feature(CPU_FTR_ARCH_300) &&
+ (pnv_deepest_stop_flag & OPAL_PM_LOSE_FULL_CONTEXT)) {
+ /*
+ * Use the default stop state for CPU-Hotplug
+ * if available.
+ */
+ if (default_stop_found) {
+ pnv_deepest_stop_psscr_val = pnv_default_stop_val;
+ pnv_deepest_stop_psscr_mask = pnv_default_stop_mask;
+ pr_warn("cpuidle-powernv: Offlined CPUs will stop with psscr = 0x%016llx\n",
+ pnv_deepest_stop_psscr_val);
+ } else { /* Fallback to snooze loop for CPU-Hotplug */
+ deepest_stop_found = false;
+ pr_warn("cpuidle-powernv: Offlined CPUs will busy wait\n");
+ }
+ }
}
/*
@@ -707,10 +1214,8 @@ static void __init pnv_probe_idle_states(void)
return;
}
- if (cpu_has_feature(CPU_FTR_ARCH_300)) {
- if (pnv_power9_idle_init())
- return;
- }
+ if (cpu_has_feature(CPU_FTR_ARCH_300))
+ pnv_power9_idle_init();
for (i = 0; i < nr_pnv_idle_states; i++)
supported_cpuidle_states |= pnv_idle_states[i].flags;
@@ -830,11 +1335,33 @@ static int pnv_parse_cpuidle_dt(void)
static int __init pnv_init_idle_states(void)
{
+ int cpu;
int rc = 0;
- supported_cpuidle_states = 0;
+
+ /* Set up PACA fields */
+ for_each_present_cpu(cpu) {
+ struct paca_struct *p = paca_ptrs[cpu];
+
+ p->idle_state = 0;
+ if (cpu == cpu_first_thread_sibling(cpu))
+ p->idle_state = (1 << threads_per_core) - 1;
+
+ if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
+ /* P7/P8 nap */
+ p->thread_idle_state = PNV_THREAD_RUNNING;
+ } else {
+ /* P9 stop */
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+ p->requested_psscr = 0;
+ atomic_set(&p->dont_stop, 0);
+#endif
+ }
+ }
/* In case we error out nr_pnv_idle_states will be zero */
nr_pnv_idle_states = 0;
+ supported_cpuidle_states = 0;
+
if (cpuidle_disable != IDLE_NO_OVERRIDE)
goto out;
rc = pnv_parse_cpuidle_dt();
@@ -842,27 +1369,40 @@ static int __init pnv_init_idle_states(void)
return rc;
pnv_probe_idle_states();
- if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) {
- patch_instruction(
- (unsigned int *)pnv_fastsleep_workaround_at_entry,
- PPC_INST_NOP);
- patch_instruction(
- (unsigned int *)pnv_fastsleep_workaround_at_exit,
- PPC_INST_NOP);
- } else {
- /*
- * OPAL_PM_SLEEP_ENABLED_ER1 is set. It indicates that
- * workaround is needed to use fastsleep. Provide sysfs
- * control to choose how this workaround has to be applied.
- */
- device_create_file(cpu_subsys.dev_root,
+ if (!cpu_has_feature(CPU_FTR_ARCH_300)) {
+ if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) {
+ power7_fastsleep_workaround_entry = false;
+ power7_fastsleep_workaround_exit = false;
+ } else {
+ /*
+ * OPAL_PM_SLEEP_ENABLED_ER1 is set. It indicates that
+ * workaround is needed to use fastsleep. Provide sysfs
+ * control to choose how this workaround has to be
+ * applied.
+ */
+ device_create_file(cpu_subsys.dev_root,
&dev_attr_fastsleep_workaround_applyonce);
- }
+ }
+
+ update_subcore_sibling_mask();
- pnv_alloc_idle_core_states();
+ if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED) {
+ ppc_md.power_save = power7_idle;
+ power7_offline_type = PNV_THREAD_NAP;
+ }
- if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED)
- ppc_md.power_save = power7_idle;
+ if ((supported_cpuidle_states & OPAL_PM_WINKLE_ENABLED) &&
+ (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT))
+ power7_offline_type = PNV_THREAD_WINKLE;
+ else if ((supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED) ||
+ (supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1))
+ power7_offline_type = PNV_THREAD_SLEEP;
+ }
+
+ if (supported_cpuidle_states & OPAL_PM_LOSE_FULL_CONTEXT) {
+ if (pnv_save_sprs_for_deep_states())
+ pnv_disable_deep_states();
+ }
out:
return 0;
diff --git a/arch/powerpc/platforms/powernv/subcore.c b/arch/powerpc/platforms/powernv/subcore.c
index 45563004feda..1d7a9fd30dd1 100644
--- a/arch/powerpc/platforms/powernv/subcore.c
+++ b/arch/powerpc/platforms/powernv/subcore.c
@@ -183,7 +183,7 @@ static void unsplit_core(void)
cpu = smp_processor_id();
if (cpu_thread_in_core(cpu) != 0) {
while (mfspr(SPRN_HID0) & mask)
- power7_idle_insn(PNV_THREAD_NAP);
+ power7_idle_type(PNV_THREAD_NAP);
per_cpu(split_state, cpu).step = SYNC_STEP_UNSPLIT;
return;
diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
index c70d17c9a6ba..f96ed93d325d 100644
--- a/arch/powerpc/xmon/xmon.c
+++ b/arch/powerpc/xmon/xmon.c
@@ -2424,7 +2424,6 @@ static void dump_one_paca(int cpu)
DUMP(p, irq_happened, "%#-*x");
DUMP(p, io_sync, "%#-*x");
DUMP(p, irq_work_pending, "%#-*x");
- DUMP(p, nap_state_lost, "%#-*x");
DUMP(p, sprg_vdso, "%#-*llx");
#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
@@ -2432,19 +2431,16 @@ static void dump_one_paca(int cpu)
#endif
#ifdef CONFIG_PPC_POWERNV
- DUMP(p, core_idle_state_ptr, "%-*px");
- DUMP(p, thread_idle_state, "%#-*x");
- DUMP(p, thread_mask, "%#-*x");
- DUMP(p, subcore_sibling_mask, "%#-*x");
- DUMP(p, requested_psscr, "%#-*llx");
- DUMP(p, stop_sprs.pid, "%#-*llx");
- DUMP(p, stop_sprs.ldbar, "%#-*llx");
- DUMP(p, stop_sprs.fscr, "%#-*llx");
- DUMP(p, stop_sprs.hfscr, "%#-*llx");
- DUMP(p, stop_sprs.mmcr1, "%#-*llx");
- DUMP(p, stop_sprs.mmcr2, "%#-*llx");
- DUMP(p, stop_sprs.mmcra, "%#-*llx");
- DUMP(p, dont_stop.counter, "%#-*x");
+ DUMP(p, idle_state, "%#-*lx");
+ if (!early_cpu_has_feature(CPU_FTR_ARCH_300)) {
+ DUMP(p, thread_idle_state, "%#-*x");
+ DUMP(p, subcore_sibling_mask, "%#-*x");
+ } else {
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+ DUMP(p, requested_psscr, "%#-*llx");
+ DUMP(p, dont_stop.counter, "%#-*x");
+#endif
+ }
#endif
DUMP(p, accounting.utime, "%#-*lx");
--
2.18.0
^ permalink raw reply related
* Re: [PATCH 2/2] powerpc/pseries: Add driver for PAPR SCM regions
From: Michael Ellerman @ 2018-10-13 12:08 UTC (permalink / raw)
To: Dan Williams, Oliver O'Halloran
Cc: Nathan Fontenot, linuxppc-dev, linux-nvdimm
In-Reply-To: <CAPcyv4jhs9f2DdOk30KU9___mWMtowQi0LP-OD1E+_5xFbxWgA@mail.gmail.com>
Dan Williams <dan.j.williams@intel.com> writes:
> On Tue, Oct 9, 2018 at 11:21 PM Oliver O'Halloran <oohall@gmail.com> wrote:
>>
>> Adds a driver that implements support for enabling and accessing PAPR
>> SCM regions. Unfortunately due to how the PAPR interface works we can't
>> use the existing of_pmem driver (yet) because:
>>
...
>> +
>> +static int papr_scm_nvdimm_init(struct papr_scm_priv *p)
>> +{
>> + struct device *dev = &p->pdev->dev;
>> + struct nd_mapping_desc mapping;
>> + struct nd_region_desc ndr_desc;
>> + unsigned long dimm_flags;
>> +
>> + p->bus_desc.ndctl = papr_scm_ndctl;
>> + p->bus_desc.module = THIS_MODULE;
>> + p->bus_desc.of_node = p->pdev->dev.of_node;
>> + p->bus_desc.attr_groups = bus_attr_groups;
>> + p->bus_desc.provider_name = kstrdup(p->pdev->name, GFP_KERNEL);
>> +
>> + if (!p->bus_desc.provider_name)
>> + return -ENOMEM;
>> +
>> + p->bus = nvdimm_bus_register(NULL, &p->bus_desc);
>> + if (!p->bus) {
>> + dev_err(dev, "Error creating nvdimm bus %pOF\n", p->dn);
>> + return -ENXIO;
>> + }
>> +
>> + dimm_flags = 0;
>> + set_bit(NDD_ALIASING, &dimm_flags);
>> +
>> + p->nvdimm = nvdimm_create(p->bus, p, papr_scm_dimm_groups,
>> + dimm_flags, PAPR_SCM_DIMM_CMD_MASK, 0, NULL);
>
> Looks good, although I'm just about to push out commits that change
> this function signature to take a 'security_ops' pointer. If you need
> a stable branch to base this on, let me know.
...
>
> Other than that looks ok to me:
>
> Acked-by: Dan Williams <dan.j.williams@intel.com>
>
> ...just the matter of what to do about function signature change.
Yeah that's a bit of a bother.
The ideal for me would be that you put the commit that changes the
signature by itself in a branch based on 4.19-rc3 (or earlier), and then
we could both just merge that.
But not sure if that will work with whatever else you're trying to sync
up with.
cheers
^ permalink raw reply
* Re: [PATCH 1/3] powerpc: Split user/kernel definitions of struct pt_regs
From: Nicholas Piggin @ 2018-10-13 12:27 UTC (permalink / raw)
To: Michael Ellerman; +Cc: linuxppc-dev
In-Reply-To: <20181013105646.5147-1-mpe@ellerman.id.au>
On Sat, 13 Oct 2018 21:56:44 +1100
Michael Ellerman <mpe@ellerman.id.au> wrote:
> We use a shared definition for struct pt_regs in uapi/asm/ptrace.h.
> That means the layout of the structure is ABI, ie. we can't change it.
>
> That would be fine if it was only used to describe the user-visible
> register state of a process, but it's also the struct we use in the
> kernel to describe the registers saved in an interrupt frame.
>
> We'd like more flexibility in the content (and possibly layout) of the
> kernel version of the struct, but currently that's not possible.
>
> So split the definition into a user-visible definition which remains
> unchanged, and a kernel internal one.
>
> At the moment they're still identical, and we check that at build
> time. That's because we have code (in ptrace etc.) that assumes that
> they are the same. We will fix that code in future patches, and then
> we can break the strict symmetry between the two structs.
>
> Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Yeah this looks much better than my int_frame thing, thanks.
Reviewed-by: Nicholas Piggin <npiggin@gmail.com>
Thanks,
Nick
^ permalink raw reply
* Re: [PATCH v2 12/18] of: overlay: check prevents multiple fragments add or delete same node
From: Joe Perches @ 2018-10-13 12:51 UTC (permalink / raw)
To: frowand.list, Rob Herring, Pantelis Antoniou, Michael Ellerman,
Benjamin Herrenschmidt, Paul Mackerras, Alan Tull, Moritz Fischer
Cc: devicetree, linux-fpga, linuxppc-dev, linux-kernel
In-Reply-To: <1539406418-18162-13-git-send-email-frowand.list@gmail.com>
On Fri, 2018-10-12 at 21:53 -0700, frowand.list@gmail.com wrote:
> From: Frank Rowand <frank.rowand@sony.com>
>
> Multiple overlay fragments adding or deleting the same node is not
> supported. Replace code comment of such, with check to detect the
> attempt and fail the overlay apply.
>
> Devicetree unittest where multiple fragments added the same node was
> added in the previous patch in the series. After applying this patch
> the unittest messages will no longer include:
>
> Duplicate name in motor-1, renamed to "controller#1"
> OF: overlay: of_overlay_apply() err=0
> ### dt-test ### of_overlay_fdt_apply() expected -22, ret=0, overlay_bad_add_dup_node
> ### dt-test ### FAIL of_unittest_overlay_high_level():2419 Adding overlay 'overlay_bad_add_dup_node' failed
>
> ...
>
> ### dt-test ### end of unittest - 210 passed, 1 failed
>
> but will instead include:
>
> OF: overlay: ERROR: multiple overlay fragments add and/or delete node /testcase-data-2/substation@100/motor-1/controller
>
> ...
>
> ### dt-test ### end of unittest - 211 passed, 0 failed
[]
> diff --git a/drivers/of/overlay.c b/drivers/of/overlay.c
[]
> @@ -523,6 +515,54 @@ static int build_changeset_symbols_node(struct overlay_changeset *ovcs,
> }
>
> /**
> + * check_changeset_dup_add_node() - changeset validation: duplicate add node
> + * @ovcs: Overlay changeset
> + *
> + * Check changeset @ovcs->cset for multiple add node entries for the same
> + * node.
> + *
> + * Returns 0 on success, -ENOMEM if memory allocation failure, or -EINVAL if
> + * invalid overlay in @ovcs->fragments[].
> + */
> +static int check_changeset_dup_add_node(struct overlay_changeset *ovcs)
> +{
> + struct of_changeset_entry *ce_1, *ce_2;
> + char *fn_1, *fn_2;
> + int name_match;
> +
> + list_for_each_entry(ce_1, &ovcs->cset.entries, node) {
> +
> + if (ce_1->action == OF_RECONFIG_ATTACH_NODE ||
> + ce_1->action == OF_RECONFIG_DETACH_NODE) {
> +
> + ce_2 = ce_1;
> + list_for_each_entry_continue(ce_2, &ovcs->cset.entries, node) {
> + if (ce_2->action == OF_RECONFIG_ATTACH_NODE ||
> + ce_2->action == OF_RECONFIG_DETACH_NODE) {
> + /* inexpensive name compare */
> + if (!of_node_cmp(ce_1->np->full_name,
> + ce_2->np->full_name)) {
A bit of odd indentation here.
This line is normally aligned to the second ( on the line above.
> + /* expensive full path name compare */
> + fn_1 = kasprintf(GFP_KERNEL, "%pOF", ce_1->np);
> + fn_2 = kasprintf(GFP_KERNEL, "%pOF", ce_2->np);
> + name_match = !strcmp(fn_1, fn_2);
> + kfree(fn_1);
> + kfree(fn_2);
> + if (name_match) {
> + pr_err("ERROR: multiple overlay fragments add and/or delete node %pOF\n",
> + ce_1->np);
> + return -EINVAL;
> + }
> + }
> + }
> + }
> + }
> + }
> +
> + return 0;
> +}
Style trivia:
Using inverted tests and continue would reduce indentation.
list_for_each_entry(ce_1, &ovcs->cset.entries, node) {
if (ce_1->action != OF_RECONFIG_ATTACH_NODE &&
ce_1->action != OF_RECONFIG_DETACH_NODE)
continue;
ce_2 = ce_1;
list_for_each_entry_continue(ce_2, &ovcs->cset.entries, node) {
if (ce_2->action != OF_RECONFIG_ATTACH_NODE &&
ce_2->action != OF_RECONFIG_DETACH_NODE)
continue;
/* inexpensive name compare */
if (of_node_cmp(ce_1->np->full_name, ce_2->np->full_name))
continue;
/* expensive full path name compare */
fn_1 = kasprintf(GFP_KERNEL, "%pOF", ce_1->np);
fn_2 = kasprintf(GFP_KERNEL, "%pOF", ce_2->np);
name_match = !strcmp(fn_1, fn_2);
kfree(fn_1);
kfree(fn_2);
if (name_match) {
pr_err("ERROR: multiple overlay fragments add and/or delete node %pOF\n",
ce_1->np);
return -EINVAL;
}
}
}
^ permalink raw reply
* [PATCH 1/6] powerpc/64s/hash: Convert SLB miss handlers to C
From: Michael Ellerman @ 2018-10-13 13:18 UTC (permalink / raw)
To: linuxppc-dev; +Cc: aneesh.kumar, npiggin
From: Nicholas Piggin <npiggin@gmail.com>
This patch moves SLB miss handlers completely to C, using the standard
exception handler macros to set up the stack and branch to C.
This can be done because the segment containing the kernel stack is
always bolted, so accessing it with relocation on will not cause an
SLB exception.
Arbitrary kernel memory must not be accessed when handling kernel
space SLB misses, so care should be taken there. However user SLB
misses can access any kernel memory, which can be used to move some
fields out of the paca (in later patches).
User SLB misses could quite easily reconcile IRQs and set up a first
class kernel environment and exit via ret_from_except, however that
doesn't seem to be necessary at the moment, so we only do that if a
bad fault is encountered.
[ Credit to Aneesh for bug fixes, error checks, and improvements to
bad address handling, etc ]
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
[mpe: Disallow tracing for all of slb.c for now.]
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
---
arch/powerpc/include/asm/asm-prototypes.h | 2 +
arch/powerpc/include/asm/exception-64s.h | 8 -
arch/powerpc/kernel/exceptions-64s.S | 202 +++----------
arch/powerpc/mm/Makefile | 4 +-
arch/powerpc/mm/slb.c | 289 +++++++++++--------
arch/powerpc/mm/slb_low.S | 335 ----------------------
6 files changed, 214 insertions(+), 626 deletions(-)
delete mode 100644 arch/powerpc/mm/slb_low.S
diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h
index 9bc98c239305..2741831482f4 100644
--- a/arch/powerpc/include/asm/asm-prototypes.h
+++ b/arch/powerpc/include/asm/asm-prototypes.h
@@ -77,6 +77,8 @@ void kernel_bad_stack(struct pt_regs *regs);
void system_reset_exception(struct pt_regs *regs);
void machine_check_exception(struct pt_regs *regs);
void emulation_assist_interrupt(struct pt_regs *regs);
+long do_slb_fault(struct pt_regs *regs, unsigned long ea);
+void do_bad_slb_fault(struct pt_regs *regs, unsigned long ea, long err);
/* signals, syscalls and interrupts */
long sys_swapcontext(struct ucontext __user *old_ctx,
diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h
index 403d73898a9a..3b4767ed3ec5 100644
--- a/arch/powerpc/include/asm/exception-64s.h
+++ b/arch/powerpc/include/asm/exception-64s.h
@@ -60,14 +60,6 @@
*/
#define MAX_MCE_DEPTH 4
-/*
- * EX_LR is only used in EXSLB and where it does not overlap with EX_DAR
- * EX_CCR similarly with DSISR, but being 4 byte registers there is a hole
- * in the save area so it's not necessary to overlap them. Could be used
- * for future savings though if another 4 byte register was to be saved.
- */
-#define EX_LR EX_DAR
-
/*
* EX_R3 is only used by the bad_stack handler. bad_stack reloads and
* saves DAR from SPRN_DAR, and EX_DAR is not used. So EX_R3 can overlap
diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index b9239dbf6d59..89d32bb79d5e 100644
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -596,28 +596,36 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
EXC_REAL_BEGIN(data_access_slb, 0x380, 0x80)
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXSLB)
- EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x380)
- mr r12,r3 /* save r3 */
- mfspr r3,SPRN_DAR
- mfspr r11,SPRN_SRR1
- crset 4*cr6+eq
- BRANCH_TO_COMMON(r10, slb_miss_common)
+EXCEPTION_PROLOG(PACA_EXSLB, data_access_slb_common, EXC_STD, KVMTEST_PR, 0x380);
EXC_REAL_END(data_access_slb, 0x380, 0x80)
EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80)
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXSLB)
- EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x380)
- mr r12,r3 /* save r3 */
- mfspr r3,SPRN_DAR
- mfspr r11,SPRN_SRR1
- crset 4*cr6+eq
- BRANCH_TO_COMMON(r10, slb_miss_common)
+EXCEPTION_RELON_PROLOG(PACA_EXSLB, data_access_slb_common, EXC_STD, NOTEST, 0x380);
EXC_VIRT_END(data_access_slb, 0x4380, 0x80)
+
TRAMP_KVM_SKIP(PACA_EXSLB, 0x380)
+EXC_COMMON_BEGIN(data_access_slb_common)
+ mfspr r10,SPRN_DAR
+ std r10,PACA_EXSLB+EX_DAR(r13)
+ EXCEPTION_PROLOG_COMMON(0x380, PACA_EXSLB)
+ ld r4,PACA_EXSLB+EX_DAR(r13)
+ std r4,_DAR(r1)
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl do_slb_fault
+ cmpdi r3,0
+ bne- 1f
+ b fast_exception_return
+1: /* Error case */
+ std r3,RESULT(r1)
+ bl save_nvgprs
+ RECONCILE_IRQ_STATE(r10, r11)
+ ld r4,_DAR(r1)
+ ld r5,RESULT(r1)
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl do_bad_slb_fault
+ b ret_from_except
+
EXC_REAL(instruction_access, 0x400, 0x80)
EXC_VIRT(instruction_access, 0x4400, 0x80, 0x400)
@@ -640,160 +648,34 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
EXC_REAL_BEGIN(instruction_access_slb, 0x480, 0x80)
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXSLB)
- EXCEPTION_PROLOG_1(PACA_EXSLB, KVMTEST_PR, 0x480)
- mr r12,r3 /* save r3 */
- mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
- mfspr r11,SPRN_SRR1
- crclr 4*cr6+eq
- BRANCH_TO_COMMON(r10, slb_miss_common)
+EXCEPTION_PROLOG(PACA_EXSLB, instruction_access_slb_common, EXC_STD, KVMTEST_PR, 0x480);
EXC_REAL_END(instruction_access_slb, 0x480, 0x80)
EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80)
- SET_SCRATCH0(r13)
- EXCEPTION_PROLOG_0(PACA_EXSLB)
- EXCEPTION_PROLOG_1(PACA_EXSLB, NOTEST, 0x480)
- mr r12,r3 /* save r3 */
- mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
- mfspr r11,SPRN_SRR1
- crclr 4*cr6+eq
- BRANCH_TO_COMMON(r10, slb_miss_common)
+EXCEPTION_RELON_PROLOG(PACA_EXSLB, instruction_access_slb_common, EXC_STD, NOTEST, 0x480);
EXC_VIRT_END(instruction_access_slb, 0x4480, 0x80)
-TRAMP_KVM(PACA_EXSLB, 0x480)
-
-
-/*
- * This handler is used by the 0x380 and 0x480 SLB miss interrupts, as well as
- * the virtual mode 0x4380 and 0x4480 interrupts if AIL is enabled.
- */
-EXC_COMMON_BEGIN(slb_miss_common)
- /*
- * r13 points to the PACA, r9 contains the saved CR,
- * r12 contains the saved r3,
- * r11 contain the saved SRR1, SRR0 is still ready for return
- * r3 has the faulting address
- * r9 - r13 are saved in paca->exslb.
- * cr6.eq is set for a D-SLB miss, clear for a I-SLB miss
- * We assume we aren't going to take any exceptions during this
- * procedure.
- */
- mflr r10
- stw r9,PACA_EXSLB+EX_CCR(r13) /* save CR in exc. frame */
- std r10,PACA_EXSLB+EX_LR(r13) /* save LR */
-
- andi. r9,r11,MSR_PR // Check for exception from userspace
- cmpdi cr4,r9,MSR_PR // And save the result in CR4 for later
-
- /*
- * Test MSR_RI before calling slb_allocate_realmode, because the
- * MSR in r11 gets clobbered. However we still want to allocate
- * SLB in case MSR_RI=0, to minimise the risk of getting stuck in
- * recursive SLB faults. So use cr5 for this, which is preserved.
- */
- andi. r11,r11,MSR_RI /* check for unrecoverable exception */
- cmpdi cr5,r11,MSR_RI
-
- crset 4*cr0+eq
-#ifdef CONFIG_PPC_BOOK3S_64
-BEGIN_MMU_FTR_SECTION
- bl slb_allocate
-END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX)
-#endif
-
- ld r10,PACA_EXSLB+EX_LR(r13)
- lwz r9,PACA_EXSLB+EX_CCR(r13) /* get saved CR */
- mtlr r10
-
- /*
- * Large address, check whether we have to allocate new contexts.
- */
- beq- 8f
- bne- cr5,2f /* if unrecoverable exception, oops */
-
- /* All done -- return from exception. */
-
- bne cr4,1f /* returning to kernel */
-
- mtcrf 0x80,r9
- mtcrf 0x08,r9 /* MSR[PR] indication is in cr4 */
- mtcrf 0x04,r9 /* MSR[RI] indication is in cr5 */
- mtcrf 0x02,r9 /* I/D indication is in cr6 */
- mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */
-
- RESTORE_CTR(r9, PACA_EXSLB)
- RESTORE_PPR_PACA(PACA_EXSLB, r9)
- mr r3,r12
- ld r9,PACA_EXSLB+EX_R9(r13)
- ld r10,PACA_EXSLB+EX_R10(r13)
- ld r11,PACA_EXSLB+EX_R11(r13)
- ld r12,PACA_EXSLB+EX_R12(r13)
- ld r13,PACA_EXSLB+EX_R13(r13)
- RFI_TO_USER
- b . /* prevent speculative execution */
-1:
- mtcrf 0x80,r9
- mtcrf 0x08,r9 /* MSR[PR] indication is in cr4 */
- mtcrf 0x04,r9 /* MSR[RI] indication is in cr5 */
- mtcrf 0x02,r9 /* I/D indication is in cr6 */
- mtcrf 0x01,r9 /* slb_allocate uses cr0 and cr7 */
-
- RESTORE_CTR(r9, PACA_EXSLB)
- RESTORE_PPR_PACA(PACA_EXSLB, r9)
- mr r3,r12
- ld r9,PACA_EXSLB+EX_R9(r13)
- ld r10,PACA_EXSLB+EX_R10(r13)
- ld r11,PACA_EXSLB+EX_R11(r13)
- ld r12,PACA_EXSLB+EX_R12(r13)
- ld r13,PACA_EXSLB+EX_R13(r13)
- RFI_TO_KERNEL
- b . /* prevent speculative execution */
-
-
-2: std r3,PACA_EXSLB+EX_DAR(r13)
- mr r3,r12
- mfspr r11,SPRN_SRR0
- mfspr r12,SPRN_SRR1
- LOAD_HANDLER(r10,unrecov_slb)
- mtspr SPRN_SRR0,r10
- ld r10,PACAKMSR(r13)
- mtspr SPRN_SRR1,r10
- RFI_TO_KERNEL
- b .
-
-8: std r3,PACA_EXSLB+EX_DAR(r13)
- mr r3,r12
- mfspr r11,SPRN_SRR0
- mfspr r12,SPRN_SRR1
- LOAD_HANDLER(r10, large_addr_slb)
- mtspr SPRN_SRR0,r10
- ld r10,PACAKMSR(r13)
- mtspr SPRN_SRR1,r10
- RFI_TO_KERNEL
- b .
+TRAMP_KVM(PACA_EXSLB, 0x480)
-EXC_COMMON_BEGIN(unrecov_slb)
- EXCEPTION_PROLOG_COMMON(0x4100, PACA_EXSLB)
- RECONCILE_IRQ_STATE(r10, r11)
+EXC_COMMON_BEGIN(instruction_access_slb_common)
+ EXCEPTION_PROLOG_COMMON(0x480, PACA_EXSLB)
+ ld r4,_NIP(r1)
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl do_slb_fault
+ cmpdi r3,0
+ bne- 1f
+ b fast_exception_return
+1: /* Error case */
+ std r3,RESULT(r1)
bl save_nvgprs
-1: addi r3,r1,STACK_FRAME_OVERHEAD
- bl unrecoverable_exception
- b 1b
-
-EXC_COMMON_BEGIN(large_addr_slb)
- EXCEPTION_PROLOG_COMMON(0x380, PACA_EXSLB)
RECONCILE_IRQ_STATE(r10, r11)
- ld r3, PACA_EXSLB+EX_DAR(r13)
- std r3, _DAR(r1)
- beq cr6, 2f
- li r10, 0x481 /* fix trap number for I-SLB miss */
- std r10, _TRAP(r1)
-2: bl save_nvgprs
- addi r3, r1, STACK_FRAME_OVERHEAD
- bl slb_miss_large_addr
+ ld r4,_NIP(r1)
+ ld r5,RESULT(r1)
+ addi r3,r1,STACK_FRAME_OVERHEAD
+ bl do_bad_slb_fault
b ret_from_except
+
EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100)
.globl hardware_interrupt_hv;
hardware_interrupt_hv:
diff --git a/arch/powerpc/mm/Makefile b/arch/powerpc/mm/Makefile
index 3c844bdd16c4..ceb352ccbc76 100644
--- a/arch/powerpc/mm/Makefile
+++ b/arch/powerpc/mm/Makefile
@@ -7,6 +7,8 @@ subdir-ccflags-$(CONFIG_PPC_WERROR) := -Werror
ccflags-$(CONFIG_PPC64) := $(NO_MINIMAL_TOC)
+CFLAGS_REMOVE_slb.o = $(CC_FLAGS_FTRACE)
+
obj-y := fault.o mem.o pgtable.o mmap.o \
init_$(BITS).o pgtable_$(BITS).o \
init-common.o mmu_context.o drmem.o
@@ -15,7 +17,7 @@ obj-$(CONFIG_PPC_MMU_NOHASH) += mmu_context_nohash.o tlb_nohash.o \
obj-$(CONFIG_PPC_BOOK3E) += tlb_low_$(BITS)e.o
hash64-$(CONFIG_PPC_NATIVE) := hash_native_64.o
obj-$(CONFIG_PPC_BOOK3E_64) += pgtable-book3e.o
-obj-$(CONFIG_PPC_BOOK3S_64) += pgtable-hash64.o hash_utils_64.o slb_low.o slb.o $(hash64-y) mmu_context_book3s64.o pgtable-book3s64.o
+obj-$(CONFIG_PPC_BOOK3S_64) += pgtable-hash64.o hash_utils_64.o slb.o $(hash64-y) mmu_context_book3s64.o pgtable-book3s64.o
obj-$(CONFIG_PPC_RADIX_MMU) += pgtable-radix.o tlb-radix.o
obj-$(CONFIG_PPC_STD_MMU_32) += ppc_mmu_32.o hash_low_32.o mmu_context_hash32.o
obj-$(CONFIG_PPC_STD_MMU) += tlb_hash$(BITS).o
diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
index 513c6596140d..76c1a9523049 100644
--- a/arch/powerpc/mm/slb.c
+++ b/arch/powerpc/mm/slb.c
@@ -14,6 +14,7 @@
* 2 of the License, or (at your option) any later version.
*/
+#include <asm/asm-prototypes.h>
#include <asm/pgtable.h>
#include <asm/mmu.h>
#include <asm/mmu_context.h>
@@ -33,7 +34,7 @@ enum slb_index {
KSTACK_INDEX = 1, /* Kernel stack map */
};
-extern void slb_allocate(unsigned long ea);
+static long slb_allocate_user(struct mm_struct *mm, unsigned long ea);
#define slb_esid_mask(ssize) \
(((ssize) == MMU_SEGSIZE_256M)? ESID_MASK: ESID_MASK_1T)
@@ -44,13 +45,19 @@ static inline unsigned long mk_esid_data(unsigned long ea, int ssize,
return (ea & slb_esid_mask(ssize)) | SLB_ESID_V | index;
}
-static inline unsigned long mk_vsid_data(unsigned long ea, int ssize,
+static inline unsigned long __mk_vsid_data(unsigned long vsid, int ssize,
unsigned long flags)
{
- return (get_kernel_vsid(ea, ssize) << slb_vsid_shift(ssize)) | flags |
+ return (vsid << slb_vsid_shift(ssize)) | flags |
((unsigned long) ssize << SLB_VSID_SSIZE_SHIFT);
}
+static inline unsigned long mk_vsid_data(unsigned long ea, int ssize,
+ unsigned long flags)
+{
+ return __mk_vsid_data(get_kernel_vsid(ea, ssize), ssize, flags);
+}
+
static inline void slb_shadow_update(unsigned long ea, int ssize,
unsigned long flags,
enum slb_index index)
@@ -353,49 +360,19 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
is_kernel_addr(exec_base))
return;
- slb_allocate(pc);
+ slb_allocate_user(mm, pc);
if (!esids_match(pc, stack))
- slb_allocate(stack);
+ slb_allocate_user(mm, stack);
if (!esids_match(pc, exec_base) &&
!esids_match(stack, exec_base))
- slb_allocate(exec_base);
-}
-
-static inline void patch_slb_encoding(unsigned int *insn_addr,
- unsigned int immed)
-{
-
- /*
- * This function patches either an li or a cmpldi instruction with
- * a new immediate value. This relies on the fact that both li
- * (which is actually addi) and cmpldi both take a 16-bit immediate
- * value, and it is situated in the same location in the instruction,
- * ie. bits 16-31 (Big endian bit order) or the lower 16 bits.
- * The signedness of the immediate operand differs between the two
- * instructions however this code is only ever patching a small value,
- * much less than 1 << 15, so we can get away with it.
- * To patch the value we read the existing instruction, clear the
- * immediate value, and or in our new value, then write the instruction
- * back.
- */
- unsigned int insn = (*insn_addr & 0xffff0000) | immed;
- patch_instruction(insn_addr, insn);
+ slb_allocate_user(mm, exec_base);
}
-extern u32 slb_miss_kernel_load_linear[];
-extern u32 slb_miss_kernel_load_io[];
-extern u32 slb_compare_rr_to_size[];
-extern u32 slb_miss_kernel_load_vmemmap[];
-
void slb_set_size(u16 size)
{
- if (mmu_slb_size == size)
- return;
-
mmu_slb_size = size;
- patch_slb_encoding(slb_compare_rr_to_size, mmu_slb_size);
}
void slb_initialize(void)
@@ -417,19 +394,9 @@ void slb_initialize(void)
#endif
if (!slb_encoding_inited) {
slb_encoding_inited = 1;
- patch_slb_encoding(slb_miss_kernel_load_linear,
- SLB_VSID_KERNEL | linear_llp);
- patch_slb_encoding(slb_miss_kernel_load_io,
- SLB_VSID_KERNEL | io_llp);
- patch_slb_encoding(slb_compare_rr_to_size,
- mmu_slb_size);
-
pr_devel("SLB: linear LLP = %04lx\n", linear_llp);
pr_devel("SLB: io LLP = %04lx\n", io_llp);
-
#ifdef CONFIG_SPARSEMEM_VMEMMAP
- patch_slb_encoding(slb_miss_kernel_load_vmemmap,
- SLB_VSID_KERNEL | vmemmap_llp);
pr_devel("SLB: vmemmap LLP = %04lx\n", vmemmap_llp);
#endif
}
@@ -458,125 +425,203 @@ void slb_initialize(void)
asm volatile("isync":::"memory");
}
-static void insert_slb_entry(unsigned long vsid, unsigned long ea,
- int bpsize, int ssize)
+static void slb_cache_update(unsigned long esid_data)
{
- unsigned long flags, vsid_data, esid_data;
- enum slb_index index;
int slb_cache_index;
if (cpu_has_feature(CPU_FTR_ARCH_300))
return; /* ISAv3.0B and later does not use slb_cache */
/*
- * We are irq disabled, hence should be safe to access PACA.
+ * Now update slb cache entries
*/
- VM_WARN_ON(!irqs_disabled());
+ slb_cache_index = local_paca->slb_cache_ptr;
+ if (slb_cache_index < SLB_CACHE_ENTRIES) {
+ /*
+ * We have space in slb cache for optimized switch_slb().
+ * Top 36 bits from esid_data as per ISA
+ */
+ local_paca->slb_cache[slb_cache_index++] = esid_data >> 28;
+ local_paca->slb_cache_ptr++;
+ } else {
+ /*
+ * Our cache is full and the current cache content strictly
+ * doesn't indicate the active SLB conents. Bump the ptr
+ * so that switch_slb() will ignore the cache.
+ */
+ local_paca->slb_cache_ptr = SLB_CACHE_ENTRIES + 1;
+ }
+}
- /*
- * We can't take a PMU exception in the following code, so hard
- * disable interrupts.
- */
- hard_irq_disable();
+static enum slb_index alloc_slb_index(void)
+{
+ enum slb_index index;
+ /* round-robin replacement of slb starting at SLB_NUM_BOLTED. */
index = get_paca()->stab_rr;
-
- /*
- * simple round-robin replacement of slb starting at SLB_NUM_BOLTED.
- */
if (index < (mmu_slb_size - 1))
index++;
else
index = SLB_NUM_BOLTED;
-
get_paca()->stab_rr = index;
- flags = SLB_VSID_USER | mmu_psize_defs[bpsize].sllp;
- vsid_data = (vsid << slb_vsid_shift(ssize)) | flags |
- ((unsigned long) ssize << SLB_VSID_SSIZE_SHIFT);
+ return index;
+}
+
+static long slb_insert_entry(unsigned long ea, unsigned long context,
+ unsigned long flags, int ssize, bool kernel)
+{
+ unsigned long vsid;
+ unsigned long vsid_data, esid_data;
+ enum slb_index index;
+
+ vsid = get_vsid(context, ea, ssize);
+ if (!vsid)
+ return -EFAULT;
+
+ /*
+ * There must not be a kernel SLB fault in alloc_slb_index or before
+ * slbmte here or the allocation bitmaps could get out of whack with
+ * the SLB.
+ *
+ * User SLB faults or preloads take this path which might get inlined
+ * into the caller, so add compiler barriers here to ensure unsafe
+ * memory accesses do not come between.
+ */
+ barrier();
+
+ index = alloc_slb_index();
+
+ vsid_data = __mk_vsid_data(vsid, ssize, flags);
esid_data = mk_esid_data(ea, ssize, index);
/*
* No need for an isync before or after this slbmte. The exception
* we enter with and the rfid we exit with are context synchronizing.
- * Also we only handle user segments here.
+ * User preloads should add isync afterwards in case the kernel
+ * accesses user memory before it returns to userspace with rfid.
*/
- asm volatile("slbmte %0, %1" : : "r" (vsid_data), "r" (esid_data)
- : "memory");
+ asm volatile("slbmte %0, %1" : : "r" (vsid_data), "r" (esid_data));
- /*
- * Now update slb cache entries
- */
- slb_cache_index = get_paca()->slb_cache_ptr;
- if (slb_cache_index < SLB_CACHE_ENTRIES) {
- /*
- * We have space in slb cache for optimized switch_slb().
- * Top 36 bits from esid_data as per ISA
- */
- get_paca()->slb_cache[slb_cache_index++] = esid_data >> 28;
- get_paca()->slb_cache_ptr++;
+ barrier();
+
+ if (!kernel)
+ slb_cache_update(esid_data);
+
+ return 0;
+}
+
+static long slb_allocate_kernel(unsigned long ea, unsigned long id)
+{
+ unsigned long context;
+ unsigned long flags;
+ int ssize;
+
+ if ((ea & ~REGION_MASK) >= (1ULL << MAX_EA_BITS_PER_CONTEXT))
+ return -EFAULT;
+
+ if (id == KERNEL_REGION_ID) {
+ flags = SLB_VSID_KERNEL | mmu_psize_defs[mmu_linear_psize].sllp;
+#ifdef CONFIG_SPARSEMEM_VMEMMAP
+ } else if (id == VMEMMAP_REGION_ID) {
+ flags = SLB_VSID_KERNEL | mmu_psize_defs[mmu_vmemmap_psize].sllp;
+#endif
+ } else if (id == VMALLOC_REGION_ID) {
+ if (ea < H_VMALLOC_END)
+ flags = get_paca()->vmalloc_sllp;
+ else
+ flags = SLB_VSID_KERNEL | mmu_psize_defs[mmu_io_psize].sllp;
} else {
- /*
- * Our cache is full and the current cache content strictly
- * doesn't indicate the active SLB conents. Bump the ptr
- * so that switch_slb() will ignore the cache.
- */
- get_paca()->slb_cache_ptr = SLB_CACHE_ENTRIES + 1;
+ return -EFAULT;
}
+
+ ssize = MMU_SEGSIZE_1T;
+ if (!mmu_has_feature(MMU_FTR_1T_SEGMENT))
+ ssize = MMU_SEGSIZE_256M;
+
+ context = id - KERNEL_REGION_CONTEXT_OFFSET;
+
+ return slb_insert_entry(ea, context, flags, ssize, true);
}
-static void handle_multi_context_slb_miss(int context_id, unsigned long ea)
+static long slb_allocate_user(struct mm_struct *mm, unsigned long ea)
{
- struct mm_struct *mm = current->mm;
- unsigned long vsid;
+ unsigned long context;
+ unsigned long flags;
int bpsize;
+ int ssize;
/*
- * We are always above 1TB, hence use high user segment size.
+ * consider this as bad access if we take a SLB miss
+ * on an address above addr limit.
*/
- vsid = get_vsid(context_id, ea, mmu_highuser_ssize);
+ if (ea >= mm->context.slb_addr_limit)
+ return -EFAULT;
+
+ context = get_ea_context(&mm->context, ea);
+ if (!context)
+ return -EFAULT;
+
+ if (unlikely(ea >= H_PGTABLE_RANGE)) {
+ WARN_ON(1);
+ return -EFAULT;
+ }
+
+ ssize = user_segment_size(ea);
+
bpsize = get_slice_psize(mm, ea);
- insert_slb_entry(vsid, ea, bpsize, mmu_highuser_ssize);
+ flags = SLB_VSID_USER | mmu_psize_defs[bpsize].sllp;
+
+ return slb_insert_entry(ea, context, flags, ssize, false);
}
-void slb_miss_large_addr(struct pt_regs *regs)
+long do_slb_fault(struct pt_regs *regs, unsigned long ea)
{
- enum ctx_state prev_state = exception_enter();
- unsigned long ea = regs->dar;
- int context;
+ unsigned long id = REGION_ID(ea);
- if (REGION_ID(ea) != USER_REGION_ID)
- goto slb_bad_addr;
+ /* IRQs are not reconciled here, so can't check irqs_disabled */
+ VM_WARN_ON(mfmsr() & MSR_EE);
- /*
- * Are we beyound what the page table layout supports ?
- */
- if ((ea & ~REGION_MASK) >= H_PGTABLE_RANGE)
- goto slb_bad_addr;
-
- /* Lower address should have been handled by asm code */
- if (ea < (1UL << MAX_EA_BITS_PER_CONTEXT))
- goto slb_bad_addr;
+ if (unlikely(!(regs->msr & MSR_RI)))
+ return -EINVAL;
/*
- * consider this as bad access if we take a SLB miss
- * on an address above addr limit.
+ * SLB kernel faults must be very careful not to touch anything
+ * that is not bolted. E.g., PACA and global variables are okay,
+ * mm->context stuff is not.
+ *
+ * SLB user faults can access all of kernel memory, but must be
+ * careful not to touch things like IRQ state because it is not
+ * "reconciled" here. The difficulty is that we must use
+ * fast_exception_return to return from kernel SLB faults without
+ * looking at possible non-bolted memory. We could test user vs
+ * kernel faults in the interrupt handler asm and do a full fault,
+ * reconcile, ret_from_except for user faults which would make them
+ * first class kernel code. But for performance it's probably nicer
+ * if they go via fast_exception_return too.
*/
- if (ea >= current->mm->context.slb_addr_limit)
- goto slb_bad_addr;
+ if (id >= KERNEL_REGION_ID) {
+ return slb_allocate_kernel(ea, id);
+ } else {
+ struct mm_struct *mm = current->mm;
- context = get_ea_context(¤t->mm->context, ea);
- if (!context)
- goto slb_bad_addr;
+ if (unlikely(!mm))
+ return -EFAULT;
- handle_multi_context_slb_miss(context, ea);
- exception_exit(prev_state);
- return;
+ return slb_allocate_user(mm, ea);
+ }
+}
-slb_bad_addr:
- if (user_mode(regs))
- _exception(SIGSEGV, regs, SEGV_BNDERR, ea);
- else
- bad_page_fault(regs, ea, SIGSEGV);
- exception_exit(prev_state);
+void do_bad_slb_fault(struct pt_regs *regs, unsigned long ea, long err)
+{
+ if (err == -EFAULT) {
+ if (user_mode(regs))
+ _exception(SIGSEGV, regs, SEGV_BNDERR, ea);
+ else
+ bad_page_fault(regs, ea, SIGSEGV);
+ } else if (err == -EINVAL) {
+ unrecoverable_exception(regs);
+ } else {
+ BUG();
+ }
}
diff --git a/arch/powerpc/mm/slb_low.S b/arch/powerpc/mm/slb_low.S
deleted file mode 100644
index 4d2e921d696e..000000000000
--- a/arch/powerpc/mm/slb_low.S
+++ /dev/null
@@ -1,335 +0,0 @@
-/*
- * Low-level SLB routines
- *
- * Copyright (C) 2004 David Gibson <dwg@au.ibm.com>, IBM
- *
- * Based on earlier C version:
- * Dave Engebretsen and Mike Corrigan {engebret|mikejc}@us.ibm.com
- * Copyright (c) 2001 Dave Engebretsen
- * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- */
-
-#include <asm/processor.h>
-#include <asm/ppc_asm.h>
-#include <asm/asm-offsets.h>
-#include <asm/cputable.h>
-#include <asm/page.h>
-#include <asm/mmu.h>
-#include <asm/pgtable.h>
-#include <asm/firmware.h>
-#include <asm/feature-fixups.h>
-
-/*
- * This macro generates asm code to compute the VSID scramble
- * function. Used in slb_allocate() and do_stab_bolted. The function
- * computed is: (protovsid*VSID_MULTIPLIER) % VSID_MODULUS
- *
- * rt = register containing the proto-VSID and into which the
- * VSID will be stored
- * rx = scratch register (clobbered)
- * rf = flags
- *
- * - rt and rx must be different registers
- * - The answer will end up in the low VSID_BITS bits of rt. The higher
- * bits may contain other garbage, so you may need to mask the
- * result.
- */
-#define ASM_VSID_SCRAMBLE(rt, rx, rf, size) \
- lis rx,VSID_MULTIPLIER_##size@h; \
- ori rx,rx,VSID_MULTIPLIER_##size@l; \
- mulld rt,rt,rx; /* rt = rt * MULTIPLIER */ \
-/* \
- * powermac get slb fault before feature fixup, so make 65 bit part \
- * the default part of feature fixup \
- */ \
-BEGIN_MMU_FTR_SECTION \
- srdi rx,rt,VSID_BITS_65_##size; \
- clrldi rt,rt,(64-VSID_BITS_65_##size); \
- add rt,rt,rx; \
- addi rx,rt,1; \
- srdi rx,rx,VSID_BITS_65_##size; \
- add rt,rt,rx; \
- rldimi rf,rt,SLB_VSID_SHIFT_##size,(64 - (SLB_VSID_SHIFT_##size + VSID_BITS_65_##size)); \
-MMU_FTR_SECTION_ELSE \
- srdi rx,rt,VSID_BITS_##size; \
- clrldi rt,rt,(64-VSID_BITS_##size); \
- add rt,rt,rx; /* add high and low bits */ \
- addi rx,rt,1; \
- srdi rx,rx,VSID_BITS_##size; /* extract 2^VSID_BITS bit */ \
- add rt,rt,rx; \
- rldimi rf,rt,SLB_VSID_SHIFT_##size,(64 - (SLB_VSID_SHIFT_##size + VSID_BITS_##size)); \
-ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_68_BIT_VA)
-
-
-/* void slb_allocate(unsigned long ea);
- *
- * Create an SLB entry for the given EA (user or kernel).
- * r3 = faulting address, r13 = PACA
- * r9, r10, r11 are clobbered by this function
- * r3 is preserved.
- * No other registers are examined or changed.
- */
-_GLOBAL(slb_allocate)
- /*
- * Check if the address falls within the range of the first context, or
- * if we may need to handle multi context. For the first context we
- * allocate the slb entry via the fast path below. For large address we
- * branch out to C-code and see if additional contexts have been
- * allocated.
- * The test here is:
- * (ea & ~REGION_MASK) >= (1ull << MAX_EA_BITS_PER_CONTEXT)
- */
- rldicr. r9,r3,4,(63 - MAX_EA_BITS_PER_CONTEXT - 4)
- bne- 8f
-
- srdi r9,r3,60 /* get region */
- srdi r10,r3,SID_SHIFT /* get esid */
- cmpldi cr7,r9,0xc /* cmp PAGE_OFFSET for later use */
-
- /* r3 = address, r10 = esid, cr7 = <> PAGE_OFFSET */
- blt cr7,0f /* user or kernel? */
-
- /* Check if hitting the linear mapping or some other kernel space
- */
- bne cr7,1f
-
- /* Linear mapping encoding bits, the "li" instruction below will
- * be patched by the kernel at boot
- */
-.globl slb_miss_kernel_load_linear
-slb_miss_kernel_load_linear:
- li r11,0
- /*
- * context = (ea >> 60) - (0xc - 1)
- * r9 = region id.
- */
- subi r9,r9,KERNEL_REGION_CONTEXT_OFFSET
-
-BEGIN_FTR_SECTION
- b .Lslb_finish_load
-END_MMU_FTR_SECTION_IFCLR(MMU_FTR_1T_SEGMENT)
- b .Lslb_finish_load_1T
-
-1:
-#ifdef CONFIG_SPARSEMEM_VMEMMAP
- cmpldi cr0,r9,0xf
- bne 1f
-/* Check virtual memmap region. To be patched at kernel boot */
-.globl slb_miss_kernel_load_vmemmap
-slb_miss_kernel_load_vmemmap:
- li r11,0
- b 6f
-1:
-#endif /* CONFIG_SPARSEMEM_VMEMMAP */
-
- /*
- * r10 contains the ESID, which is the original faulting EA shifted
- * right by 28 bits. We need to compare that with (H_VMALLOC_END >> 28)
- * which is 0xd00038000. That can't be used as an immediate, even if we
- * ignored the 0xd, so we have to load it into a register, and we only
- * have one register free. So we must load all of (H_VMALLOC_END >> 28)
- * into a register and compare ESID against that.
- */
- lis r11,(H_VMALLOC_END >> 32)@h // r11 = 0xffffffffd0000000
- ori r11,r11,(H_VMALLOC_END >> 32)@l // r11 = 0xffffffffd0003800
- // Rotate left 4, then mask with 0xffffffff0
- rldic r11,r11,4,28 // r11 = 0xd00038000
- cmpld r10,r11 // if r10 >= r11
- bge 5f // goto io_mapping
-
- /*
- * vmalloc mapping gets the encoding from the PACA as the mapping
- * can be demoted from 64K -> 4K dynamically on some machines.
- */
- lhz r11,PACAVMALLOCSLLP(r13)
- b 6f
-5:
- /* IO mapping */
-.globl slb_miss_kernel_load_io
-slb_miss_kernel_load_io:
- li r11,0
-6:
- /*
- * context = (ea >> 60) - (0xc - 1)
- * r9 = region id.
- */
- subi r9,r9,KERNEL_REGION_CONTEXT_OFFSET
-
-BEGIN_FTR_SECTION
- b .Lslb_finish_load
-END_MMU_FTR_SECTION_IFCLR(MMU_FTR_1T_SEGMENT)
- b .Lslb_finish_load_1T
-
-0: /*
- * For userspace addresses, make sure this is region 0.
- */
- cmpdi r9, 0
- bne- 8f
- /*
- * user space make sure we are within the allowed limit
- */
- ld r11,PACA_SLB_ADDR_LIMIT(r13)
- cmpld r3,r11
- bge- 8f
-
- /* when using slices, we extract the psize off the slice bitmaps
- * and then we need to get the sllp encoding off the mmu_psize_defs
- * array.
- *
- * XXX This is a bit inefficient especially for the normal case,
- * so we should try to implement a fast path for the standard page
- * size using the old sllp value so we avoid the array. We cannot
- * really do dynamic patching unfortunately as processes might flip
- * between 4k and 64k standard page size
- */
-#ifdef CONFIG_PPC_MM_SLICES
- /* r10 have esid */
- cmpldi r10,16
- /* below SLICE_LOW_TOP */
- blt 5f
- /*
- * Handle hpsizes,
- * r9 is get_paca()->context.high_slices_psize[index], r11 is mask_index
- */
- srdi r11,r10,(SLICE_HIGH_SHIFT - SLICE_LOW_SHIFT + 1) /* index */
- addi r9,r11,PACAHIGHSLICEPSIZE
- lbzx r9,r13,r9 /* r9 is hpsizes[r11] */
- /* r11 = (r10 >> (SLICE_HIGH_SHIFT - SLICE_LOW_SHIFT)) & 0x1 */
- rldicl r11,r10,(64 - (SLICE_HIGH_SHIFT - SLICE_LOW_SHIFT)),63
- b 6f
-
-5:
- /*
- * Handle lpsizes
- * r9 is get_paca()->context.low_slices_psize[index], r11 is mask_index
- */
- srdi r11,r10,1 /* index */
- addi r9,r11,PACALOWSLICESPSIZE
- lbzx r9,r13,r9 /* r9 is lpsizes[r11] */
- rldicl r11,r10,0,63 /* r11 = r10 & 0x1 */
-6:
- sldi r11,r11,2 /* index * 4 */
- /* Extract the psize and multiply to get an array offset */
- srd r9,r9,r11
- andi. r9,r9,0xf
- mulli r9,r9,MMUPSIZEDEFSIZE
-
- /* Now get to the array and obtain the sllp
- */
- ld r11,PACATOC(r13)
- ld r11,mmu_psize_defs@got(r11)
- add r11,r11,r9
- ld r11,MMUPSIZESLLP(r11)
- ori r11,r11,SLB_VSID_USER
-#else
- /* paca context sllp already contains the SLB_VSID_USER bits */
- lhz r11,PACACONTEXTSLLP(r13)
-#endif /* CONFIG_PPC_MM_SLICES */
-
- ld r9,PACACONTEXTID(r13)
-BEGIN_FTR_SECTION
- cmpldi r10,0x1000
- bge .Lslb_finish_load_1T
-END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
- b .Lslb_finish_load
-
-8: /* invalid EA - return an error indication */
- crset 4*cr0+eq /* indicate failure */
- blr
-
-/*
- * Finish loading of an SLB entry and return
- *
- * r3 = EA, r9 = context, r10 = ESID, r11 = flags, clobbers r9, cr7 = <> PAGE_OFFSET
- */
-.Lslb_finish_load:
- rldimi r10,r9,ESID_BITS,0
- ASM_VSID_SCRAMBLE(r10,r9,r11,256M)
- /* r3 = EA, r11 = VSID data */
- /*
- * Find a slot, round robin. Previously we tried to find a
- * free slot first but that took too long. Unfortunately we
- * dont have any LRU information to help us choose a slot.
- */
-
- mr r9,r3
-
- /* slb_finish_load_1T continues here. r9=EA with non-ESID bits clear */
-7: ld r10,PACASTABRR(r13)
- addi r10,r10,1
- /* This gets soft patched on boot. */
-.globl slb_compare_rr_to_size
-slb_compare_rr_to_size:
- cmpldi r10,0
-
- blt+ 4f
- li r10,SLB_NUM_BOLTED
-
-4:
- std r10,PACASTABRR(r13)
-
-3:
- rldimi r9,r10,0,36 /* r9 = EA[0:35] | entry */
- oris r10,r9,SLB_ESID_V@h /* r10 = r9 | SLB_ESID_V */
-
- /* r9 = ESID data, r11 = VSID data */
-
- /*
- * No need for an isync before or after this slbmte. The exception
- * we enter with and the rfid we exit with are context synchronizing.
- */
- slbmte r11,r10
-
- /* we're done for kernel addresses */
- crclr 4*cr0+eq /* set result to "success" */
- bgelr cr7
-
- /* Update the slb cache */
- lhz r9,PACASLBCACHEPTR(r13) /* offset = paca->slb_cache_ptr */
- cmpldi r9,SLB_CACHE_ENTRIES
- bge 1f
-
- /* still room in the slb cache */
- sldi r11,r9,2 /* r11 = offset * sizeof(u32) */
- srdi r10,r10,28 /* get the 36 bits of the ESID */
- add r11,r11,r13 /* r11 = (u32 *)paca + offset */
- stw r10,PACASLBCACHE(r11) /* paca->slb_cache[offset] = esid */
- addi r9,r9,1 /* offset++ */
- b 2f
-1: /* offset >= SLB_CACHE_ENTRIES */
- li r9,SLB_CACHE_ENTRIES+1
-2:
- sth r9,PACASLBCACHEPTR(r13) /* paca->slb_cache_ptr = offset */
- crclr 4*cr0+eq /* set result to "success" */
- blr
-
-/*
- * Finish loading of a 1T SLB entry (for the kernel linear mapping) and return.
- *
- * r3 = EA, r9 = context, r10 = ESID(256MB), r11 = flags, clobbers r9
- */
-.Lslb_finish_load_1T:
- srdi r10,r10,(SID_SHIFT_1T - SID_SHIFT) /* get 1T ESID */
- rldimi r10,r9,ESID_BITS_1T,0
- ASM_VSID_SCRAMBLE(r10,r9,r11,1T)
-
- li r10,MMU_SEGSIZE_1T
- rldimi r11,r10,SLB_VSID_SSIZE_SHIFT,0 /* insert segment size */
-
- /* r3 = EA, r11 = VSID data */
- clrrdi r9,r3,SID_SHIFT_1T /* clear out non-ESID bits */
- b 7b
-
-
-_ASM_NOKPROBE_SYMBOL(slb_allocate)
-_ASM_NOKPROBE_SYMBOL(slb_miss_kernel_load_linear)
-_ASM_NOKPROBE_SYMBOL(slb_miss_kernel_load_io)
-_ASM_NOKPROBE_SYMBOL(slb_compare_rr_to_size)
-#ifdef CONFIG_SPARSEMEM_VMEMMAP
-_ASM_NOKPROBE_SYMBOL(slb_miss_kernel_load_vmemmap)
-#endif
--
2.17.1
^ permalink raw reply related
* [PATCH 2/6] powerpc/64s/hash: Add SLB allocation status bitmaps
From: Michael Ellerman @ 2018-10-13 13:18 UTC (permalink / raw)
To: linuxppc-dev; +Cc: aneesh.kumar, npiggin
In-Reply-To: <20181013131836.26764-1-mpe@ellerman.id.au>
From: Nicholas Piggin <npiggin@gmail.com>
Add 32-entry bitmaps to track the allocation status of the first 32
SLB entries, and whether they are user or kernel entries. These are
used to allocate free SLB entries first, before resorting to the round
robin allocator.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
---
arch/powerpc/include/asm/paca.h | 6 ++-
arch/powerpc/kernel/asm-offsets.c | 2 +-
arch/powerpc/mm/slb.c | 64 ++++++++++++++++++++++++-------
arch/powerpc/xmon/xmon.c | 4 +-
4 files changed, 59 insertions(+), 17 deletions(-)
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index c6d01f0aa898..97e8a57a4998 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -113,7 +113,10 @@ struct paca_struct {
* on the linear mapping */
/* SLB related definitions */
u16 vmalloc_sllp;
- u16 slb_cache_ptr;
+ u8 slb_cache_ptr;
+ u8 stab_rr; /* stab/slb round-robin counter */
+ u32 slb_used_bitmap; /* Bitmaps for first 32 SLB entries. */
+ u32 slb_kern_bitmap;
u32 slb_cache[SLB_CACHE_ENTRIES];
#endif /* CONFIG_PPC_BOOK3S_64 */
@@ -160,7 +163,6 @@ struct paca_struct {
*/
struct task_struct *__current; /* Pointer to current */
u64 kstack; /* Saved Kernel stack addr */
- u64 stab_rr; /* stab/slb round-robin counter */
u64 saved_r1; /* r1 save for RTAS calls or PM or EE=0 */
u64 saved_msr; /* MSR saved here by enter_rtas */
u16 trap_save; /* Used when bad stack is encountered */
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index 92156c61d21c..10ef2e4db2fd 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -178,7 +178,6 @@ int main(void)
OFFSET(PACAKSAVE, paca_struct, kstack);
OFFSET(PACACURRENT, paca_struct, __current);
OFFSET(PACASAVEDMSR, paca_struct, saved_msr);
- OFFSET(PACASTABRR, paca_struct, stab_rr);
OFFSET(PACAR1, paca_struct, saved_r1);
OFFSET(PACATOC, paca_struct, kernel_toc);
OFFSET(PACAKBASE, paca_struct, kernelbase);
@@ -217,6 +216,7 @@ int main(void)
#ifdef CONFIG_PPC_BOOK3S_64
OFFSET(PACASLBCACHE, paca_struct, slb_cache);
OFFSET(PACASLBCACHEPTR, paca_struct, slb_cache_ptr);
+ OFFSET(PACASTABRR, paca_struct, stab_rr);
OFFSET(PACAVMALLOCSLLP, paca_struct, vmalloc_sllp);
#ifdef CONFIG_PPC_MM_SLICES
OFFSET(MMUPSIZESLLP, mmu_psize_def, sllp);
diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
index 76c1a9523049..ed61639fe4f4 100644
--- a/arch/powerpc/mm/slb.c
+++ b/arch/powerpc/mm/slb.c
@@ -122,6 +122,9 @@ void slb_restore_bolted_realmode(void)
{
__slb_restore_bolted_realmode();
get_paca()->slb_cache_ptr = 0;
+
+ get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1;
+ get_paca()->slb_used_bitmap = get_paca()->slb_kern_bitmap;
}
/*
@@ -129,9 +132,6 @@ void slb_restore_bolted_realmode(void)
*/
void slb_flush_all_realmode(void)
{
- /*
- * This flushes all SLB entries including 0, so it must be realmode.
- */
asm volatile("slbmte %0,%0; slbia" : : "r" (0));
}
@@ -177,6 +177,9 @@ void slb_flush_and_rebolt(void)
: "memory");
get_paca()->slb_cache_ptr = 0;
+
+ get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1;
+ get_paca()->slb_used_bitmap = get_paca()->slb_kern_bitmap;
}
void slb_save_contents(struct slb_entry *slb_ptr)
@@ -209,7 +212,7 @@ void slb_dump_contents(struct slb_entry *slb_ptr)
return;
pr_err("SLB contents of cpu 0x%x\n", smp_processor_id());
- pr_err("Last SLB entry inserted at slot %lld\n", get_paca()->stab_rr);
+ pr_err("Last SLB entry inserted at slot %d\n", get_paca()->stab_rr);
for (i = 0; i < mmu_slb_size; i++) {
e = slb_ptr->esid;
@@ -342,10 +345,13 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
"isync"
:: "r"(ksp_vsid_data),
"r"(ksp_esid_data));
+
+ get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1;
}
get_paca()->slb_cache_ptr = 0;
}
+ get_paca()->slb_used_bitmap = get_paca()->slb_kern_bitmap;
copy_mm_to_paca(mm);
@@ -402,6 +408,8 @@ void slb_initialize(void)
}
get_paca()->stab_rr = SLB_NUM_BOLTED - 1;
+ get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1;
+ get_paca()->slb_used_bitmap = get_paca()->slb_kern_bitmap;
lflags = SLB_VSID_KERNEL | linear_llp;
@@ -453,17 +461,47 @@ static void slb_cache_update(unsigned long esid_data)
}
}
-static enum slb_index alloc_slb_index(void)
+static enum slb_index alloc_slb_index(bool kernel)
{
enum slb_index index;
- /* round-robin replacement of slb starting at SLB_NUM_BOLTED. */
- index = get_paca()->stab_rr;
- if (index < (mmu_slb_size - 1))
- index++;
- else
- index = SLB_NUM_BOLTED;
- get_paca()->stab_rr = index;
+ /*
+ * The allocation bitmaps can become out of synch with the SLB
+ * when the _switch code does slbie when bolting a new stack
+ * segment and it must not be anywhere else in the SLB. This leaves
+ * a kernel allocated entry that is unused in the SLB. With very
+ * large systems or small segment sizes, the bitmaps could slowly
+ * fill with these entries. They will eventually be cleared out
+ * by the round robin allocator in that case, so it's probably not
+ * worth accounting for.
+ */
+
+ /*
+ * SLBs beyond 32 entries are allocated with stab_rr only
+ * POWER7/8/9 have 32 SLB entries, this could be expanded if a
+ * future CPU has more.
+ */
+ if (local_paca->slb_used_bitmap != U32_MAX) {
+ index = ffz(local_paca->slb_used_bitmap);
+ local_paca->slb_used_bitmap |= 1U << index;
+ if (kernel)
+ local_paca->slb_kern_bitmap |= 1U << index;
+ } else {
+ /* round-robin replacement of slb starting at SLB_NUM_BOLTED. */
+ index = local_paca->stab_rr;
+ if (index < (mmu_slb_size - 1))
+ index++;
+ else
+ index = SLB_NUM_BOLTED;
+ local_paca->stab_rr = index;
+ if (index < 32) {
+ if (kernel)
+ local_paca->slb_kern_bitmap |= 1U << index;
+ else
+ local_paca->slb_kern_bitmap &= ~(1U << index);
+ }
+ }
+ BUG_ON(index < SLB_NUM_BOLTED);
return index;
}
@@ -490,7 +528,7 @@ static long slb_insert_entry(unsigned long ea, unsigned long context,
*/
barrier();
- index = alloc_slb_index();
+ index = alloc_slb_index(kernel);
vsid_data = __mk_vsid_data(vsid, ssize, flags);
esid_data = mk_esid_data(ea, ssize, index);
diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c
index 8345defa0e43..58e67b67a97c 100644
--- a/arch/powerpc/xmon/xmon.c
+++ b/arch/powerpc/xmon/xmon.c
@@ -2394,7 +2394,9 @@ static void dump_one_paca(int cpu)
}
}
DUMP(p, vmalloc_sllp, "%#-*x");
- DUMP(p, stab_rr, "%#-*llx");
+ DUMP(p, stab_rr, "%#-*x");
+ DUMP(p, slb_used_bitmap, "%#-*x");
+ DUMP(p, slb_kern_bitmap, "%#-*x");
if (!early_cpu_has_feature(CPU_FTR_ARCH_300)) {
DUMP(p, slb_cache_ptr, "%#-*x");
--
2.17.1
^ permalink raw reply related
* [PATCH 3/6] powerpc/64s/hash: Provide arch_setup_exec() hooks for hash slice setup
From: Michael Ellerman @ 2018-10-13 13:18 UTC (permalink / raw)
To: linuxppc-dev; +Cc: aneesh.kumar, npiggin
In-Reply-To: <20181013131836.26764-1-mpe@ellerman.id.au>
From: Nicholas Piggin <npiggin@gmail.com>
This will be used by the SLB code in the next patch, but for now this
sets the slb_addr_limit to the correct size for 32-bit tasks.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
---
arch/powerpc/include/asm/book3s/64/mmu-hash.h | 2 ++
arch/powerpc/include/asm/slice.h | 1 +
arch/powerpc/include/asm/thread_info.h | 6 ++++++
arch/powerpc/kernel/process.c | 9 +++++++++
arch/powerpc/mm/mmu_context_book3s64.c | 5 +++++
arch/powerpc/mm/slice.c | 14 ++++++++++++++
6 files changed, 37 insertions(+)
diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
index e0e4ce8f77d6..14e552ea5e52 100644
--- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
@@ -487,6 +487,8 @@ int htab_remove_mapping(unsigned long vstart, unsigned long vend,
extern void pseries_add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages);
extern void demote_segment_4k(struct mm_struct *mm, unsigned long addr);
+extern void hash__setup_new_exec(void);
+
#ifdef CONFIG_PPC_PSERIES
void hpte_init_pseries(void);
#else
diff --git a/arch/powerpc/include/asm/slice.h b/arch/powerpc/include/asm/slice.h
index e40406cf5628..a595461c9cb0 100644
--- a/arch/powerpc/include/asm/slice.h
+++ b/arch/powerpc/include/asm/slice.h
@@ -32,6 +32,7 @@ void slice_set_range_psize(struct mm_struct *mm, unsigned long start,
unsigned long len, unsigned int psize);
void slice_init_new_context_exec(struct mm_struct *mm);
+void slice_setup_new_exec(void);
#endif /* __ASSEMBLY__ */
diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h
index 3185f8ac1182..916a3d67b592 100644
--- a/arch/powerpc/include/asm/thread_info.h
+++ b/arch/powerpc/include/asm/thread_info.h
@@ -72,6 +72,12 @@ static inline struct thread_info *current_thread_info(void)
}
extern int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src);
+
+#ifdef CONFIG_PPC_BOOK3S_64
+void arch_setup_new_exec(void);
+#define arch_setup_new_exec arch_setup_new_exec
+#endif
+
#endif /* __ASSEMBLY__ */
/*
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index f9d1cca28cce..96cd9cd1a119 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -1490,6 +1490,15 @@ void flush_thread(void)
#endif /* CONFIG_HAVE_HW_BREAKPOINT */
}
+#ifdef CONFIG_PPC_BOOK3S_64
+void arch_setup_new_exec(void)
+{
+ if (radix_enabled())
+ return;
+ hash__setup_new_exec();
+}
+#endif
+
int set_thread_uses_vas(void)
{
#ifdef CONFIG_PPC_BOOK3S_64
diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c
index dbd8f762140b..f7352c66b6b8 100644
--- a/arch/powerpc/mm/mmu_context_book3s64.c
+++ b/arch/powerpc/mm/mmu_context_book3s64.c
@@ -84,6 +84,11 @@ static int hash__init_new_context(struct mm_struct *mm)
return index;
}
+void hash__setup_new_exec(void)
+{
+ slice_setup_new_exec();
+}
+
static int radix__init_new_context(struct mm_struct *mm)
{
unsigned long rts_field;
diff --git a/arch/powerpc/mm/slice.c b/arch/powerpc/mm/slice.c
index 205fe557ca10..546dd07c8083 100644
--- a/arch/powerpc/mm/slice.c
+++ b/arch/powerpc/mm/slice.c
@@ -757,6 +757,20 @@ void slice_init_new_context_exec(struct mm_struct *mm)
bitmap_fill(mask->high_slices, SLICE_NUM_HIGH);
}
+#ifdef CONFIG_PPC_BOOK3S_64
+void slice_setup_new_exec(void)
+{
+ struct mm_struct *mm = current->mm;
+
+ slice_dbg("slice_setup_new_exec(mm=%p)\n", mm);
+
+ if (!is_32bit_task())
+ return;
+
+ mm->context.slb_addr_limit = DEFAULT_MAP_WINDOW;
+}
+#endif
+
void slice_set_range_psize(struct mm_struct *mm, unsigned long start,
unsigned long len, unsigned int psize)
{
--
2.17.1
^ permalink raw reply related
* [PATCH 4/6] powerpc/64s/hash: Add a SLB preload cache
From: Michael Ellerman @ 2018-10-13 13:18 UTC (permalink / raw)
To: linuxppc-dev; +Cc: aneesh.kumar, npiggin
In-Reply-To: <20181013131836.26764-1-mpe@ellerman.id.au>
From: Nicholas Piggin <npiggin@gmail.com>
When switching processes, currently all user SLBEs are cleared, and a
few (exec_base, pc, and stack) are preloaded. In trivial testing with
small apps, this tends to miss the heap and low 256MB segments, and it
will also miss commonly accessed segments on large memory workloads.
Add a simple round-robin preload cache that just inserts the last SLB
miss into the head of the cache and preloads those at context switch
time. Every 256 context switches, the oldest entry is removed from the
cache to shrink the cache and require fewer slbmte if they are unused.
Much more could go into this, including into the SLB entry reclaim
side to track some LRU information etc, which would require a study of
large memory workloads. But this is a simple thing we can do now that
is an obvious win for common workloads.
With the full series, process switching speed on the context_switch
benchmark on POWER9/hash (with kernel speculation security masures
disabled) increases from 140K/s to 178K/s (27%).
POWER8 does not change much (within 1%), it's unclear why it does not
see a big gain like POWER9.
Booting to busybox init with 256MB segments has SLB misses go down
from 945 to 69, and with 1T segments 900 to 21. These could almost all
be eliminated by preloading a bit more carefully with ELF binary
loading.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
---
arch/powerpc/include/asm/processor.h | 1 +
arch/powerpc/include/asm/thread_info.h | 5 +
arch/powerpc/kernel/process.c | 7 +
arch/powerpc/mm/mmu_context_book3s64.c | 4 +
arch/powerpc/mm/slb.c | 208 +++++++++++++++++++------
5 files changed, 181 insertions(+), 44 deletions(-)
diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index 3fefb8a65b17..7d04d60a39c9 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -273,6 +273,7 @@ struct thread_struct {
#endif /* CONFIG_HAVE_HW_BREAKPOINT */
struct arch_hw_breakpoint hw_brk; /* info on the hardware breakpoint */
unsigned long trap_nr; /* last trap # on this thread */
+ u8 load_slb; /* Ages out SLB preload cache entries */
u8 load_fp;
#ifdef CONFIG_ALTIVEC
u8 load_vec;
diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h
index 916a3d67b592..544cac0474cb 100644
--- a/arch/powerpc/include/asm/thread_info.h
+++ b/arch/powerpc/include/asm/thread_info.h
@@ -29,6 +29,7 @@
#include <asm/page.h>
#include <asm/accounting.h>
+#define SLB_PRELOAD_NR 16U
/*
* low level task data.
*/
@@ -44,6 +45,10 @@ struct thread_info {
#if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC32)
struct cpu_accounting_data accounting;
#endif
+ unsigned char slb_preload_nr;
+ unsigned char slb_preload_tail;
+ u32 slb_preload_esid[SLB_PRELOAD_NR];
+
/* low level flags - has atomic operations done on it */
unsigned long flags ____cacheline_aligned_in_smp;
};
diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 96cd9cd1a119..7ad304a3cc7d 100644
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -1727,6 +1727,8 @@ int copy_thread(unsigned long clone_flags, unsigned long usp,
return 0;
}
+void preload_new_slb_context(unsigned long start, unsigned long sp);
+
/*
* Set up a thread for executing a new program
*/
@@ -1734,6 +1736,10 @@ void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp)
{
#ifdef CONFIG_PPC64
unsigned long load_addr = regs->gpr[2]; /* saved by ELF_PLAT_INIT */
+
+#ifdef CONFIG_PPC_BOOK3S_64
+ preload_new_slb_context(start, sp);
+#endif
#endif
/*
@@ -1824,6 +1830,7 @@ void start_thread(struct pt_regs *regs, unsigned long start, unsigned long sp)
#ifdef CONFIG_VSX
current->thread.used_vsr = 0;
#endif
+ current->thread.load_slb = 0;
current->thread.load_fp = 0;
memset(¤t->thread.fp_state, 0, sizeof(current->thread.fp_state));
current->thread.fp_save_area = NULL;
diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c
index f7352c66b6b8..510f103d7813 100644
--- a/arch/powerpc/mm/mmu_context_book3s64.c
+++ b/arch/powerpc/mm/mmu_context_book3s64.c
@@ -53,6 +53,8 @@ int hash__alloc_context_id(void)
}
EXPORT_SYMBOL_GPL(hash__alloc_context_id);
+void slb_setup_new_exec(void);
+
static int hash__init_new_context(struct mm_struct *mm)
{
int index;
@@ -87,6 +89,8 @@ static int hash__init_new_context(struct mm_struct *mm)
void hash__setup_new_exec(void)
{
slice_setup_new_exec();
+
+ slb_setup_new_exec();
}
static int radix__init_new_context(struct mm_struct *mm)
diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
index ed61639fe4f4..3b7d8af09724 100644
--- a/arch/powerpc/mm/slb.c
+++ b/arch/powerpc/mm/slb.c
@@ -257,41 +257,148 @@ void slb_vmalloc_update(void)
slb_flush_and_rebolt();
}
-/* Helper function to compare esids. There are four cases to handle.
- * 1. The system is not 1T segment size capable. Use the GET_ESID compare.
- * 2. The system is 1T capable, both addresses are < 1T, use the GET_ESID compare.
- * 3. The system is 1T capable, only one of the two addresses is > 1T. This is not a match.
- * 4. The system is 1T capable, both addresses are > 1T, use the GET_ESID_1T macro to compare.
- */
-static inline int esids_match(unsigned long addr1, unsigned long addr2)
+static bool preload_hit(struct thread_info *ti, unsigned long esid)
{
- int esid_1t_count;
+ unsigned char i;
- /* System is not 1T segment size capable. */
- if (!mmu_has_feature(MMU_FTR_1T_SEGMENT))
- return (GET_ESID(addr1) == GET_ESID(addr2));
+ for (i = 0; i < ti->slb_preload_nr; i++) {
+ unsigned char idx;
+
+ idx = (ti->slb_preload_tail + i) % SLB_PRELOAD_NR;
+ if (esid == ti->slb_preload_esid[idx])
+ return true;
+ }
+ return false;
+}
+
+static bool preload_add(struct thread_info *ti, unsigned long ea)
+{
+ unsigned char idx;
+ unsigned long esid;
+
+ if (mmu_has_feature(MMU_FTR_1T_SEGMENT)) {
+ /* EAs are stored >> 28 so 256MB segments don't need clearing */
+ if (ea & ESID_MASK_1T)
+ ea &= ESID_MASK_1T;
+ }
- esid_1t_count = (((addr1 >> SID_SHIFT_1T) != 0) +
- ((addr2 >> SID_SHIFT_1T) != 0));
+ esid = ea >> SID_SHIFT;
- /* both addresses are < 1T */
- if (esid_1t_count == 0)
- return (GET_ESID(addr1) == GET_ESID(addr2));
+ if (preload_hit(ti, esid))
+ return false;
- /* One address < 1T, the other > 1T. Not a match */
- if (esid_1t_count == 1)
- return 0;
+ idx = (ti->slb_preload_tail + ti->slb_preload_nr) % SLB_PRELOAD_NR;
+ ti->slb_preload_esid[idx] = esid;
+ if (ti->slb_preload_nr == SLB_PRELOAD_NR)
+ ti->slb_preload_tail = (ti->slb_preload_tail + 1) % SLB_PRELOAD_NR;
+ else
+ ti->slb_preload_nr++;
- /* Both addresses are > 1T. */
- return (GET_ESID_1T(addr1) == GET_ESID_1T(addr2));
+ return true;
}
+static void preload_age(struct thread_info *ti)
+{
+ if (!ti->slb_preload_nr)
+ return;
+ ti->slb_preload_nr--;
+ ti->slb_preload_tail = (ti->slb_preload_tail + 1) % SLB_PRELOAD_NR;
+}
+
+void slb_setup_new_exec(void)
+{
+ struct thread_info *ti = current_thread_info();
+ struct mm_struct *mm = current->mm;
+ unsigned long exec = 0x10000000;
+
+ WARN_ON(irqs_disabled());
+
+ /*
+ * preload cache can only be used to determine whether a SLB
+ * entry exists if it does not start to overflow.
+ */
+ if (ti->slb_preload_nr + 2 > SLB_PRELOAD_NR)
+ return;
+
+ hard_irq_disable();
+
+ /*
+ * We have no good place to clear the slb preload cache on exec,
+ * flush_thread is about the earliest arch hook but that happens
+ * after we switch to the mm and have aleady preloaded the SLBEs.
+ *
+ * For the most part that's probably okay to use entries from the
+ * previous exec, they will age out if unused. It may turn out to
+ * be an advantage to clear the cache before switching to it,
+ * however.
+ */
+
+ /*
+ * preload some userspace segments into the SLB.
+ * Almost all 32 and 64bit PowerPC executables are linked at
+ * 0x10000000 so it makes sense to preload this segment.
+ */
+ if (!is_kernel_addr(exec)) {
+ if (preload_add(ti, exec))
+ slb_allocate_user(mm, exec);
+ }
+
+ /* Libraries and mmaps. */
+ if (!is_kernel_addr(mm->mmap_base)) {
+ if (preload_add(ti, mm->mmap_base))
+ slb_allocate_user(mm, mm->mmap_base);
+ }
+
+ /* see switch_slb */
+ asm volatile("isync" : : : "memory");
+
+ local_irq_enable();
+}
+
+void preload_new_slb_context(unsigned long start, unsigned long sp)
+{
+ struct thread_info *ti = current_thread_info();
+ struct mm_struct *mm = current->mm;
+ unsigned long heap = mm->start_brk;
+
+ WARN_ON(irqs_disabled());
+
+ /* see above */
+ if (ti->slb_preload_nr + 3 > SLB_PRELOAD_NR)
+ return;
+
+ hard_irq_disable();
+
+ /* Userspace entry address. */
+ if (!is_kernel_addr(start)) {
+ if (preload_add(ti, start))
+ slb_allocate_user(mm, start);
+ }
+
+ /* Top of stack, grows down. */
+ if (!is_kernel_addr(sp)) {
+ if (preload_add(ti, sp))
+ slb_allocate_user(mm, sp);
+ }
+
+ /* Bottom of heap, grows up. */
+ if (heap && !is_kernel_addr(heap)) {
+ if (preload_add(ti, heap))
+ slb_allocate_user(mm, heap);
+ }
+
+ /* see switch_slb */
+ asm volatile("isync" : : : "memory");
+
+ local_irq_enable();
+}
+
+
/* Flush all user entries from the segment table of the current processor. */
void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
{
- unsigned long pc = KSTK_EIP(tsk);
- unsigned long stack = KSTK_ESP(tsk);
- unsigned long exec_base;
+ struct thread_info *ti = task_thread_info(tsk);
+ unsigned char i;
/*
* We need interrupts hard-disabled here, not just soft-disabled,
@@ -300,6 +407,7 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
* which would update the slb_cache/slb_cache_ptr fields in the PACA.
*/
hard_irq_disable();
+ asm volatile("isync" : : : "memory");
if (cpu_has_feature(CPU_FTR_ARCH_300)) {
/*
* SLBIA IH=3 invalidates all Class=1 SLBEs and their
@@ -307,16 +415,14 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
* switch_slb wants. So ARCH_300 does not use the slb
* cache.
*/
- asm volatile("isync ; " PPC_SLBIA(3)" ; isync");
+ asm volatile(PPC_SLBIA(3));
} else {
unsigned long offset = get_paca()->slb_cache_ptr;
if (!mmu_has_feature(MMU_FTR_NO_SLBIE_B) &&
offset <= SLB_CACHE_ENTRIES) {
unsigned long slbie_data = 0;
- int i;
- asm volatile("isync" : : : "memory");
for (i = 0; i < offset; i++) {
/* EA */
slbie_data = (unsigned long)
@@ -331,7 +437,6 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
if (!cpu_has_feature(CPU_FTR_ARCH_207S) && offset == 1)
asm volatile("slbie %0" : : "r" (slbie_data));
- asm volatile("isync" : : : "memory");
} else {
struct slb_shadow *p = get_slb_shadow();
unsigned long ksp_esid_data =
@@ -339,8 +444,7 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
unsigned long ksp_vsid_data =
be64_to_cpu(p->save_area[KSTACK_INDEX].vsid);
- asm volatile("isync\n"
- PPC_SLBIA(1) "\n"
+ asm volatile(PPC_SLBIA(1) "\n"
"slbmte %0,%1\n"
"isync"
:: "r"(ksp_vsid_data),
@@ -356,24 +460,35 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
copy_mm_to_paca(mm);
/*
- * preload some userspace segments into the SLB.
- * Almost all 32 and 64bit PowerPC executables are linked at
- * 0x10000000 so it makes sense to preload this segment.
+ * We gradually age out SLBs after a number of context switches to
+ * reduce reload overhead of unused entries (like we do with FP/VEC
+ * reload). Each time we wrap 256 switches, take an entry out of the
+ * SLB preload cache.
*/
- exec_base = 0x10000000;
+ tsk->thread.load_slb++;
+ if (!tsk->thread.load_slb) {
+ unsigned long pc = KSTK_EIP(tsk);
- if (is_kernel_addr(pc) || is_kernel_addr(stack) ||
- is_kernel_addr(exec_base))
- return;
+ preload_age(ti);
+ preload_add(ti, pc);
+ }
+
+ for (i = 0; i < ti->slb_preload_nr; i++) {
+ unsigned char idx;
+ unsigned long ea;
- slb_allocate_user(mm, pc);
+ idx = (ti->slb_preload_tail + i) % SLB_PRELOAD_NR;
+ ea = (unsigned long)ti->slb_preload_esid[idx] << SID_SHIFT;
- if (!esids_match(pc, stack))
- slb_allocate_user(mm, stack);
+ slb_allocate_user(mm, ea);
+ }
- if (!esids_match(pc, exec_base) &&
- !esids_match(stack, exec_base))
- slb_allocate_user(mm, exec_base);
+ /*
+ * Synchronize slbmte preloads with possible subsequent user memory
+ * address accesses by the kernel (user mode won't happen until
+ * rfid, which is safe).
+ */
+ asm volatile("isync" : : : "memory");
}
void slb_set_size(u16 size)
@@ -642,11 +757,16 @@ long do_slb_fault(struct pt_regs *regs, unsigned long ea)
return slb_allocate_kernel(ea, id);
} else {
struct mm_struct *mm = current->mm;
+ long err;
if (unlikely(!mm))
return -EFAULT;
- return slb_allocate_user(mm, ea);
+ err = slb_allocate_user(mm, ea);
+ if (!err)
+ preload_add(current_thread_info(), ea);
+
+ return err;
}
}
--
2.17.1
^ permalink raw reply related
* [PATCH 5/6] powerpc/64s/hash: Simplify slb_flush_and_rebolt()
From: Michael Ellerman @ 2018-10-13 13:18 UTC (permalink / raw)
To: linuxppc-dev; +Cc: aneesh.kumar, npiggin
In-Reply-To: <20181013131836.26764-1-mpe@ellerman.id.au>
From: Nicholas Piggin <npiggin@gmail.com>
slb_flush_and_rebolt() is misleading, it is called in virtual mode, so
it can not possibly change the stack, so it should not be touching the
shadow area. And since vmalloc is no longer bolted, it should not
change any bolted mappings at all.
Change the name to slb_flush_and_restore_bolted(), and have it just
load the kernel stack from what's currently in the shadow SLB area.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
---
arch/powerpc/include/asm/book3s/64/mmu-hash.h | 2 +-
arch/powerpc/kernel/swsusp_asm64.S | 2 +-
arch/powerpc/mm/hash_utils_64.c | 4 +-
arch/powerpc/mm/slb.c | 46 +++++++------------
arch/powerpc/mm/slice.c | 2 +-
5 files changed, 21 insertions(+), 35 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
index 14e552ea5e52..60cda8fb0677 100644
--- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
@@ -503,7 +503,7 @@ struct slb_entry {
};
extern void slb_initialize(void);
-extern void slb_flush_and_rebolt(void);
+void slb_flush_and_restore_bolted(void);
void slb_flush_all_realmode(void);
void __slb_restore_bolted_realmode(void);
void slb_restore_bolted_realmode(void);
diff --git a/arch/powerpc/kernel/swsusp_asm64.S b/arch/powerpc/kernel/swsusp_asm64.S
index f83bf6f72cb0..185216becb8b 100644
--- a/arch/powerpc/kernel/swsusp_asm64.S
+++ b/arch/powerpc/kernel/swsusp_asm64.S
@@ -262,7 +262,7 @@ END_FW_FTR_SECTION_IFCLR(FW_FEATURE_LPAR)
addi r1,r1,-128
#ifdef CONFIG_PPC_BOOK3S_64
- bl slb_flush_and_rebolt
+ bl slb_flush_and_restore_bolted
#endif
bl do_after_copyback
addi r1,r1,128
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 854edc3722e0..0cc7fbc3bd1c 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -1125,7 +1125,7 @@ void demote_segment_4k(struct mm_struct *mm, unsigned long addr)
if ((get_paca_psize(addr) != MMU_PAGE_4K) && (current->mm == mm)) {
copy_mm_to_paca(mm);
- slb_flush_and_rebolt();
+ slb_flush_and_restore_bolted();
}
}
#endif /* CONFIG_PPC_64K_PAGES */
@@ -1197,7 +1197,7 @@ static void check_paca_psize(unsigned long ea, struct mm_struct *mm,
if (user_region) {
if (psize != get_paca_psize(ea)) {
copy_mm_to_paca(mm);
- slb_flush_and_rebolt();
+ slb_flush_and_restore_bolted();
}
} else if (get_paca()->vmalloc_sllp !=
mmu_psize_defs[mmu_vmalloc_psize].sllp) {
diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
index 3b7d8af09724..d8d9c9bd15d3 100644
--- a/arch/powerpc/mm/slb.c
+++ b/arch/powerpc/mm/slb.c
@@ -115,8 +115,6 @@ void __slb_restore_bolted_realmode(void)
/*
* Insert the bolted entries into an empty SLB.
- * This is not the same as rebolt because the bolted segments are not
- * changed, just loaded from the shadow area.
*/
void slb_restore_bolted_realmode(void)
{
@@ -135,12 +133,15 @@ void slb_flush_all_realmode(void)
asm volatile("slbmte %0,%0; slbia" : : "r" (0));
}
-void slb_flush_and_rebolt(void)
+/*
+ * This flushes non-bolted entries, it can be run in virtual mode. Must
+ * be called with interrupts disabled.
+ */
+void slb_flush_and_restore_bolted(void)
{
- /* If you change this make sure you change SLB_NUM_BOLTED
- * and PR KVM appropriately too. */
- unsigned long linear_llp, lflags;
- unsigned long ksp_esid_data, ksp_vsid_data;
+ struct slb_shadow *p = get_slb_shadow();
+
+ BUILD_BUG_ON(SLB_NUM_BOLTED != 2);
WARN_ON(!irqs_disabled());
@@ -150,30 +151,12 @@ void slb_flush_and_rebolt(void)
*/
hard_irq_disable();
- linear_llp = mmu_psize_defs[mmu_linear_psize].sllp;
- lflags = SLB_VSID_KERNEL | linear_llp;
-
- ksp_esid_data = mk_esid_data(get_paca()->kstack, mmu_kernel_ssize, KSTACK_INDEX);
- if ((ksp_esid_data & ~0xfffffffUL) <= PAGE_OFFSET) {
- ksp_esid_data &= ~SLB_ESID_V;
- ksp_vsid_data = 0;
- slb_shadow_clear(KSTACK_INDEX);
- } else {
- /* Update stack entry; others don't change */
- slb_shadow_update(get_paca()->kstack, mmu_kernel_ssize, lflags, KSTACK_INDEX);
- ksp_vsid_data =
- be64_to_cpu(get_slb_shadow()->save_area[KSTACK_INDEX].vsid);
- }
-
- /* We need to do this all in asm, so we're sure we don't touch
- * the stack between the slbia and rebolting it. */
asm volatile("isync\n"
"slbia\n"
- /* Slot 1 - kernel stack */
- "slbmte %0,%1\n"
- "isync"
- :: "r"(ksp_vsid_data),
- "r"(ksp_esid_data)
+ "slbmte %0, %1\n"
+ "isync\n"
+ :: "r" (be64_to_cpu(p->save_area[KSTACK_INDEX].vsid)),
+ "r" (be64_to_cpu(p->save_area[KSTACK_INDEX].esid))
: "memory");
get_paca()->slb_cache_ptr = 0;
@@ -254,7 +237,10 @@ void slb_dump_contents(struct slb_entry *slb_ptr)
void slb_vmalloc_update(void)
{
- slb_flush_and_rebolt();
+ /*
+ * vmalloc is not bolted, so just have to flush non-bolted.
+ */
+ slb_flush_and_restore_bolted();
}
static bool preload_hit(struct thread_info *ti, unsigned long esid)
diff --git a/arch/powerpc/mm/slice.c b/arch/powerpc/mm/slice.c
index 546dd07c8083..50ba3d0456a5 100644
--- a/arch/powerpc/mm/slice.c
+++ b/arch/powerpc/mm/slice.c
@@ -219,7 +219,7 @@ static void slice_flush_segments(void *parm)
copy_mm_to_paca(current->active_mm);
local_irq_save(flags);
- slb_flush_and_rebolt();
+ slb_flush_and_restore_bolted();
local_irq_restore(flags);
#endif
}
--
2.17.1
^ permalink raw reply related
* [PATCH 6/6] powerpc/64s/hash: Add some SLB debugging tests
From: Michael Ellerman @ 2018-10-13 13:18 UTC (permalink / raw)
To: linuxppc-dev; +Cc: aneesh.kumar, npiggin
In-Reply-To: <20181013131836.26764-1-mpe@ellerman.id.au>
From: Nicholas Piggin <npiggin@gmail.com>
This adds CONFIG_DEBUG_VM checks to ensure:
- The kernel stack is in the SLB after it's flushed and bolted.
- We don't insert an SLB for an address that is aleady in the SLB.
- The kernel SLB miss handler does not take an SLB miss.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
---
arch/powerpc/include/asm/paca.h | 3 ++
arch/powerpc/mm/slb.c | 53 +++++++++++++++++++++++++++++++--
2 files changed, 53 insertions(+), 3 deletions(-)
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 97e8a57a4998..e843bc5d1a0f 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -115,6 +115,9 @@ struct paca_struct {
u16 vmalloc_sllp;
u8 slb_cache_ptr;
u8 stab_rr; /* stab/slb round-robin counter */
+#ifdef CONFIG_DEBUG_VM
+ u8 in_kernel_slb_handler;
+#endif
u32 slb_used_bitmap; /* Bitmaps for first 32 SLB entries. */
u32 slb_kern_bitmap;
u32 slb_cache[SLB_CACHE_ENTRIES];
diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c
index d8d9c9bd15d3..4b6e7a21a7c5 100644
--- a/arch/powerpc/mm/slb.c
+++ b/arch/powerpc/mm/slb.c
@@ -58,6 +58,30 @@ static inline unsigned long mk_vsid_data(unsigned long ea, int ssize,
return __mk_vsid_data(get_kernel_vsid(ea, ssize), ssize, flags);
}
+static void assert_slb_exists(unsigned long ea)
+{
+#ifdef CONFIG_DEBUG_VM
+ unsigned long tmp;
+
+ WARN_ON_ONCE(mfmsr() & MSR_EE);
+
+ asm volatile("slbfee. %0, %1" : "=r"(tmp) : "r"(ea) : "cr0");
+ WARN_ON(tmp == 0);
+#endif
+}
+
+static void assert_slb_notexists(unsigned long ea)
+{
+#ifdef CONFIG_DEBUG_VM
+ unsigned long tmp;
+
+ WARN_ON_ONCE(mfmsr() & MSR_EE);
+
+ asm volatile("slbfee. %0, %1" : "=r"(tmp) : "r"(ea) : "cr0");
+ WARN_ON(tmp != 0);
+#endif
+}
+
static inline void slb_shadow_update(unsigned long ea, int ssize,
unsigned long flags,
enum slb_index index)
@@ -90,6 +114,7 @@ static inline void create_shadowed_slbe(unsigned long ea, int ssize,
*/
slb_shadow_update(ea, ssize, flags, index);
+ assert_slb_notexists(ea);
asm volatile("slbmte %0,%1" :
: "r" (mk_vsid_data(ea, ssize, flags)),
"r" (mk_esid_data(ea, ssize, index))
@@ -111,6 +136,8 @@ void __slb_restore_bolted_realmode(void)
: "r" (be64_to_cpu(p->save_area[index].vsid)),
"r" (be64_to_cpu(p->save_area[index].esid)));
}
+
+ assert_slb_exists(local_paca->kstack);
}
/*
@@ -158,6 +185,7 @@ void slb_flush_and_restore_bolted(void)
:: "r" (be64_to_cpu(p->save_area[KSTACK_INDEX].vsid)),
"r" (be64_to_cpu(p->save_area[KSTACK_INDEX].esid))
: "memory");
+ assert_slb_exists(get_paca()->kstack);
get_paca()->slb_cache_ptr = 0;
@@ -410,9 +438,17 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm)
unsigned long slbie_data = 0;
for (i = 0; i < offset; i++) {
- /* EA */
- slbie_data = (unsigned long)
+ unsigned long ea;
+
+ ea = (unsigned long)
get_paca()->slb_cache[i] << SID_SHIFT;
+ /*
+ * Could assert_slb_exists here, but hypervisor
+ * or machine check could have come in and
+ * removed the entry at this point.
+ */
+
+ slbie_data = ea;
slbie_data |= user_segment_size(slbie_data)
<< SLBIE_SSIZE_SHIFT;
slbie_data |= SLBIE_C; /* user slbs have C=1 */
@@ -640,6 +676,7 @@ static long slb_insert_entry(unsigned long ea, unsigned long context,
* User preloads should add isync afterwards in case the kernel
* accesses user memory before it returns to userspace with rfid.
*/
+ assert_slb_notexists(ea);
asm volatile("slbmte %0, %1" : : "r" (vsid_data), "r" (esid_data));
barrier();
@@ -740,7 +777,17 @@ long do_slb_fault(struct pt_regs *regs, unsigned long ea)
* if they go via fast_exception_return too.
*/
if (id >= KERNEL_REGION_ID) {
- return slb_allocate_kernel(ea, id);
+ long err;
+#ifdef CONFIG_DEBUG_VM
+ /* Catch recursive kernel SLB faults. */
+ BUG_ON(local_paca->in_kernel_slb_handler);
+ local_paca->in_kernel_slb_handler = 1;
+#endif
+ err = slb_allocate_kernel(ea, id);
+#ifdef CONFIG_DEBUG_VM
+ local_paca->in_kernel_slb_handler = 0;
+#endif
+ return err;
} else {
struct mm_struct *mm = current->mm;
long err;
--
2.17.1
^ permalink raw reply related
* [PATCH 4/8] pci: consolidate PCI config entry in drivers/pci
From: Christoph Hellwig @ 2018-10-13 15:10 UTC (permalink / raw)
To: Masahiro Yamada
Cc: linux-arch, linux-scsi, linux-kbuild, linux-pci, linux-kernel,
Dominik Brodowski, Alexandre Bounine, linuxppc-dev
In-Reply-To: <20181013151016.31674-1-hch@lst.de>
There is no good reason to duplicate the PCI menu in every architecture.
Instead provide a selectable HAS_PCI symbol that indicates availability
of PCI support and the handle the rest in drivers/pci.
Note that for powerpc we now select HAS_PCI globally instead of the
convoluted mess of conditional or or non-conditional support per board,
similar to what we do e.g. on x86. For alpha PCI is selected for the
non-jensen configs as it was the default before, and a lot of code does
not compile without PCI enabled. On other architectures with limited
PCI support that wasn't as complicated I've left the selection as-is.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
arch/alpha/Kconfig | 15 ++-------
arch/arc/Kconfig | 20 ------------
arch/arc/plat-axs10x/Kconfig | 2 +-
arch/arc/plat-hsdk/Kconfig | 2 +-
arch/arm/Kconfig | 19 ++----------
arch/arm/mach-ks8695/Kconfig | 10 +++---
arch/arm/mach-pxa/Kconfig | 2 +-
arch/arm64/Kconfig | 10 +-----
arch/hexagon/Kconfig | 3 --
arch/ia64/Kconfig | 9 +-----
arch/m68k/Kconfig.bus | 11 -------
arch/m68k/Kconfig.cpu | 1 +
arch/microblaze/Kconfig | 6 +---
arch/mips/Kconfig | 43 +++++++++-----------------
arch/mips/alchemy/Kconfig | 6 ++--
arch/mips/ath25/Kconfig | 2 +-
arch/mips/ath79/Kconfig | 8 ++---
arch/mips/bcm63xx/Kconfig | 14 ++++-----
arch/mips/lantiq/Kconfig | 2 +-
arch/mips/loongson64/Kconfig | 6 ++--
arch/mips/pmcs-msp71xx/Kconfig | 10 +++---
arch/mips/ralink/Kconfig | 8 ++---
arch/mips/sibyte/Kconfig | 10 +++---
arch/mips/txx9/Kconfig | 8 ++---
arch/mips/vr41xx/Kconfig | 8 ++---
arch/parisc/Kconfig | 1 +
arch/powerpc/Kconfig | 25 ++++-----------
arch/powerpc/platforms/44x/Kconfig | 2 +-
arch/powerpc/platforms/512x/Kconfig | 2 +-
arch/powerpc/platforms/52xx/Kconfig | 2 +-
arch/powerpc/platforms/83xx/Kconfig | 2 +-
arch/powerpc/platforms/85xx/Kconfig | 2 +-
arch/powerpc/platforms/86xx/Kconfig | 4 +--
arch/powerpc/platforms/Kconfig | 2 +-
arch/powerpc/platforms/Kconfig.cputype | 4 +--
arch/powerpc/platforms/ps3/Kconfig | 2 +-
arch/riscv/Kconfig | 17 +---------
arch/s390/Kconfig | 23 +++++---------
arch/sh/Kconfig | 19 ++----------
arch/sh/boards/Kconfig | 30 +++++++++---------
arch/sparc/Kconfig | 15 +--------
arch/um/Kconfig | 3 --
arch/unicore32/Kconfig | 11 +------
arch/x86/Kconfig | 12 +------
arch/xtensa/Kconfig | 16 +---------
drivers/Kconfig | 4 +++
drivers/parisc/Kconfig | 11 -------
drivers/pci/Kconfig | 12 +++++++
48 files changed, 139 insertions(+), 317 deletions(-)
diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig
index 5b4f88363453..ca40746ffa18 100644
--- a/arch/alpha/Kconfig
+++ b/arch/alpha/Kconfig
@@ -6,6 +6,8 @@ config ALPHA
select ARCH_MIGHT_HAVE_PC_SERIO
select ARCH_NO_PREEMPT
select ARCH_USE_CMPXCHG_LOCKREF
+ select HAS_PCI if !ALPHA_JENSEN
+ select PCI if !ALPHA_JENSEN
select HAVE_AOUT
select HAVE_IDE
select HAVE_OPROFILE
@@ -15,6 +17,7 @@ config ALPHA
select NEED_SG_DMA_LENGTH
select VIRT_TO_BUS
select GENERIC_IRQ_PROBE
+ select GENERIC_PCI_IOMAP if PCI
select AUTO_IRQ_AFFINITY if SMP
select GENERIC_IRQ_SHOW
select ARCH_WANT_IPC_PARSE_VERSION
@@ -319,17 +322,6 @@ config ISA_DMA_API
bool
default y
-config PCI
- bool
- depends on !ALPHA_JENSEN
- select GENERIC_PCI_IOMAP
- default y
- help
- Find out whether you have a PCI motherboard. PCI is the name of a
- bus system, i.e. the way the CPU talks to the other stuff inside
- your box. Other bus systems are ISA, EISA, MicroChannel (MCA) or
- VESA. If you have PCI, say Y, otherwise N.
-
config PCI_DOMAINS
bool
default y
@@ -681,7 +673,6 @@ config HZ
default 1200 if HZ_1200
default 1024
-source "drivers/pci/Kconfig"
source "drivers/eisa/Kconfig"
source "drivers/pcmcia/Kconfig"
diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig
index a045f3086047..55a6953e9239 100644
--- a/arch/arc/Kconfig
+++ b/arch/arc/Kconfig
@@ -53,9 +53,6 @@ config ARC
config ARCH_HAS_CACHE_LINE_SIZE
def_bool y
-config MIGHT_HAVE_PCI
- bool
-
config TRACE_IRQFLAGS_SUPPORT
def_bool y
@@ -555,24 +552,7 @@ config FORCE_MAX_ZONEORDER
default "12" if ARC_HUGEPAGE_16M
default "11"
-menu "Bus Support"
-
-config PCI
- bool "PCI support" if MIGHT_HAVE_PCI
- help
- PCI is the name of a bus system, i.e., the way the CPU talks to
- the other stuff inside your box. Find out if your board/platform
- has PCI.
-
- Note: PCIe support for Synopsys Device will be available only
- when HAPS DX is configured with PCIe RC bitmap. If you have PCI,
- say Y, otherwise N.
-
config PCI_SYSCALL
def_bool PCI
-source "drivers/pci/Kconfig"
-
-endmenu
-
source "kernel/power/Kconfig"
diff --git a/arch/arc/plat-axs10x/Kconfig b/arch/arc/plat-axs10x/Kconfig
index 4e0df7b7a248..33e759233be8 100644
--- a/arch/arc/plat-axs10x/Kconfig
+++ b/arch/arc/plat-axs10x/Kconfig
@@ -11,7 +11,7 @@ menuconfig ARC_PLAT_AXS10X
select DW_APB_ICTL
select GPIO_DWAPB
select OF_GPIO
- select MIGHT_HAVE_PCI
+ select HAS_PCI
select GENERIC_IRQ_CHIP
select GPIOLIB
select AXS101 if ISA_ARCOMPACT
diff --git a/arch/arc/plat-hsdk/Kconfig b/arch/arc/plat-hsdk/Kconfig
index 9356753c2ed8..c213ac5a45b5 100644
--- a/arch/arc/plat-hsdk/Kconfig
+++ b/arch/arc/plat-hsdk/Kconfig
@@ -11,4 +11,4 @@ menuconfig ARC_SOC_HSDK
select ARC_HAS_ACCL_REGS
select CLK_HSDK
select RESET_HSDK
- select MIGHT_HAVE_PCI
+ select HAS_PCI
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index e8cd55a5b04c..a166ccdfdf86 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -149,9 +149,6 @@ config ARM_DMA_IOMMU_ALIGNMENT
endif
-config MIGHT_HAVE_PCI
- bool
-
config SYS_SUPPORTS_APM_EMULATION
bool
@@ -335,7 +332,7 @@ config ARCH_MULTIPLATFORM
select COMMON_CLK
select GENERIC_CLOCKEVENTS
select GENERIC_IRQ_MULTI_HANDLER
- select MIGHT_HAVE_PCI
+ select HAS_PCI
select PCI_DOMAINS if PCI
select SPARSE_IRQ
select USE_OF
@@ -451,7 +448,7 @@ config ARCH_IXP4XX
select DMABOUNCE if PCI
select GENERIC_CLOCKEVENTS
select GPIOLIB
- select MIGHT_HAVE_PCI
+ select HAS_PCI
select NEED_MACH_IO_H
select USB_EHCI_BIG_ENDIAN_DESC
select USB_EHCI_BIG_ENDIAN_MMIO
@@ -464,7 +461,7 @@ config ARCH_DOVE
select GENERIC_CLOCKEVENTS
select GENERIC_IRQ_MULTI_HANDLER
select GPIOLIB
- select MIGHT_HAVE_PCI
+ select HAS_PCI
select MVEBU_MBUS
select PINCTRL
select PINCTRL_DOVE
@@ -1231,14 +1228,6 @@ config ISA_DMA
config ISA_DMA_API
bool
-config PCI
- bool "PCI support" if MIGHT_HAVE_PCI
- help
- Find out whether you have a PCI motherboard. PCI is the name of a
- bus system, i.e. the way the CPU talks to the other stuff inside
- your box. Other bus systems are ISA, EISA, MicroChannel (MCA) or
- VESA. If you have PCI, say Y, otherwise N.
-
config PCI_DOMAINS
bool "Support for multiple PCI domains"
depends on PCI
@@ -1267,8 +1256,6 @@ config PCI_HOST_ITE8152
default y
select DMABOUNCE
-source "drivers/pci/Kconfig"
-
source "drivers/pcmcia/Kconfig"
endmenu
diff --git a/arch/arm/mach-ks8695/Kconfig b/arch/arm/mach-ks8695/Kconfig
index a545976bdbd6..b14541bf31bd 100644
--- a/arch/arm/mach-ks8695/Kconfig
+++ b/arch/arm/mach-ks8695/Kconfig
@@ -4,7 +4,7 @@ menu "Kendin/Micrel KS8695 Implementations"
config MACH_KS8695
bool "KS8695 development board"
- select MIGHT_HAVE_PCI
+ select HAS_PCI
help
Say 'Y' here if you want your kernel to run on the original
Kendin-Micrel KS8695 development board.
@@ -52,7 +52,7 @@ config MACH_CM4002
config MACH_CM4008
bool "OpenGear CM4008"
- select MIGHT_HAVE_PCI
+ select HAS_PCI
help
Say 'Y' here if you want your kernel to support the OpenGear
CM4008 Console Server. See http://www.opengear.com for more
@@ -60,7 +60,7 @@ config MACH_CM4008
config MACH_CM41xx
bool "OpenGear CM41xx"
- select MIGHT_HAVE_PCI
+ select HAS_PCI
help
Say 'Y' here if you want your kernel to support the OpenGear
CM4016 or CM4048 Console Servers. See http://www.opengear.com for
@@ -68,7 +68,7 @@ config MACH_CM41xx
config MACH_IM4004
bool "OpenGear IM4004"
- select MIGHT_HAVE_PCI
+ select HAS_PCI
help
Say 'Y' here if you want your kernel to support the OpenGear
IM4004 Secure Access Server. See http://www.opengear.com for
@@ -76,7 +76,7 @@ config MACH_IM4004
config MACH_IM42xx
bool "OpenGear IM42xx"
- select MIGHT_HAVE_PCI
+ select HAS_PCI
help
Say 'Y' here if you want your kernel to support the OpenGear
IM4216 or IM4248 Console Servers. See http://www.opengear.com for
diff --git a/arch/arm/mach-pxa/Kconfig b/arch/arm/mach-pxa/Kconfig
index a68b34183107..f8d8716636ad 100644
--- a/arch/arm/mach-pxa/Kconfig
+++ b/arch/arm/mach-pxa/Kconfig
@@ -125,7 +125,7 @@ config MACH_ARMCORE
bool "CompuLab CM-X255/CM-X270 modules"
select ARCH_HAS_DMA_SET_COHERENT_MASK if PCI
select IWMMXT
- select MIGHT_HAVE_PCI
+ select HAS_PCI
select NEED_MACH_IO_H if PCI
select PXA25x
select PXA27x
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 1b1a0e95c751..c6a3f705d47b 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -98,6 +98,7 @@ config ARM64
select GENERIC_TIME_VSYSCALL
select HANDLE_DOMAIN_IRQ
select HARDIRQS_SW_RESEND
+ select HAS_PCI
select HAVE_ACPI_APEI if (ACPI && EFI)
select HAVE_ALIGNED_STRUCT_PAGE if SLUB
select HAVE_ARCH_AUDITSYSCALL
@@ -284,13 +285,6 @@ source "arch/arm64/Kconfig.platforms"
menu "Bus support"
-config PCI
- bool "PCI support"
- help
- This feature enables support for PCI bus system. If you say Y
- here, the kernel will include drivers and infrastructure code
- to support PCI bus devices.
-
config PCI_DOMAINS
def_bool PCI
@@ -300,8 +294,6 @@ config PCI_DOMAINS_GENERIC
config PCI_SYSCALL
def_bool PCI
-source "drivers/pci/Kconfig"
-
endmenu
menu "Kernel Features"
diff --git a/arch/hexagon/Kconfig b/arch/hexagon/Kconfig
index 89a4b22f34d9..1bc856c61ed8 100644
--- a/arch/hexagon/Kconfig
+++ b/arch/hexagon/Kconfig
@@ -44,9 +44,6 @@ config FRAME_POINTER
config LOCKDEP_SUPPORT
def_bool y
-config PCI
- def_bool n
-
config EARLY_PRINTK
def_bool y
diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig
index 8b4a0c1748c0..272614773c97 100644
--- a/arch/ia64/Kconfig
+++ b/arch/ia64/Kconfig
@@ -50,6 +50,7 @@ config IA64
select HAVE_MOD_ARCH_SPECIFIC
select MODULES_USE_ELF_RELA
select ARCH_USE_CMPXCHG_LOCKREF
+ select HAS_PCI if !IA64_HP_SIM
select HAVE_ARCH_AUDITSYSCALL
select NEED_DMA_MAP_STATE
select NEED_SG_DMA_LENGTH
@@ -546,20 +547,12 @@ if !IA64_HP_SIM
menu "Bus options (PCI, PCMCIA)"
-config PCI
- bool "PCI support"
- help
- Real IA-64 machines all have PCI/PCI-X/PCI Express busses. Say Y
- here unless you are using a simulator without PCI support.
-
config PCI_DOMAINS
def_bool PCI
config PCI_SYSCALL
def_bool PCI
-source "drivers/pci/Kconfig"
-
source "drivers/pcmcia/Kconfig"
endmenu
diff --git a/arch/m68k/Kconfig.bus b/arch/m68k/Kconfig.bus
index aef698fa50e5..8cb0604b195b 100644
--- a/arch/m68k/Kconfig.bus
+++ b/arch/m68k/Kconfig.bus
@@ -63,17 +63,6 @@ source "drivers/zorro/Kconfig"
endif
-config PCI
- bool "PCI support"
- depends on M54xx
- help
- Enable the PCI bus. Support for the PCI bus hardware built into the
- ColdFire 547x and 548x processors.
-
-if PCI
-source "drivers/pci/Kconfig"
-endif
-
if !MMU
config ISA_DMA_API
diff --git a/arch/m68k/Kconfig.cpu b/arch/m68k/Kconfig.cpu
index 21f00349af52..aea0bc963ab8 100644
--- a/arch/m68k/Kconfig.cpu
+++ b/arch/m68k/Kconfig.cpu
@@ -299,6 +299,7 @@ config M53xx
bool
config M54xx
+ select HAS_PCI
bool
endif # COLDFIRE
diff --git a/arch/microblaze/Kconfig b/arch/microblaze/Kconfig
index ace5c5bf1836..5ad416e17d9e 100644
--- a/arch/microblaze/Kconfig
+++ b/arch/microblaze/Kconfig
@@ -29,6 +29,7 @@ config MICROBLAZE
select HAVE_FUNCTION_GRAPH_TRACER
select HAVE_FUNCTION_TRACER
select NO_BOOTMEM
+ select HAS_PCI
select HAVE_MEMBLOCK
select HAVE_MEMBLOCK_NODE_MAP
select HAVE_OPROFILE
@@ -268,9 +269,6 @@ endmenu
menu "Bus Options"
-config PCI
- bool "PCI support"
-
config PCI_DOMAINS
def_bool PCI
@@ -284,6 +282,4 @@ config PCI_XILINX
bool "Xilinx PCI host bridge support"
depends on PCI
-source "drivers/pci/Kconfig"
-
endmenu
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index 35511999156a..83361add48ca 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -74,6 +74,7 @@ config MIPS
select IRQ_FORCED_THREADING
select MODULES_USE_ELF_RELA if MODULES && 64BIT
select MODULES_USE_ELF_REL if MODULES
+ select PCI_DOMAINS if PCI
select PERF_USE_VMALLOC
select RTC_LIB if !MACH_LOONGSON64
select SYSCTL_EXCEPTION_TRACE
@@ -96,7 +97,7 @@ config MIPS_GENERIC
select CPU_MIPSR2_IRQ_EI
select CSRC_R4K
select DMA_PERDEV_COHERENT
- select HW_HAS_PCI
+ select HAS_PCI
select IRQ_MIPS_CPU
select LIBFDT
select MIPS_AUTO_PFN_OFFSET
@@ -256,7 +257,7 @@ config BCM47XX
select CEVT_R4K
select CSRC_R4K
select DMA_NONCOHERENT
- select HW_HAS_PCI
+ select HAS_PCI
select IRQ_MIPS_CPU
select SYS_HAS_CPU_MIPS32_R1
select NO_EXCEPT_FILL
@@ -299,7 +300,7 @@ config MIPS_COBALT
select CSRC_R4K
select CEVT_GT641XX
select DMA_NONCOHERENT
- select HW_HAS_PCI
+ select HAS_PCI
select I8253
select I8259
select IRQ_MIPS_CPU
@@ -422,7 +423,7 @@ config LASAT
select CSRC_R4K
select DMA_NONCOHERENT
select SYS_HAS_EARLY_PRINTK
- select HW_HAS_PCI
+ select HAS_PCI
select IRQ_MIPS_CPU
select PCI_GT64XXX_PCI0
select MIPS_NILE4
@@ -502,7 +503,7 @@ config MIPS_MALTA
select HAVE_PCSPKR_PLATFORM
select IRQ_MIPS_CPU
select MIPS_GIC
- select HW_HAS_PCI
+ select HAS_PCI
select I8253
select I8259
select MIPS_BONITO64
@@ -556,7 +557,7 @@ config MACH_PIC32
config NEC_MARKEINS
bool "NEC EMMA2RH Mark-eins board"
select SOC_EMMA2RH
- select HW_HAS_PCI
+ select HAS_PCI
help
This enables support for the NEC Electronics Mark-eins boards.
@@ -673,7 +674,7 @@ config SGI_IP27
select BOOT_ELF64
select DEFAULT_SGI_PARTITION
select SYS_HAS_EARLY_PRINTK
- select HW_HAS_PCI
+ select HAS_PCI
select NR_CPUS_DEFAULT_64
select SYS_HAS_CPU_R10000
select SYS_SUPPORTS_64BIT_KERNEL
@@ -733,7 +734,7 @@ config SGI_IP32
select CEVT_R4K
select CSRC_R4K
select DMA_NONCOHERENT
- select HW_HAS_PCI
+ select HAS_PCI
select IRQ_MIPS_CPU
select R5000_CPU_SCACHE
select RM7000_CPU_SCACHE
@@ -843,7 +844,7 @@ config SNI_RM
select GENERIC_ISA_DMA
select HAVE_PCSPKR_PLATFORM
select HW_HAS_EISA
- select HW_HAS_PCI
+ select HAS_PCI
select IRQ_MIPS_CPU
select I8253
select I8259
@@ -876,7 +877,7 @@ config MIKROTIK_RB532
select CEVT_R4K
select CSRC_R4K
select DMA_NONCOHERENT
- select HW_HAS_PCI
+ select HAS_PCI
select IRQ_MIPS_CPU
select SYS_HAS_CPU_MIPS32_R1
select SYS_SUPPORTS_32BIT_KERNEL
@@ -903,7 +904,7 @@ config CAVIUM_OCTEON_SOC
select SYS_SUPPORTS_HOTPLUG_CPU if CPU_BIG_ENDIAN
select SYS_HAS_EARLY_PRINTK
select SYS_HAS_CPU_CAVIUM_OCTEON
- select HW_HAS_PCI
+ select HAS_PCI
select ZONE_DMA32
select HOLES_IN_ZONE
select GPIOLIB
@@ -936,7 +937,7 @@ config NLM_XLR_BOARD
select NLM_COMMON
select SYS_HAS_CPU_XLR
select SYS_SUPPORTS_SMP
- select HW_HAS_PCI
+ select HAS_PCI
select SWAP_IO_SPACE
select SYS_SUPPORTS_32BIT_KERNEL
select SYS_SUPPORTS_64BIT_KERNEL
@@ -962,7 +963,7 @@ config NLM_XLP_BOARD
select NLM_COMMON
select SYS_HAS_CPU_XLP
select SYS_SUPPORTS_SMP
- select HW_HAS_PCI
+ select HAS_PCI
select SYS_SUPPORTS_32BIT_KERNEL
select SYS_SUPPORTS_64BIT_KERNEL
select PHYS_ADDR_T_64BIT
@@ -997,7 +998,7 @@ config MIPS_PARAVIRT
select SYS_HAS_CPU_MIPS32_R2
select SYS_HAS_CPU_MIPS64_R2
select SYS_HAS_CPU_CAVIUM_OCTEON
- select HW_HAS_PCI
+ select HAS_PCI
select SWAP_IO_SPACE
help
This option supports guest running under ????
@@ -2992,18 +2993,6 @@ menu "Bus options (PCI, PCMCIA, EISA, ISA, TC)"
config HW_HAS_EISA
bool
-config HW_HAS_PCI
- bool
-
-config PCI
- bool "Support for PCI controller"
- depends on HW_HAS_PCI
- select PCI_DOMAINS
- help
- Find out whether you have a PCI motherboard. PCI is the name of a
- bus system, i.e. the way the CPU talks to the other stuff inside
- your box. Other bus systems are ISA, EISA, or VESA. If you have PCI,
- say Y, otherwise N.
config HT_PCI
bool "Support for HT-linked PCI"
@@ -3030,8 +3019,6 @@ config PCI_DRIVERS_LEGACY
def_bool !PCI_DRIVERS_GENERIC
select NO_GENERIC_PCI_IOPORT_MAP
-source "drivers/pci/Kconfig"
-
#
# ISA support is now enabled via select. Too many systems still have the one
# or other ISA chip on the board that users don't know about so don't expect
diff --git a/arch/mips/alchemy/Kconfig b/arch/mips/alchemy/Kconfig
index 7d73f7f4202b..78e015a9b165 100644
--- a/arch/mips/alchemy/Kconfig
+++ b/arch/mips/alchemy/Kconfig
@@ -14,7 +14,7 @@ choice
config MIPS_MTX1
bool "4G Systems MTX-1 board"
- select HW_HAS_PCI
+ select HAS_PCI
select ALCHEMY_GPIOINT_AU1000
select SYS_SUPPORTS_LITTLE_ENDIAN
select SYS_HAS_EARLY_PRINTK
@@ -22,7 +22,7 @@ config MIPS_MTX1
config MIPS_DB1XXX
bool "Alchemy DB1XXX / PB1XXX boards"
select GPIOLIB
- select HW_HAS_PCI
+ select HAS_PCI
select SYS_SUPPORTS_LITTLE_ENDIAN
select SYS_HAS_EARLY_PRINTK
help
@@ -40,7 +40,7 @@ config MIPS_XXS1500
config MIPS_GPR
bool "Trapeze ITS GPR board"
select ALCHEMY_GPIOINT_AU1000
- select HW_HAS_PCI
+ select HAS_PCI
select SYS_SUPPORTS_LITTLE_ENDIAN
select SYS_HAS_EARLY_PRINTK
diff --git a/arch/mips/ath25/Kconfig b/arch/mips/ath25/Kconfig
index 2c1dfd06c366..59b8fd9b6179 100644
--- a/arch/mips/ath25/Kconfig
+++ b/arch/mips/ath25/Kconfig
@@ -13,6 +13,6 @@ config PCI_AR2315
bool "Atheros AR2315 PCI controller support"
depends on SOC_AR2315
select ARCH_HAS_PHYS_TO_DMA
- select HW_HAS_PCI
+ select HAS_PCI
select PCI
default y
diff --git a/arch/mips/ath79/Kconfig b/arch/mips/ath79/Kconfig
index 9547cf1ea38d..7c4af384f191 100644
--- a/arch/mips/ath79/Kconfig
+++ b/arch/mips/ath79/Kconfig
@@ -75,11 +75,11 @@ config ATH79_MACH_UBNT_XM
endmenu
config SOC_AR71XX
- select HW_HAS_PCI
+ select HAS_PCI
def_bool n
config SOC_AR724X
- select HW_HAS_PCI
+ select HAS_PCI
select PCI_AR724X if PCI
def_bool n
@@ -90,12 +90,12 @@ config SOC_AR933X
def_bool n
config SOC_AR934X
- select HW_HAS_PCI
+ select HAS_PCI
select PCI_AR724X if PCI
def_bool n
config SOC_QCA955X
- select HW_HAS_PCI
+ select HAS_PCI
select PCI_AR724X if PCI
def_bool n
diff --git a/arch/mips/bcm63xx/Kconfig b/arch/mips/bcm63xx/Kconfig
index 96ed735a4f4a..dc82ad30f4d2 100644
--- a/arch/mips/bcm63xx/Kconfig
+++ b/arch/mips/bcm63xx/Kconfig
@@ -5,17 +5,17 @@ menu "CPU support"
config BCM63XX_CPU_3368
bool "support 3368 CPU"
select SYS_HAS_CPU_BMIPS4350
- select HW_HAS_PCI
+ select HAS_PCI
config BCM63XX_CPU_6328
bool "support 6328 CPU"
select SYS_HAS_CPU_BMIPS4350
- select HW_HAS_PCI
+ select HAS_PCI
config BCM63XX_CPU_6338
bool "support 6338 CPU"
select SYS_HAS_CPU_BMIPS32_3300
- select HW_HAS_PCI
+ select HAS_PCI
config BCM63XX_CPU_6345
bool "support 6345 CPU"
@@ -24,22 +24,22 @@ config BCM63XX_CPU_6345
config BCM63XX_CPU_6348
bool "support 6348 CPU"
select SYS_HAS_CPU_BMIPS32_3300
- select HW_HAS_PCI
+ select HAS_PCI
config BCM63XX_CPU_6358
bool "support 6358 CPU"
select SYS_HAS_CPU_BMIPS4350
- select HW_HAS_PCI
+ select HAS_PCI
config BCM63XX_CPU_6362
bool "support 6362 CPU"
select SYS_HAS_CPU_BMIPS4350
- select HW_HAS_PCI
+ select HAS_PCI
config BCM63XX_CPU_6368
bool "support 6368 CPU"
select SYS_HAS_CPU_BMIPS4350
- select HW_HAS_PCI
+ select HAS_PCI
endmenu
source "arch/mips/bcm63xx/boards/Kconfig"
diff --git a/arch/mips/lantiq/Kconfig b/arch/mips/lantiq/Kconfig
index 8e3a1fc2bc39..3c6916c6e0a8 100644
--- a/arch/mips/lantiq/Kconfig
+++ b/arch/mips/lantiq/Kconfig
@@ -19,7 +19,7 @@ config SOC_AMAZON_SE
config SOC_XWAY
bool "XWAY"
select SOC_TYPE_XWAY
- select HW_HAS_PCI
+ select HAS_PCI
select MFD_SYSCON
select MFD_CORE
diff --git a/arch/mips/loongson64/Kconfig b/arch/mips/loongson64/Kconfig
index c865b4b9b775..816ef9d6101d 100644
--- a/arch/mips/loongson64/Kconfig
+++ b/arch/mips/loongson64/Kconfig
@@ -15,7 +15,7 @@ config LEMOTE_FULOONG2E
select DMA_NONCOHERENT
select BOOT_ELF32
select BOARD_SCACHE
- select HW_HAS_PCI
+ select HAS_PCI
select I8259
select ISA
select IRQ_MIPS_CPU
@@ -46,7 +46,7 @@ config LEMOTE_MACH2F
select DMA_NONCOHERENT
select GENERIC_ISA_DMA_SUPPORT_BROKEN
select HAVE_CLK
- select HW_HAS_PCI
+ select HAS_PCI
select I8259
select IRQ_MIPS_CPU
select ISA
@@ -74,7 +74,7 @@ config LOONGSON_MACH3X
select CSRC_R4K
select CEVT_R4K
select CPU_HAS_WB
- select HW_HAS_PCI
+ select HAS_PCI
select ISA
select HT_PCI
select I8259
diff --git a/arch/mips/pmcs-msp71xx/Kconfig b/arch/mips/pmcs-msp71xx/Kconfig
index d319bc0c3df6..c00af0d5e795 100644
--- a/arch/mips/pmcs-msp71xx/Kconfig
+++ b/arch/mips/pmcs-msp71xx/Kconfig
@@ -6,25 +6,25 @@ choice
config PMC_MSP4200_EVAL
bool "PMC-Sierra MSP4200 Eval Board"
select IRQ_MSP_SLP
- select HW_HAS_PCI
+ select HAS_PCI
select MIPS_L1_CACHE_SHIFT_4
config PMC_MSP4200_GW
bool "PMC-Sierra MSP4200 VoIP Gateway"
select IRQ_MSP_SLP
- select HW_HAS_PCI
+ select HAS_PCI
config PMC_MSP7120_EVAL
bool "PMC-Sierra MSP7120 Eval Board"
select SYS_SUPPORTS_MULTITHREADING
select IRQ_MSP_CIC
- select HW_HAS_PCI
+ select HAS_PCI
config PMC_MSP7120_GW
bool "PMC-Sierra MSP7120 Residential Gateway"
select SYS_SUPPORTS_MULTITHREADING
select IRQ_MSP_CIC
- select HW_HAS_PCI
+ select HAS_PCI
select MSP_HAS_USB
select MSP_ETH
@@ -32,7 +32,7 @@ config PMC_MSP7120_FPGA
bool "PMC-Sierra MSP7120 FPGA"
select SYS_SUPPORTS_MULTITHREADING
select IRQ_MSP_CIC
- select HW_HAS_PCI
+ select HAS_PCI
endchoice
diff --git a/arch/mips/ralink/Kconfig b/arch/mips/ralink/Kconfig
index 1f9cb0e3c79a..8b7413fe04a8 100644
--- a/arch/mips/ralink/Kconfig
+++ b/arch/mips/ralink/Kconfig
@@ -27,18 +27,18 @@ choice
config SOC_RT288X
bool "RT288x"
select MIPS_L1_CACHE_SHIFT_4
- select HW_HAS_PCI
+ select HAS_PCI
config SOC_RT305X
bool "RT305x"
config SOC_RT3883
bool "RT3883"
- select HW_HAS_PCI
+ select HAS_PCI
config SOC_MT7620
bool "MT7620/8"
- select HW_HAS_PCI
+ select HAS_PCI
config SOC_MT7621
bool "MT7621"
@@ -50,7 +50,7 @@ choice
select MIPS_GIC
select COMMON_CLK
select CLKSRC_MIPS_GIC
- select HW_HAS_PCI
+ select HAS_PCI
endchoice
choice
diff --git a/arch/mips/sibyte/Kconfig b/arch/mips/sibyte/Kconfig
index 7ec278d72096..0583331ba54e 100644
--- a/arch/mips/sibyte/Kconfig
+++ b/arch/mips/sibyte/Kconfig
@@ -3,7 +3,7 @@ config SIBYTE_SB1250
bool
select CEVT_SB1250
select CSRC_SB1250
- select HW_HAS_PCI
+ select HAS_PCI
select IRQ_MIPS_CPU
select SIBYTE_ENABLE_LDT_IF_PCI
select SIBYTE_HAS_ZBUS_PROFILING
@@ -23,7 +23,7 @@ config SIBYTE_BCM1125
bool
select CEVT_SB1250
select CSRC_SB1250
- select HW_HAS_PCI
+ select HAS_PCI
select IRQ_MIPS_CPU
select SIBYTE_BCM112X
select SIBYTE_HAS_ZBUS_PROFILING
@@ -33,7 +33,7 @@ config SIBYTE_BCM1125H
bool
select CEVT_SB1250
select CSRC_SB1250
- select HW_HAS_PCI
+ select HAS_PCI
select IRQ_MIPS_CPU
select SIBYTE_BCM112X
select SIBYTE_ENABLE_LDT_IF_PCI
@@ -52,7 +52,7 @@ config SIBYTE_BCM1x80
bool
select CEVT_BCM1480
select CSRC_BCM1480
- select HW_HAS_PCI
+ select HAS_PCI
select IRQ_MIPS_CPU
select SIBYTE_HAS_ZBUS_PROFILING
select SIBYTE_SB1xxx_SOC
@@ -62,7 +62,7 @@ config SIBYTE_BCM1x55
bool
select CEVT_BCM1480
select CSRC_BCM1480
- select HW_HAS_PCI
+ select HAS_PCI
select IRQ_MIPS_CPU
select SIBYTE_SB1xxx_SOC
select SIBYTE_HAS_ZBUS_PROFILING
diff --git a/arch/mips/txx9/Kconfig b/arch/mips/txx9/Kconfig
index d2509c93f0ee..cf5a49c513d4 100644
--- a/arch/mips/txx9/Kconfig
+++ b/arch/mips/txx9/Kconfig
@@ -59,7 +59,7 @@ config SOC_TX3927
bool
select CEVT_TXX9
select HAS_TXX9_SERIAL
- select HW_HAS_PCI
+ select HAS_PCI
select IRQ_TXX9
select GPIO_TXX9
@@ -67,7 +67,7 @@ config SOC_TX4927
bool
select CEVT_TXX9
select HAS_TXX9_SERIAL
- select HW_HAS_PCI
+ select HAS_PCI
select IRQ_TXX9
select PCI_TX4927
select GPIO_TXX9
@@ -77,7 +77,7 @@ config SOC_TX4938
bool
select CEVT_TXX9
select HAS_TXX9_SERIAL
- select HW_HAS_PCI
+ select HAS_PCI
select IRQ_TXX9
select PCI_TX4927
select GPIO_TXX9
@@ -87,7 +87,7 @@ config SOC_TX4939
bool
select CEVT_TXX9
select HAS_TXX9_SERIAL
- select HW_HAS_PCI
+ select HAS_PCI
select PCI_TX4927
select HAS_TXX9_ACLC
diff --git a/arch/mips/vr41xx/Kconfig b/arch/mips/vr41xx/Kconfig
index 992c988b83b0..01f37c35fa19 100644
--- a/arch/mips/vr41xx/Kconfig
+++ b/arch/mips/vr41xx/Kconfig
@@ -30,7 +30,7 @@ config TANBAC_TB022X
select CSRC_R4K
select DMA_NONCOHERENT
select IRQ_MIPS_CPU
- select HW_HAS_PCI
+ select HAS_PCI
select SYS_SUPPORTS_32BIT_KERNEL
select SYS_SUPPORTS_LITTLE_ENDIAN
help
@@ -46,7 +46,7 @@ config VICTOR_MPC30X
select CSRC_R4K
select DMA_NONCOHERENT
select IRQ_MIPS_CPU
- select HW_HAS_PCI
+ select HAS_PCI
select PCI_VR41XX
select SYS_SUPPORTS_32BIT_KERNEL
select SYS_SUPPORTS_LITTLE_ENDIAN
@@ -57,7 +57,7 @@ config ZAO_CAPCELLA
select CSRC_R4K
select DMA_NONCOHERENT
select IRQ_MIPS_CPU
- select HW_HAS_PCI
+ select HAS_PCI
select PCI_VR41XX
select SYS_SUPPORTS_32BIT_KERNEL
select SYS_SUPPORTS_LITTLE_ENDIAN
@@ -99,6 +99,6 @@ endchoice
config PCI_VR41XX
bool "Add PCI control unit support of NEC VR4100 series"
- depends on MACH_VR41XX && HW_HAS_PCI
+ depends on MACH_VR41XX && HAS_PCI
default y
select PCI
diff --git a/arch/parisc/Kconfig b/arch/parisc/Kconfig
index 8e6d83f79e72..6b946eea90ee 100644
--- a/arch/parisc/Kconfig
+++ b/arch/parisc/Kconfig
@@ -19,6 +19,7 @@ config PARISC
select NO_BOOTMEM
select BUG
select BUILDTIME_EXTABLE_SORT
+ select HAS_PCI
select HAVE_PERF_EVENTS
select HAVE_KERNEL_BZIP2
select HAVE_KERNEL_GZIP
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 6fa6f92edb7f..0713375e1127 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -168,10 +168,16 @@ config PPC
select GENERIC_CPU_VULNERABILITIES if PPC_BARRIER_NOSPEC
select GENERIC_IRQ_SHOW
select GENERIC_IRQ_SHOW_LEVEL
+ select GENERIC_PCI_IOMAP if PCI
select GENERIC_SMP_IDLE_THREAD
select GENERIC_STRNCPY_FROM_USER
select GENERIC_STRNLEN_USER
select GENERIC_TIME_VSYSCALL
+ # Platforms that what PCI turned unconditionally just do select PCI
+ # in their config node, otherwise allow the user to enable it.
+ # This means we can enable PCI even on platforms that don't support
+ # it, which is harmless except for wasted memory.
+ select HAS_PCI
select HAVE_ARCH_AUDITSYSCALL
select HAVE_ARCH_JUMP_LABEL
select HAVE_ARCH_KGDB
@@ -945,23 +951,6 @@ config FSL_GTM
config MCA
bool
-# Platforms that what PCI turned unconditionally just do select PCI
-# in their config node. Platforms that want to choose at config
-# time should select PPC_PCI_CHOICE
-config PPC_PCI_CHOICE
- bool
-
-config PCI
- bool "PCI support" if PPC_PCI_CHOICE
- default y if !40x && !CPM2 && !PPC_8xx && !PPC_83xx \
- && !PPC_85xx && !PPC_86xx && !GAMECUBE_COMMON
- select GENERIC_PCI_IOMAP
- help
- Find out whether your system includes a PCI bus. PCI is the name of
- a bus system, i.e. the way the CPU talks to the other stuff inside
- your box. If you say Y here, the kernel will include drivers and
- infrastructure code to support PCI bus devices.
-
config PCI_DOMAINS
def_bool PCI
@@ -982,8 +971,6 @@ config PCI_8260
select PPC_INDIRECT_PCI
default y
-source "drivers/pci/Kconfig"
-
source "drivers/pcmcia/Kconfig"
config HAS_RAPIDIO
diff --git a/arch/powerpc/platforms/44x/Kconfig b/arch/powerpc/platforms/44x/Kconfig
index 70856a213663..000efce8191b 100644
--- a/arch/powerpc/platforms/44x/Kconfig
+++ b/arch/powerpc/platforms/44x/Kconfig
@@ -272,7 +272,7 @@ config XILINX_VIRTEX440_GENERIC_BOARD
config XILINX_ML510
bool "Xilinx ML510 extra support"
depends on XILINX_VIRTEX440_GENERIC_BOARD
- select PPC_PCI_CHOICE
+ select HAS_PCI
select XILINX_PCI if PCI
select PPC_INDIRECT_PCI if PCI
select PPC_I8259 if PCI
diff --git a/arch/powerpc/platforms/512x/Kconfig b/arch/powerpc/platforms/512x/Kconfig
index b59eab6cbb1b..afc0a7f62cd9 100644
--- a/arch/powerpc/platforms/512x/Kconfig
+++ b/arch/powerpc/platforms/512x/Kconfig
@@ -5,7 +5,7 @@ config PPC_MPC512x
select COMMON_CLK
select FSL_SOC
select IPIC
- select PPC_PCI_CHOICE
+ select HAS_PCI
select FSL_PCI if PCI
select USB_EHCI_BIG_ENDIAN_MMIO if USB_EHCI_HCD
select USB_EHCI_BIG_ENDIAN_DESC if USB_EHCI_HCD
diff --git a/arch/powerpc/platforms/52xx/Kconfig b/arch/powerpc/platforms/52xx/Kconfig
index 55a587070342..557379857de5 100644
--- a/arch/powerpc/platforms/52xx/Kconfig
+++ b/arch/powerpc/platforms/52xx/Kconfig
@@ -3,7 +3,7 @@ config PPC_MPC52xx
bool "52xx-based boards"
depends on 6xx
select COMMON_CLK
- select PPC_PCI_CHOICE
+ select HAS_PCI
config PPC_MPC5200_SIMPLE
bool "Generic support for simple MPC5200 based boards"
diff --git a/arch/powerpc/platforms/83xx/Kconfig b/arch/powerpc/platforms/83xx/Kconfig
index 071f53b0c0a0..32c70d6dee1e 100644
--- a/arch/powerpc/platforms/83xx/Kconfig
+++ b/arch/powerpc/platforms/83xx/Kconfig
@@ -3,7 +3,7 @@ menuconfig PPC_83xx
bool "83xx-based boards"
depends on 6xx
select PPC_UDBG_16550
- select PPC_PCI_CHOICE
+ select HAS_PCI
select FSL_PCI if PCI
select FSL_SOC
select IPIC
diff --git a/arch/powerpc/platforms/85xx/Kconfig b/arch/powerpc/platforms/85xx/Kconfig
index 68920d42b4bc..8438ac3f5c70 100644
--- a/arch/powerpc/platforms/85xx/Kconfig
+++ b/arch/powerpc/platforms/85xx/Kconfig
@@ -5,7 +5,7 @@ menuconfig FSL_SOC_BOOKE
select FSL_SOC
select PPC_UDBG_16550
select MPIC
- select PPC_PCI_CHOICE
+ select HAS_PCI
select FSL_PCI if PCI
select SERIAL_8250_EXTENDED if SERIAL_8250
select SERIAL_8250_SHARE_IRQ if SERIAL_8250
diff --git a/arch/powerpc/platforms/86xx/Kconfig b/arch/powerpc/platforms/86xx/Kconfig
index bcd179d3ed92..bbdf5b8d871a 100644
--- a/arch/powerpc/platforms/86xx/Kconfig
+++ b/arch/powerpc/platforms/86xx/Kconfig
@@ -70,7 +70,7 @@ endif
config MPC8641
bool
- select PPC_PCI_CHOICE
+ select HAS_PCI
select FSL_PCI if PCI
select PPC_UDBG_16550
select MPIC
@@ -79,7 +79,7 @@ config MPC8641
config MPC8610
bool
- select PPC_PCI_CHOICE
+ select HAS_PCI
select FSL_PCI if PCI
select PPC_UDBG_16550
select MPIC
diff --git a/arch/powerpc/platforms/Kconfig b/arch/powerpc/platforms/Kconfig
index 14ef17e10ec9..6887739b6568 100644
--- a/arch/powerpc/platforms/Kconfig
+++ b/arch/powerpc/platforms/Kconfig
@@ -285,7 +285,7 @@ config CPM2
bool "Enable support for the CPM2 (Communications Processor Module)"
depends on (FSL_SOC_BOOKE && PPC32) || 8260
select CPM
- select PPC_PCI_CHOICE
+ select HAS_PCI
select GPIOLIB
help
The CPM2 (Communications Processor Module) is a coprocessor on
diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype
index 6c6a7c72cae4..0b0a2cceea64 100644
--- a/arch/powerpc/platforms/Kconfig.cputype
+++ b/arch/powerpc/platforms/Kconfig.cputype
@@ -40,14 +40,14 @@ config 40x
select PPC_DCR_NATIVE
select PPC_UDBG_16550
select 4xx_SOC
- select PPC_PCI_CHOICE
+ select HAS_PCI
config 44x
bool "AMCC 44x, 46x or 47x"
select PPC_DCR_NATIVE
select PPC_UDBG_16550
select 4xx_SOC
- select PPC_PCI_CHOICE
+ select HAS_PCI
select PHYS_64BIT
config E200
diff --git a/arch/powerpc/platforms/ps3/Kconfig b/arch/powerpc/platforms/ps3/Kconfig
index 6f7525555b19..804a8924d421 100644
--- a/arch/powerpc/platforms/ps3/Kconfig
+++ b/arch/powerpc/platforms/ps3/Kconfig
@@ -6,7 +6,7 @@ config PPC_PS3
select USB_OHCI_LITTLE_ENDIAN
select USB_OHCI_BIG_ENDIAN_MMIO
select USB_EHCI_BIG_ENDIAN_MMIO
- select PPC_PCI_CHOICE
+ select HAS_PCI
help
This option enables support for the Sony PS3 game console
and other platforms using the PS3 hypervisor. Enabling this
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index a344980287a5..6eb31a0698ac 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -38,6 +38,7 @@ config RISCV
select RISCV_ISA_A if SMP
select SPARSE_IRQ
select SYSCTL_EXCEPTION_TRACE
+ select HAS_PCI
select HAVE_ARCH_TRACEHOOK
select MODULES_USE_ELF_RELA if MODULES
select THREAD_INFO_IN_TASK
@@ -216,28 +217,12 @@ source "kernel/Kconfig.hz"
endmenu
-menu "Bus support"
-
-config PCI
- bool "PCI support"
- select PCI_MSI
- help
- This feature enables support for PCI bus system. If you say Y
- here, the kernel will include drivers and infrastructure code
- to support PCI bus devices.
-
- If you don't know what to do here, say Y.
-
config PCI_DOMAINS
def_bool PCI
config PCI_DOMAINS_GENERIC
def_bool PCI
-source "drivers/pci/Kconfig"
-
-endmenu
-
menu "Power management options"
source kernel/power/Kconfig
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 9a9c7a6fe925..3491a084cf55 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -121,6 +121,7 @@ config S390
select HAVE_ARCH_AUDITSYSCALL
select HAVE_ARCH_JUMP_LABEL
select CPU_NO_EFFICIENT_FFS if !HAVE_MARCH_Z9_109_FEATURES
+ select HAS_PCI
select HAVE_ARCH_SECCOMP_FILTER
select HAVE_ARCH_SOFT_DIRTY
select HAVE_ARCH_TRACEHOOK
@@ -165,10 +166,15 @@ config S390
select HAVE_RSEQ
select HAVE_SYSCALL_TRACEPOINTS
select HAVE_VIRT_CPU_ACCOUNTING
+ select IOMMU_HELPER if PCI
+ select IOMMU_SUPPORT if PCI
select MODULES_USE_ELF_RELA
+ select NEED_DMA_MAP_STATE if PCI
+ select NEED_SG_DMA_LENGTH if PCI
select NO_BOOTMEM
select OLD_SIGACTION
select OLD_SIGSUSPEND3
+ select PCI_MSI if PCI
select SPARSE_IRQ
select SYSCTL_EXCEPTION_TRACE
select THREAD_INFO_IN_TASK
@@ -698,17 +704,6 @@ config QDIO
If unsure, say Y.
-menuconfig PCI
- bool "PCI support"
- select PCI_MSI
- select IOMMU_HELPER
- select IOMMU_SUPPORT
- select NEED_DMA_MAP_STATE
- select NEED_SG_DMA_LENGTH
-
- help
- Enable PCI support.
-
if PCI
config PCI_NR_FUNCTIONS
@@ -719,13 +714,11 @@ config PCI_NR_FUNCTIONS
This allows you to specify the maximum number of PCI functions which
this kernel will support.
-source "drivers/pci/Kconfig"
-
-endif # PCI
-
config PCI_DOMAINS
def_bool PCI
+endif # PCI
+
config HAS_IOMEM
def_bool PCI
diff --git a/arch/sh/Kconfig b/arch/sh/Kconfig
index 1fb7b6d72baf..2ff6855811a5 100644
--- a/arch/sh/Kconfig
+++ b/arch/sh/Kconfig
@@ -41,13 +41,16 @@ config SUPERH
select GENERIC_IDLE_POLL_SETUP
select GENERIC_CLOCKEVENTS
select GENERIC_CMOS_UPDATE if SH_SH03 || SH_DREAMCAST
+ select GENERIC_PCI_IOMAP if PCI
select GENERIC_SCHED_CLOCK
select GENERIC_STRNCPY_FROM_USER
select GENERIC_STRNLEN_USER
select HAVE_MOD_ARCH_SPECIFIC if DWARF_UNWINDER
select MODULES_USE_ELF_RELA
+ select NO_GENERIC_PCI_IOPORT_MAP if PCI
select OLD_SIGSUSPEND
select OLD_SIGACTION
+ select PCI_DOMAINS if PCI
select HAVE_ARCH_AUDITSYSCALL
select HAVE_FUTEX_CMPXCHG if FUTEX
select HAVE_NMI
@@ -131,9 +134,6 @@ config SYS_SUPPORTS_SMP
config SYS_SUPPORTS_NUMA
bool
-config SYS_SUPPORTS_PCI
- bool
-
config STACKTRACE_SUPPORT
def_bool y
@@ -858,22 +858,9 @@ config MAPLE
Dreamcast with a serial line terminal or a remote network
connection.
-config PCI
- bool "PCI support"
- depends on SYS_SUPPORTS_PCI
- select PCI_DOMAINS
- select GENERIC_PCI_IOMAP
- select NO_GENERIC_PCI_IOPORT_MAP
- help
- Find out whether you have a PCI motherboard. PCI is the name of a
- bus system, i.e. the way the CPU talks to the other stuff inside
- your box. If you have PCI, say Y, otherwise N.
-
config PCI_DOMAINS
bool
-source "drivers/pci/Kconfig"
-
source "drivers/pcmcia/Kconfig"
endmenu
diff --git a/arch/sh/boards/Kconfig b/arch/sh/boards/Kconfig
index 6394b4f0a69b..7d74abbbf7f4 100644
--- a/arch/sh/boards/Kconfig
+++ b/arch/sh/boards/Kconfig
@@ -101,7 +101,7 @@ config SH_7751_SOLUTION_ENGINE
config SH_7780_SOLUTION_ENGINE
bool "SolutionEngine7780"
select SOLUTION_ENGINE
- select SYS_SUPPORTS_PCI
+ select HAS_PCI
depends on CPU_SUBTYPE_SH7780
help
Select 7780 SolutionEngine if configuring for a Renesas SH7780
@@ -129,7 +129,7 @@ config SH_HP6XX
config SH_DREAMCAST
bool "Dreamcast"
- select SYS_SUPPORTS_PCI
+ select HAS_PCI
depends on CPU_SUBTYPE_SH7091
help
Select Dreamcast if configuring for a SEGA Dreamcast.
@@ -139,7 +139,7 @@ config SH_SH03
bool "Interface CTP/PCI-SH03"
depends on CPU_SUBTYPE_SH7751
select CPU_HAS_IPR_IRQ
- select SYS_SUPPORTS_PCI
+ select HAS_PCI
help
CTP/PCI-SH03 is a CPU module computer that is produced
by Interface Corporation.
@@ -149,7 +149,7 @@ config SH_SECUREEDGE5410
bool "SecureEdge5410"
depends on CPU_SUBTYPE_SH7751R
select CPU_HAS_IPR_IRQ
- select SYS_SUPPORTS_PCI
+ select HAS_PCI
help
Select SecureEdge5410 if configuring for a SnapGear SH board.
This includes both the OEM SecureEdge products as well as the
@@ -158,7 +158,7 @@ config SH_SECUREEDGE5410
config SH_RTS7751R2D
bool "RTS7751R2D"
depends on CPU_SUBTYPE_SH7751R
- select SYS_SUPPORTS_PCI
+ select HAS_PCI
select IO_TRAPPED if MMU
help
Select RTS7751R2D if configuring for a Renesas Technology
@@ -176,7 +176,7 @@ config SH_RSK
config SH_SDK7780
bool "SDK7780R3"
depends on CPU_SUBTYPE_SH7780
- select SYS_SUPPORTS_PCI
+ select HAS_PCI
help
Select SDK7780 if configuring for a Renesas SH7780 SDK7780R3
evaluation board.
@@ -184,7 +184,7 @@ config SH_SDK7780
config SH_SDK7786
bool "SDK7786"
depends on CPU_SUBTYPE_SH7786
- select SYS_SUPPORTS_PCI
+ select HAS_PCI
select NO_IOPORT_MAP if !PCI
select HAVE_SRAM_POOL
select REGULATOR_FIXED_VOLTAGE if REGULATOR
@@ -195,7 +195,7 @@ config SH_SDK7786
config SH_HIGHLANDER
bool "Highlander"
depends on CPU_SUBTYPE_SH7780 || CPU_SUBTYPE_SH7785
- select SYS_SUPPORTS_PCI
+ select HAS_PCI
select IO_TRAPPED if MMU
config SH_SH7757LCR
@@ -207,7 +207,7 @@ config SH_SH7757LCR
config SH_SH7785LCR
bool "SH7785LCR"
depends on CPU_SUBTYPE_SH7785
- select SYS_SUPPORTS_PCI
+ select HAS_PCI
config SH_SH7785LCR_29BIT_PHYSMAPS
bool "SH7785LCR 29bit physmaps"
@@ -229,7 +229,7 @@ config SH_URQUELL
bool "Urquell"
depends on CPU_SUBTYPE_SH7786
select GPIOLIB
- select SYS_SUPPORTS_PCI
+ select HAS_PCI
select NO_IOPORT_MAP if !PCI
config SH_MIGOR
@@ -302,7 +302,7 @@ config SH_SH4202_MICRODEV
config SH_LANDISK
bool "LANDISK"
depends on CPU_SUBTYPE_SH7751R
- select SYS_SUPPORTS_PCI
+ select HAS_PCI
help
I-O DATA DEVICE, INC. "LANDISK Series" support.
@@ -310,7 +310,7 @@ config SH_TITAN
bool "TITAN"
depends on CPU_SUBTYPE_SH7751R
select CPU_HAS_IPR_IRQ
- select SYS_SUPPORTS_PCI
+ select HAS_PCI
help
Select Titan if you are configuring for a Nimble Microsystems
NetEngine NP51R.
@@ -325,7 +325,7 @@ config SH_SHMIN
config SH_LBOX_RE2
bool "L-BOX RE2"
depends on CPU_SUBTYPE_SH7751R
- select SYS_SUPPORTS_PCI
+ select HAS_PCI
help
Select L-BOX RE2 if configuring for the NTT COMWARE L-BOX RE2.
@@ -346,7 +346,7 @@ config SH_MAGIC_PANEL_R2
config SH_CAYMAN
bool "Hitachi Cayman"
depends on CPU_SUBTYPE_SH5_101 || CPU_SUBTYPE_SH5_103
- select SYS_SUPPORTS_PCI
+ select HAS_PCI
select ARCH_MIGHT_HAVE_PC_SERIO
config SH_POLARIS
@@ -380,7 +380,7 @@ config SH_APSH4A3A
config SH_APSH4AD0A
bool "AP-SH4AD-0A"
select SH_ALPHA_BOARD
- select SYS_SUPPORTS_PCI
+ select HAS_PCI
select REGULATOR_FIXED_VOLTAGE if REGULATOR
depends on CPU_SUBTYPE_SH7786
help
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig
index e6f2a38d2e61..2a4943f9765e 100644
--- a/arch/sparc/Kconfig
+++ b/arch/sparc/Kconfig
@@ -16,6 +16,7 @@ config SPARC
select ARCH_MIGHT_HAVE_PC_SERIO
select OF
select OF_PROMTREE
+ select HAS_PCI
select HAVE_IDE
select HAVE_OPROFILE
select HAVE_ARCH_KGDB if !SMP || SPARC64
@@ -474,18 +475,6 @@ config SUN_LDOMS
Say Y here is you want to support virtual devices via
Logical Domains.
-config PCI
- bool "Support for PCI and PS/2 keyboard/mouse"
- help
- Find out whether your system includes a PCI bus. PCI is the name of
- a bus system, i.e. the way the CPU talks to the other stuff inside
- your box. If you say Y here, the kernel will include drivers and
- infrastructure code to support PCI bus devices.
-
- CONFIG_PCI is needed for all JavaStation's (including MrCoffee),
- CP-1200, JavaEngine-1, Corona, Red October, and Serengeti SGSC.
- All of these platforms are extremely obscure, so say N if unsure.
-
config PCI_DOMAINS
def_bool PCI if SPARC64
@@ -520,8 +509,6 @@ config SPARC_GRPCI2
help
Say Y here to include the GRPCI2 Host Bridge Driver.
-source "drivers/pci/Kconfig"
-
source "drivers/pcmcia/Kconfig"
config SUN_OPENPROMFS
diff --git a/arch/um/Kconfig b/arch/um/Kconfig
index 6b9938919f0b..de982541a059 100644
--- a/arch/um/Kconfig
+++ b/arch/um/Kconfig
@@ -31,9 +31,6 @@ config ISA
config SBUS
bool
-config PCI
- bool
-
config PCMCIA
bool
diff --git a/arch/unicore32/Kconfig b/arch/unicore32/Kconfig
index 60eae744d8fd..d90a78f1143a 100644
--- a/arch/unicore32/Kconfig
+++ b/arch/unicore32/Kconfig
@@ -4,6 +4,7 @@ config UNICORE32
select ARCH_HAS_DEVMEM_IS_ALLOWED
select ARCH_MIGHT_HAVE_PC_PARPORT
select ARCH_MIGHT_HAVE_PC_SERIO
+ select HAS_PCI
select HAVE_MEMBLOCK
select HAVE_GENERIC_DMA_COHERENT
select HAVE_KERNEL_GZIP
@@ -119,16 +120,6 @@ endmenu
menu "Bus support"
-config PCI
- bool "PCI Support"
- help
- Find out whether you have a PCI motherboard. PCI is the name of a
- bus system, i.e. the way the CPU talks to the other stuff inside
- your box. Other bus systems are ISA, EISA, MicroChannel (MCA) or
- VESA. If you have PCI, say Y, otherwise N.
-
-source "drivers/pci/Kconfig"
-
source "drivers/pcmcia/Kconfig"
endmenu
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 1a0be022f91d..7cbc656f47e7 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -113,6 +113,7 @@ config X86
select GENERIC_STRNLEN_USER
select GENERIC_TIME_VSYSCALL
select HARDLOCKUP_CHECK_TIMESTAMP if X86_64
+ select HAS_PCI
select HAVE_ACPI_APEI if ACPI
select HAVE_ACPI_APEI_NMI if ACPI
select HAVE_ALIGNED_STRUCT_PAGE if SLUB
@@ -2567,15 +2568,6 @@ endmenu
menu "Bus options (PCI etc.)"
-config PCI
- bool "PCI support"
- default y
- ---help---
- Find out whether you have a PCI motherboard. PCI is the name of a
- bus system, i.e. the way the CPU talks to the other stuff inside
- your box. Other bus systems are ISA, EISA, MicroChannel (MCA) or
- VESA. If you have PCI, say Y, otherwise N.
-
choice
prompt "PCI access mode"
depends on X86_32 && PCI
@@ -2658,8 +2650,6 @@ config PCI_CNB20LE_QUIRK
You should say N unless you know you need this.
-source "drivers/pci/Kconfig"
-
config ISA_BUS
bool "ISA bus support on modern systems" if EXPERT
help
diff --git a/arch/xtensa/Kconfig b/arch/xtensa/Kconfig
index b9ad83a0ee5d..22abeb236863 100644
--- a/arch/xtensa/Kconfig
+++ b/arch/xtensa/Kconfig
@@ -20,6 +20,7 @@ config XTENSA
select GENERIC_PCI_IOMAP
select GENERIC_SCHED_CLOCK
select GENERIC_STRNCPY_FROM_USER if KASAN
+ select HAS_PCI
select HAVE_ARCH_KASAN if MMU
select HAVE_DEBUG_KMEMLEAK
select HAVE_DMA_CONTIGUOUS
@@ -384,21 +385,6 @@ config XTENSA_CALIBRATE_CCOUNT
config SERIAL_CONSOLE
def_bool n
-menu "Bus options"
-
-config PCI
- bool "PCI support"
- default y
- help
- Find out whether you have a PCI motherboard. PCI is the name of a
- bus system, i.e. the way the CPU talks to the other stuff inside
- your box. Other bus systems are ISA, EISA, MicroChannel (MCA) or
- VESA. If you have PCI, say Y, otherwise N.
-
-source "drivers/pci/Kconfig"
-
-endmenu
-
menu "Platform options"
choice
diff --git a/drivers/Kconfig b/drivers/Kconfig
index ab4d43923c4d..059573823387 100644
--- a/drivers/Kconfig
+++ b/drivers/Kconfig
@@ -1,7 +1,11 @@
# SPDX-License-Identifier: GPL-2.0
menu "Device Drivers"
+# Keep I/O buses first
+
source "drivers/amba/Kconfig"
+source "drivers/pci/Kconfig"
+
source "drivers/base/Kconfig"
diff --git a/drivers/parisc/Kconfig b/drivers/parisc/Kconfig
index 5a48b5606110..5bbfea1a019c 100644
--- a/drivers/parisc/Kconfig
+++ b/drivers/parisc/Kconfig
@@ -63,17 +63,6 @@ config ISA
If you want to plug an ISA card into your EISA bus, say Y here.
Most people should say N.
-config PCI
- bool "PCI support"
- help
- All recent HP machines have PCI slots, and you should say Y here
- if you have a recent machine. If you are convinced you do not have
- PCI slots in your machine (eg a 712), then you may say "N" here.
- Beware that some GSC cards have a Dino onboard and PCI inside them,
- so it may be safest to say "Y" anyway.
-
-source "drivers/pci/Kconfig"
-
config GSC_DINO
bool "GSCtoPCI/Dino PCI support"
depends on PCI && GSC
diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig
index 56ff8f6d31fc..229a518b68cd 100644
--- a/drivers/pci/Kconfig
+++ b/drivers/pci/Kconfig
@@ -3,6 +3,18 @@
# PCI configuration
#
+config HAS_PCI
+ bool
+
+menuconfig PCI
+ bool "PCI support"
+ depends on HAS_PCI
+
+ help
+ This option enables support for the PCI local bus, including
+ support for PCI-X and the fundations for PCI Express support.
+ Say 'Y' here unless you know what you are doing.
+
source "drivers/pci/pcie/Kconfig"
config PCI_MSI
--
2.19.1
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox