* [PATCH v3 4/6] powerpc/fsl_booke/64: do not clear the BSS for the second pass
From: Jason Yan @ 2020-02-06 2:58 UTC (permalink / raw)
To: mpe, linuxppc-dev, diana.craciun, christophe.leroy, benh, paulus,
npiggin, keescook, kernel-hardening, oss
Cc: Jason Yan, linux-kernel, zhaohongjiang
In-Reply-To: <20200206025825.22934-1-yanaijie@huawei.com>
The BSS section has already cleared out in the first pass. No need to
clear it again. This can save some time when booting with KASLR
enabled.
Signed-off-by: Jason Yan <yanaijie@huawei.com>
Cc: Scott Wood <oss@buserror.net>
Cc: Diana Craciun <diana.craciun@nxp.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Christophe Leroy <christophe.leroy@c-s.fr>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Kees Cook <keescook@chromium.org>
---
arch/powerpc/kernel/head_64.S | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S
index 744624140fb8..8c644e7c3eaf 100644
--- a/arch/powerpc/kernel/head_64.S
+++ b/arch/powerpc/kernel/head_64.S
@@ -914,6 +914,13 @@ start_here_multiplatform:
bl relative_toc
tovirt(r2,r2)
+ /* Do not clear the BSS for the second pass if randomized */
+ LOAD_REG_ADDR(r3, kernstart_virt_addr)
+ lwz r3,0(r3)
+ LOAD_REG_IMMEDIATE(r4, KERNELBASE)
+ cmpw r3,r4
+ bne 4f
+
/* Clear out the BSS. It may have been done in prom_init,
* already but that's irrelevant since prom_init will soon
* be detached from the kernel completely. Besides, we need
--
2.17.2
^ permalink raw reply related
* [PATCH v3 3/6] powerpc/fsl_booke/64: implement KASLR for fsl_booke64
From: Jason Yan @ 2020-02-06 2:58 UTC (permalink / raw)
To: mpe, linuxppc-dev, diana.craciun, christophe.leroy, benh, paulus,
npiggin, keescook, kernel-hardening, oss
Cc: Jason Yan, linux-kernel, zhaohongjiang
In-Reply-To: <20200206025825.22934-1-yanaijie@huawei.com>
The implementation for Freescale BookE64 is similar as BookE32. One
difference is that Freescale BookE64 set up a TLB mapping of 1G during
booting. Another difference is that ppc64 needs the kernel to be
64K-aligned. So we can randomize the kernel in this 1G mapping and make
it 64K-aligned. This can save some code to creat another TLB map at
early boot. The disadvantage is that we only have about 1G/64K = 16384
slots to put the kernel in.
To support secondary cpu boot up, a variable __kaslr_offset was added in
first_256B section. This can help secondary cpu get the kaslr offset
before the 1:1 mapping has been setup.
Signed-off-by: Jason Yan <yanaijie@huawei.com>
Cc: Scott Wood <oss@buserror.net>
Cc: Diana Craciun <diana.craciun@nxp.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Christophe Leroy <christophe.leroy@c-s.fr>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Kees Cook <keescook@chromium.org>
---
arch/powerpc/Kconfig | 2 +-
arch/powerpc/kernel/exceptions-64e.S | 10 +++++++++
arch/powerpc/kernel/head_64.S | 7 ++++++
arch/powerpc/kernel/setup_64.c | 4 +++-
arch/powerpc/mm/mmu_decl.h | 16 +++++++-------
arch/powerpc/mm/nohash/kaslr_booke.c | 33 +++++++++++++++++++++++++---
6 files changed, 59 insertions(+), 13 deletions(-)
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index c150a9d49343..754aeb96bb1c 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -568,7 +568,7 @@ config RELOCATABLE
config RANDOMIZE_BASE
bool "Randomize the address of the kernel image"
- depends on (FSL_BOOKE && FLATMEM && PPC32)
+ depends on (PPC_FSL_BOOK3E && FLATMEM)
depends on RELOCATABLE
help
Randomizes the virtual address at which the kernel image is
diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
index 1b9b174bee86..c1c05b8684ca 100644
--- a/arch/powerpc/kernel/exceptions-64e.S
+++ b/arch/powerpc/kernel/exceptions-64e.S
@@ -1378,6 +1378,7 @@ skpinv: addi r6,r6,1 /* Increment */
1: mflr r6
addi r6,r6,(2f - 1b)
tovirt(r6,r6)
+ add r6,r6,r19
lis r7,MSR_KERNEL@h
ori r7,r7,MSR_KERNEL@l
mtspr SPRN_SRR0,r6
@@ -1400,6 +1401,7 @@ skpinv: addi r6,r6,1 /* Increment */
/* We translate LR and return */
tovirt(r8,r8)
+ add r8,r8,r19
mtlr r8
blr
@@ -1528,6 +1530,7 @@ a2_tlbinit_code_end:
*/
_GLOBAL(start_initialization_book3e)
mflr r28
+ li r19, 0
/* First, we need to setup some initial TLBs to map the kernel
* text, data and bss at PAGE_OFFSET. We don't have a real mode
@@ -1570,6 +1573,12 @@ _GLOBAL(book3e_secondary_core_init)
cmplwi r4,0
bne 2f
+ li r19, 0
+#ifdef CONFIG_RANDOMIZE_BASE
+ LOAD_REG_ADDR_PIC(r19, __kaslr_offset)
+ lwz r19,0(r19)
+ rlwinm r19,r19,0,0,5
+#endif
/* Setup TLB for this core */
bl initial_tlb_book3e
@@ -1602,6 +1611,7 @@ _GLOBAL(book3e_secondary_core_init)
lis r3,PAGE_OFFSET@highest
sldi r3,r3,32
or r28,r28,r3
+ add r28,r28,r19
1: mtlr r28
blr
diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S
index ad79fddb974d..744624140fb8 100644
--- a/arch/powerpc/kernel/head_64.S
+++ b/arch/powerpc/kernel/head_64.S
@@ -104,6 +104,13 @@ __secondary_hold_acknowledge:
.8byte 0x0
#ifdef CONFIG_RELOCATABLE
+#ifdef CONFIG_RANDOMIZE_BASE
+ . = 0x58
+ .globl __kaslr_offset
+__kaslr_offset:
+DEFINE_FIXED_SYMBOL(__kaslr_offset)
+ .long 0
+#endif
/* This flag is set to 1 by a loader if the kernel should run
* at the loaded address instead of the linked address. This
* is used by kexec-tools to keep the the kdump kernel in the
diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c
index 6104917a282d..a16b970a8d1a 100644
--- a/arch/powerpc/kernel/setup_64.c
+++ b/arch/powerpc/kernel/setup_64.c
@@ -66,7 +66,7 @@
#include <asm/feature-fixups.h>
#include <asm/kup.h>
#include <asm/early_ioremap.h>
-
+#include <mm/mmu_decl.h>
#include "setup.h"
int spinning_secondaries;
@@ -300,6 +300,8 @@ void __init early_setup(unsigned long dt_ptr)
/* Enable early debugging if any specified (see udbg.h) */
udbg_early_init();
+ kaslr_early_init(__va(dt_ptr), 0);
+
udbg_printf(" -> %s(), dt_ptr: 0x%lx\n", __func__, dt_ptr);
/*
diff --git a/arch/powerpc/mm/mmu_decl.h b/arch/powerpc/mm/mmu_decl.h
index 3e1c85c7d10b..bbd721d1e3d7 100644
--- a/arch/powerpc/mm/mmu_decl.h
+++ b/arch/powerpc/mm/mmu_decl.h
@@ -147,14 +147,6 @@ void reloc_kernel_entry(void *fdt, long addr);
extern void loadcam_entry(unsigned int index);
extern void loadcam_multi(int first_idx, int num, int tmp_idx);
-#ifdef CONFIG_RANDOMIZE_BASE
-void kaslr_early_init(void *dt_ptr, phys_addr_t size);
-void kaslr_late_init(void);
-#else
-static inline void kaslr_early_init(void *dt_ptr, phys_addr_t size) {}
-static inline void kaslr_late_init(void) {}
-#endif
-
struct tlbcam {
u32 MAS0;
u32 MAS1;
@@ -164,6 +156,14 @@ struct tlbcam {
};
#endif
+#ifdef CONFIG_RANDOMIZE_BASE
+void kaslr_early_init(void *dt_ptr, phys_addr_t size);
+void kaslr_late_init(void);
+#else
+static inline void kaslr_early_init(void *dt_ptr, phys_addr_t size) {}
+static inline void kaslr_late_init(void) {}
+#endif
+
#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_FSL_BOOKE) || defined(CONFIG_PPC_8xx)
/* 6xx have BATS */
/* FSL_BOOKE have TLBCAM */
diff --git a/arch/powerpc/mm/nohash/kaslr_booke.c b/arch/powerpc/mm/nohash/kaslr_booke.c
index 07b036e98353..c6f5c1db1394 100644
--- a/arch/powerpc/mm/nohash/kaslr_booke.c
+++ b/arch/powerpc/mm/nohash/kaslr_booke.c
@@ -231,7 +231,7 @@ static __init unsigned long get_usable_address(const void *fdt,
unsigned long pa;
unsigned long pa_end;
- for (pa = offset; (long)pa > (long)start; pa -= SZ_16K) {
+ for (pa = offset; (long)pa > (long)start; pa -= SZ_64K) {
pa_end = pa + regions.kernel_size;
if (overlaps_region(fdt, pa, pa_end))
continue;
@@ -265,14 +265,14 @@ static unsigned long __init kaslr_legal_offset(void *dt_ptr, unsigned long rando
{
unsigned long koffset = 0;
unsigned long start;
- unsigned long index;
unsigned long offset;
+#ifdef CONFIG_PPC32
/*
* Decide which 64M we want to start
* Only use the low 8 bits of the random seed
*/
- index = random & 0xFF;
+ unsigned long index = random & 0xFF;
index %= regions.linear_sz / SZ_64M;
/* Decide offset inside 64M */
@@ -287,6 +287,15 @@ static unsigned long __init kaslr_legal_offset(void *dt_ptr, unsigned long rando
break;
index--;
}
+#else
+ /* Decide kernel offset inside 1G */
+ offset = random % (SZ_1G - regions.kernel_size);
+ offset = round_down(offset, SZ_64K);
+
+ start = memstart_addr;
+ offset = memstart_addr + offset;
+ koffset = get_usable_address(dt_ptr, start, offset);
+#endif
if (koffset != 0)
koffset -= memstart_addr;
@@ -325,6 +334,7 @@ static unsigned long __init kaslr_choose_location(void *dt_ptr, phys_addr_t size
else
pr_warn("KASLR: No safe seed for randomizing the kernel base.\n");
+#ifdef CONFIG_PPC32
ram = min_t(phys_addr_t, __max_low_memory, size);
ram = map_mem_in_cams(ram, CONFIG_LOWMEM_CAM_NUM, true);
linear_sz = min_t(unsigned long, ram, SZ_512M);
@@ -332,6 +342,7 @@ static unsigned long __init kaslr_choose_location(void *dt_ptr, phys_addr_t size
/* If the linear size is smaller than 64M, do not randmize */
if (linear_sz < SZ_64M)
return 0;
+#endif
/* check for a reserved-memory node and record its cell sizes */
regions.reserved_mem = fdt_path_offset(dt_ptr, "/reserved-memory");
@@ -363,6 +374,17 @@ notrace void __init kaslr_early_init(void *dt_ptr, phys_addr_t size)
unsigned long offset;
unsigned long kernel_sz;
+#ifdef CONFIG_PPC64
+ unsigned int *__kaslr_offset = (unsigned int *)(KERNELBASE + 0x58);
+ unsigned int *__run_at_load = (unsigned int *)(KERNELBASE + 0x5c);
+
+ if (*__run_at_load == 1)
+ return;
+
+ /* Setup flat device-tree pointer */
+ initial_boot_params = dt_ptr;
+#endif
+
kernel_sz = (unsigned long)_end - (unsigned long)_stext;
offset = kaslr_choose_location(dt_ptr, size, kernel_sz);
@@ -372,6 +394,7 @@ notrace void __init kaslr_early_init(void *dt_ptr, phys_addr_t size)
kernstart_virt_addr += offset;
kernstart_addr += offset;
+#ifdef CONFIG_PPC32
is_second_reloc = 1;
if (offset >= SZ_64M) {
@@ -381,6 +404,10 @@ notrace void __init kaslr_early_init(void *dt_ptr, phys_addr_t size)
/* Create kernel map to relocate in */
create_kaslr_tlb_entry(1, tlb_virt, tlb_phys);
}
+#else
+ *__kaslr_offset = kernstart_virt_addr - KERNELBASE;
+ *__run_at_load = 1;
+#endif
/* Copy the kernel to it's new location and run */
memcpy((void *)kernstart_virt_addr, (void *)_stext, kernel_sz);
--
2.17.2
^ permalink raw reply related
* [PATCH v3 1/6] powerpc/fsl_booke/kaslr: refactor kaslr_legal_offset() and kaslr_early_init()
From: Jason Yan @ 2020-02-06 2:58 UTC (permalink / raw)
To: mpe, linuxppc-dev, diana.craciun, christophe.leroy, benh, paulus,
npiggin, keescook, kernel-hardening, oss
Cc: Jason Yan, linux-kernel, zhaohongjiang
In-Reply-To: <20200206025825.22934-1-yanaijie@huawei.com>
Some code refactor in kaslr_legal_offset() and kaslr_early_init(). No
functional change. This is a preparation for KASLR fsl_booke64.
Signed-off-by: Jason Yan <yanaijie@huawei.com>
Cc: Scott Wood <oss@buserror.net>
Cc: Diana Craciun <diana.craciun@nxp.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Christophe Leroy <christophe.leroy@c-s.fr>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Kees Cook <keescook@chromium.org>
---
arch/powerpc/mm/nohash/kaslr_booke.c | 40 ++++++++++++++--------------
1 file changed, 20 insertions(+), 20 deletions(-)
diff --git a/arch/powerpc/mm/nohash/kaslr_booke.c b/arch/powerpc/mm/nohash/kaslr_booke.c
index 4a75f2d9bf0e..07b036e98353 100644
--- a/arch/powerpc/mm/nohash/kaslr_booke.c
+++ b/arch/powerpc/mm/nohash/kaslr_booke.c
@@ -25,6 +25,7 @@ struct regions {
unsigned long pa_start;
unsigned long pa_end;
unsigned long kernel_size;
+ unsigned long linear_sz;
unsigned long dtb_start;
unsigned long dtb_end;
unsigned long initrd_start;
@@ -260,11 +261,23 @@ static __init void get_cell_sizes(const void *fdt, int node, int *addr_cells,
*size_cells = fdt32_to_cpu(*prop);
}
-static unsigned long __init kaslr_legal_offset(void *dt_ptr, unsigned long index,
- unsigned long offset)
+static unsigned long __init kaslr_legal_offset(void *dt_ptr, unsigned long random)
{
unsigned long koffset = 0;
unsigned long start;
+ unsigned long index;
+ unsigned long offset;
+
+ /*
+ * Decide which 64M we want to start
+ * Only use the low 8 bits of the random seed
+ */
+ index = random & 0xFF;
+ index %= regions.linear_sz / SZ_64M;
+
+ /* Decide offset inside 64M */
+ offset = random % (SZ_64M - regions.kernel_size);
+ offset = round_down(offset, SZ_16K);
while ((long)index >= 0) {
offset = memstart_addr + index * SZ_64M + offset;
@@ -289,10 +302,9 @@ static inline __init bool kaslr_disabled(void)
static unsigned long __init kaslr_choose_location(void *dt_ptr, phys_addr_t size,
unsigned long kernel_sz)
{
- unsigned long offset, random;
+ unsigned long random;
unsigned long ram, linear_sz;
u64 seed;
- unsigned long index;
kaslr_get_cmdline(dt_ptr);
if (kaslr_disabled())
@@ -333,22 +345,12 @@ static unsigned long __init kaslr_choose_location(void *dt_ptr, phys_addr_t size
regions.dtb_start = __pa(dt_ptr);
regions.dtb_end = __pa(dt_ptr) + fdt_totalsize(dt_ptr);
regions.kernel_size = kernel_sz;
+ regions.linear_sz = linear_sz;
get_initrd_range(dt_ptr);
get_crash_kernel(dt_ptr, ram);
- /*
- * Decide which 64M we want to start
- * Only use the low 8 bits of the random seed
- */
- index = random & 0xFF;
- index %= linear_sz / SZ_64M;
-
- /* Decide offset inside 64M */
- offset = random % (SZ_64M - kernel_sz);
- offset = round_down(offset, SZ_16K);
-
- return kaslr_legal_offset(dt_ptr, index, offset);
+ return kaslr_legal_offset(dt_ptr, random);
}
/*
@@ -358,8 +360,6 @@ static unsigned long __init kaslr_choose_location(void *dt_ptr, phys_addr_t size
*/
notrace void __init kaslr_early_init(void *dt_ptr, phys_addr_t size)
{
- unsigned long tlb_virt;
- phys_addr_t tlb_phys;
unsigned long offset;
unsigned long kernel_sz;
@@ -375,8 +375,8 @@ notrace void __init kaslr_early_init(void *dt_ptr, phys_addr_t size)
is_second_reloc = 1;
if (offset >= SZ_64M) {
- tlb_virt = round_down(kernstart_virt_addr, SZ_64M);
- tlb_phys = round_down(kernstart_addr, SZ_64M);
+ unsigned long tlb_virt = round_down(kernstart_virt_addr, SZ_64M);
+ phys_addr_t tlb_phys = round_down(kernstart_addr, SZ_64M);
/* Create kernel map to relocate in */
create_kaslr_tlb_entry(1, tlb_virt, tlb_phys);
--
2.17.2
^ permalink raw reply related
* [PATCH v3 6/6] powerpc/fsl_booke/kaslr: rename kaslr-booke32.rst to kaslr-booke.rst and add 64bit part
From: Jason Yan @ 2020-02-06 2:58 UTC (permalink / raw)
To: mpe, linuxppc-dev, diana.craciun, christophe.leroy, benh, paulus,
npiggin, keescook, kernel-hardening, oss
Cc: Jason Yan, linux-kernel, zhaohongjiang
In-Reply-To: <20200206025825.22934-1-yanaijie@huawei.com>
Now we support both 32 and 64 bit KASLR for fsl booke. Add document for
64 bit part and rename kaslr-booke32.rst to kaslr-booke.rst.
Signed-off-by: Jason Yan <yanaijie@huawei.com>
Cc: Scott Wood <oss@buserror.net>
Cc: Diana Craciun <diana.craciun@nxp.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Christophe Leroy <christophe.leroy@c-s.fr>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Kees Cook <keescook@chromium.org>
---
.../{kaslr-booke32.rst => kaslr-booke.rst} | 35 ++++++++++++++++---
1 file changed, 31 insertions(+), 4 deletions(-)
rename Documentation/powerpc/{kaslr-booke32.rst => kaslr-booke.rst} (59%)
diff --git a/Documentation/powerpc/kaslr-booke32.rst b/Documentation/powerpc/kaslr-booke.rst
similarity index 59%
rename from Documentation/powerpc/kaslr-booke32.rst
rename to Documentation/powerpc/kaslr-booke.rst
index 8b259fdfdf03..42121fed8249 100644
--- a/Documentation/powerpc/kaslr-booke32.rst
+++ b/Documentation/powerpc/kaslr-booke.rst
@@ -1,15 +1,18 @@
.. SPDX-License-Identifier: GPL-2.0
-===========================
-KASLR for Freescale BookE32
-===========================
+=========================
+KASLR for Freescale BookE
+=========================
The word KASLR stands for Kernel Address Space Layout Randomization.
This document tries to explain the implementation of the KASLR for
-Freescale BookE32. KASLR is a security feature that deters exploit
+Freescale BookE. KASLR is a security feature that deters exploit
attempts relying on knowledge of the location of kernel internals.
+KASLR for Freescale BookE32
+-------------------------
+
Since CONFIG_RELOCATABLE has already supported, what we need to do is
map or copy kernel to a proper place and relocate. Freescale Book-E
parts expect lowmem to be mapped by fixed TLB entries(TLB1). The TLB1
@@ -38,5 +41,29 @@ bit of the entropy to decide the index of the 64M zone. Then we chose a
kernstart_virt_addr
+
+KASLR for Freescale BookE64
+---------------------------
+
+The implementation for Freescale BookE64 is similar as BookE32. One
+difference is that Freescale BookE64 set up a TLB mapping of 1G during
+booting. Another difference is that ppc64 needs the kernel to be
+64K-aligned. So we can randomize the kernel in this 1G mapping and make
+it 64K-aligned. This can save some code to creat another TLB map at early
+boot. The disadvantage is that we only have about 1G/64K = 16384 slots to
+put the kernel in::
+
+ KERNELBASE
+
+ 64K |--> kernel <--|
+ | | |
+ +--+--+--+ +--+--+--+--+--+--+--+--+--+ +--+--+
+ | | | |....| | | | | | | | | |....| | |
+ +--+--+--+ +--+--+--+--+--+--+--+--+--+ +--+--+
+ | | 1G
+ |-----> offset <-----|
+
+ kernstart_virt_addr
+
To enable KASLR, set CONFIG_RANDOMIZE_BASE = y. If KASLR is enable and you
want to disable it at runtime, add "nokaslr" to the kernel cmdline.
--
2.17.2
^ permalink raw reply related
* [PATCH v3 5/6] powerpc/fsl_booke/64: clear the original kernel if randomized
From: Jason Yan @ 2020-02-06 2:58 UTC (permalink / raw)
To: mpe, linuxppc-dev, diana.craciun, christophe.leroy, benh, paulus,
npiggin, keescook, kernel-hardening, oss
Cc: Jason Yan, linux-kernel, zhaohongjiang
In-Reply-To: <20200206025825.22934-1-yanaijie@huawei.com>
The original kernel still exists in the memory, clear it now.
Signed-off-by: Jason Yan <yanaijie@huawei.com>
Cc: Scott Wood <oss@buserror.net>
Cc: Diana Craciun <diana.craciun@nxp.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Christophe Leroy <christophe.leroy@c-s.fr>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Kees Cook <keescook@chromium.org>
---
arch/powerpc/mm/nohash/kaslr_booke.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/arch/powerpc/mm/nohash/kaslr_booke.c b/arch/powerpc/mm/nohash/kaslr_booke.c
index c6f5c1db1394..ed1277059368 100644
--- a/arch/powerpc/mm/nohash/kaslr_booke.c
+++ b/arch/powerpc/mm/nohash/kaslr_booke.c
@@ -378,8 +378,10 @@ notrace void __init kaslr_early_init(void *dt_ptr, phys_addr_t size)
unsigned int *__kaslr_offset = (unsigned int *)(KERNELBASE + 0x58);
unsigned int *__run_at_load = (unsigned int *)(KERNELBASE + 0x5c);
- if (*__run_at_load == 1)
+ if (*__run_at_load == 1) {
+ kaslr_late_init();
return;
+ }
/* Setup flat device-tree pointer */
initial_boot_params = dt_ptr;
--
2.17.2
^ permalink raw reply related
* [PATCH v6 00/11] Introduces new functions for tracking lockless pagetable walks
From: Leonardo Bras @ 2020-02-06 3:08 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Arnd Bergmann, Andrew Morton, Aneesh Kumar K.V, Nicholas Piggin,
Christophe Leroy, Steven Price, Robin Murphy, Leonardo Bras,
Mahesh Salgaonkar, Balbir Singh, Reza Arbab, Thomas Gleixner,
Allison Randal, Greg Kroah-Hartman, Mike Rapoport,
Michal Suchanek
Cc: linux-arch, linux-mm, linuxppc-dev, linux-kernel, kvm-ppc
Patches 1-2: Introduces new arch-generic functions to use before
and after lockless pagetable walks, instead of local_irq_*, and
applies them to generic code. It makes lockless pagetable walks
more explicit and improves documentation about it.
Patches 3-9: Introduces a powerpc-specific version of the above
functions with the option to not touch irq config. Then apply them
to all powerpc code that do lockless pagetable walks.
Patches 10-11: Introduces a percpu counting method to keep track of
the lockless page table walks, then uses this info to reduce the
waiting time on serialize_against_pte_lookup().
Use case:
If a process (qemu) with a lot of CPUs (128) try to munmap() a large
chunk of memory (496GB) mapped with THP, it takes an average of 275
seconds, which can cause a lot of problems to the load (in qemu case,
the guest will lock for this time).
Trying to find the source of this bug, I found out most of this time is
spent on serialize_against_pte_lookup(). This function will take a lot
of time in smp_call_function_many() if there is more than a couple CPUs
running the user process. Since it has to happen to all THP mapped, it
will take a very long time for large amounts of memory.
By the docs, serialize_against_pte_lookup() is needed in order to avoid
pmd_t to pte_t casting inside find_current_mm_pte(), or any lockless
pagetable walk, to happen concurrently with THP splitting/collapsing.
It does so by calling a do_nothing() on each CPU in mm->cpu_bitmap[],
after interrupts are re-enabled.
Since, interrupts are (usually) disabled during lockless pagetable
walk, and serialize_against_pte_lookup will only return after
interrupts are enabled, it is protected.
Percpu count-based method:
So, by what I could understand, if there is no lockless pagetable walk
running on given cpu, there is no need to call
serialize_against_pte_lookup() there.
To reduce the cost of running serialize_against_pte_lookup(), I
propose a percpu-counter that keeps track of how many
lockless pagetable walks are currently running on each cpu, and if there
is none, just skip smp_call_function_many() for that cpu.
- Every percpu-counter can be changed only by it's own CPU
- It makes use of the original memory barrier in the functions
- Any counter can be read by any CPU
Due to not locking nor using atomic variables, the impact on the
lockless pagetable walk is intended to be minimum.
The related functions are:
begin_lockless_pgtbl_walk()
Insert before starting any lockless pgtable walk
end_lockless_pgtbl_walk()
Insert after the end of any lockless pgtable walk
(Mostly after the ptep is last used)
Results:
On my workload (qemu), I could see munmap's time reduction from 275
seconds to 430ms.
Bonus:
I documented some lockless pagetable walks in which it's not
necessary to keep track, given they work on init_mm or guest pgd.
Also fixed some misplaced local_irq_{restore, enable}.
Changes since v5:
Changed counting approach from atomic variables to percpu variables
Counting method only affects powepc, arch-generic only toggle irqs
Changed commit order, so the counting method is introduced at the end
Removed config option, always enabled in powerpc
Rebased on top of v5.5
Link: http://patchwork.ozlabs.org/project/linuxppc-dev/list/?series=133907
Changes since v4:
Rebased on top of v5.4-rc1
Declared real generic functions instead of dummies
start_lockless_pgtbl_walk renamed to begin_lockless_pgtbl_walk
Interrupt {dis,en}able is now inside of {begin,end}_lockless_pgtbl_walk
Power implementation has option to not {dis,en}able interrupt
More documentation inside the funtions.
Some irq masks variables renamed
Removed some proxy mm_structs
Few typos fixed
Link: http://patchwork.ozlabs.org/project/linuxppc-dev/list/?series=133015
Changes since v3:
Explain (comments) why some lockless pgtbl walks don't need
local_irq_disable (real mode + MSR_EE=0)
Explain (comments) places where counting method is not needed (guest pgd,
which is not touched by THP)
Fixes some misplaced local_irq_restore()
Link: http://patchwork.ozlabs.org/project/linuxppc-dev/list/?series=132417
Changes since v2:
Rebased to v5.3
Adds support on __get_user_pages_fast
Adds usage decription to *_lockless_pgtbl_walk()
Better style to dummy functions
Link: http://patchwork.ozlabs.org/project/linuxppc-dev/list/?series=131839
Changes since v1:
Isolated atomic operations in functions *_lockless_pgtbl_walk()
Fixed behavior of decrementing before last ptep was used
Link: http://patchwork.ozlabs.org/patch/1163093/
Special thanks for:
Aneesh Kumar, Nick Piggin, Paul Mackerras, Michael Ellerman, Fabiano Rosas,
Dipankar Sarma and Oliver O'Halloran.
Leonardo Bras (11):
asm-generic/pgtable: Adds generic functions to track lockless pgtable
walks
mm/gup: Use functions to track lockless pgtbl walks on gup_pgd_range
powerpc/mm: Adds arch-specificic functions to track lockless pgtable
walks
powerpc/mce_power: Use functions to track lockless pgtbl walks
powerpc/perf: Use functions to track lockless pgtbl walks
powerpc/mm/book3s64/hash: Use functions to track lockless pgtbl walks
powerpc/kvm/e500: Use functions to track lockless pgtbl walks
powerpc/kvm/book3s_hv: Use functions to track lockless pgtbl walks
powerpc/kvm/book3s_64: Use functions to track lockless pgtbl walks
powerpc/mm: Adds counting method to track lockless pagetable walks
powerpc/mm/book3s64/pgtable: Uses counting method to skip serializing
arch/powerpc/include/asm/book3s/64/pgtable.h | 6 +
arch/powerpc/kernel/mce_power.c | 6 +-
arch/powerpc/kvm/book3s_64_mmu_hv.c | 6 +-
arch/powerpc/kvm/book3s_64_mmu_radix.c | 34 +++++-
arch/powerpc/kvm/book3s_64_vio_hv.c | 6 +-
arch/powerpc/kvm/book3s_hv_nested.c | 22 +++-
arch/powerpc/kvm/book3s_hv_rm_mmu.c | 28 +++--
arch/powerpc/kvm/e500_mmu_host.c | 9 +-
arch/powerpc/mm/book3s64/hash_tlb.c | 6 +-
arch/powerpc/mm/book3s64/hash_utils.c | 27 +++--
arch/powerpc/mm/book3s64/pgtable.c | 120 ++++++++++++++++++-
arch/powerpc/perf/callchain.c | 6 +-
include/asm-generic/pgtable.h | 51 ++++++++
mm/gup.c | 10 +-
14 files changed, 288 insertions(+), 49 deletions(-)
--
2.24.1
^ permalink raw reply
* [PATCH v6 01/11] asm-generic/pgtable: Adds generic functions to track lockless pgtable walks
From: Leonardo Bras @ 2020-02-06 3:08 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Arnd Bergmann, Andrew Morton, Aneesh Kumar K.V, Nicholas Piggin,
Christophe Leroy, Steven Price, Robin Murphy, Leonardo Bras,
Mahesh Salgaonkar, Balbir Singh, Reza Arbab, Thomas Gleixner,
Allison Randal, Greg Kroah-Hartman, Mike Rapoport,
Michal Suchanek
Cc: linux-arch, linux-mm, linuxppc-dev, linux-kernel, kvm-ppc
In-Reply-To: <20200206030900.147032-1-leonardo@linux.ibm.com>
It's necessary to track lockless pagetable walks, in order to avoid doing
THP splitting/collapsing during them.
The default solution is to disable irq before lockless pagetable walks and
enable it after it's finished.
On code, this means you can find local_irq_disable() and local_irq_enable()
around some pieces of code, usually without comments on why it is needed.
This patch proposes a set of generic functions to be called before starting
and after finishing a lockless pagetable walk. It is supposed to make clear
that a lockless pagetable walk happens there, and also carries information
on why the irq disable/enable is needed.
begin_lockless_pgtbl_walk()
Insert before starting any lockless pgtable walk
end_lockless_pgtbl_walk()
Insert after the end of any lockless pgtable walk
(Mostly after the ptep is last used)
A memory barrier was also added just to make sure there is no speculative
read outside the interrupt disabled area. Other than that, it is not
supposed to have any change of behavior from current code.
It is planned to allow arch-specific versions, so that additional steps can
be added while keeping the code clean.
Signed-off-by: Leonardo Bras <leonardo@linux.ibm.com>
---
include/asm-generic/pgtable.h | 51 +++++++++++++++++++++++++++++++++++
1 file changed, 51 insertions(+)
diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h
index e2e2bef07dd2..8d368d3c0974 100644
--- a/include/asm-generic/pgtable.h
+++ b/include/asm-generic/pgtable.h
@@ -1222,6 +1222,57 @@ static inline bool arch_has_pfn_modify_check(void)
#endif
#endif
+#ifndef __HAVE_ARCH_LOCKLESS_PGTBL_WALK_CONTROL
+/*
+ * begin_lockless_pgtbl_walk: Must be inserted before a function call that does
+ * lockless pagetable walks, such as __find_linux_pte()
+ */
+static inline
+unsigned long begin_lockless_pgtbl_walk(void)
+{
+ unsigned long irq_mask;
+
+ /*
+ * Interrupts must be disabled during the lockless page table walk.
+ * That's because the deleting or splitting involves flushing TLBs,
+ * which in turn issues interrupts, that will block when disabled.
+ */
+ local_irq_save(irq_mask);
+
+ /*
+ * This memory barrier pairs with any code that is either trying to
+ * delete page tables, or split huge pages. Without this barrier,
+ * the page tables could be read speculatively outside of interrupt
+ * disabling.
+ */
+ smp_mb();
+
+ return irq_mask;
+}
+
+/*
+ * end_lockless_pgtbl_walk: Must be inserted after the last use of a pointer
+ * returned by a lockless pagetable walk, such as __find_linux_pte()
+ */
+static inline void end_lockless_pgtbl_walk(unsigned long irq_mask)
+{
+ /*
+ * This memory barrier pairs with any code that is either trying to
+ * delete page tables, or split huge pages. Without this barrier,
+ * the page tables could be read speculatively outside of interrupt
+ * disabling.
+ */
+ smp_mb();
+
+ /*
+ * Interrupts must be disabled during the lockless page table walk.
+ * That's because the deleting or splitting involves flushing TLBs,
+ * which in turn issues interrupts, that will block when disabled.
+ */
+ local_irq_restore(irq_mask);
+}
+#endif
+
/*
* On some architectures it depends on the mm if the p4d/pud or pmd
* layer of the page table hierarchy is folded or not.
--
2.24.1
^ permalink raw reply related
* [PATCH v6 02/11] mm/gup: Use functions to track lockless pgtbl walks on gup_pgd_range
From: Leonardo Bras @ 2020-02-06 3:08 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Arnd Bergmann, Andrew Morton, Aneesh Kumar K.V, Nicholas Piggin,
Christophe Leroy, Steven Price, Robin Murphy, Leonardo Bras,
Mahesh Salgaonkar, Balbir Singh, Reza Arbab, Thomas Gleixner,
Allison Randal, Greg Kroah-Hartman, Mike Rapoport,
Michal Suchanek
Cc: linux-arch, linux-mm, linuxppc-dev, linux-kernel, kvm-ppc
In-Reply-To: <20200206030900.147032-1-leonardo@linux.ibm.com>
As described, gup_pgd_range is a lockless pagetable walk. So, in order to
track against THP split/collapse, it disables/enables irq around it.
To make use of the new tracking functions, it replaces irq disable/enable
by {begin,end}_lockless_pgtbl_walk().
As local_irq_{save,restore} is present inside
{begin,end}_lockless_pgtbl_walk, there should be no change in the
workings.
Signed-off-by: Leonardo Bras <leonardo@linux.ibm.com>
---
mm/gup.c | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/mm/gup.c b/mm/gup.c
index 1b521e0ac1de..04e6f46993b6 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -2369,7 +2369,7 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
struct page **pages)
{
unsigned long len, end;
- unsigned long flags;
+ unsigned long irq_mask;
int nr = 0;
start = untagged_addr(start) & PAGE_MASK;
@@ -2395,9 +2395,9 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
if (IS_ENABLED(CONFIG_HAVE_FAST_GUP) &&
gup_fast_permitted(start, end)) {
- local_irq_save(flags);
+ irq_mask = begin_lockless_pgtbl_walk();
gup_pgd_range(start, end, write ? FOLL_WRITE : 0, pages, &nr);
- local_irq_restore(flags);
+ end_lockless_pgtbl_walk(irq_mask);
}
return nr;
@@ -2450,9 +2450,9 @@ static int internal_get_user_pages_fast(unsigned long start, int nr_pages,
if (IS_ENABLED(CONFIG_HAVE_FAST_GUP) &&
gup_fast_permitted(start, end)) {
- local_irq_disable();
+ begin_lockless_pgtbl_walk();
gup_pgd_range(addr, end, gup_flags, pages, &nr);
- local_irq_enable();
+ end_lockless_pgtbl_walk(IRQS_ENABLED);
ret = nr;
}
--
2.24.1
^ permalink raw reply related
* [PATCH v6 03/11] powerpc/mm: Adds arch-specificic functions to track lockless pgtable walks
From: Leonardo Bras @ 2020-02-06 3:08 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Arnd Bergmann, Andrew Morton, Aneesh Kumar K.V, Nicholas Piggin,
Christophe Leroy, Steven Price, Robin Murphy, Leonardo Bras,
Mahesh Salgaonkar, Balbir Singh, Reza Arbab, Thomas Gleixner,
Allison Randal, Greg Kroah-Hartman, Mike Rapoport,
Michal Suchanek
Cc: linux-arch, linux-mm, linuxppc-dev, linux-kernel, kvm-ppc
In-Reply-To: <20200206030900.147032-1-leonardo@linux.ibm.com>
On powerpc, we need to do some lockless pagetable walks from functions
that already have disabled interrupts, specially from real mode with
MSR[EE=0].
In these contexts, disabling/enabling interrupts can be very troubling.
So, this arch-specific implementation features functions with an extra
argument that allows interrupt enable/disable to be skipped:
__begin_lockless_pgtbl_walk() and __end_lockless_pgtbl_walk().
Functions similar to the generic ones are also exported, by calling
the above functions with parameter {en,dis}able_irq = true.
Signed-off-by: Leonardo Bras <leonardo@linux.ibm.com>
---
arch/powerpc/include/asm/book3s/64/pgtable.h | 6 ++
arch/powerpc/mm/book3s64/pgtable.c | 86 +++++++++++++++++++-
2 files changed, 91 insertions(+), 1 deletion(-)
diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h
index 201a69e6a355..78f6ffb1bb3e 100644
--- a/arch/powerpc/include/asm/book3s/64/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable.h
@@ -1375,5 +1375,11 @@ static inline bool pgd_is_leaf(pgd_t pgd)
return !!(pgd_raw(pgd) & cpu_to_be64(_PAGE_PTE));
}
+#define __HAVE_ARCH_LOCKLESS_PGTBL_WALK_CONTROL
+unsigned long begin_lockless_pgtbl_walk(void);
+unsigned long __begin_lockless_pgtbl_walk(bool disable_irq);
+void end_lockless_pgtbl_walk(unsigned long irq_mask);
+void __end_lockless_pgtbl_walk(unsigned long irq_mask, bool enable_irq);
+
#endif /* __ASSEMBLY__ */
#endif /* _ASM_POWERPC_BOOK3S_64_PGTABLE_H_ */
diff --git a/arch/powerpc/mm/book3s64/pgtable.c b/arch/powerpc/mm/book3s64/pgtable.c
index 2bf7e1b4fd82..535613030363 100644
--- a/arch/powerpc/mm/book3s64/pgtable.c
+++ b/arch/powerpc/mm/book3s64/pgtable.c
@@ -82,6 +82,7 @@ static void do_nothing(void *unused)
{
}
+
/*
* Serialize against find_current_mm_pte which does lock-less
* lookup in page tables with local interrupts disabled. For huge pages
@@ -98,6 +99,89 @@ void serialize_against_pte_lookup(struct mm_struct *mm)
smp_call_function_many(mm_cpumask(mm), do_nothing, NULL, 1);
}
+/* begin_lockless_pgtbl_walk: Must be inserted before a function call that does
+ * lockless pagetable walks, such as __find_linux_pte().
+ * This version allows setting disable_irq=false, so irqs are not touched, which
+ * is quite useful for running when ints are already disabled (like real-mode)
+ */
+inline
+unsigned long __begin_lockless_pgtbl_walk(bool disable_irq)
+{
+ unsigned long irq_mask = 0;
+
+ /*
+ * Interrupts must be disabled during the lockless page table walk.
+ * That's because the deleting or splitting involves flushing TLBs,
+ * which in turn issues interrupts, that will block when disabled.
+ *
+ * When this function is called from realmode with MSR[EE=0],
+ * it's not needed to touch irq, since it's already disabled.
+ */
+ if (disable_irq)
+ local_irq_save(irq_mask);
+
+ /*
+ * This memory barrier pairs with any code that is either trying to
+ * delete page tables, or split huge pages. Without this barrier,
+ * the page tables could be read speculatively outside of interrupt
+ * disabling or reference counting.
+ */
+ smp_mb();
+
+ return irq_mask;
+}
+EXPORT_SYMBOL(__begin_lockless_pgtbl_walk);
+
+/* begin_lockless_pgtbl_walk: Must be inserted before a function call that does
+ * lockless pagetable walks, such as __find_linux_pte().
+ * This version is used by generic code, and always assume irqs will be disabled
+ */
+unsigned long begin_lockless_pgtbl_walk(void)
+{
+ return __begin_lockless_pgtbl_walk(true);
+}
+EXPORT_SYMBOL(begin_lockless_pgtbl_walk);
+
+/*
+ * __end_lockless_pgtbl_walk: Must be inserted after the last use of a pointer
+ * returned by a lockless pagetable walk, such as __find_linux_pte()
+ * This version allows setting enable_irq=false, so irqs are not touched, which
+ * is quite useful for running when ints are already disabled (like real-mode)
+ */
+inline void __end_lockless_pgtbl_walk(unsigned long irq_mask, bool enable_irq)
+{
+ /*
+ * This memory barrier pairs with any code that is either trying to
+ * delete page tables, or split huge pages. Without this barrier,
+ * the page tables could be read speculatively outside of interrupt
+ * disabling or reference counting.
+ */
+ smp_mb();
+
+ /*
+ * Interrupts must be disabled during the lockless page table walk.
+ * That's because the deleting or splitting involves flushing TLBs,
+ * which in turn issues interrupts, that will block when disabled.
+ *
+ * When this function is called from realmode with MSR[EE=0],
+ * it's not needed to touch irq, since it's already disabled.
+ */
+ if (enable_irq)
+ local_irq_restore(irq_mask);
+}
+EXPORT_SYMBOL(__end_lockless_pgtbl_walk);
+
+/*
+ * end_lockless_pgtbl_walk: Must be inserted after the last use of a pointer
+ * returned by a lockless pagetable walk, such as __find_linux_pte()
+ * This version is used by generic code, and always assume irqs will be enabled
+ */
+void end_lockless_pgtbl_walk(unsigned long irq_mask)
+{
+ __end_lockless_pgtbl_walk(irq_mask, true);
+}
+EXPORT_SYMBOL(end_lockless_pgtbl_walk);
+
/*
* We use this to invalidate a pmdp entry before switching from a
* hugepte to regular pmd entry.
@@ -487,7 +571,7 @@ static int __init setup_disable_tlbie(char *str)
tlbie_capable = false;
tlbie_enabled = false;
- return 1;
+ return 1;
}
__setup("disable_tlbie", setup_disable_tlbie);
--
2.24.1
^ permalink raw reply related
* [PATCH v6 04/11] powerpc/mce_power: Use functions to track lockless pgtbl walks
From: Leonardo Bras @ 2020-02-06 3:08 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Arnd Bergmann, Andrew Morton, Aneesh Kumar K.V, Nicholas Piggin,
Christophe Leroy, Steven Price, Robin Murphy, Leonardo Bras,
Mahesh Salgaonkar, Balbir Singh, Reza Arbab, Thomas Gleixner,
Allison Randal, Greg Kroah-Hartman, Mike Rapoport,
Michal Suchanek
Cc: linux-arch, linux-mm, linuxppc-dev, linux-kernel, kvm-ppc
In-Reply-To: <20200206030900.147032-1-leonardo@linux.ibm.com>
Applies the new functions used for tracking lockless pgtable walks on
addr_to_pfn().
local_irq_{save,restore} is already inside {begin,end}_lockless_pgtbl_walk,
so there is no need to repeat it here.
Signed-off-by: Leonardo Bras <leonardo@linux.ibm.com>
---
arch/powerpc/kernel/mce_power.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/arch/powerpc/kernel/mce_power.c b/arch/powerpc/kernel/mce_power.c
index 1cbf7f1a4e3d..a9e38ef4e437 100644
--- a/arch/powerpc/kernel/mce_power.c
+++ b/arch/powerpc/kernel/mce_power.c
@@ -29,7 +29,7 @@ unsigned long addr_to_pfn(struct pt_regs *regs, unsigned long addr)
{
pte_t *ptep;
unsigned int shift;
- unsigned long pfn, flags;
+ unsigned long pfn, irq_mask;
struct mm_struct *mm;
if (user_mode(regs))
@@ -37,7 +37,7 @@ unsigned long addr_to_pfn(struct pt_regs *regs, unsigned long addr)
else
mm = &init_mm;
- local_irq_save(flags);
+ irq_mask = begin_lockless_pgtbl_walk();
ptep = __find_linux_pte(mm->pgd, addr, NULL, &shift);
if (!ptep || pte_special(*ptep)) {
@@ -53,7 +53,7 @@ unsigned long addr_to_pfn(struct pt_regs *regs, unsigned long addr)
}
out:
- local_irq_restore(flags);
+ end_lockless_pgtbl_walk(irq_mask);
return pfn;
}
--
2.24.1
^ permalink raw reply related
* [PATCH v6 05/11] powerpc/perf: Use functions to track lockless pgtbl walks
From: Leonardo Bras @ 2020-02-06 3:08 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Arnd Bergmann, Andrew Morton, Aneesh Kumar K.V, Nicholas Piggin,
Christophe Leroy, Steven Price, Robin Murphy, Leonardo Bras,
Mahesh Salgaonkar, Balbir Singh, Reza Arbab, Thomas Gleixner,
Allison Randal, Greg Kroah-Hartman, Mike Rapoport,
Michal Suchanek
Cc: linux-arch, linux-mm, linuxppc-dev, linux-kernel, kvm-ppc
In-Reply-To: <20200206030900.147032-1-leonardo@linux.ibm.com>
Applies the new functions used for tracking lockless pgtable walks on
read_user_stack_slow.
local_irq_{save,restore} is already inside {begin,end}_lockless_pgtbl_walk,
so there is no need to repeat it here.
Variable that saves the irq mask was renamed from flags to irq_mask so it
doesn't lose meaning now it's not directly passed to local_irq_* functions.
Signed-off-by: Leonardo Bras <leonardo@linux.ibm.com>
---
arch/powerpc/perf/callchain.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/arch/powerpc/perf/callchain.c b/arch/powerpc/perf/callchain.c
index cbc251981209..fd9979e69f36 100644
--- a/arch/powerpc/perf/callchain.c
+++ b/arch/powerpc/perf/callchain.c
@@ -116,14 +116,14 @@ static int read_user_stack_slow(void __user *ptr, void *buf, int nb)
unsigned shift;
unsigned long addr = (unsigned long) ptr;
unsigned long offset;
- unsigned long pfn, flags;
+ unsigned long pfn, irq_mask;
void *kaddr;
pgdir = current->mm->pgd;
if (!pgdir)
return -EFAULT;
- local_irq_save(flags);
+ irq_mask = begin_lockless_pgtbl_walk();
ptep = find_current_mm_pte(pgdir, addr, NULL, &shift);
if (!ptep)
goto err_out;
@@ -145,7 +145,7 @@ static int read_user_stack_slow(void __user *ptr, void *buf, int nb)
memcpy(buf, kaddr + offset, nb);
ret = 0;
err_out:
- local_irq_restore(flags);
+ end_lockless_pgtbl_walk(irq_mask);
return ret;
}
--
2.24.1
^ permalink raw reply related
* [PATCH v6 06/11] powerpc/mm/book3s64/hash: Use functions to track lockless pgtbl walks
From: Leonardo Bras @ 2020-02-06 3:08 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Arnd Bergmann, Andrew Morton, Aneesh Kumar K.V, Nicholas Piggin,
Christophe Leroy, Steven Price, Robin Murphy, Leonardo Bras,
Mahesh Salgaonkar, Balbir Singh, Reza Arbab, Thomas Gleixner,
Allison Randal, Greg Kroah-Hartman, Mike Rapoport,
Michal Suchanek
Cc: linux-arch, linux-mm, linuxppc-dev, linux-kernel, kvm-ppc
In-Reply-To: <20200206030900.147032-1-leonardo@linux.ibm.com>
Applies the new tracking functions to all hash-related functions that do
lockless pagetable walks.
hash_page_mm: Adds comment that explain that there is no need to
local_int_disable/save given that it is only called from DataAccess
interrupt, so interrupts are already disabled.
local_irq_{save,restore} is already inside {begin,end}_lockless_pgtbl_walk,
so there is no need to repeat it here.
Variable that saves the irq mask was renamed from flags to irq_mask so it
doesn't lose meaning now it's not directly passed to local_irq_* functions.
Signed-off-by: Leonardo Bras <leonardo@linux.ibm.com>
---
arch/powerpc/mm/book3s64/hash_tlb.c | 6 +++---
arch/powerpc/mm/book3s64/hash_utils.c | 27 +++++++++++++++++----------
2 files changed, 20 insertions(+), 13 deletions(-)
diff --git a/arch/powerpc/mm/book3s64/hash_tlb.c b/arch/powerpc/mm/book3s64/hash_tlb.c
index 4a70d8dd39cd..86547c4151f6 100644
--- a/arch/powerpc/mm/book3s64/hash_tlb.c
+++ b/arch/powerpc/mm/book3s64/hash_tlb.c
@@ -194,7 +194,7 @@ void __flush_hash_table_range(struct mm_struct *mm, unsigned long start,
{
bool is_thp;
int hugepage_shift;
- unsigned long flags;
+ unsigned long irq_mask;
start = _ALIGN_DOWN(start, PAGE_SIZE);
end = _ALIGN_UP(end, PAGE_SIZE);
@@ -209,7 +209,7 @@ void __flush_hash_table_range(struct mm_struct *mm, unsigned long start,
* to being hashed). This is not the most performance oriented
* way to do things but is fine for our needs here.
*/
- local_irq_save(flags);
+ irq_mask = begin_lockless_pgtbl_walk();
arch_enter_lazy_mmu_mode();
for (; start < end; start += PAGE_SIZE) {
pte_t *ptep = find_current_mm_pte(mm->pgd, start, &is_thp,
@@ -229,7 +229,7 @@ void __flush_hash_table_range(struct mm_struct *mm, unsigned long start,
hpte_need_flush(mm, start, ptep, pte, hugepage_shift);
}
arch_leave_lazy_mmu_mode();
- local_irq_restore(flags);
+ end_lockless_pgtbl_walk(irq_mask);
}
void flush_tlb_pmd_range(struct mm_struct *mm, pmd_t *pmd, unsigned long addr)
diff --git a/arch/powerpc/mm/book3s64/hash_utils.c b/arch/powerpc/mm/book3s64/hash_utils.c
index 523d4d39d11e..e6d4ab42173b 100644
--- a/arch/powerpc/mm/book3s64/hash_utils.c
+++ b/arch/powerpc/mm/book3s64/hash_utils.c
@@ -1341,12 +1341,16 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea,
ea &= ~((1ul << mmu_psize_defs[psize].shift) - 1);
#endif /* CONFIG_PPC_64K_PAGES */
- /* Get PTE and page size from page tables */
+ /* Get PTE and page size from page tables :
+ * Called in from DataAccess interrupt (data_access_common: 0x300),
+ * interrupts are disabled here.
+ */
+ __begin_lockless_pgtbl_walk(false);
ptep = find_linux_pte(pgdir, ea, &is_thp, &hugeshift);
if (ptep == NULL || !pte_present(*ptep)) {
DBG_LOW(" no PTE !\n");
rc = 1;
- goto bail;
+ goto bail_pgtbl_walk;
}
/* Add _PAGE_PRESENT to the required access perm */
@@ -1359,7 +1363,7 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea,
if (!check_pte_access(access, pte_val(*ptep))) {
DBG_LOW(" no access !\n");
rc = 1;
- goto bail;
+ goto bail_pgtbl_walk;
}
if (hugeshift) {
@@ -1383,7 +1387,7 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea,
if (current->mm == mm)
check_paca_psize(ea, mm, psize, user_region);
- goto bail;
+ goto bail_pgtbl_walk;
}
#ifndef CONFIG_PPC_64K_PAGES
@@ -1457,6 +1461,8 @@ int hash_page_mm(struct mm_struct *mm, unsigned long ea,
#endif
DBG_LOW(" -> rc=%d\n", rc);
+bail_pgtbl_walk:
+ __end_lockless_pgtbl_walk(0, false);
bail:
exception_exit(prev_state);
return rc;
@@ -1545,7 +1551,7 @@ static void hash_preload(struct mm_struct *mm, unsigned long ea,
unsigned long vsid;
pgd_t *pgdir;
pte_t *ptep;
- unsigned long flags;
+ unsigned long irq_mask;
int rc, ssize, update_flags = 0;
unsigned long access = _PAGE_PRESENT | _PAGE_READ | (is_exec ? _PAGE_EXEC : 0);
@@ -1567,11 +1573,12 @@ static void hash_preload(struct mm_struct *mm, unsigned long ea,
vsid = get_user_vsid(&mm->context, ea, ssize);
if (!vsid)
return;
+
/*
* Hash doesn't like irqs. Walking linux page table with irq disabled
* saves us from holding multiple locks.
*/
- local_irq_save(flags);
+ irq_mask = begin_lockless_pgtbl_walk();
/*
* THP pages use update_mmu_cache_pmd. We don't do
@@ -1616,7 +1623,7 @@ static void hash_preload(struct mm_struct *mm, unsigned long ea,
mm_ctx_user_psize(&mm->context),
pte_val(*ptep));
out_exit:
- local_irq_restore(flags);
+ end_lockless_pgtbl_walk(irq_mask);
}
/*
@@ -1679,16 +1686,16 @@ u16 get_mm_addr_key(struct mm_struct *mm, unsigned long address)
{
pte_t *ptep;
u16 pkey = 0;
- unsigned long flags;
+ unsigned long irq_mask;
if (!mm || !mm->pgd)
return 0;
- local_irq_save(flags);
+ irq_mask = begin_lockless_pgtbl_walk();
ptep = find_linux_pte(mm->pgd, address, NULL, NULL);
if (ptep)
pkey = pte_to_pkey_bits(pte_val(READ_ONCE(*ptep)));
- local_irq_restore(flags);
+ end_lockless_pgtbl_walk(irq_mask);
return pkey;
}
--
2.24.1
^ permalink raw reply related
* [PATCH v6 07/11] powerpc/kvm/e500: Use functions to track lockless pgtbl walks
From: Leonardo Bras @ 2020-02-06 3:08 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Arnd Bergmann, Andrew Morton, Aneesh Kumar K.V, Nicholas Piggin,
Christophe Leroy, Steven Price, Robin Murphy, Leonardo Bras,
Mahesh Salgaonkar, Balbir Singh, Reza Arbab, Thomas Gleixner,
Allison Randal, Greg Kroah-Hartman, Mike Rapoport,
Michal Suchanek
Cc: linux-arch, linux-mm, linuxppc-dev, linux-kernel, kvm-ppc
In-Reply-To: <20200206030900.147032-1-leonardo@linux.ibm.com>
Applies the new functions used for tracking lockless pgtable walks on
kvmppc_e500_shadow_map().
Fixes the place where local_irq_restore() is called: previously, if ptep
was NULL, local_irq_restore() would never be called.
local_irq_{save,restore} is already inside {begin,end}_lockless_pgtbl_walk,
so there is no need to repeat it here.
Variable that saves the irq mask was renamed from flags to irq_mask so it
doesn't lose meaning now it's not directly passed to local_irq_* functions.
Signed-off-by: Leonardo Bras <leonardo@linux.ibm.com>
---
arch/powerpc/kvm/e500_mmu_host.c | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/arch/powerpc/kvm/e500_mmu_host.c b/arch/powerpc/kvm/e500_mmu_host.c
index 425d13806645..3dcf11f77256 100644
--- a/arch/powerpc/kvm/e500_mmu_host.c
+++ b/arch/powerpc/kvm/e500_mmu_host.c
@@ -336,7 +336,7 @@ static inline int kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500,
pte_t *ptep;
unsigned int wimg = 0;
pgd_t *pgdir;
- unsigned long flags;
+ unsigned long irq_mask;
/* used to check for invalidations in progress */
mmu_seq = kvm->mmu_notifier_seq;
@@ -473,7 +473,7 @@ static inline int kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500,
* We are holding kvm->mmu_lock so a notifier invalidate
* can't run hence pfn won't change.
*/
- local_irq_save(flags);
+ irq_mask = begin_lockless_pgtbl_walk();
ptep = find_linux_pte(pgdir, hva, NULL, NULL);
if (ptep) {
pte_t pte = READ_ONCE(*ptep);
@@ -481,15 +481,16 @@ static inline int kvmppc_e500_shadow_map(struct kvmppc_vcpu_e500 *vcpu_e500,
if (pte_present(pte)) {
wimg = (pte_val(pte) >> PTE_WIMGE_SHIFT) &
MAS2_WIMGE_MASK;
- local_irq_restore(flags);
} else {
- local_irq_restore(flags);
+ end_lockless_pgtbl_walk(irq_mask);
pr_err_ratelimited("%s: pte not present: gfn %lx,pfn %lx\n",
__func__, (long)gfn, pfn);
ret = -EINVAL;
goto out;
}
}
+ end_lockless_pgtbl_walk(irq_mask);
+
kvmppc_e500_ref_setup(ref, gtlbe, pfn, wimg);
kvmppc_e500_setup_stlbe(&vcpu_e500->vcpu, gtlbe, tsize,
--
2.24.1
^ permalink raw reply related
* [PATCH v6 08/11] powerpc/kvm/book3s_hv: Use functions to track lockless pgtbl walks
From: Leonardo Bras @ 2020-02-06 3:08 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Arnd Bergmann, Andrew Morton, Aneesh Kumar K.V, Nicholas Piggin,
Christophe Leroy, Steven Price, Robin Murphy, Leonardo Bras,
Mahesh Salgaonkar, Balbir Singh, Reza Arbab, Thomas Gleixner,
Allison Randal, Greg Kroah-Hartman, Mike Rapoport,
Michal Suchanek
Cc: linux-arch, linux-mm, linuxppc-dev, linux-kernel, kvm-ppc
In-Reply-To: <20200206030900.147032-1-leonardo@linux.ibm.com>
Applies the new functions for tracking all book3s_hv related
functions that do lockless pagetable walks.
Adds comments explaining that some lockless pagetable walks don't need
protection due to guest pgd not being a target of THP collapse/split, or
due to being called from Realmode + MSR_EE = 0
kvmppc_do_h_enter: Fixes where local_irq_restore() must be placed (after
the last usage of ptep).
Given that some of these functions can be called in real mode, and others
always are, we use __{begin,end}_lockless_pgtbl_walk so we can decide when
to disable interrupts.
Signed-off-by: Leonardo Bras <leonardo@linux.ibm.com>
---
arch/powerpc/kvm/book3s_hv_nested.c | 22 ++++++++++++++++++++--
arch/powerpc/kvm/book3s_hv_rm_mmu.c | 28 ++++++++++++++++++----------
2 files changed, 38 insertions(+), 12 deletions(-)
diff --git a/arch/powerpc/kvm/book3s_hv_nested.c b/arch/powerpc/kvm/book3s_hv_nested.c
index dc97e5be76f6..a398061d5778 100644
--- a/arch/powerpc/kvm/book3s_hv_nested.c
+++ b/arch/powerpc/kvm/book3s_hv_nested.c
@@ -803,7 +803,11 @@ static void kvmhv_update_nest_rmap_rc(struct kvm *kvm, u64 n_rmap,
if (!gp)
return;
- /* Find the pte */
+ /* Find the pte:
+ * We are walking the nested guest (partition-scoped) page table here.
+ * We can do this without disabling irq because the Linux MM
+ * subsystem doesn't do THP splits and collapses on this tree.
+ */
ptep = __find_linux_pte(gp->shadow_pgtable, gpa, NULL, &shift);
/*
* If the pte is present and the pfn is still the same, update the pte.
@@ -853,7 +857,11 @@ static void kvmhv_remove_nest_rmap(struct kvm *kvm, u64 n_rmap,
if (!gp)
return;
- /* Find and invalidate the pte */
+ /* Find and invalidate the pte:
+ * We are walking the nested guest (partition-scoped) page table here.
+ * We can do this without disabling irq because the Linux MM
+ * subsystem doesn't do THP splits and collapses on this tree.
+ */
ptep = __find_linux_pte(gp->shadow_pgtable, gpa, NULL, &shift);
/* Don't spuriously invalidate ptes if the pfn has changed */
if (ptep && pte_present(*ptep) && ((pte_val(*ptep) & mask) == hpa))
@@ -921,6 +929,11 @@ static bool kvmhv_invalidate_shadow_pte(struct kvm_vcpu *vcpu,
int shift;
spin_lock(&kvm->mmu_lock);
+ /*
+ * We are walking the nested guest (partition-scoped) page table here.
+ * We can do this without disabling irq because the Linux MM
+ * subsystem doesn't do THP splits and collapses on this tree.
+ */
ptep = __find_linux_pte(gp->shadow_pgtable, gpa, NULL, &shift);
if (!shift)
shift = PAGE_SHIFT;
@@ -1362,6 +1375,11 @@ static long int __kvmhv_nested_page_fault(struct kvm_run *run,
/* See if can find translation in our partition scoped tables for L1 */
pte = __pte(0);
spin_lock(&kvm->mmu_lock);
+ /*
+ * We are walking the secondary (partition-scoped) page table here.
+ * We can do this without disabling irq because the Linux MM
+ * subsystem doesn't do THP splits and collapses on this tree.
+ */
pte_p = __find_linux_pte(kvm->arch.pgtable, gpa, NULL, &shift);
if (!shift)
shift = PAGE_SHIFT;
diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
index 220305454c23..fd4d8f174f09 100644
--- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c
+++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
@@ -210,7 +210,7 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
pte_t *ptep;
unsigned int writing;
unsigned long mmu_seq;
- unsigned long rcbits, irq_flags = 0;
+ unsigned long rcbits, irq_mask = 0;
if (kvm_is_radix(kvm))
return H_FUNCTION;
@@ -252,8 +252,8 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
* If we had a page table table change after lookup, we would
* retry via mmu_notifier_retry.
*/
- if (!realmode)
- local_irq_save(irq_flags);
+ irq_mask = __begin_lockless_pgtbl_walk(!realmode);
+
/*
* If called in real mode we have MSR_EE = 0. Otherwise
* we disable irq above.
@@ -272,8 +272,7 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
* to <= host page size, if host is using hugepage
*/
if (host_pte_size < psize) {
- if (!realmode)
- local_irq_restore(flags);
+ __end_lockless_pgtbl_walk(irq_mask, !realmode);
return H_PARAMETER;
}
pte = kvmppc_read_update_linux_pte(ptep, writing);
@@ -287,8 +286,6 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
pa |= gpa & ~PAGE_MASK;
}
}
- if (!realmode)
- local_irq_restore(irq_flags);
ptel &= HPTE_R_KEY | HPTE_R_PP0 | (psize-1);
ptel |= pa;
@@ -302,8 +299,10 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
/*If we had host pte mapping then Check WIMG */
if (ptep && !hpte_cache_flags_ok(ptel, is_ci)) {
- if (is_ci)
+ if (is_ci) {
+ __end_lockless_pgtbl_walk(irq_mask, !realmode);
return H_PARAMETER;
+ }
/*
* Allow guest to map emulated device memory as
* uncacheable, but actually make it cacheable.
@@ -311,6 +310,7 @@ long kvmppc_do_h_enter(struct kvm *kvm, unsigned long flags,
ptel &= ~(HPTE_R_W|HPTE_R_I|HPTE_R_G);
ptel |= HPTE_R_M;
}
+ __end_lockless_pgtbl_walk(irq_mask, !realmode);
/* Find and lock the HPTEG slot to use */
do_insert:
@@ -907,11 +907,19 @@ static int kvmppc_get_hpa(struct kvm_vcpu *vcpu, unsigned long gpa,
/* Translate to host virtual address */
hva = __gfn_to_hva_memslot(memslot, gfn);
- /* Try to find the host pte for that virtual address */
+ /* Try to find the host pte for that virtual address :
+ * Called by hcall_real_table (real mode + MSR_EE=0)
+ * Interrupts are disabled here.
+ */
+ __begin_lockless_pgtbl_walk(false);
ptep = __find_linux_pte(vcpu->arch.pgdir, hva, NULL, &shift);
- if (!ptep)
+ if (!ptep) {
+ __end_lockless_pgtbl_walk(0, false);
return H_TOO_HARD;
+ }
pte = kvmppc_read_update_linux_pte(ptep, writing);
+ __end_lockless_pgtbl_walk(0, false);
+
if (!pte_present(pte))
return H_TOO_HARD;
--
2.24.1
^ permalink raw reply related
* [PATCH v6 09/11] powerpc/kvm/book3s_64: Use functions to track lockless pgtbl walks
From: Leonardo Bras @ 2020-02-06 3:08 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Arnd Bergmann, Andrew Morton, Aneesh Kumar K.V, Nicholas Piggin,
Christophe Leroy, Steven Price, Robin Murphy, Leonardo Bras,
Mahesh Salgaonkar, Balbir Singh, Reza Arbab, Thomas Gleixner,
Allison Randal, Greg Kroah-Hartman, Mike Rapoport,
Michal Suchanek
Cc: linux-arch, linux-mm, linuxppc-dev, linux-kernel, kvm-ppc
In-Reply-To: <20200206030900.147032-1-leonardo@linux.ibm.com>
Applies the new tracking functions to all book3s_64 related
functions that do lockless pagetable walks.
Adds comments explaining that some lockless pagetable walks don't need
protection due to guest pgd not being a target of THP collapse/split, or
due to being called from Realmode + MSR_EE = 0.
Given that some of these functions always are called in realmode, we use
__{begin,end}_lockless_pgtbl_walk so we can decide when to disable
interrupts.
local_irq_{save,restore} is already inside {begin,end}_lockless_pgtbl_walk,
so there is no need to repeat it here.
Variable that saves the irq mask was renamed from flags to irq_mask so it
doesn't lose meaning now it's not directly passed to local_irq_* functions.
There are also a function that uses local_irq_{en,dis}able, so the return
value of begin_lockless_pgtbl_walk() is ignored and we pass IRQS_ENABLED to
end_lockless_pgtbl_walk() to mimic the effect of local_irq_enable().
Signed-off-by: Leonardo Bras <leonardo@linux.ibm.com>
---
arch/powerpc/kvm/book3s_64_mmu_hv.c | 6 ++---
arch/powerpc/kvm/book3s_64_mmu_radix.c | 34 +++++++++++++++++++++++---
arch/powerpc/kvm/book3s_64_vio_hv.c | 6 ++++-
3 files changed, 39 insertions(+), 7 deletions(-)
diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c
index 6c372f5c61b6..e7ce29a5df60 100644
--- a/arch/powerpc/kvm/book3s_64_mmu_hv.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c
@@ -605,19 +605,19 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu,
/* if the guest wants write access, see if that is OK */
if (!writing && hpte_is_writable(r)) {
pte_t *ptep, pte;
- unsigned long flags;
+ unsigned long irq_mask;
/*
* We need to protect against page table destruction
* hugepage split and collapse.
*/
- local_irq_save(flags);
+ irq_mask = begin_lockless_pgtbl_walk();
ptep = find_current_mm_pte(mm->pgd, hva, NULL, NULL);
if (ptep) {
pte = kvmppc_read_update_linux_pte(ptep, 1);
if (__pte_write(pte))
write_ok = 1;
}
- local_irq_restore(flags);
+ end_lockless_pgtbl_walk(irq_mask);
}
}
diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c
index 803940d79b73..cda2e455baf2 100644
--- a/arch/powerpc/kvm/book3s_64_mmu_radix.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c
@@ -813,20 +813,20 @@ int kvmppc_book3s_instantiate_page(struct kvm_vcpu *vcpu,
* Read the PTE from the process' radix tree and use that
* so we get the shift and attribute bits.
*/
- local_irq_disable();
+ begin_lockless_pgtbl_walk();
ptep = __find_linux_pte(vcpu->arch.pgdir, hva, NULL, &shift);
/*
* If the PTE disappeared temporarily due to a THP
* collapse, just return and let the guest try again.
*/
if (!ptep) {
- local_irq_enable();
+ end_lockless_pgtbl_walk(IRQS_ENABLED);
if (page)
put_page(page);
return RESUME_GUEST;
}
pte = *ptep;
- local_irq_enable();
+ end_lockless_pgtbl_walk(IRQS_ENABLED);
/* If we're logging dirty pages, always map single pages */
large_enable = !(memslot->flags & KVM_MEM_LOG_DIRTY_PAGES);
@@ -980,10 +980,16 @@ int kvm_unmap_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
return 0;
}
+ /*
+ * We are walking the secondary (partition-scoped) page table here.
+ * We can do this without disabling irq because the Linux MM
+ * subsystem doesn't do THP splits and collapses on this tree.
+ */
ptep = __find_linux_pte(kvm->arch.pgtable, gpa, NULL, &shift);
if (ptep && pte_present(*ptep))
kvmppc_unmap_pte(kvm, ptep, gpa, shift, memslot,
kvm->arch.lpid);
+
return 0;
}
@@ -1000,6 +1006,11 @@ int kvm_age_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
return ref;
+ /*
+ * We are walking the secondary (partition-scoped) page table here.
+ * We can do this without disabling irq because the Linux MM
+ * subsystem doesn't do THP splits and collapses on this tree.
+ */
ptep = __find_linux_pte(kvm->arch.pgtable, gpa, NULL, &shift);
if (ptep && pte_present(*ptep) && pte_young(*ptep)) {
old = kvmppc_radix_update_pte(kvm, ptep, _PAGE_ACCESSED, 0,
@@ -1027,6 +1038,11 @@ int kvm_test_age_radix(struct kvm *kvm, struct kvm_memory_slot *memslot,
if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
return ref;
+ /*
+ * We are walking the secondary (partition-scoped) page table here.
+ * We can do this without disabling irq because the Linux MM
+ * subsystem doesn't do THP splits and collapses on this tree.
+ */
ptep = __find_linux_pte(kvm->arch.pgtable, gpa, NULL, &shift);
if (ptep && pte_present(*ptep) && pte_young(*ptep))
ref = 1;
@@ -1047,6 +1063,11 @@ static int kvm_radix_test_clear_dirty(struct kvm *kvm,
if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
return ret;
+ /*
+ * We are walking the secondary (partition-scoped) page table here.
+ * We can do this without disabling irq because the Linux MM
+ * subsystem doesn't do THP splits and collapses on this tree.
+ */
ptep = __find_linux_pte(kvm->arch.pgtable, gpa, NULL, &shift);
if (ptep && pte_present(*ptep) && pte_dirty(*ptep)) {
ret = 1;
@@ -1063,6 +1084,7 @@ static int kvm_radix_test_clear_dirty(struct kvm *kvm,
1UL << shift);
spin_unlock(&kvm->mmu_lock);
}
+
return ret;
}
@@ -1108,6 +1130,12 @@ void kvmppc_radix_flush_memslot(struct kvm *kvm,
gpa = memslot->base_gfn << PAGE_SHIFT;
spin_lock(&kvm->mmu_lock);
for (n = memslot->npages; n; --n) {
+ /*
+ * We are walking the secondary (partition-scoped) page table
+ * here.
+ * We can do this without disabling irq because the Linux MM
+ * subsystem doesn't do THP splits and collapses on this tree.
+ */
ptep = __find_linux_pte(kvm->arch.pgtable, gpa, NULL, &shift);
if (ptep && pte_present(*ptep))
kvmppc_unmap_pte(kvm, ptep, gpa, shift, memslot,
diff --git a/arch/powerpc/kvm/book3s_64_vio_hv.c b/arch/powerpc/kvm/book3s_64_vio_hv.c
index ab6eeb8e753e..83c70c1557e4 100644
--- a/arch/powerpc/kvm/book3s_64_vio_hv.c
+++ b/arch/powerpc/kvm/book3s_64_vio_hv.c
@@ -453,10 +453,14 @@ static long kvmppc_rm_ua_to_hpa(struct kvm_vcpu *vcpu,
* to exit which will agains result in the below page table walk
* to finish.
*/
+ __begin_lockless_pgtbl_walk(false);
ptep = __find_linux_pte(vcpu->arch.pgdir, ua, NULL, &shift);
- if (!ptep || !pte_present(*ptep))
+ if (!ptep || !pte_present(*ptep)) {
+ __end_lockless_pgtbl_walk(0, false);
return -ENXIO;
+ }
pte = *ptep;
+ __end_lockless_pgtbl_walk(0, false);
if (!shift)
shift = PAGE_SHIFT;
--
2.24.1
^ permalink raw reply related
* [PATCH v6 10/11] powerpc/mm: Adds counting method to track lockless pagetable walks
From: Leonardo Bras @ 2020-02-06 3:08 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Arnd Bergmann, Andrew Morton, Aneesh Kumar K.V, Nicholas Piggin,
Christophe Leroy, Steven Price, Robin Murphy, Leonardo Bras,
Mahesh Salgaonkar, Balbir Singh, Reza Arbab, Thomas Gleixner,
Allison Randal, Greg Kroah-Hartman, Mike Rapoport,
Michal Suchanek
Cc: linux-arch, linux-mm, linuxppc-dev, linux-kernel, kvm-ppc
In-Reply-To: <20200206030900.147032-1-leonardo@linux.ibm.com>
Implements an additional feature to track lockless pagetable walks,
using a per-cpu counter: lockless_pgtbl_walk_counter.
Before a lockless pagetable walk, preemption is disabled and the
current cpu's counter is increased.
When the lockless pagetable walk finishes, the current cpu counter
is decreased and the preemption is enabled.
With that, it's possible to know in which cpus are happening lockless
pagetable walks, and optimize serialize_against_pte_lookup().
Implementation notes:
- Every counter can be changed only by it's CPU
- It makes use of the original memory barrier in the functions
- Any counter can be read by any CPU
Due to not locking nor using atomic variables, the impact on the
lockless pagetable walk is intended to be minimum.
Signed-off-by: Leonardo Bras <leonardo@linux.ibm.com>
---
arch/powerpc/mm/book3s64/pgtable.c | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/arch/powerpc/mm/book3s64/pgtable.c b/arch/powerpc/mm/book3s64/pgtable.c
index 535613030363..bb138b628f86 100644
--- a/arch/powerpc/mm/book3s64/pgtable.c
+++ b/arch/powerpc/mm/book3s64/pgtable.c
@@ -83,6 +83,7 @@ static void do_nothing(void *unused)
}
+static DEFINE_PER_CPU(int, lockless_pgtbl_walk_counter);
/*
* Serialize against find_current_mm_pte which does lock-less
* lookup in page tables with local interrupts disabled. For huge pages
@@ -120,6 +121,15 @@ unsigned long __begin_lockless_pgtbl_walk(bool disable_irq)
if (disable_irq)
local_irq_save(irq_mask);
+ /*
+ * Counts this instance of lockless pagetable walk for this cpu.
+ * Disables preempt to make sure there is no cpu change between
+ * begin/end lockless pagetable walk, so that percpu counting
+ * works fine.
+ */
+ preempt_disable();
+ (*this_cpu_ptr(&lockless_pgtbl_walk_counter))++;
+
/*
* This memory barrier pairs with any code that is either trying to
* delete page tables, or split huge pages. Without this barrier,
@@ -158,6 +168,14 @@ inline void __end_lockless_pgtbl_walk(unsigned long irq_mask, bool enable_irq)
*/
smp_mb();
+ /*
+ * Removes this instance of lockless pagetable walk for this cpu.
+ * Enables preempt only after end lockless pagetable walk,
+ * so that percpu counting works fine.
+ */
+ (*this_cpu_ptr(&lockless_pgtbl_walk_counter))--;
+ preempt_enable();
+
/*
* Interrupts must be disabled during the lockless page table walk.
* That's because the deleting or splitting involves flushing TLBs,
--
2.24.1
^ permalink raw reply related
* [PATCH v6 11/11] powerpc/mm/book3s64/pgtable: Uses counting method to skip serializing
From: Leonardo Bras @ 2020-02-06 3:09 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Arnd Bergmann, Andrew Morton, Aneesh Kumar K.V, Nicholas Piggin,
Christophe Leroy, Steven Price, Robin Murphy, Leonardo Bras,
Mahesh Salgaonkar, Balbir Singh, Reza Arbab, Thomas Gleixner,
Allison Randal, Greg Kroah-Hartman, Mike Rapoport,
Michal Suchanek
Cc: linux-arch, linux-mm, linuxppc-dev, linux-kernel, kvm-ppc
In-Reply-To: <20200206030900.147032-1-leonardo@linux.ibm.com>
For each cpu in cpumask, checks if it's running a lockless pagetable
walk. Then, run serialize_against_pte_lookup() only on these cpus.
serialize_agains_pte_lookup() can take a long while when there are a
lot of cpus in cpumask.
This method is intended to reduce this waiting, while not impacting
too much on the lockless pagetable walk.
Signed-off-by: Leonardo Bras <leonardo@linux.ibm.com>
---
arch/powerpc/mm/book3s64/pgtable.c | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git a/arch/powerpc/mm/book3s64/pgtable.c b/arch/powerpc/mm/book3s64/pgtable.c
index bb138b628f86..4822ff1aac4b 100644
--- a/arch/powerpc/mm/book3s64/pgtable.c
+++ b/arch/powerpc/mm/book3s64/pgtable.c
@@ -96,8 +96,22 @@ static DEFINE_PER_CPU(int, lockless_pgtbl_walk_counter);
*/
void serialize_against_pte_lookup(struct mm_struct *mm)
{
+ int cpu;
+ struct cpumask cm;
+
smp_mb();
- smp_call_function_many(mm_cpumask(mm), do_nothing, NULL, 1);
+
+ /*
+ * Fills a new cpumask only with cpus that are currently doing a
+ * lockless pagetable walk. This reduces time spent in this function.
+ */
+ cpumask_clear(&cm);
+ for_each_cpu(cpu, mm_cpumask((mm))) {
+ if (per_cpu(lockless_pgtbl_walk_counter, cpu) > 0)
+ cpumask_set_cpu(cpu, &cm);
+ }
+
+ smp_call_function_many(&cm, do_nothing, NULL, 1);
}
/* begin_lockless_pgtbl_walk: Must be inserted before a function call that does
--
2.24.1
^ permalink raw reply related
* Re: [PATCH V13] mm/debug: Add tests validating architecture page table helpers
From: kbuild test robot @ 2020-02-05 23:10 UTC (permalink / raw)
To: Anshuman Khandual
Cc: Mark Rutland, uclinux-h8-devel, linux-ia64, linux-sh,
Peter Zijlstra, James Hogan, Tetsuo Handa, Heiko Carstens,
Michal Hocko, linux-mm, Paul Mackerras, sparclinux, linux-hexagon,
linux-riscv, Thomas Gleixner, linux-s390, linux-c6x-dev,
Yoshinori Sato, Max Filippov, x86, Russell King - ARM Linux,
Matthew Wilcox, Steven Price, Jason Gunthorpe, Geert Uytterhoeven,
Gerald Schaefer, Mark Salter, Matt Turner, linux-snps-arc,
linux-arm-kernel, Ingo Molnar, linux-xtensa, Kees Cook,
Anshuman Khandual, Masahiro Yamada, linux-alpha,
Aurelien Jacquiot, linux-m68k, Mark Brown, Ivan Kokshaysky,
Paul Walmsley, Kirill A . Shutemov, Dan Williams, Guan Xuetao,
Vlastimil Babka, Richard Henderson, Chris Zankel, Michal Simek,
kbuild-all, Brian Cain, Ard Biesheuvel, Greg Kroah-Hartman,
Dave Hansen, linux-mips, Ralf Baechle, linux-kernel, Paul Burton,
Mike Rapoport, Palmer Dabbelt, Vineet Gupta, Martin Schwidefsky,
Qian Cai, Andrew Morton, linuxppc-dev, David S. Miller
In-Reply-To: <1580897674-16456-1-git-send-email-anshuman.khandual@arm.com>
[-- Attachment #1: Type: text/plain, Size: 3061 bytes --]
Hi Anshuman,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on powerpc/next]
[also build test ERROR on s390/features linus/master arc/for-next v5.5]
[cannot apply to mmotm/master tip/x86/core arm64/for-next/core next-20200205]
[if your patch is applied to the wrong git tree, please drop us a note to help
improve the system. BTW, we also suggest to use '--base' option to specify the
base tree in git format-patch, please see https://stackoverflow.com/a/37406982]
url: https://github.com/0day-ci/linux/commits/Anshuman-Khandual/mm-debug-Add-tests-validating-architecture-page-table-helpers/20200205-215507
base: https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: ia64-allmodconfig (attached as .config)
compiler: ia64-linux-gcc (GCC) 7.5.0
reproduce:
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
GCC_VERSION=7.5.0 make.cross ARCH=ia64
If you fix the issue, kindly add following tag
Reported-by: kbuild test robot <lkp@intel.com>
All error/warnings (new ones prefixed by >>):
In file included from include/asm-generic/pgtable-nopud.h:8:0,
from arch/ia64/include/asm/pgtable.h:586,
from include/linux/mm.h:99,
from include/linux/highmem.h:8,
from mm/debug_vm_pgtable.c:14:
mm/debug_vm_pgtable.c: In function 'pud_clear_tests':
>> include/asm-generic/pgtable-nop4d-hack.h:47:32: error: implicit declaration of function '__pgd'; did you mean '__p4d'? [-Werror=implicit-function-declaration]
#define __pud(x) ((pud_t) { __pgd(x) })
^
>> mm/debug_vm_pgtable.c:141:8: note: in expansion of macro '__pud'
pud = __pud(pud_val(pud) | RANDOM_ORVALUE);
^~~~~
>> include/asm-generic/pgtable-nop4d-hack.h:47:22: warning: missing braces around initializer [-Wmissing-braces]
#define __pud(x) ((pud_t) { __pgd(x) })
^
>> mm/debug_vm_pgtable.c:141:8: note: in expansion of macro '__pud'
pud = __pud(pud_val(pud) | RANDOM_ORVALUE);
^~~~~
cc1: some warnings being treated as errors
vim +47 include/asm-generic/pgtable-nop4d-hack.h
30ec842660bd0d Kirill A. Shutemov 2017-03-09 45
30ec842660bd0d Kirill A. Shutemov 2017-03-09 46 #define pud_val(x) (pgd_val((x).pgd))
30ec842660bd0d Kirill A. Shutemov 2017-03-09 @47 #define __pud(x) ((pud_t) { __pgd(x) })
30ec842660bd0d Kirill A. Shutemov 2017-03-09 48
:::::: The code at line 47 was first introduced by commit
:::::: 30ec842660bd0d056d4a7028ac5bd4a82b113d4f asm-generic: introduce __ARCH_USE_5LEVEL_HACK
:::::: TO: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
:::::: CC: Linus Torvalds <torvalds@linux-foundation.org>
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org Intel Corporation
[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 55724 bytes --]
^ permalink raw reply
* [PATCH 1/1] vfio-pci/nvlink2: Allow fallback to ibm,mmio-atsd[0]
From: Sam Bobroff @ 2020-02-06 3:17 UTC (permalink / raw)
To: kvm, linuxppc-dev; +Cc: aik
Older versions of skiboot only provide a single value in the device
tree property "ibm,mmio-atsd", even when multiple Address Translation
Shoot Down (ATSD) registers are present. This prevents NVLink2 devices
(other than the first) from being used with vfio-pci because vfio-pci
expects to be able to assign a dedicated ATSD register to each NVLink2
device.
However, ATSD registers can be shared among devices. This change
allows vfio-pci to fall back to sharing the register at index 0 if
necessary.
Signed-off-by: Sam Bobroff <sbobroff@linux.ibm.com>
---
drivers/vfio/pci/vfio_pci_nvlink2.c | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/drivers/vfio/pci/vfio_pci_nvlink2.c b/drivers/vfio/pci/vfio_pci_nvlink2.c
index f2983f0f84be..851ba673882b 100644
--- a/drivers/vfio/pci/vfio_pci_nvlink2.c
+++ b/drivers/vfio/pci/vfio_pci_nvlink2.c
@@ -420,8 +420,17 @@ int vfio_pci_ibm_npu2_init(struct vfio_pci_device *vdev)
if (of_property_read_u64_index(hose->dn, "ibm,mmio-atsd", nvlink_index,
&mmio_atsd)) {
- dev_warn(&vdev->pdev->dev, "No available ATSD found\n");
- mmio_atsd = 0;
+ dev_warn(&vdev->pdev->dev,
+ "No ibm,mmio-atsd[%d] found: trying ibm,mmio-atsd[0]\n",
+ nvlink_index);
+ if (of_property_read_u64_index(hose->dn, "ibm,mmio-atsd", 0,
+ &mmio_atsd)) {
+ dev_warn(&vdev->pdev->dev, "No available ATSD found\n");
+ mmio_atsd = 0;
+ } else {
+ dev_warn(&vdev->pdev->dev,
+ "Using fallback ibm,mmio-atsd[0] for ATSD.\n");
+ }
}
if (of_property_read_u64(npu_node, "ibm,device-tgt-addr", &tgt)) {
--
2.22.0.216.g00a2a96fc9
^ permalink raw reply related
* Re: [PATCH v6 02/11] mm/gup: Use functions to track lockless pgtbl walks on gup_pgd_range
From: Leonardo Bras @ 2020-02-06 3:25 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Arnd Bergmann, Andrew Morton, Aneesh Kumar K.V, Nicholas Piggin,
Christophe Leroy, Steven Price, Robin Murphy, Mahesh Salgaonkar,
Balbir Singh, Reza Arbab, Thomas Gleixner, Allison Randal,
Greg Kroah-Hartman, Mike Rapoport, Michal Suchanek
Cc: linux-arch, linux-mm, linuxppc-dev, linux-kernel, kvm-ppc
In-Reply-To: <20200206030900.147032-3-leonardo@linux.ibm.com>
[-- Attachment #1: Type: text/plain, Size: 367 bytes --]
On Thu, 2020-02-06 at 00:08 -0300, Leonardo Bras wrote:
> gup_pgd_range(addr, end, gup_flags, pages, &nr);
> - local_irq_enable();
> + end_lockless_pgtbl_walk(IRQS_ENABLED);
> ret = nr;
> }
>
Just noticed IRQS_ENABLED is not available on other archs than ppc64.
I will fix this for v7.
[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply
* Re: [yyu168-linux_cet:cet 55/58] powerpc64le-linux-ld: warning: discarding dynamic section .rela___ksymtab+jiffies_to_timeval
From: Michael Ellerman @ 2020-02-06 3:26 UTC (permalink / raw)
To: H.J. Lu, Benjamin Herrenschmidt, Paul Mackerras, linuxppc-dev
Cc: Yu-cheng Yu, kbuild-all, Kees Cook
In-Reply-To: <CAMe9rOrJEVL8Qw3fgm9FeDjwGZNxGoZ-OO6bfmV=gtK=g68cvQ@mail.gmail.com>
"H.J. Lu" <hjl.tools@gmail.com> writes:
> On Tue, Feb 4, 2020 at 3:37 PM kbuild test robot <lkp@intel.com> wrote:
>>
>> tree: https://github.com/yyu168/linux_cet.git cet
>> head: bba707cc4715c1036b6561ab38b16747f9c49cfa
>> commit: 71bb971dd76eeacd351690f28864ad5c5bec3691 [55/58] Discard .note.gnu.property sections in generic NOTES
>> config: powerpc-rhel-kconfig (attached as .config)
>> compiler: powerpc64le-linux-gcc (GCC) 7.5.0
>> reproduce:
>> wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
>> chmod +x ~/bin/make.cross
>> git checkout 71bb971dd76eeacd351690f28864ad5c5bec3691
>> # save the attached .config to linux build tree
>> GCC_VERSION=7.5.0 make.cross ARCH=powerpc
>>
>> If you fix the issue, kindly add following tag
>> Reported-by: kbuild test robot <lkp@intel.com>
>>
>> All warnings (new ones prefixed by >>):
>>
>> powerpc64le-linux-ld: warning: discarding dynamic section .rela___ksymtab_gpl+__wait_rcu_gp
>
> arch/powerpc/kernel/vmlinux.lds.S has
>
> .rela.dyn : AT(ADDR(.rela.dyn) - (0xc000000000000000 -0x00000000))
> {
> __rela_dyn_start = .;
> *(.rela*) <<<<<<<< Keep .rela* sections
> }
The above is inside #ifdef CONFIG_RELOCATABLE
> ...
> /DISCARD/ : {
> *(*.EMB.apuinfo)
> *(.glink .iplt .plt .rela* .comment)
> ^^^^ Discard .rela* sections. But it is ignored.
> *(.gnu.version*)
> *(.gnu.attributes)
> *(.eh_frame)
> }
But that is not #ifdef'ed at all.
> With my
>
> ommit 71bb971dd76eeacd351690f28864ad5c5bec3691
> Author: H.J. Lu <hjl.tools@gmail.com>
> Date: Thu Jan 30 12:39:09 2020 -0800
>
> Discard .note.gnu.property sections in generic NOTES
>
> With the command-line option, -mx86-used-note=yes, the x86 assembler
> in binutils 2.32 and above generates a program property note in a note
> section, .note.gnu.property, to encode used x86 ISAs and features. But
> kernel linker script only contains a single NOTE segment:
>
> /DISCARD/ : { *(.note.gnu.property) }
>
> is placed before
>
> .rela.dyn : AT(ADDR(.rela.dyn) - (0xc000000000000000 -0x00000000))
> {
> __rela_dyn_start = .;
> *(.rela*) <<<<<<<< Keep .rela* sections
> }
>
> Then .rela* in
>
> /DISCARD/ : {
> *(*.EMB.apuinfo)
> *(.glink .iplt .plt .rela* .comment)
> *(.gnu.version*)
> *(.gnu.attributes)
> *(.eh_frame)
> }
>
> is honored. Can someone from POWERPC comment on it?
Hmm OK. I'm not really a toolchain person.
The comment on DISCARDS says:
* Some archs want to discard exit text/data at runtime rather than
* link time due to cross-section references such as alt instructions,
* bug table, eh_frame, etc. DISCARDS must be the last of output
* section definitions so that such archs put those in earlier section
* definitions.
*/
But I guess you're changing those semantics in your series.
This seems to fix the warning for me?
diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S
index b4c89a1acebb..076b3e8a849d 100644
--- a/arch/powerpc/kernel/vmlinux.lds.S
+++ b/arch/powerpc/kernel/vmlinux.lds.S
@@ -365,9 +365,12 @@ SECTIONS
DISCARDS
/DISCARD/ : {
*(*.EMB.apuinfo)
- *(.glink .iplt .plt .rela* .comment)
+ *(.glink .iplt .plt .comment)
*(.gnu.version*)
*(.gnu.attributes)
*(.eh_frame)
+#ifndef CONFIG_RELOCATABLE
+ *(.rela*)
+#endif
}
}
cheers
^ permalink raw reply related
* Re: [PATCH 1/6] powerpc/eeh: Add sysfs files in late probe
From: Sam Bobroff @ 2020-02-06 4:13 UTC (permalink / raw)
To: Oliver O'Halloran; +Cc: tyreld, linuxppc-dev
In-Reply-To: <20200203083521.16549-2-oohall@gmail.com>
[-- Attachment #1: Type: text/plain, Size: 6492 bytes --]
On Mon, Feb 03, 2020 at 07:35:16PM +1100, Oliver O'Halloran wrote:
> Move creating the EEH specific sysfs files into eeh_add_device_late()
> rather than being open-coded all over the place. Calling the function is
> generally done immediately after calling eeh_add_device_late() anyway. The
> two cases where it's not done there (OF based PCI probing and the pseries
> VFs) don't seem to have any issues with the re-ordering.
I haven't tested it explicitly, but I suspect the re-ordering will
actually improve things: in some error cases it will no longer add sysfs
files for devices that have failed to init, because bailing out in
eeh_add_device_late() (or eeh_probve_device()) will now prevent
eeh_sysfs_add_device() from being called.
Nice cleanup.
Reviewed-by: Sam Bobroff <sbobroff@linux.ibm.com>
> Signed-off-by: Oliver O'Halloran <oohall@gmail.com>
> ---
> arch/powerpc/include/asm/eeh.h | 3 ---
> arch/powerpc/kernel/eeh.c | 24 +-----------------------
> arch/powerpc/kernel/of_platform.c | 3 ---
> arch/powerpc/kernel/pci-common.c | 3 ---
> arch/powerpc/platforms/powernv/eeh-powernv.c | 1 -
> arch/powerpc/platforms/pseries/eeh_pseries.c | 3 +--
> 6 files changed, 2 insertions(+), 35 deletions(-)
>
> diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h
> index 6f9b2a1..5a34907 100644
> --- a/arch/powerpc/include/asm/eeh.h
> +++ b/arch/powerpc/include/asm/eeh.h
> @@ -305,7 +305,6 @@ void eeh_add_device_early(struct pci_dn *);
> void eeh_add_device_tree_early(struct pci_dn *);
> void eeh_add_device_late(struct pci_dev *);
> void eeh_add_device_tree_late(struct pci_bus *);
> -void eeh_add_sysfs_files(struct pci_bus *);
> void eeh_remove_device(struct pci_dev *);
> int eeh_unfreeze_pe(struct eeh_pe *pe);
> int eeh_pe_reset_and_recover(struct eeh_pe *pe);
> @@ -368,8 +367,6 @@ static inline void eeh_add_device_late(struct pci_dev *dev) { }
>
> static inline void eeh_add_device_tree_late(struct pci_bus *bus) { }
>
> -static inline void eeh_add_sysfs_files(struct pci_bus *bus) { }
> -
> static inline void eeh_remove_device(struct pci_dev *dev) { }
>
> #define EEH_POSSIBLE_ERROR(val, type) (0)
> diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c
> index 17cb3e9..0878912 100644
> --- a/arch/powerpc/kernel/eeh.c
> +++ b/arch/powerpc/kernel/eeh.c
> @@ -1210,6 +1210,7 @@ void eeh_add_device_late(struct pci_dev *dev)
> dev->dev.archdata.edev = edev;
>
> eeh_addr_cache_insert_dev(dev);
> + eeh_sysfs_add_device(dev);
> }
>
> /**
> @@ -1238,29 +1239,6 @@ void eeh_add_device_tree_late(struct pci_bus *bus)
> EXPORT_SYMBOL_GPL(eeh_add_device_tree_late);
>
> /**
> - * eeh_add_sysfs_files - Add EEH sysfs files for the indicated PCI bus
> - * @bus: PCI bus
> - *
> - * This routine must be used to add EEH sysfs files for PCI
> - * devices which are attached to the indicated PCI bus. The PCI bus
> - * is added after system boot through hotplug or dlpar.
> - */
> -void eeh_add_sysfs_files(struct pci_bus *bus)
> -{
> - struct pci_dev *dev;
> -
> - list_for_each_entry(dev, &bus->devices, bus_list) {
> - eeh_sysfs_add_device(dev);
> - if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
> - struct pci_bus *subbus = dev->subordinate;
> - if (subbus)
> - eeh_add_sysfs_files(subbus);
> - }
> - }
> -}
> -EXPORT_SYMBOL_GPL(eeh_add_sysfs_files);
> -
> -/**
> * eeh_remove_device - Undo EEH setup for the indicated pci device
> * @dev: pci device to be removed
> *
> diff --git a/arch/powerpc/kernel/of_platform.c b/arch/powerpc/kernel/of_platform.c
> index 427fc22..cb68800 100644
> --- a/arch/powerpc/kernel/of_platform.c
> +++ b/arch/powerpc/kernel/of_platform.c
> @@ -86,9 +86,6 @@ static int of_pci_phb_probe(struct platform_device *dev)
> /* Add probed PCI devices to the device model */
> pci_bus_add_devices(phb->bus);
>
> - /* sysfs files should only be added after devices are added */
> - eeh_add_sysfs_files(phb->bus);
> -
So for this case, the sysfs files are added by pci_bus_add_devices(),
via...
pci_bus_add_devices() (loops over devices) ->
pci_bus_add_device() ->
pcibios_bus_add_device() ->
ppc_md.pcibios_bus_add_device() ->
{pseries,pnv}_pcibios_bus_add_device() ->
eeh_add_device_late() ->
eeh_sysfs_add_device().
> return 0;
> }
>
> diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c
> index c6c0341..3d2b1cf 100644
> --- a/arch/powerpc/kernel/pci-common.c
> +++ b/arch/powerpc/kernel/pci-common.c
> @@ -1404,9 +1404,6 @@ void pcibios_finish_adding_to_bus(struct pci_bus *bus)
>
> /* Add new devices to global lists. Register in proc, sysfs. */
> pci_bus_add_devices(bus);
> -
> - /* sysfs files should only be added after devices are added */
> - eeh_add_sysfs_files(bus);
As above.
> }
> EXPORT_SYMBOL_GPL(pcibios_finish_adding_to_bus);
>
> diff --git a/arch/powerpc/platforms/powernv/eeh-powernv.c b/arch/powerpc/platforms/powernv/eeh-powernv.c
> index 6f300ab..ef727ec 100644
> --- a/arch/powerpc/platforms/powernv/eeh-powernv.c
> +++ b/arch/powerpc/platforms/powernv/eeh-powernv.c
> @@ -48,7 +48,6 @@ void pnv_pcibios_bus_add_device(struct pci_dev *pdev)
> dev_dbg(&pdev->dev, "EEH: Setting up device\n");
> eeh_add_device_early(pdn);
> eeh_add_device_late(pdev);
> - eeh_sysfs_add_device(pdev);
So for this case, the sysfs files are added by eeh_add_device_late(),
via...
eeh_add_device_late() ->
eeh_sysfs_add_device().
> }
>
> static int pnv_eeh_init(void)
> diff --git a/arch/powerpc/platforms/pseries/eeh_pseries.c b/arch/powerpc/platforms/pseries/eeh_pseries.c
> index 893ba3f..95bbf91 100644
> --- a/arch/powerpc/platforms/pseries/eeh_pseries.c
> +++ b/arch/powerpc/platforms/pseries/eeh_pseries.c
> @@ -68,7 +68,6 @@ void pseries_pcibios_bus_add_device(struct pci_dev *pdev)
> }
> #endif
> eeh_add_device_early(pdn);
> - eeh_add_device_late(pdev);
> #ifdef CONFIG_PCI_IOV
> if (pdev->is_virtfn) {
> struct eeh_dev *edev = pdn_to_eeh_dev(pdn);
> @@ -78,7 +77,7 @@ void pseries_pcibios_bus_add_device(struct pci_dev *pdev)
> eeh_add_to_parent_pe(edev); /* Add as VF PE type */
> }
> #endif
> - eeh_sysfs_add_device(pdev);
> + eeh_add_device_late(pdev);
This is just a re-ordering.
> }
>
> /*
> --
> 2.9.5
>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 488 bytes --]
^ permalink raw reply
* Re: [PATCH 1/1] vfio-pci/nvlink2: Allow fallback to ibm,mmio-atsd[0]
From: Alexey Kardashevskiy @ 2020-02-06 4:23 UTC (permalink / raw)
To: Sam Bobroff, kvm, linuxppc-dev, Alex Williamson
In-Reply-To: <426f75e09ac1a6879a6d51f592bf683c698b4bda.1580959044.git.sbobroff@linux.ibm.com>
On 06/02/2020 14:17, Sam Bobroff wrote:
> Older versions of skiboot only provide a single value in the device
> tree property "ibm,mmio-atsd", even when multiple Address Translation
> Shoot Down (ATSD) registers are present. This prevents NVLink2 devices
> (other than the first) from being used with vfio-pci because vfio-pci
> expects to be able to assign a dedicated ATSD register to each NVLink2
> device.
>
> However, ATSD registers can be shared among devices. This change
> allows vfio-pci to fall back to sharing the register at index 0 if
> necessary.
>
> Signed-off-by: Sam Bobroff <sbobroff@linux.ibm.com>
> ---
> drivers/vfio/pci/vfio_pci_nvlink2.c | 13 +++++++++++--
> 1 file changed, 11 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/vfio/pci/vfio_pci_nvlink2.c b/drivers/vfio/pci/vfio_pci_nvlink2.c
> index f2983f0f84be..851ba673882b 100644
> --- a/drivers/vfio/pci/vfio_pci_nvlink2.c
> +++ b/drivers/vfio/pci/vfio_pci_nvlink2.c
> @@ -420,8 +420,17 @@ int vfio_pci_ibm_npu2_init(struct vfio_pci_device *vdev)
>
> if (of_property_read_u64_index(hose->dn, "ibm,mmio-atsd", nvlink_index,
> &mmio_atsd)) {
> - dev_warn(&vdev->pdev->dev, "No available ATSD found\n");
> - mmio_atsd = 0;
> + dev_warn(&vdev->pdev->dev,
> + "No ibm,mmio-atsd[%d] found: trying ibm,mmio-atsd[0]\n",
> + nvlink_index);
We do not really need this warning (nvlink_index doesn't matter that
much, we can work out from the device tree what happened), warnings
below are enough (if you really want, you can print nvlink_index there).
Either way,
Reviewed-by: Alexey Kardashevskiy <aik@ozlabs.ru>
> + if (of_property_read_u64_index(hose->dn, "ibm,mmio-atsd", 0,
> + &mmio_atsd)) {
> + dev_warn(&vdev->pdev->dev, "No available ATSD found\n");
> + mmio_atsd = 0;
> + } else {
> + dev_warn(&vdev->pdev->dev,
> + "Using fallback ibm,mmio-atsd[0] for ATSD.\n");
> + }
> }
>
> if (of_property_read_u64(npu_node, "ibm,device-tgt-addr", &tgt)) {
>
--
Alexey
^ permalink raw reply
* Re: [PATCH 2/6] powerpc/eeh: Remove eeh_add_device_tree_late()
From: Sam Bobroff @ 2020-02-06 4:23 UTC (permalink / raw)
To: Oliver O'Halloran; +Cc: tyreld, linuxppc-dev
In-Reply-To: <20200203083521.16549-3-oohall@gmail.com>
[-- Attachment #1: Type: text/plain, Size: 3995 bytes --]
On Mon, Feb 03, 2020 at 07:35:17PM +1100, Oliver O'Halloran wrote:
> On pseries and PowerNV pcibios_bus_add_device() calls eeh_add_device_late()
> so there's no need to do a separate tree traversal to bind the eeh_dev and
> pci_dev together setting up the PHB at boot. As a result we can remove
> eeh_add_device_tree_late().
>
> Signed-off-by: Oliver O'Halloran <oohall@gmail.com>
... with pcibios_bus_add_device() being called from
pci_bus_add_devices(), in this case.
Looks good.
Reviewed-by: Sam Bobroff <sbobroff@linux.ibm.com>
> ---
> arch/powerpc/include/asm/eeh.h | 3 ---
> arch/powerpc/kernel/eeh.c | 25 -------------------------
> arch/powerpc/kernel/of_platform.c | 3 ---
> arch/powerpc/kernel/pci-common.c | 3 ---
> 4 files changed, 34 deletions(-)
>
> diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h
> index 5a34907..5d10781 100644
> --- a/arch/powerpc/include/asm/eeh.h
> +++ b/arch/powerpc/include/asm/eeh.h
> @@ -304,7 +304,6 @@ void eeh_addr_cache_init(void);
> void eeh_add_device_early(struct pci_dn *);
> void eeh_add_device_tree_early(struct pci_dn *);
> void eeh_add_device_late(struct pci_dev *);
> -void eeh_add_device_tree_late(struct pci_bus *);
> void eeh_remove_device(struct pci_dev *);
> int eeh_unfreeze_pe(struct eeh_pe *pe);
> int eeh_pe_reset_and_recover(struct eeh_pe *pe);
> @@ -365,8 +364,6 @@ static inline void eeh_add_device_tree_early(struct pci_dn *pdn) { }
>
> static inline void eeh_add_device_late(struct pci_dev *dev) { }
>
> -static inline void eeh_add_device_tree_late(struct pci_bus *bus) { }
> -
> static inline void eeh_remove_device(struct pci_dev *dev) { }
>
> #define EEH_POSSIBLE_ERROR(val, type) (0)
> diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c
> index 0878912..9cb3370 100644
> --- a/arch/powerpc/kernel/eeh.c
> +++ b/arch/powerpc/kernel/eeh.c
> @@ -1214,31 +1214,6 @@ void eeh_add_device_late(struct pci_dev *dev)
> }
>
> /**
> - * eeh_add_device_tree_late - Perform EEH initialization for the indicated PCI bus
> - * @bus: PCI bus
> - *
> - * This routine must be used to perform EEH initialization for PCI
> - * devices which are attached to the indicated PCI bus. The PCI bus
> - * is added after system boot through hotplug or dlpar.
> - */
> -void eeh_add_device_tree_late(struct pci_bus *bus)
> -{
> - struct pci_dev *dev;
> -
> - if (eeh_has_flag(EEH_FORCE_DISABLED))
> - return;
> - list_for_each_entry(dev, &bus->devices, bus_list) {
> - eeh_add_device_late(dev);
> - if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
> - struct pci_bus *subbus = dev->subordinate;
> - if (subbus)
> - eeh_add_device_tree_late(subbus);
> - }
> - }
> -}
> -EXPORT_SYMBOL_GPL(eeh_add_device_tree_late);
> -
> -/**
> * eeh_remove_device - Undo EEH setup for the indicated pci device
> * @dev: pci device to be removed
> *
> diff --git a/arch/powerpc/kernel/of_platform.c b/arch/powerpc/kernel/of_platform.c
> index cb68800..64edac81 100644
> --- a/arch/powerpc/kernel/of_platform.c
> +++ b/arch/powerpc/kernel/of_platform.c
> @@ -80,9 +80,6 @@ static int of_pci_phb_probe(struct platform_device *dev)
> */
> pcibios_claim_one_bus(phb->bus);
>
> - /* Finish EEH setup */
> - eeh_add_device_tree_late(phb->bus);
> -
> /* Add probed PCI devices to the device model */
> pci_bus_add_devices(phb->bus);
>
> diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c
> index 3d2b1cf..8983afa 100644
> --- a/arch/powerpc/kernel/pci-common.c
> +++ b/arch/powerpc/kernel/pci-common.c
> @@ -1399,9 +1399,6 @@ void pcibios_finish_adding_to_bus(struct pci_bus *bus)
> pci_assign_unassigned_bus_resources(bus);
> }
>
> - /* Fixup EEH */
> - eeh_add_device_tree_late(bus);
> -
> /* Add new devices to global lists. Register in proc, sysfs. */
> pci_bus_add_devices(bus);
> }
> --
> 2.9.5
>
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 488 bytes --]
^ permalink raw reply
* Re: Latest Git kernel: avahi-daemon[2410]: ioctl(): Inappropriate ioctl for device
From: Michael Ellerman @ 2020-02-06 4:35 UTC (permalink / raw)
To: Christian Zigotzky, Jakub Kicinski
Cc: DTML, Darren Stevens, mad skateman, netdev@vger.kernel.org,
Linux Kernel Mailing List, linuxppc-dev, contact@a-eon.com,
R.T.Dickinson, Christoph Hellwig
In-Reply-To: <C11859E1-BE71-494F-81E2-9B27E27E60EE@xenosoft.de>
Christian Zigotzky <chzigotzky@xenosoft.de> writes:
> Kernel 5.5 PowerPC is also affected.
I don't know what you mean by that. What sha are you talking about?
I have a system with avahi running and everything's fine.
# grep use- /etc/avahi/avahi-daemon.conf
use-ipv4=yes
use-ipv6=yes
# systemctl status -l --no-pager avahi-daemon
● avahi-daemon.service - Avahi mDNS/DNS-SD Stack
Loaded: loaded (/lib/systemd/system/avahi-daemon.service; enabled; vendor preset: enabled)
Active: active (running) since Thu 2020-02-06 14:55:34 AEDT; 38min ago
Main PID: 1884 (avahi-daemon)
Status: "avahi-daemon 0.7 starting up."
CGroup: /system.slice/avahi-daemon.service
├─1884 avahi-daemon: running [mpe-ubuntu-le.local]
└─1888 avahi-daemon: chroot helper
Feb 06 14:55:34 mpe-ubuntu-le avahi-daemon[1884]: Registering new address record for fe80::5054:ff:fe66:2a19 on eth0.*.
Feb 06 14:55:34 mpe-ubuntu-le avahi-daemon[1884]: Registering new address record for 10.61.141.81 on eth0.IPv4.
Feb 06 14:55:34 mpe-ubuntu-le avahi-daemon[1884]: Registering new address record for ::1 on lo.*.
Feb 06 14:55:34 mpe-ubuntu-le avahi-daemon[1884]: Registering new address record for 127.0.0.1 on lo.IPv4.
Feb 06 14:55:34 mpe-ubuntu-le systemd[1]: Started Avahi mDNS/DNS-SD Stack.
Feb 06 14:55:35 mpe-ubuntu-le avahi-daemon[1884]: Server startup complete. Host name is mpe-ubuntu-le.local. Local service cookie is 3972418141.
Feb 06 14:55:38 mpe-ubuntu-le avahi-daemon[1884]: Leaving mDNS multicast group on interface eth0.IPv6 with address fe80::5054:ff:fe66:2a19.
Feb 06 14:55:38 mpe-ubuntu-le avahi-daemon[1884]: Joining mDNS multicast group on interface eth0.IPv6 with address fd69:d75f:b8b5:61:5054:ff:fe66:2a19.
Feb 06 14:55:38 mpe-ubuntu-le avahi-daemon[1884]: Registering new address record for fd69:d75f:b8b5:61:5054:ff:fe66:2a19 on eth0.*.
Feb 06 14:55:38 mpe-ubuntu-le avahi-daemon[1884]: Withdrawing address record for fe80::5054:ff:fe66:2a19 on eth0.
# uname -r
5.5.0-gcc-8.2.0
The key question is what ioctl is it complaining about. You should be
able to find that via strace.
cheers
> Christian Zigotzky wrote:
>
> Hi All,
>
> The issue with the avahi-daemon still exist in the latest Git kernel. It's a PowerPC issue. I compiled the latest Git kernel on a PC today and there aren't any issues with the avahi daemon. Another Power Mac user reported the same issue on his G5. I tested with the AmigaOne X1000 and X5000 in the last days.
>
> I bisected today but I think the result isn't correct because it found the other problem with ordering of PCSCSI definition in esp_rev enum. I don't know how to bisect if there is another issue at the same time. Maybe "git bisect skip"?
>
> 2086faae3c55a652cfbd369e18ecdb703aacc493 is the first bad commit
> commit 2086faae3c55a652cfbd369e18ecdb703aacc493
> Author: Kars de Jong <jongk@linux-m68k.org>
> Date: Tue Nov 19 21:20:20 2019 +0100
>
> scsi: esp_scsi: Correct ordering of PCSCSI definition in esp_rev enum
>
> The order of the definitions in the esp_rev enum is important. The values
> are used in comparisons for chip features.
>
> Add a comment to the enum explaining this.
>
> Also, the actual values for the enum fields are irrelevant, so remove the
> explicit values (suggested by Geert Uytterhoeven). This makes adding a new
> field in the middle of the enum easier.
>
> Finally, move the PCSCSI definition to the right place in the enum. In its
> previous location, at the end of the enum, the wrong values are written to
> the CONFIG3 register when used with FAST-SCSI targets.
>
> Link: https://lore.kernel.org/r/20191119202021.28720-2-jongk@linux-m68k.org
> Signed-off-by: Kars de Jong <jongk@linux-m68k.org>
> Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
>
> :040000 040000 cdc128596e33fb60406b5de9b17b79623c187c1a 48ceab06439f95285e8b30181e75f9a68c25fcb5 M drivers
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox