* [PATCH v2 05/13] nios2: add support for folded p4d page tables
From: Mike Rapoport @ 2020-02-16 8:18 UTC (permalink / raw)
To: linux-kernel
Cc: Rich Felker, linux-ia64, Geert Uytterhoeven, linux-sh, linux-mm,
Paul Mackerras, linux-hexagon, Will Deacon, kvmarm, Jonas Bonn,
linux-arch, Brian Cain, Marc Zyngier, Russell King, Ley Foon Tan,
Mike Rapoport, Catalin Marinas, Julien Thierry, uclinux-h8-devel,
Fenghua Yu, Arnd Bergmann, Suzuki K Poulose, kvm-ppc,
Stefan Kristiansson, openrisc, Stafford Horne, Guan Xuetao,
linux-arm-kernel, Tony Luck, Yoshinori Sato, James Morse,
nios2-dev, Andrew Morton, linuxppc-dev, Mike Rapoport
In-Reply-To: <20200216081843.28670-1-rppt@kernel.org>
From: Mike Rapoport <rppt@linux.ibm.com>
Implement primitives necessary for the 4th level folding, add walks of p4d
level where appropriate and remove usage of __ARCH_USE_5LEVEL_HACK.
Signed-off-by: Mike Rapoport <rppt@linux.ibm.com>
---
arch/nios2/include/asm/pgtable.h | 3 +--
arch/nios2/mm/fault.c | 9 +++++++--
arch/nios2/mm/ioremap.c | 6 +++++-
3 files changed, 13 insertions(+), 5 deletions(-)
diff --git a/arch/nios2/include/asm/pgtable.h b/arch/nios2/include/asm/pgtable.h
index 99985d8b7166..54305aa09b74 100644
--- a/arch/nios2/include/asm/pgtable.h
+++ b/arch/nios2/include/asm/pgtable.h
@@ -22,7 +22,6 @@
#include <asm/tlbflush.h>
#include <asm/pgtable-bits.h>
-#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
#define FIRST_USER_ADDRESS 0UL
@@ -100,7 +99,7 @@ extern pte_t invalid_pte_table[PAGE_SIZE/sizeof(pte_t)];
*/
static inline void set_pmd(pmd_t *pmdptr, pmd_t pmdval)
{
- pmdptr->pud.pgd.pgd = pmdval.pud.pgd.pgd;
+ *pmdptr = pmdval;
}
/* to find an entry in a page-table-directory */
diff --git a/arch/nios2/mm/fault.c b/arch/nios2/mm/fault.c
index 6a2e716b959f..d3da995665c3 100644
--- a/arch/nios2/mm/fault.c
+++ b/arch/nios2/mm/fault.c
@@ -245,6 +245,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause,
*/
int offset = pgd_index(address);
pgd_t *pgd, *pgd_k;
+ p4d_t *p4d, *p4d_k;
pud_t *pud, *pud_k;
pmd_t *pmd, *pmd_k;
pte_t *pte_k;
@@ -256,8 +257,12 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause,
goto no_context;
set_pgd(pgd, *pgd_k);
- pud = pud_offset(pgd, address);
- pud_k = pud_offset(pgd_k, address);
+ p4d = p4d_offset(pgd, address);
+ p4d_k = p4d_offset(pgd_k, address);
+ if (!p4d_present(*p4d_k))
+ goto no_context;
+ pud = pud_offset(p4d, address);
+ pud_k = pud_offset(p4d_k, address);
if (!pud_present(*pud_k))
goto no_context;
pmd = pmd_offset(pud, address);
diff --git a/arch/nios2/mm/ioremap.c b/arch/nios2/mm/ioremap.c
index 819bdfcc2e71..fe821efb9a99 100644
--- a/arch/nios2/mm/ioremap.c
+++ b/arch/nios2/mm/ioremap.c
@@ -86,11 +86,15 @@ static int remap_area_pages(unsigned long address, unsigned long phys_addr,
if (address >= end)
BUG();
do {
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
error = -ENOMEM;
- pud = pud_alloc(&init_mm, dir, address);
+ p4d = p4d_alloc(&init_mm, dir, address);
+ if (!p4d)
+ break;
+ pud = pud_alloc(&init_mm, p4d, address);
if (!pud)
break;
pmd = pmd_alloc(&init_mm, pud, address);
--
2.24.0
^ permalink raw reply related
* [PATCH v2 04/13] ia64: add support for folded p4d page tables
From: Mike Rapoport @ 2020-02-16 8:18 UTC (permalink / raw)
To: linux-kernel
Cc: Rich Felker, linux-ia64, Geert Uytterhoeven, linux-sh, linux-mm,
Paul Mackerras, linux-hexagon, Will Deacon, kvmarm, Jonas Bonn,
linux-arch, Brian Cain, Marc Zyngier, Russell King, Ley Foon Tan,
Mike Rapoport, Catalin Marinas, Julien Thierry, uclinux-h8-devel,
Fenghua Yu, Arnd Bergmann, Suzuki K Poulose, kvm-ppc,
Stefan Kristiansson, openrisc, Stafford Horne, Guan Xuetao,
linux-arm-kernel, Tony Luck, Yoshinori Sato, James Morse,
nios2-dev, Andrew Morton, linuxppc-dev, Mike Rapoport
In-Reply-To: <20200216081843.28670-1-rppt@kernel.org>
From: Mike Rapoport <rppt@linux.ibm.com>
Implement primitives necessary for the 4th level folding, add walks of p4d
level where appropriate, remove usage of __ARCH_USE_5LEVEL_HACK and replace
5level-fixup.h with pgtable-nop4d.h
Signed-off-by: Mike Rapoport <rppt@linux.ibm.com>
---
arch/ia64/include/asm/pgalloc.h | 4 ++--
arch/ia64/include/asm/pgtable.h | 17 ++++++++---------
arch/ia64/mm/fault.c | 7 ++++++-
arch/ia64/mm/hugetlbpage.c | 18 ++++++++++++------
arch/ia64/mm/init.c | 28 ++++++++++++++++++++++++----
5 files changed, 52 insertions(+), 22 deletions(-)
diff --git a/arch/ia64/include/asm/pgalloc.h b/arch/ia64/include/asm/pgalloc.h
index f4c491044882..2a3050345099 100644
--- a/arch/ia64/include/asm/pgalloc.h
+++ b/arch/ia64/include/asm/pgalloc.h
@@ -36,9 +36,9 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
#if CONFIG_PGTABLE_LEVELS == 4
static inline void
-pgd_populate(struct mm_struct *mm, pgd_t * pgd_entry, pud_t * pud)
+p4d_populate(struct mm_struct *mm, p4d_t * p4d_entry, pud_t * pud)
{
- pgd_val(*pgd_entry) = __pa(pud);
+ p4d_val(*p4d_entry) = __pa(pud);
}
static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
diff --git a/arch/ia64/include/asm/pgtable.h b/arch/ia64/include/asm/pgtable.h
index d602e7c622db..c87f789bc914 100644
--- a/arch/ia64/include/asm/pgtable.h
+++ b/arch/ia64/include/asm/pgtable.h
@@ -283,12 +283,12 @@ extern unsigned long VMALLOC_END;
#define pud_page(pud) virt_to_page((pud_val(pud) + PAGE_OFFSET))
#if CONFIG_PGTABLE_LEVELS == 4
-#define pgd_none(pgd) (!pgd_val(pgd))
-#define pgd_bad(pgd) (!ia64_phys_addr_valid(pgd_val(pgd)))
-#define pgd_present(pgd) (pgd_val(pgd) != 0UL)
-#define pgd_clear(pgdp) (pgd_val(*(pgdp)) = 0UL)
-#define pgd_page_vaddr(pgd) ((unsigned long) __va(pgd_val(pgd) & _PFN_MASK))
-#define pgd_page(pgd) virt_to_page((pgd_val(pgd) + PAGE_OFFSET))
+#define p4d_none(p4d) (!p4d_val(p4d))
+#define p4d_bad(p4d) (!ia64_phys_addr_valid(p4d_val(p4d)))
+#define p4d_present(p4d) (p4d_val(p4d) != 0UL)
+#define p4d_clear(p4dp) (p4d_val(*(p4dp)) = 0UL)
+#define p4d_page_vaddr(p4d) ((unsigned long) __va(p4d_val(p4d) & _PFN_MASK))
+#define p4d_page(p4d) virt_to_page((p4d_val(p4d) + PAGE_OFFSET))
#endif
/*
@@ -388,7 +388,7 @@ pgd_offset (const struct mm_struct *mm, unsigned long address)
#if CONFIG_PGTABLE_LEVELS == 4
/* Find an entry in the second-level page table.. */
#define pud_offset(dir,addr) \
- ((pud_t *) pgd_page_vaddr(*(dir)) + (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)))
+ ((pud_t *) p4d_page_vaddr(*(dir)) + (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)))
#endif
/* Find an entry in the third-level page table.. */
@@ -582,10 +582,9 @@ extern struct page *zero_page_memmap_ptr;
#if CONFIG_PGTABLE_LEVELS == 3
-#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopud.h>
#endif
-#include <asm-generic/5level-fixup.h>
+#include <asm-generic/pgtable-nop4d.h>
#include <asm-generic/pgtable.h>
#endif /* _ASM_IA64_PGTABLE_H */
diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c
index c2f299fe9e04..ec994135cb74 100644
--- a/arch/ia64/mm/fault.c
+++ b/arch/ia64/mm/fault.c
@@ -29,6 +29,7 @@ static int
mapped_kernel_page_is_present (unsigned long address)
{
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *ptep, pte;
@@ -37,7 +38,11 @@ mapped_kernel_page_is_present (unsigned long address)
if (pgd_none(*pgd) || pgd_bad(*pgd))
return 0;
- pud = pud_offset(pgd, address);
+ p4d = p4d_offset(pgd, address);
+ if (p4d_none(*p4d) || p4d_bad(*p4d))
+ return 0;
+
+ pud = pud_offset(p4d, address);
if (pud_none(*pud) || pud_bad(*pud))
return 0;
diff --git a/arch/ia64/mm/hugetlbpage.c b/arch/ia64/mm/hugetlbpage.c
index d16e419fd712..32352a73df0c 100644
--- a/arch/ia64/mm/hugetlbpage.c
+++ b/arch/ia64/mm/hugetlbpage.c
@@ -30,12 +30,14 @@ huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz)
{
unsigned long taddr = htlbpage_to_page(addr);
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *pte = NULL;
pgd = pgd_offset(mm, taddr);
- pud = pud_alloc(mm, pgd, taddr);
+ p4d = p4d_offset(pgd, taddr);
+ pud = pud_alloc(mm, p4d, taddr);
if (pud) {
pmd = pmd_alloc(mm, pud, taddr);
if (pmd)
@@ -49,17 +51,21 @@ huge_pte_offset (struct mm_struct *mm, unsigned long addr, unsigned long sz)
{
unsigned long taddr = htlbpage_to_page(addr);
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *pte = NULL;
pgd = pgd_offset(mm, taddr);
if (pgd_present(*pgd)) {
- pud = pud_offset(pgd, taddr);
- if (pud_present(*pud)) {
- pmd = pmd_offset(pud, taddr);
- if (pmd_present(*pmd))
- pte = pte_offset_map(pmd, taddr);
+ p4d = p4d_offset(pgd, addr);
+ if (p4d_present(*p4d)) {
+ pud = pud_offset(p4d, taddr);
+ if (pud_present(*pud)) {
+ pmd = pmd_offset(pud, taddr);
+ if (pmd_present(*pmd))
+ pte = pte_offset_map(pmd, taddr);
+ }
}
}
diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c
index b01d68a2d5d9..4808f58220ac 100644
--- a/arch/ia64/mm/init.c
+++ b/arch/ia64/mm/init.c
@@ -208,6 +208,7 @@ static struct page * __init
put_kernel_page (struct page *page, unsigned long address, pgprot_t pgprot)
{
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
@@ -215,7 +216,10 @@ put_kernel_page (struct page *page, unsigned long address, pgprot_t pgprot)
pgd = pgd_offset_k(address); /* note: this is NOT pgd_offset()! */
{
- pud = pud_alloc(&init_mm, pgd, address);
+ p4d = p4d_alloc(&init_mm, pgd, address);
+ if (!p4d)
+ goto out;
+ pud = pud_alloc(&init_mm, p4d, address);
if (!pud)
goto out;
pmd = pmd_alloc(&init_mm, pud, address);
@@ -382,6 +386,7 @@ int vmemmap_find_next_valid_pfn(int node, int i)
do {
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
@@ -392,7 +397,13 @@ int vmemmap_find_next_valid_pfn(int node, int i)
continue;
}
- pud = pud_offset(pgd, end_address);
+ p4d = p4d_offset(pgd, end_address);
+ if (p4d_none(*p4d)) {
+ end_address += P4D_SIZE;
+ continue;
+ }
+
+ pud = pud_offset(p4d, end_address);
if (pud_none(*pud)) {
end_address += PUD_SIZE;
continue;
@@ -430,6 +441,7 @@ int __init create_mem_map_page_table(u64 start, u64 end, void *arg)
struct page *map_start, *map_end;
int node;
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
@@ -444,12 +456,20 @@ int __init create_mem_map_page_table(u64 start, u64 end, void *arg)
for (address = start_page; address < end_page; address += PAGE_SIZE) {
pgd = pgd_offset_k(address);
if (pgd_none(*pgd)) {
+ p4d = memblock_alloc_node(PAGE_SIZE, PAGE_SIZE, node);
+ if (!p4d)
+ goto err_alloc;
+ pgd_populate(&init_mm, pgd, p4d);
+ }
+ p4d = p4d_offset(pgd, address);
+
+ if (p4d_none(*p4d)) {
pud = memblock_alloc_node(PAGE_SIZE, PAGE_SIZE, node);
if (!pud)
goto err_alloc;
- pgd_populate(&init_mm, pgd, pud);
+ p4d_populate(&init_mm, p4d, pud);
}
- pud = pud_offset(pgd, address);
+ pud = pud_offset(p4d, address);
if (pud_none(*pud)) {
pmd = memblock_alloc_node(PAGE_SIZE, PAGE_SIZE, node);
--
2.24.0
^ permalink raw reply related
* [PATCH v2 03/13] hexagon: remove __ARCH_USE_5LEVEL_HACK
From: Mike Rapoport @ 2020-02-16 8:18 UTC (permalink / raw)
To: linux-kernel
Cc: Rich Felker, linux-ia64, Geert Uytterhoeven, linux-sh, linux-mm,
Paul Mackerras, linux-hexagon, Will Deacon, kvmarm, Jonas Bonn,
linux-arch, Brian Cain, Marc Zyngier, Russell King, Ley Foon Tan,
Mike Rapoport, Catalin Marinas, Julien Thierry, uclinux-h8-devel,
Fenghua Yu, Arnd Bergmann, Suzuki K Poulose, kvm-ppc,
Stefan Kristiansson, openrisc, Stafford Horne, Guan Xuetao,
linux-arm-kernel, Tony Luck, Yoshinori Sato, James Morse,
nios2-dev, Andrew Morton, linuxppc-dev, Mike Rapoport
In-Reply-To: <20200216081843.28670-1-rppt@kernel.org>
From: Mike Rapoport <rppt@linux.ibm.com>
The hexagon architecture has 2 level page tables and as such most of the
page table folding is already implemented in asm-generic/pgtable-nopmd.h.
Fixup the only place in arch/hexagon to unfold the p4d level and remove
__ARCH_USE_5LEVEL_HACK.
Signed-off-by: Mike Rapoport <rppt@linux.ibm.com>
---
arch/hexagon/include/asm/fixmap.h | 4 ++--
arch/hexagon/include/asm/pgtable.h | 1 -
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/arch/hexagon/include/asm/fixmap.h b/arch/hexagon/include/asm/fixmap.h
index 933dac167504..97b1b062e750 100644
--- a/arch/hexagon/include/asm/fixmap.h
+++ b/arch/hexagon/include/asm/fixmap.h
@@ -16,7 +16,7 @@
#include <asm-generic/fixmap.h>
#define kmap_get_fixmap_pte(vaddr) \
- pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), \
- (vaddr)), (vaddr)), (vaddr))
+ pte_offset_kernel(pmd_offset(pud_offset(p4d_offset(pgd_offset_k(vaddr), \
+ (vaddr)), (vaddr)), (vaddr)), (vaddr))
#endif
diff --git a/arch/hexagon/include/asm/pgtable.h b/arch/hexagon/include/asm/pgtable.h
index 2fec20ad939e..83b544936eed 100644
--- a/arch/hexagon/include/asm/pgtable.h
+++ b/arch/hexagon/include/asm/pgtable.h
@@ -12,7 +12,6 @@
* Page table definitions for Qualcomm Hexagon processor.
*/
#include <asm/page.h>
-#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
/* A handy thing to have if one has the RAM. Declared in head.S */
--
2.24.0
^ permalink raw reply related
* [PATCH v2 02/13] h8300: remove usage of __ARCH_USE_5LEVEL_HACK
From: Mike Rapoport @ 2020-02-16 8:18 UTC (permalink / raw)
To: linux-kernel
Cc: Rich Felker, linux-ia64, Geert Uytterhoeven, linux-sh, linux-mm,
Paul Mackerras, linux-hexagon, Will Deacon, kvmarm, Jonas Bonn,
linux-arch, Brian Cain, Marc Zyngier, Russell King, Ley Foon Tan,
Mike Rapoport, Catalin Marinas, Julien Thierry, uclinux-h8-devel,
Fenghua Yu, Arnd Bergmann, Suzuki K Poulose, kvm-ppc,
Stefan Kristiansson, openrisc, Stafford Horne, Guan Xuetao,
linux-arm-kernel, Tony Luck, Yoshinori Sato, James Morse,
nios2-dev, Andrew Morton, linuxppc-dev, Mike Rapoport
In-Reply-To: <20200216081843.28670-1-rppt@kernel.org>
From: Mike Rapoport <rppt@linux.ibm.com>
h8300 is a nommu architecture and does not require fixup for upper layers
of the page tables because it is already handled by the generic nommu
implementation.
Remove definition of __ARCH_USE_5LEVEL_HACK in
arch/h8300/include/asm/pgtable.h
Signed-off-by: Mike Rapoport <rppt@linux.ibm.com>
---
arch/h8300/include/asm/pgtable.h | 1 -
1 file changed, 1 deletion(-)
diff --git a/arch/h8300/include/asm/pgtable.h b/arch/h8300/include/asm/pgtable.h
index 4d00152fab58..f00828720dc4 100644
--- a/arch/h8300/include/asm/pgtable.h
+++ b/arch/h8300/include/asm/pgtable.h
@@ -1,7 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _H8300_PGTABLE_H
#define _H8300_PGTABLE_H
-#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopud.h>
#include <asm-generic/pgtable.h>
extern void paging_init(void);
--
2.24.0
^ permalink raw reply related
* [PATCH v2 01/13] arm/arm64: add support for folded p4d page tables
From: Mike Rapoport @ 2020-02-16 8:18 UTC (permalink / raw)
To: linux-kernel
Cc: Rich Felker, linux-ia64, Geert Uytterhoeven, linux-sh, linux-mm,
Paul Mackerras, linux-hexagon, Will Deacon, kvmarm, Jonas Bonn,
linux-arch, Brian Cain, Marc Zyngier, Russell King, Ley Foon Tan,
Mike Rapoport, Catalin Marinas, Julien Thierry, uclinux-h8-devel,
Fenghua Yu, Arnd Bergmann, Suzuki K Poulose, kvm-ppc,
Stefan Kristiansson, openrisc, Stafford Horne, Guan Xuetao,
linux-arm-kernel, Tony Luck, Yoshinori Sato, James Morse,
nios2-dev, Andrew Morton, linuxppc-dev, Mike Rapoport
In-Reply-To: <20200216081843.28670-1-rppt@kernel.org>
From: Mike Rapoport <rppt@linux.ibm.com>
Implement primitives necessary for the 4th level folding, add walks of p4d
level where appropriate, replace 5level-fixup.h with pgtable-nop4d.h and
remove __ARCH_USE_5LEVEL_HACK.
Since arm and arm64 share kvm memory management bits, make the conversion
for both variants at once to avoid breaking the builds in the middle.
Signed-off-by: Mike Rapoport <rppt@linux.ibm.com>
---
arch/arm/include/asm/kvm_mmu.h | 5 +-
arch/arm/include/asm/pgtable.h | 1 -
arch/arm/include/asm/stage2_pgtable.h | 15 +-
arch/arm/lib/uaccess_with_memcpy.c | 9 +-
arch/arm/mach-sa1100/assabet.c | 2 +-
arch/arm/mm/dump.c | 29 +++-
arch/arm/mm/fault-armv.c | 7 +-
arch/arm/mm/fault.c | 28 +++-
arch/arm/mm/idmap.c | 3 +-
arch/arm/mm/init.c | 2 +-
arch/arm/mm/ioremap.c | 12 +-
arch/arm/mm/mm.h | 2 +-
arch/arm/mm/mmu.c | 35 +++-
arch/arm/mm/pgd.c | 40 ++++-
arch/arm64/include/asm/kvm_mmu.h | 10 +-
arch/arm64/include/asm/pgalloc.h | 10 +-
arch/arm64/include/asm/pgtable-types.h | 5 +-
arch/arm64/include/asm/pgtable.h | 37 +++--
arch/arm64/include/asm/stage2_pgtable.h | 48 ++++--
arch/arm64/kernel/hibernate.c | 44 ++++-
arch/arm64/mm/fault.c | 9 +-
arch/arm64/mm/hugetlbpage.c | 15 +-
arch/arm64/mm/kasan_init.c | 26 ++-
arch/arm64/mm/mmu.c | 52 ++++--
arch/arm64/mm/pageattr.c | 7 +-
virt/kvm/arm/mmu.c | 209 ++++++++++++++++++++----
26 files changed, 522 insertions(+), 140 deletions(-)
diff --git a/arch/arm/include/asm/kvm_mmu.h b/arch/arm/include/asm/kvm_mmu.h
index 0d84d50bf9ba..8c511bb99e4c 100644
--- a/arch/arm/include/asm/kvm_mmu.h
+++ b/arch/arm/include/asm/kvm_mmu.h
@@ -68,7 +68,8 @@ void kvm_clear_hyp_idmap(void);
#define kvm_mk_pmd(ptep) __pmd(__pa(ptep) | PMD_TYPE_TABLE)
#define kvm_mk_pud(pmdp) __pud(__pa(pmdp) | PMD_TYPE_TABLE)
-#define kvm_mk_pgd(pudp) ({ BUILD_BUG(); 0; })
+#define kvm_mk_p4d(pudp) ({ BUILD_BUG(); __p4d(0); })
+#define kvm_mk_pgd(p4dp) ({ BUILD_BUG(); 0; })
#define kvm_pfn_pte(pfn, prot) pfn_pte(pfn, prot)
#define kvm_pfn_pmd(pfn, prot) pfn_pmd(pfn, prot)
@@ -194,10 +195,12 @@ static inline bool kvm_page_empty(void *ptr)
#define kvm_pte_table_empty(kvm, ptep) kvm_page_empty(ptep)
#define kvm_pmd_table_empty(kvm, pmdp) kvm_page_empty(pmdp)
#define kvm_pud_table_empty(kvm, pudp) false
+#define kvm_p4d_table_empty(kvm, p4dp) false
#define hyp_pte_table_empty(ptep) kvm_page_empty(ptep)
#define hyp_pmd_table_empty(pmdp) kvm_page_empty(pmdp)
#define hyp_pud_table_empty(pudp) false
+#define hyp_p4d_table_empty(p4dp) false
struct kvm;
diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h
index eabcb48a7840..9e3464842dfc 100644
--- a/arch/arm/include/asm/pgtable.h
+++ b/arch/arm/include/asm/pgtable.h
@@ -17,7 +17,6 @@
#else
-#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopud.h>
#include <asm/memory.h>
#include <asm/pgtable-hwdef.h>
diff --git a/arch/arm/include/asm/stage2_pgtable.h b/arch/arm/include/asm/stage2_pgtable.h
index aaceec7855ec..7ed66e216a5e 100644
--- a/arch/arm/include/asm/stage2_pgtable.h
+++ b/arch/arm/include/asm/stage2_pgtable.h
@@ -19,8 +19,17 @@
#define stage2_pgd_none(kvm, pgd) pgd_none(pgd)
#define stage2_pgd_clear(kvm, pgd) pgd_clear(pgd)
#define stage2_pgd_present(kvm, pgd) pgd_present(pgd)
-#define stage2_pgd_populate(kvm, pgd, pud) pgd_populate(NULL, pgd, pud)
-#define stage2_pud_offset(kvm, pgd, address) pud_offset(pgd, address)
+#define stage2_pgd_populate(kvm, pgd, p4d) pgd_populate(NULL, pgd, p4d)
+
+#define stage2_p4d_offset(kvm, pgd, address) p4d_offset(pgd, address)
+#define stage2_p4d_free(kvm, p4d) do { } while (0)
+
+#define stage2_p4d_none(kvm, p4d) p4d_none(p4d)
+#define stage2_p4d_clear(kvm, p4d) p4d_clear(p4d)
+#define stage2_p4d_present(kvm, p4d) p4d_present(p4d)
+#define stage2_p4d_populate(kvm, p4d, pud) p4d_populate(NULL, p4d, pud)
+
+#define stage2_pud_offset(kvm, p4d, address) pud_offset(p4d, address)
#define stage2_pud_free(kvm, pud) do { } while (0)
#define stage2_pud_none(kvm, pud) pud_none(pud)
@@ -41,6 +50,7 @@ stage2_pgd_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)
return (boundary - 1 < end - 1) ? boundary : end;
}
+#define stage2_p4d_addr_end(kvm, addr, end) (end)
#define stage2_pud_addr_end(kvm, addr, end) (end)
static inline phys_addr_t
@@ -56,6 +66,7 @@ stage2_pmd_addr_end(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)
#define stage2_pte_table_empty(kvm, ptep) kvm_page_empty(ptep)
#define stage2_pmd_table_empty(kvm, pmdp) kvm_page_empty(pmdp)
#define stage2_pud_table_empty(kvm, pudp) false
+#define stage2_p4d_table_empty(kvm, p4dp) false
static inline bool kvm_stage2_has_pud(struct kvm *kvm)
{
diff --git a/arch/arm/lib/uaccess_with_memcpy.c b/arch/arm/lib/uaccess_with_memcpy.c
index c9450982a155..cabf1119c256 100644
--- a/arch/arm/lib/uaccess_with_memcpy.c
+++ b/arch/arm/lib/uaccess_with_memcpy.c
@@ -24,6 +24,7 @@ pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp)
{
unsigned long addr = (unsigned long)_addr;
pgd_t *pgd;
+ p4d_t *p4d;
pmd_t *pmd;
pte_t *pte;
pud_t *pud;
@@ -33,7 +34,11 @@ pin_page_for_write(const void __user *_addr, pte_t **ptep, spinlock_t **ptlp)
if (unlikely(pgd_none(*pgd) || pgd_bad(*pgd)))
return 0;
- pud = pud_offset(pgd, addr);
+ p4d = p4d_offset(pgd, addr);
+ if (unlikely(p4d_none(*p4d) || p4d_bad(*p4d)))
+ return 0;
+
+ pud = pud_offset(p4d, addr);
if (unlikely(pud_none(*pud) || pud_bad(*pud)))
return 0;
@@ -154,7 +159,7 @@ arm_copy_to_user(void __user *to, const void *from, unsigned long n)
}
return n;
}
-
+
static unsigned long noinline
__clear_user_memset(void __user *addr, unsigned long n)
{
diff --git a/arch/arm/mach-sa1100/assabet.c b/arch/arm/mach-sa1100/assabet.c
index d96a101e5504..0631a7b02678 100644
--- a/arch/arm/mach-sa1100/assabet.c
+++ b/arch/arm/mach-sa1100/assabet.c
@@ -633,7 +633,7 @@ static void __init map_sa1100_gpio_regs( void )
int prot = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_DOMAIN(DOMAIN_IO);
pmd_t *pmd;
- pmd = pmd_offset(pud_offset(pgd_offset_k(virt), virt), virt);
+ pmd = pmd_offset(pud_offset(p4d_offset(pgd_offset_k(virt), virt), virt), virt);
*pmd = __pmd(phys | prot);
flush_pmd_entry(pmd);
}
diff --git a/arch/arm/mm/dump.c b/arch/arm/mm/dump.c
index 7d6291f23251..677549d6854c 100644
--- a/arch/arm/mm/dump.c
+++ b/arch/arm/mm/dump.c
@@ -207,6 +207,7 @@ struct pg_level {
static struct pg_level pg_level[] = {
{
}, { /* pgd */
+ }, { /* p4d */
}, { /* pud */
}, { /* pmd */
.bits = section_bits,
@@ -308,7 +309,7 @@ static void walk_pte(struct pg_state *st, pmd_t *pmd, unsigned long start,
for (i = 0; i < PTRS_PER_PTE; i++, pte++) {
addr = start + i * PAGE_SIZE;
- note_page(st, addr, 4, pte_val(*pte), domain);
+ note_page(st, addr, 5, pte_val(*pte), domain);
}
}
@@ -350,14 +351,14 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start)
addr += SECTION_SIZE;
pmd++;
domain = get_domain_name(pmd);
- note_page(st, addr, 3, pmd_val(*pmd), domain);
+ note_page(st, addr, 4, pmd_val(*pmd), domain);
}
}
}
-static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)
+static void walk_pud(struct pg_state *st, p4d_t *p4d, unsigned long start)
{
- pud_t *pud = pud_offset(pgd, 0);
+ pud_t *pud = pud_offset(p4d, 0);
unsigned long addr;
unsigned i;
@@ -366,7 +367,23 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)
if (!pud_none(*pud)) {
walk_pmd(st, pud, addr);
} else {
- note_page(st, addr, 2, pud_val(*pud), NULL);
+ note_page(st, addr, 3, pud_val(*pud), NULL);
+ }
+ }
+}
+
+static void walk_p4d(struct pg_state *st, pgd_t *pgd, unsigned long start)
+{
+ p4d_t *p4d = p4d_offset(pgd, 0);
+ unsigned long addr;
+ unsigned i;
+
+ for (i = 0; i < PTRS_PER_P4D; i++, p4d++) {
+ addr = start + i * P4D_SIZE;
+ if (!p4d_none(*p4d)) {
+ walk_pud(st, p4d, addr);
+ } else {
+ note_page(st, addr, 2, p4d_val(*p4d), NULL);
}
}
}
@@ -381,7 +398,7 @@ static void walk_pgd(struct pg_state *st, struct mm_struct *mm,
for (i = 0; i < PTRS_PER_PGD; i++, pgd++) {
addr = start + i * PGDIR_SIZE;
if (!pgd_none(*pgd)) {
- walk_pud(st, pgd, addr);
+ walk_p4d(st, pgd, addr);
} else {
note_page(st, addr, 1, pgd_val(*pgd), NULL);
}
diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c
index ae857f41f68d..489aaafa6ebd 100644
--- a/arch/arm/mm/fault-armv.c
+++ b/arch/arm/mm/fault-armv.c
@@ -91,6 +91,7 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address,
{
spinlock_t *ptl;
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
@@ -100,7 +101,11 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address,
if (pgd_none_or_clear_bad(pgd))
return 0;
- pud = pud_offset(pgd, address);
+ p4d = p4d_offset(pgd, address);
+ if (p4d_none_or_clear_bad(p4d))
+ return 0;
+
+ pud = pud_offset(p4d, address);
if (pud_none_or_clear_bad(pud))
return 0;
diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index bd0f4821f7e1..c2bd35a822e3 100644
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -43,6 +43,7 @@ void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)
printk("%s[%08lx] *pgd=%08llx", lvl, addr, (long long)pgd_val(*pgd));
do {
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
@@ -55,7 +56,19 @@ void show_pte(const char *lvl, struct mm_struct *mm, unsigned long addr)
break;
}
- pud = pud_offset(pgd, addr);
+ p4d = p4d_offset(pgd, addr);
+ if (PTRS_PER_P4D != 1)
+ pr_cont(", *p4d=%08llx", (long long)p4d_val(*p4d));
+
+ if (p4d_none(*p4d))
+ break;
+
+ if (p4d_bad(*p4d)) {
+ pr_cont("(bad)");
+ break;
+ }
+
+ pud = pud_offset(p4d, addr);
if (PTRS_PER_PUD != 1)
pr_cont(", *pud=%08llx", (long long)pud_val(*pud));
@@ -408,6 +421,7 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
{
unsigned int index;
pgd_t *pgd, *pgd_k;
+ p4d_t *p4d, *p4d_k;
pud_t *pud, *pud_k;
pmd_t *pmd, *pmd_k;
@@ -427,8 +441,16 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
if (!pgd_present(*pgd))
set_pgd(pgd, *pgd_k);
- pud = pud_offset(pgd, addr);
- pud_k = pud_offset(pgd_k, addr);
+ p4d = p4d_offset(pgd, addr);
+ p4d_k = p4d_offset(pgd_k, addr);
+
+ if (p4d_none(*p4d_k))
+ goto bad_area;
+ if (!p4d_present(*p4d))
+ set_p4d(p4d, *p4d_k);
+
+ pud = pud_offset(p4d, addr);
+ pud_k = pud_offset(p4d_k, addr);
if (pud_none(*pud_k))
goto bad_area;
diff --git a/arch/arm/mm/idmap.c b/arch/arm/mm/idmap.c
index a033f6134a64..cd54411ef1b8 100644
--- a/arch/arm/mm/idmap.c
+++ b/arch/arm/mm/idmap.c
@@ -68,7 +68,8 @@ static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end,
static void idmap_add_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
unsigned long prot)
{
- pud_t *pud = pud_offset(pgd, addr);
+ p4d_t *p4d = p4d_offset(pgd, addr);
+ pud_t *pud = pud_offset(p4d, addr);
unsigned long next;
do {
diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c
index 054be44d1cdb..963b5284d284 100644
--- a/arch/arm/mm/init.c
+++ b/arch/arm/mm/init.c
@@ -571,7 +571,7 @@ static inline void section_update(unsigned long addr, pmdval_t mask,
{
pmd_t *pmd;
- pmd = pmd_offset(pud_offset(pgd_offset(mm, addr), addr), addr);
+ pmd = pmd_off_k(addr);
#ifdef CONFIG_ARM_LPAE
pmd[0] = __pmd((pmd_val(pmd[0]) & mask) | prot);
diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c
index 72286f9a4d30..75529d76d28c 100644
--- a/arch/arm/mm/ioremap.c
+++ b/arch/arm/mm/ioremap.c
@@ -142,12 +142,14 @@ static void unmap_area_sections(unsigned long virt, unsigned long size)
{
unsigned long addr = virt, end = virt + (size & ~(SZ_1M - 1));
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmdp;
flush_cache_vunmap(addr, end);
pgd = pgd_offset_k(addr);
- pud = pud_offset(pgd, addr);
+ p4d = p4d_offset(pgd, addr);
+ pud = pud_offset(p4d, addr);
pmdp = pmd_offset(pud, addr);
do {
pmd_t pmd = *pmdp;
@@ -190,6 +192,7 @@ remap_area_sections(unsigned long virt, unsigned long pfn,
{
unsigned long addr = virt, end = virt + size;
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
@@ -200,7 +203,8 @@ remap_area_sections(unsigned long virt, unsigned long pfn,
unmap_area_sections(virt, size);
pgd = pgd_offset_k(addr);
- pud = pud_offset(pgd, addr);
+ p4d = p4d_offset(pgd, addr);
+ pud = pud_offset(p4d, addr);
pmd = pmd_offset(pud, addr);
do {
pmd[0] = __pmd(__pfn_to_phys(pfn) | type->prot_sect);
@@ -222,6 +226,7 @@ remap_area_supersections(unsigned long virt, unsigned long pfn,
{
unsigned long addr = virt, end = virt + size;
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
@@ -232,7 +237,8 @@ remap_area_supersections(unsigned long virt, unsigned long pfn,
unmap_area_sections(virt, size);
pgd = pgd_offset_k(virt);
- pud = pud_offset(pgd, addr);
+ p4d = p4d_offset(pgd, addr);
+ pud = pud_offset(p4d, addr);
pmd = pmd_offset(pud, addr);
do {
unsigned long super_pmd_val, i;
diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h
index 88c121ac14b3..4f1f72b75890 100644
--- a/arch/arm/mm/mm.h
+++ b/arch/arm/mm/mm.h
@@ -38,7 +38,7 @@ static inline pte_t get_top_pte(unsigned long va)
static inline pmd_t *pmd_off_k(unsigned long virt)
{
- return pmd_offset(pud_offset(pgd_offset_k(virt), virt), virt);
+ return pmd_offset(pud_offset(p4d_offset(pgd_offset_k(virt), virt), virt), virt);
}
struct mem_type {
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index 5d0d0f86e790..afd97342b634 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -375,7 +375,8 @@ static pte_t *pte_offset_late_fixmap(pmd_t *dir, unsigned long addr)
static inline pmd_t * __init fixmap_pmd(unsigned long addr)
{
pgd_t *pgd = pgd_offset_k(addr);
- pud_t *pud = pud_offset(pgd, addr);
+ p4d_t *p4d = p4d_offset(pgd, addr);
+ pud_t *pud = pud_offset(p4d, addr);
pmd_t *pmd = pmd_offset(pud, addr);
return pmd;
@@ -827,12 +828,12 @@ static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
} while (pmd++, addr = next, addr != end);
}
-static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
+static void __init alloc_init_pud(p4d_t *p4d, unsigned long addr,
unsigned long end, phys_addr_t phys,
const struct mem_type *type,
void *(*alloc)(unsigned long sz), bool ng)
{
- pud_t *pud = pud_offset(pgd, addr);
+ pud_t *pud = pud_offset(p4d, addr);
unsigned long next;
do {
@@ -842,6 +843,21 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
} while (pud++, addr = next, addr != end);
}
+static void __init alloc_init_p4d(pgd_t *pgd, unsigned long addr,
+ unsigned long end, phys_addr_t phys,
+ const struct mem_type *type,
+ void *(*alloc)(unsigned long sz), bool ng)
+{
+ p4d_t *p4d = p4d_offset(pgd, addr);
+ unsigned long next;
+
+ do {
+ next = p4d_addr_end(addr, end);
+ alloc_init_pud(p4d, addr, next, phys, type, alloc, ng);
+ phys += next - addr;
+ } while (p4d++, addr = next, addr != end);
+}
+
#ifndef CONFIG_ARM_LPAE
static void __init create_36bit_mapping(struct mm_struct *mm,
struct map_desc *md,
@@ -889,7 +905,8 @@ static void __init create_36bit_mapping(struct mm_struct *mm,
pgd = pgd_offset(mm, addr);
end = addr + length;
do {
- pud_t *pud = pud_offset(pgd, addr);
+ p4d_t *p4d = p4d_offset(pgd, addr);
+ pud_t *pud = pud_offset(p4d, addr);
pmd_t *pmd = pmd_offset(pud, addr);
int i;
@@ -940,7 +957,7 @@ static void __init __create_mapping(struct mm_struct *mm, struct map_desc *md,
do {
unsigned long next = pgd_addr_end(addr, end);
- alloc_init_pud(pgd, addr, next, phys, type, alloc, ng);
+ alloc_init_p4d(pgd, addr, next, phys, type, alloc, ng);
phys += next - addr;
addr = next;
@@ -976,7 +993,13 @@ void __init create_mapping_late(struct mm_struct *mm, struct map_desc *md,
bool ng)
{
#ifdef CONFIG_ARM_LPAE
- pud_t *pud = pud_alloc(mm, pgd_offset(mm, md->virtual), md->virtual);
+ p4d_t *p4d;
+ pud_t *pud;
+
+ p4d = p4d_alloc(mm, pgd_offset(mm, md->virtual), md->virtual);
+ if (!WARN_ON(!p4d))
+ return;
+ pud = pud_alloc(mm, p4d, md->virtual);
if (WARN_ON(!pud))
return;
pmd_alloc(mm, pud, 0);
diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c
index 478bd2c6aa50..c5e1b27046a8 100644
--- a/arch/arm/mm/pgd.c
+++ b/arch/arm/mm/pgd.c
@@ -30,6 +30,7 @@
pgd_t *pgd_alloc(struct mm_struct *mm)
{
pgd_t *new_pgd, *init_pgd;
+ p4d_t *new_p4d, *init_p4d;
pud_t *new_pud, *init_pud;
pmd_t *new_pmd, *init_pmd;
pte_t *new_pte, *init_pte;
@@ -53,8 +54,12 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
/*
* Allocate PMD table for modules and pkmap mappings.
*/
- new_pud = pud_alloc(mm, new_pgd + pgd_index(MODULES_VADDR),
+ new_p4d = p4d_alloc(mm, new_pgd + pgd_index(MODULES_VADDR),
MODULES_VADDR);
+ if (!new_p4d)
+ goto no_p4d;
+
+ new_pud = pud_alloc(mm, new_p4d, MODULES_VADDR);
if (!new_pud)
goto no_pud;
@@ -69,7 +74,11 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
* contains the machine vectors. The vectors are always high
* with LPAE.
*/
- new_pud = pud_alloc(mm, new_pgd, 0);
+ new_p4d = p4d_alloc(mm, new_pgd, 0);
+ if (!new_p4d)
+ goto no_p4d;
+
+ new_pud = pud_alloc(mm, new_p4d, 0);
if (!new_pud)
goto no_pud;
@@ -91,7 +100,8 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
pmd_val(*new_pmd) |= PMD_DOMAIN(DOMAIN_VECTORS);
#endif
- init_pud = pud_offset(init_pgd, 0);
+ init_p4d = p4d_offset(init_pgd, 0);
+ init_pud = pud_offset(init_p4d, 0);
init_pmd = pmd_offset(init_pud, 0);
init_pte = pte_offset_map(init_pmd, 0);
set_pte_ext(new_pte + 0, init_pte[0], 0);
@@ -108,6 +118,8 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
no_pmd:
pud_free(mm, new_pud);
no_pud:
+ p4d_free(mm, new_p4d);
+no_p4d:
__pgd_free(new_pgd);
no_pgd:
return NULL;
@@ -116,6 +128,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
{
pgd_t *pgd;
+ p4d_t *p4d;
pud_t *pud;
pmd_t *pmd;
pgtable_t pte;
@@ -127,7 +140,11 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
if (pgd_none_or_clear_bad(pgd))
goto no_pgd;
- pud = pud_offset(pgd, 0);
+ p4d = p4d_offset(pgd, 0);
+ if (p4d_none_or_clear_bad(p4d))
+ goto no_p4d;
+
+ pud = pud_offset(p4d, 0);
if (pud_none_or_clear_bad(pud))
goto no_pud;
@@ -144,8 +161,11 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
pmd_free(mm, pmd);
mm_dec_nr_pmds(mm);
no_pud:
- pgd_clear(pgd);
+ p4d_clear(p4d);
pud_free(mm, pud);
+no_p4d:
+ pgd_clear(pgd);
+ p4d_free(mm, p4d);
no_pgd:
#ifdef CONFIG_ARM_LPAE
/*
@@ -156,15 +176,21 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
continue;
if (pgd_val(*pgd) & L_PGD_SWAPPER)
continue;
- pud = pud_offset(pgd, 0);
+ p4d = p4d_offset(pgd, 0);
+ if (p4d_none_or_clear_bad(p4d))
+ continue;
+ pud = pud_offset(p4d, 0);
if (pud_none_or_clear_bad(pud))
continue;
pmd = pmd_offset(pud, 0);
pud_clear(pud);
pmd_free(mm, pmd);
mm_dec_nr_pmds(mm);
- pgd_clear(pgd);
+ p4d_clear(p4d);
pud_free(mm, pud);
+ mm_dec_nr_puds(mm);
+ pgd_clear(pgd);
+ p4d_free(mm, p4d);
}
#endif
__pgd_free(pgd_base);
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index 53d846f1bfe7..1f9bf19ac553 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -172,8 +172,8 @@ void kvm_clear_hyp_idmap(void);
__pmd(__phys_to_pmd_val(__pa(ptep)) | PMD_TYPE_TABLE)
#define kvm_mk_pud(pmdp) \
__pud(__phys_to_pud_val(__pa(pmdp)) | PMD_TYPE_TABLE)
-#define kvm_mk_pgd(pudp) \
- __pgd(__phys_to_pgd_val(__pa(pudp)) | PUD_TYPE_TABLE)
+#define kvm_mk_p4d(pmdp) \
+ __p4d(__phys_to_p4d_val(__pa(pmdp)) | PUD_TYPE_TABLE)
#define kvm_set_pud(pudp, pud) set_pud(pudp, pud)
@@ -299,6 +299,12 @@ static inline bool kvm_s2pud_young(pud_t pud)
#define hyp_pud_table_empty(pudp) kvm_page_empty(pudp)
#endif
+#ifdef __PAGETABLE_P4D_FOLDED
+#define hyp_p4d_table_empty(p4dp) (0)
+#else
+#define hyp_p4d_table_empty(p4dp) kvm_page_empty(p4dp)
+#endif
+
struct kvm;
#define kvm_flush_dcache_to_poc(a,l) __flush_dcache_area((a), (l))
diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
index 172d76fa0245..58e93583ddb6 100644
--- a/arch/arm64/include/asm/pgalloc.h
+++ b/arch/arm64/include/asm/pgalloc.h
@@ -73,17 +73,17 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pudp)
free_page((unsigned long)pudp);
}
-static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pudp, pgdval_t prot)
+static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot)
{
- set_pgd(pgdp, __pgd(__phys_to_pgd_val(pudp) | prot));
+ set_p4d(p4dp, __p4d(__phys_to_p4d_val(pudp) | prot));
}
-static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgdp, pud_t *pudp)
+static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp)
{
- __pgd_populate(pgdp, __pa(pudp), PUD_TYPE_TABLE);
+ __p4d_populate(p4dp, __pa(pudp), PUD_TYPE_TABLE);
}
#else
-static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pudp, pgdval_t prot)
+static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot)
{
BUILD_BUG();
}
diff --git a/arch/arm64/include/asm/pgtable-types.h b/arch/arm64/include/asm/pgtable-types.h
index acb0751a6606..b8f158ae2527 100644
--- a/arch/arm64/include/asm/pgtable-types.h
+++ b/arch/arm64/include/asm/pgtable-types.h
@@ -14,6 +14,7 @@
typedef u64 pteval_t;
typedef u64 pmdval_t;
typedef u64 pudval_t;
+typedef u64 p4dval_t;
typedef u64 pgdval_t;
/*
@@ -44,13 +45,11 @@ typedef struct { pteval_t pgprot; } pgprot_t;
#define __pgprot(x) ((pgprot_t) { (x) } )
#if CONFIG_PGTABLE_LEVELS == 2
-#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopmd.h>
#elif CONFIG_PGTABLE_LEVELS == 3
-#define __ARCH_USE_5LEVEL_HACK
#include <asm-generic/pgtable-nopud.h>
#elif CONFIG_PGTABLE_LEVELS == 4
-#include <asm-generic/5level-fixup.h>
+#include <asm-generic/pgtable-nop4d.h>
#endif
#endif /* __ASM_PGTABLE_TYPES_H */
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 538c85e62f86..c23c5a4e6dc6 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -298,6 +298,11 @@ static inline pte_t pgd_pte(pgd_t pgd)
return __pte(pgd_val(pgd));
}
+static inline pte_t p4d_pte(p4d_t p4d)
+{
+ return __pte(p4d_val(p4d));
+}
+
static inline pte_t pud_pte(pud_t pud)
{
return __pte(pud_val(pud));
@@ -401,6 +406,9 @@ static inline pmd_t pmd_mkdevmap(pmd_t pmd)
#define set_pmd_at(mm, addr, pmdp, pmd) set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd))
+#define __p4d_to_phys(p4d) __pte_to_phys(p4d_pte(p4d))
+#define __phys_to_p4d_val(phys) __phys_to_pte_val(phys)
+
#define __pgd_to_phys(pgd) __pte_to_phys(pgd_pte(pgd))
#define __phys_to_pgd_val(phys) __phys_to_pte_val(phys)
@@ -588,49 +596,50 @@ static inline phys_addr_t pud_page_paddr(pud_t pud)
#define pud_ERROR(pud) __pud_error(__FILE__, __LINE__, pud_val(pud))
-#define pgd_none(pgd) (!pgd_val(pgd))
-#define pgd_bad(pgd) (!(pgd_val(pgd) & 2))
-#define pgd_present(pgd) (pgd_val(pgd))
+#define p4d_none(p4d) (!p4d_val(p4d))
+#define p4d_bad(p4d) (!(p4d_val(p4d) & 2))
+#define p4d_present(p4d) (p4d_val(p4d))
-static inline void set_pgd(pgd_t *pgdp, pgd_t pgd)
+static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
{
- if (in_swapper_pgdir(pgdp)) {
- set_swapper_pgd(pgdp, pgd);
+ if (in_swapper_pgdir(p4dp)) {
+ set_swapper_pgd((pgd_t *)p4dp, __pgd(p4d_val(p4d)));
return;
}
- WRITE_ONCE(*pgdp, pgd);
+ WRITE_ONCE(*p4dp, p4d);
dsb(ishst);
isb();
}
-static inline void pgd_clear(pgd_t *pgdp)
+static inline void p4d_clear(p4d_t *p4dp)
{
- set_pgd(pgdp, __pgd(0));
+ set_p4d(p4dp, __p4d(0));
}
-static inline phys_addr_t pgd_page_paddr(pgd_t pgd)
+static inline phys_addr_t p4d_page_paddr(p4d_t p4d)
{
- return __pgd_to_phys(pgd);
+ return __p4d_to_phys(p4d);
}
/* Find an entry in the frst-level page table. */
#define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))
-#define pud_offset_phys(dir, addr) (pgd_page_paddr(READ_ONCE(*(dir))) + pud_index(addr) * sizeof(pud_t))
+#define pud_offset_phys(dir, addr) (p4d_page_paddr(READ_ONCE(*(dir))) + pud_index(addr) * sizeof(pud_t))
#define pud_offset(dir, addr) ((pud_t *)__va(pud_offset_phys((dir), (addr))))
#define pud_set_fixmap(addr) ((pud_t *)set_fixmap_offset(FIX_PUD, addr))
-#define pud_set_fixmap_offset(pgd, addr) pud_set_fixmap(pud_offset_phys(pgd, addr))
+#define pud_set_fixmap_offset(p4d, addr) pud_set_fixmap(pud_offset_phys(p4d, addr))
#define pud_clear_fixmap() clear_fixmap(FIX_PUD)
-#define pgd_page(pgd) pfn_to_page(__phys_to_pfn(__pgd_to_phys(pgd)))
+#define p4d_page(p4d) pfn_to_page(__phys_to_pfn(__p4d_to_phys(p4d)))
/* use ONLY for statically allocated translation tables */
#define pud_offset_kimg(dir,addr) ((pud_t *)__phys_to_kimg(pud_offset_phys((dir), (addr))))
#else
+#define p4d_page_paddr(p4d) ({ BUILD_BUG(); 0;})
#define pgd_page_paddr(pgd) ({ BUILD_BUG(); 0;})
/* Match pud_offset folding in <asm/generic/pgtable-nopud.h> */
diff --git a/arch/arm64/include/asm/stage2_pgtable.h b/arch/arm64/include/asm/stage2_pgtable.h
index 326aac658b9d..9a364aeae5fb 100644
--- a/arch/arm64/include/asm/stage2_pgtable.h
+++ b/arch/arm64/include/asm/stage2_pgtable.h
@@ -68,41 +68,67 @@ static inline bool kvm_stage2_has_pud(struct kvm *kvm)
#define S2_PUD_SIZE (1UL << S2_PUD_SHIFT)
#define S2_PUD_MASK (~(S2_PUD_SIZE - 1))
-static inline bool stage2_pgd_none(struct kvm *kvm, pgd_t pgd)
+#define stage2_pgd_none(kvm, pgd) pgd_none(pgd)
+#define stage2_pgd_clear(kvm, pgd) pgd_clear(pgd)
+#define stage2_pgd_present(kvm, pgd) pgd_present(pgd)
+#define stage2_pgd_populate(kvm, pgd, p4d) pgd_populate(NULL, pgd, p4d)
+
+static inline p4d_t *stage2_p4d_offset(struct kvm *kvm,
+ pgd_t *pgd, unsigned long address)
+{
+ return p4d_offset(pgd, address);
+}
+
+static inline void stage2_p4d_free(struct kvm *kvm, p4d_t *p4d)
+{
+}
+
+static inline bool stage2_p4d_table_empty(struct kvm *kvm, p4d_t *p4dp)
+{
+ return false;
+}
+
+static inline phys_addr_t stage2_p4d_addr_end(struct kvm *kvm,
+ phys_addr_t addr, phys_addr_t end)
+{
+ return end;
+}
+
+static inline bool stage2_p4d_none(struct kvm *kvm, p4d_t p4d)
{
if (kvm_stage2_has_pud(kvm))
- return pgd_none(pgd);
+ return p4d_none(p4d);
else
return 0;
}
-static inline void stage2_pgd_clear(struct kvm *kvm, pgd_t *pgdp)
+static inline void stage2_p4d_clear(struct kvm *kvm, p4d_t *p4dp)
{
if (kvm_stage2_has_pud(kvm))
- pgd_clear(pgdp);
+ p4d_clear(p4dp);
}
-static inline bool stage2_pgd_present(struct kvm *kvm, pgd_t pgd)
+static inline bool stage2_p4d_present(struct kvm *kvm, p4d_t p4d)
{
if (kvm_stage2_has_pud(kvm))
- return pgd_present(pgd);
+ return p4d_present(p4d);
else
return 1;
}
-static inline void stage2_pgd_populate(struct kvm *kvm, pgd_t *pgd, pud_t *pud)
+static inline void stage2_p4d_populate(struct kvm *kvm, p4d_t *p4d, pud_t *pud)
{
if (kvm_stage2_has_pud(kvm))
- pgd_populate(NULL, pgd, pud);
+ p4d_populate(NULL, p4d, pud);
}
static inline pud_t *stage2_pud_offset(struct kvm *kvm,
- pgd_t *pgd, unsigned long address)
+ p4d_t *p4d, unsigned long address)
{
if (kvm_stage2_has_pud(kvm))
- return pud_offset(pgd, address);
+ return pud_offset(p4d, address);
else
- return (pud_t *)pgd;
+ return (pud_t *)p4d;
}
static inline void stage2_pud_free(struct kvm *kvm, pud_t *pud)
diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c
index 590963c9c609..a370b1afeae0 100644
--- a/arch/arm64/kernel/hibernate.c
+++ b/arch/arm64/kernel/hibernate.c
@@ -187,6 +187,7 @@ static int trans_pgd_map_page(pgd_t *trans_pgd, void *page,
pgprot_t pgprot)
{
pgd_t *pgdp;
+ p4d_t *p4dp;
pud_t *pudp;
pmd_t *pmdp;
pte_t *ptep;
@@ -199,7 +200,15 @@ static int trans_pgd_map_page(pgd_t *trans_pgd, void *page,
pgd_populate(&init_mm, pgdp, pudp);
}
- pudp = pud_offset(pgdp, dst_addr);
+ p4dp = p4d_offset(pgdp, dst_addr);
+ if (p4d_none(READ_ONCE(*p4dp))) {
+ pudp = (void *)get_safe_page(GFP_ATOMIC);
+ if (!pudp)
+ return -ENOMEM;
+ p4d_populate(&init_mm, p4dp, pudp);
+ }
+
+ pudp = pud_offset(p4dp, dst_addr);
if (pud_none(READ_ONCE(*pudp))) {
pmdp = (void *)get_safe_page(GFP_ATOMIC);
if (!pmdp)
@@ -422,7 +431,7 @@ static int copy_pmd(pud_t *dst_pudp, pud_t *src_pudp, unsigned long start,
return 0;
}
-static int copy_pud(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long start,
+static int copy_pud(p4d_t *dst_p4dp, p4d_t *src_p4dp, unsigned long start,
unsigned long end)
{
pud_t *dst_pudp;
@@ -430,15 +439,15 @@ static int copy_pud(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long start,
unsigned long next;
unsigned long addr = start;
- if (pgd_none(READ_ONCE(*dst_pgdp))) {
+ if (p4d_none(READ_ONCE(*dst_p4dp))) {
dst_pudp = (pud_t *)get_safe_page(GFP_ATOMIC);
if (!dst_pudp)
return -ENOMEM;
- pgd_populate(&init_mm, dst_pgdp, dst_pudp);
+ p4d_populate(&init_mm, dst_p4dp, dst_pudp);
}
- dst_pudp = pud_offset(dst_pgdp, start);
+ dst_pudp = pud_offset(dst_p4dp, start);
- src_pudp = pud_offset(src_pgdp, start);
+ src_pudp = pud_offset(src_p4dp, start);
do {
pud_t pud = READ_ONCE(*src_pudp);
@@ -457,6 +466,27 @@ static int copy_pud(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long start,
return 0;
}
+static int copy_p4d(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long start,
+ unsigned long end)
+{
+ p4d_t *dst_p4dp;
+ p4d_t *src_p4dp;
+ unsigned long next;
+ unsigned long addr = start;
+
+ dst_p4dp = p4d_offset(dst_pgdp, start);
+ src_p4dp = p4d_offset(src_pgdp, start);
+ do {
+ next = p4d_addr_end(addr, end);
+ if (p4d_none(READ_ONCE(*src_p4dp)))
+ continue;
+ if (copy_pud(dst_p4dp, src_p4dp, addr, next))
+ return -ENOMEM;
+ } while (dst_p4dp++, src_p4dp++, addr = next, addr != end);
+
+ return 0;
+}
+
static int copy_page_tables(pgd_t *dst_pgdp, unsigned long start,
unsigned long end)
{
@@ -469,7 +499,7 @@ static int copy_page_tables(pgd_t *dst_pgdp, unsigned long start,
next = pgd_addr_end(addr, end);
if (pgd_none(READ_ONCE(*src_pgdp)))
continue;
- if (copy_pud(dst_pgdp, src_pgdp, addr, next))
+ if (copy_p4d(dst_pgdp, src_pgdp, addr, next))
return -ENOMEM;
} while (dst_pgdp++, src_pgdp++, addr = next, addr != end);
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 85566d32958f..fa6e7960f7d1 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -145,6 +145,7 @@ static void show_pte(unsigned long addr)
pr_alert("[%016lx] pgd=%016llx", addr, pgd_val(pgd));
do {
+ p4d_t *p4dp, p4d;
pud_t *pudp, pud;
pmd_t *pmdp, pmd;
pte_t *ptep, pte;
@@ -152,7 +153,13 @@ static void show_pte(unsigned long addr)
if (pgd_none(pgd) || pgd_bad(pgd))
break;
- pudp = pud_offset(pgdp, addr);
+ p4dp = p4d_offset(pgdp, addr);
+ p4d = READ_ONCE(*p4dp);
+ pr_cont(", p4d=%016llx", p4d_val(p4d));
+ if (p4d_none(p4d) || p4d_bad(p4d))
+ break;
+
+ pudp = pud_offset(p4dp, addr);
pud = READ_ONCE(*pudp);
pr_cont(", pud=%016llx", pud_val(pud));
if (pud_none(pud) || pud_bad(pud))
diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c
index bbeb6a5a6ba6..b8a9f26f3790 100644
--- a/arch/arm64/mm/hugetlbpage.c
+++ b/arch/arm64/mm/hugetlbpage.c
@@ -67,11 +67,13 @@ static int find_num_contig(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, size_t *pgsize)
{
pgd_t *pgdp = pgd_offset(mm, addr);
+ p4d_t *p4dp;
pud_t *pudp;
pmd_t *pmdp;
*pgsize = PAGE_SIZE;
- pudp = pud_offset(pgdp, addr);
+ p4dp = p4d_offset(pgdp, addr);
+ pudp = pud_offset(p4dp, addr);
pmdp = pmd_offset(pudp, addr);
if ((pte_t *)pmdp == ptep) {
*pgsize = PMD_SIZE;
@@ -217,12 +219,14 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
unsigned long addr, unsigned long sz)
{
pgd_t *pgdp;
+ p4d_t *p4dp;
pud_t *pudp;
pmd_t *pmdp;
pte_t *ptep = NULL;
pgdp = pgd_offset(mm, addr);
- pudp = pud_alloc(mm, pgdp, addr);
+ p4dp = p4d_offset(pgdp, addr);
+ pudp = pud_alloc(mm, p4dp, addr);
if (!pudp)
return NULL;
@@ -259,6 +263,7 @@ pte_t *huge_pte_offset(struct mm_struct *mm,
unsigned long addr, unsigned long sz)
{
pgd_t *pgdp;
+ p4d_t *p4dp;
pud_t *pudp, pud;
pmd_t *pmdp, pmd;
@@ -266,7 +271,11 @@ pte_t *huge_pte_offset(struct mm_struct *mm,
if (!pgd_present(READ_ONCE(*pgdp)))
return NULL;
- pudp = pud_offset(pgdp, addr);
+ p4dp = p4d_offset(pgdp, addr);
+ if (!p4d_present(READ_ONCE(*p4dp)))
+ return NULL;
+
+ pudp = pud_offset(p4dp, addr);
pud = READ_ONCE(*pudp);
if (sz != PUD_SIZE && pud_none(pud))
return NULL;
diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c
index f87a32484ea8..2339811f317b 100644
--- a/arch/arm64/mm/kasan_init.c
+++ b/arch/arm64/mm/kasan_init.c
@@ -84,17 +84,17 @@ static pmd_t *__init kasan_pmd_offset(pud_t *pudp, unsigned long addr, int node,
return early ? pmd_offset_kimg(pudp, addr) : pmd_offset(pudp, addr);
}
-static pud_t *__init kasan_pud_offset(pgd_t *pgdp, unsigned long addr, int node,
+static pud_t *__init kasan_pud_offset(p4d_t *p4dp, unsigned long addr, int node,
bool early)
{
- if (pgd_none(READ_ONCE(*pgdp))) {
+ if (p4d_none(READ_ONCE(*p4dp))) {
phys_addr_t pud_phys = early ?
__pa_symbol(kasan_early_shadow_pud)
: kasan_alloc_zeroed_page(node);
- __pgd_populate(pgdp, pud_phys, PMD_TYPE_TABLE);
+ __p4d_populate(p4dp, pud_phys, PMD_TYPE_TABLE);
}
- return early ? pud_offset_kimg(pgdp, addr) : pud_offset(pgdp, addr);
+ return early ? pud_offset_kimg(p4dp, addr) : pud_offset(p4dp, addr);
}
static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr,
@@ -126,11 +126,11 @@ static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr,
} while (pmdp++, addr = next, addr != end && pmd_none(READ_ONCE(*pmdp)));
}
-static void __init kasan_pud_populate(pgd_t *pgdp, unsigned long addr,
+static void __init kasan_pud_populate(p4d_t *p4dp, unsigned long addr,
unsigned long end, int node, bool early)
{
unsigned long next;
- pud_t *pudp = kasan_pud_offset(pgdp, addr, node, early);
+ pud_t *pudp = kasan_pud_offset(p4dp, addr, node, early);
do {
next = pud_addr_end(addr, end);
@@ -138,6 +138,18 @@ static void __init kasan_pud_populate(pgd_t *pgdp, unsigned long addr,
} while (pudp++, addr = next, addr != end && pud_none(READ_ONCE(*pudp)));
}
+static void __init kasan_p4d_populate(pgd_t *pgdp, unsigned long addr,
+ unsigned long end, int node, bool early)
+{
+ unsigned long next;
+ p4d_t *p4dp = p4d_offset(pgdp, addr);
+
+ do {
+ next = p4d_addr_end(addr, end);
+ kasan_pud_populate(p4dp, addr, next, node, early);
+ } while (p4dp++, addr = next, addr != end);
+}
+
static void __init kasan_pgd_populate(unsigned long addr, unsigned long end,
int node, bool early)
{
@@ -147,7 +159,7 @@ static void __init kasan_pgd_populate(unsigned long addr, unsigned long end,
pgdp = pgd_offset_k(addr);
do {
next = pgd_addr_end(addr, end);
- kasan_pud_populate(pgdp, addr, next, node, early);
+ kasan_p4d_populate(pgdp, addr, next, node, early);
} while (pgdp++, addr = next, addr != end);
}
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 128f70852bf3..ad4be3e8e0c1 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -289,18 +289,19 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
{
unsigned long next;
pud_t *pudp;
- pgd_t pgd = READ_ONCE(*pgdp);
+ p4d_t *p4dp = p4d_offset(pgdp, addr);
+ p4d_t p4d = READ_ONCE(*p4dp);
- if (pgd_none(pgd)) {
+ if (p4d_none(p4d)) {
phys_addr_t pud_phys;
BUG_ON(!pgtable_alloc);
pud_phys = pgtable_alloc(PUD_SHIFT);
- __pgd_populate(pgdp, pud_phys, PUD_TYPE_TABLE);
- pgd = READ_ONCE(*pgdp);
+ __p4d_populate(p4dp, pud_phys, PUD_TYPE_TABLE);
+ p4d = READ_ONCE(*p4dp);
}
- BUG_ON(pgd_bad(pgd));
+ BUG_ON(p4d_bad(p4d));
- pudp = pud_set_fixmap_offset(pgdp, addr);
+ pudp = pud_set_fixmap_offset(p4dp, addr);
do {
pud_t old_pud = READ_ONCE(*pudp);
@@ -647,6 +648,7 @@ static void __init map_kernel(pgd_t *pgdp)
READ_ONCE(*pgd_offset_k(FIXADDR_START)));
} else if (CONFIG_PGTABLE_LEVELS > 3) {
pgd_t *bm_pgdp;
+ p4d_t *bm_p4dp;
pud_t *bm_pudp;
/*
* The fixmap shares its top level pgd entry with the kernel
@@ -656,7 +658,8 @@ static void __init map_kernel(pgd_t *pgdp)
*/
BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
bm_pgdp = pgd_offset_raw(pgdp, FIXADDR_START);
- bm_pudp = pud_set_fixmap_offset(bm_pgdp, FIXADDR_START);
+ bm_p4dp = p4d_offset(bm_pgdp, FIXADDR_START);
+ bm_pudp = pud_set_fixmap_offset(bm_p4dp, FIXADDR_START);
pud_populate(&init_mm, bm_pudp, lm_alias(bm_pmd));
pud_clear_fixmap();
} else {
@@ -690,6 +693,7 @@ void __init paging_init(void)
int kern_addr_valid(unsigned long addr)
{
pgd_t *pgdp;
+ p4d_t *p4dp;
pud_t *pudp, pud;
pmd_t *pmdp, pmd;
pte_t *ptep, pte;
@@ -701,7 +705,11 @@ int kern_addr_valid(unsigned long addr)
if (pgd_none(READ_ONCE(*pgdp)))
return 0;
- pudp = pud_offset(pgdp, addr);
+ p4dp = p4d_offset(pgdp, addr);
+ if (p4d_none(READ_ONCE(*p4dp)))
+ return 0;
+
+ pudp = pud_offset(p4dp, addr);
pud = READ_ONCE(*pudp);
if (pud_none(pud))
return 0;
@@ -738,6 +746,7 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
unsigned long addr = start;
unsigned long next;
pgd_t *pgdp;
+ p4d_t *p4dp;
pud_t *pudp;
pmd_t *pmdp;
@@ -748,7 +757,11 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
if (!pgdp)
return -ENOMEM;
- pudp = vmemmap_pud_populate(pgdp, addr, node);
+ p4dp = vmemmap_p4d_populate(pgdp, addr, node);
+ if (!p4dp)
+ return -ENOMEM;
+
+ pudp = vmemmap_pud_populate(p4dp, addr, node);
if (!pudp)
return -ENOMEM;
@@ -777,11 +790,12 @@ void vmemmap_free(unsigned long start, unsigned long end,
static inline pud_t * fixmap_pud(unsigned long addr)
{
pgd_t *pgdp = pgd_offset_k(addr);
- pgd_t pgd = READ_ONCE(*pgdp);
+ p4d_t *p4dp = p4d_offset(pgdp, addr);
+ p4d_t p4d = READ_ONCE(*p4dp);
- BUG_ON(pgd_none(pgd) || pgd_bad(pgd));
+ BUG_ON(p4d_none(p4d) || p4d_bad(p4d));
- return pud_offset_kimg(pgdp, addr);
+ return pud_offset_kimg(p4dp, addr);
}
static inline pmd_t * fixmap_pmd(unsigned long addr)
@@ -807,25 +821,27 @@ static inline pte_t * fixmap_pte(unsigned long addr)
*/
void __init early_fixmap_init(void)
{
- pgd_t *pgdp, pgd;
+ pgd_t *pgdp;
+ p4d_t *p4dp, p4d;
pud_t *pudp;
pmd_t *pmdp;
unsigned long addr = FIXADDR_START;
pgdp = pgd_offset_k(addr);
- pgd = READ_ONCE(*pgdp);
+ p4dp = p4d_offset(pgdp, addr);
+ p4d = READ_ONCE(*p4dp);
if (CONFIG_PGTABLE_LEVELS > 3 &&
- !(pgd_none(pgd) || pgd_page_paddr(pgd) == __pa_symbol(bm_pud))) {
+ !(p4d_none(p4d) || p4d_page_paddr(p4d) == __pa_symbol(bm_pud))) {
/*
* We only end up here if the kernel mapping and the fixmap
* share the top level pgd entry, which should only happen on
* 16k/4 levels configurations.
*/
BUG_ON(!IS_ENABLED(CONFIG_ARM64_16K_PAGES));
- pudp = pud_offset_kimg(pgdp, addr);
+ pudp = pud_offset_kimg(p4dp, addr);
} else {
- if (pgd_none(pgd))
- __pgd_populate(pgdp, __pa_symbol(bm_pud), PUD_TYPE_TABLE);
+ if (p4d_none(p4d))
+ __p4d_populate(p4dp, __pa_symbol(bm_pud), PUD_TYPE_TABLE);
pudp = fixmap_pud(addr);
}
if (pud_none(READ_ONCE(*pudp)))
diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c
index 250c49008d73..5a310991ff73 100644
--- a/arch/arm64/mm/pageattr.c
+++ b/arch/arm64/mm/pageattr.c
@@ -198,6 +198,7 @@ void __kernel_map_pages(struct page *page, int numpages, int enable)
bool kernel_page_present(struct page *page)
{
pgd_t *pgdp;
+ p4d_t *p4dp;
pud_t *pudp, pud;
pmd_t *pmdp, pmd;
pte_t *ptep;
@@ -210,7 +211,11 @@ bool kernel_page_present(struct page *page)
if (pgd_none(READ_ONCE(*pgdp)))
return false;
- pudp = pud_offset(pgdp, addr);
+ p4dp = p4d_offset(pgdp, addr);
+ if (p4d_none(READ_ONCE(*p4dp)))
+ return false;
+
+ pudp = pud_offset(p4dp, addr);
pud = READ_ONCE(*pudp);
if (pud_none(pud))
return false;
diff --git a/virt/kvm/arm/mmu.c b/virt/kvm/arm/mmu.c
index 19c961ac4e3c..3d250fa3d2b9 100644
--- a/virt/kvm/arm/mmu.c
+++ b/virt/kvm/arm/mmu.c
@@ -158,13 +158,22 @@ static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc)
static void clear_stage2_pgd_entry(struct kvm *kvm, pgd_t *pgd, phys_addr_t addr)
{
- pud_t *pud_table __maybe_unused = stage2_pud_offset(kvm, pgd, 0UL);
+ p4d_t *p4d_table __maybe_unused = stage2_p4d_offset(kvm, pgd, 0UL);
stage2_pgd_clear(kvm, pgd);
kvm_tlb_flush_vmid_ipa(kvm, addr);
- stage2_pud_free(kvm, pud_table);
+ stage2_p4d_free(kvm, p4d_table);
put_page(virt_to_page(pgd));
}
+static void clear_stage2_p4d_entry(struct kvm *kvm, p4d_t *p4d, phys_addr_t addr)
+{
+ pud_t *pud_table __maybe_unused = stage2_pud_offset(kvm, p4d, 0);
+ stage2_p4d_clear(kvm, p4d);
+ kvm_tlb_flush_vmid_ipa(kvm, addr);
+ stage2_pud_free(kvm, pud_table);
+ put_page(virt_to_page(p4d));
+}
+
static void clear_stage2_pud_entry(struct kvm *kvm, pud_t *pud, phys_addr_t addr)
{
pmd_t *pmd_table __maybe_unused = stage2_pmd_offset(kvm, pud, 0);
@@ -208,12 +217,20 @@ static inline void kvm_pud_populate(pud_t *pudp, pmd_t *pmdp)
dsb(ishst);
}
-static inline void kvm_pgd_populate(pgd_t *pgdp, pud_t *pudp)
+static inline void kvm_p4d_populate(p4d_t *p4dp, pud_t *pudp)
{
- WRITE_ONCE(*pgdp, kvm_mk_pgd(pudp));
+ WRITE_ONCE(*p4dp, kvm_mk_p4d(pudp));
dsb(ishst);
}
+static inline void kvm_pgd_populate(pgd_t *pgdp, p4d_t *p4dp)
+{
+#ifndef __PAGETABLE_P4D_FOLDED
+ WRITE_ONCE(*pgdp, kvm_mk_pgd(p4dp));
+ dsb(ishst);
+#endif
+}
+
/*
* Unmapping vs dcache management:
*
@@ -293,13 +310,13 @@ static void unmap_stage2_pmds(struct kvm *kvm, pud_t *pud,
clear_stage2_pud_entry(kvm, pud, start_addr);
}
-static void unmap_stage2_puds(struct kvm *kvm, pgd_t *pgd,
+static void unmap_stage2_puds(struct kvm *kvm, p4d_t *p4d,
phys_addr_t addr, phys_addr_t end)
{
phys_addr_t next, start_addr = addr;
pud_t *pud, *start_pud;
- start_pud = pud = stage2_pud_offset(kvm, pgd, addr);
+ start_pud = pud = stage2_pud_offset(kvm, p4d, addr);
do {
next = stage2_pud_addr_end(kvm, addr, end);
if (!stage2_pud_none(kvm, *pud)) {
@@ -317,6 +334,23 @@ static void unmap_stage2_puds(struct kvm *kvm, pgd_t *pgd,
} while (pud++, addr = next, addr != end);
if (stage2_pud_table_empty(kvm, start_pud))
+ clear_stage2_p4d_entry(kvm, p4d, start_addr);
+}
+
+static void unmap_stage2_p4ds(struct kvm *kvm, pgd_t *pgd,
+ phys_addr_t addr, phys_addr_t end)
+{
+ phys_addr_t next, start_addr = addr;
+ p4d_t *p4d, *start_p4d;
+
+ start_p4d = p4d = stage2_p4d_offset(kvm, pgd, addr);
+ do {
+ next = stage2_p4d_addr_end(kvm, addr, end);
+ if (!stage2_p4d_none(kvm, *p4d))
+ unmap_stage2_puds(kvm, p4d, addr, next);
+ } while (p4d++, addr = next, addr != end);
+
+ if (stage2_p4d_table_empty(kvm, start_p4d))
clear_stage2_pgd_entry(kvm, pgd, start_addr);
}
@@ -351,7 +385,7 @@ static void unmap_stage2_range(struct kvm *kvm, phys_addr_t start, u64 size)
break;
next = stage2_pgd_addr_end(kvm, addr, end);
if (!stage2_pgd_none(kvm, *pgd))
- unmap_stage2_puds(kvm, pgd, addr, next);
+ unmap_stage2_p4ds(kvm, pgd, addr, next);
/*
* If the range is too large, release the kvm->mmu_lock
* to prevent starvation and lockup detector warnings.
@@ -391,13 +425,13 @@ static void stage2_flush_pmds(struct kvm *kvm, pud_t *pud,
} while (pmd++, addr = next, addr != end);
}
-static void stage2_flush_puds(struct kvm *kvm, pgd_t *pgd,
+static void stage2_flush_puds(struct kvm *kvm, p4d_t *p4d,
phys_addr_t addr, phys_addr_t end)
{
pud_t *pud;
phys_addr_t next;
- pud = stage2_pud_offset(kvm, pgd, addr);
+ pud = stage2_pud_offset(kvm, p4d, addr);
do {
next = stage2_pud_addr_end(kvm, addr, end);
if (!stage2_pud_none(kvm, *pud)) {
@@ -409,6 +443,20 @@ static void stage2_flush_puds(struct kvm *kvm, pgd_t *pgd,
} while (pud++, addr = next, addr != end);
}
+static void stage2_flush_p4ds(struct kvm *kvm, pgd_t *pgd,
+ phys_addr_t addr, phys_addr_t end)
+{
+ p4d_t *p4d;
+ phys_addr_t next;
+
+ p4d = stage2_p4d_offset(kvm, pgd, addr);
+ do {
+ next = stage2_p4d_addr_end(kvm, addr, end);
+ if (!stage2_p4d_none(kvm, *p4d))
+ stage2_flush_puds(kvm, p4d, addr, next);
+ } while (p4d++, addr = next, addr != end);
+}
+
static void stage2_flush_memslot(struct kvm *kvm,
struct kvm_memory_slot *memslot)
{
@@ -421,7 +469,7 @@ static void stage2_flush_memslot(struct kvm *kvm,
do {
next = stage2_pgd_addr_end(kvm, addr, end);
if (!stage2_pgd_none(kvm, *pgd))
- stage2_flush_puds(kvm, pgd, addr, next);
+ stage2_flush_p4ds(kvm, pgd, addr, next);
} while (pgd++, addr = next, addr != end);
}
@@ -451,12 +499,21 @@ static void stage2_flush_vm(struct kvm *kvm)
static void clear_hyp_pgd_entry(pgd_t *pgd)
{
- pud_t *pud_table __maybe_unused = pud_offset(pgd, 0UL);
+ p4d_t *p4d_table __maybe_unused = p4d_offset(pgd, 0UL);
pgd_clear(pgd);
- pud_free(NULL, pud_table);
+ p4d_free(NULL, p4d_table);
put_page(virt_to_page(pgd));
}
+static void clear_hyp_p4d_entry(p4d_t *p4d)
+{
+ pud_t *pud_table __maybe_unused = pud_offset(p4d, 0);
+ VM_BUG_ON(p4d_huge(*p4d));
+ p4d_clear(p4d);
+ pud_free(NULL, pud_table);
+ put_page(virt_to_page(p4d));
+}
+
static void clear_hyp_pud_entry(pud_t *pud)
{
pmd_t *pmd_table __maybe_unused = pmd_offset(pud, 0);
@@ -508,12 +565,12 @@ static void unmap_hyp_pmds(pud_t *pud, phys_addr_t addr, phys_addr_t end)
clear_hyp_pud_entry(pud);
}
-static void unmap_hyp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end)
+static void unmap_hyp_puds(p4d_t *p4d, phys_addr_t addr, phys_addr_t end)
{
phys_addr_t next;
pud_t *pud, *start_pud;
- start_pud = pud = pud_offset(pgd, addr);
+ start_pud = pud = pud_offset(p4d, addr);
do {
next = pud_addr_end(addr, end);
/* Hyp doesn't use huge puds */
@@ -522,6 +579,23 @@ static void unmap_hyp_puds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end)
} while (pud++, addr = next, addr != end);
if (hyp_pud_table_empty(start_pud))
+ clear_hyp_p4d_entry(p4d);
+}
+
+static void unmap_hyp_p4ds(pgd_t *pgd, phys_addr_t addr, phys_addr_t end)
+{
+ phys_addr_t next;
+ p4d_t *p4d, *start_p4d;
+
+ start_p4d = p4d = p4d_offset(pgd, addr);
+ do {
+ next = p4d_addr_end(addr, end);
+ /* Hyp doesn't use huge p4ds */
+ if (!p4d_none(*p4d))
+ unmap_hyp_puds(p4d, addr, next);
+ } while (p4d++, addr = next, addr != end);
+
+ if (hyp_p4d_table_empty(start_p4d))
clear_hyp_pgd_entry(pgd);
}
@@ -545,7 +619,7 @@ static void __unmap_hyp_range(pgd_t *pgdp, unsigned long ptrs_per_pgd,
do {
next = pgd_addr_end(addr, end);
if (!pgd_none(*pgd))
- unmap_hyp_puds(pgd, addr, next);
+ unmap_hyp_p4ds(pgd, addr, next);
} while (pgd++, addr = next, addr != end);
}
@@ -655,7 +729,7 @@ static int create_hyp_pmd_mappings(pud_t *pud, unsigned long start,
return 0;
}
-static int create_hyp_pud_mappings(pgd_t *pgd, unsigned long start,
+static int create_hyp_pud_mappings(p4d_t *p4d, unsigned long start,
unsigned long end, unsigned long pfn,
pgprot_t prot)
{
@@ -666,7 +740,7 @@ static int create_hyp_pud_mappings(pgd_t *pgd, unsigned long start,
addr = start;
do {
- pud = pud_offset(pgd, addr);
+ pud = pud_offset(p4d, addr);
if (pud_none_or_clear_bad(pud)) {
pmd = pmd_alloc_one(NULL, addr);
@@ -688,12 +762,45 @@ static int create_hyp_pud_mappings(pgd_t *pgd, unsigned long start,
return 0;
}
+static int create_hyp_p4d_mappings(pgd_t *pgd, unsigned long start,
+ unsigned long end, unsigned long pfn,
+ pgprot_t prot)
+{
+ p4d_t *p4d;
+ pud_t *pud;
+ unsigned long addr, next;
+ int ret;
+
+ addr = start;
+ do {
+ p4d = p4d_offset(pgd, addr);
+
+ if (p4d_none(*p4d)) {
+ pud = pud_alloc_one(NULL, addr);
+ if (!pud) {
+ kvm_err("Cannot allocate Hyp pud\n");
+ return -ENOMEM;
+ }
+ kvm_p4d_populate(p4d, pud);
+ get_page(virt_to_page(p4d));
+ }
+
+ next = p4d_addr_end(addr, end);
+ ret = create_hyp_pud_mappings(p4d, addr, next, pfn, prot);
+ if (ret)
+ return ret;
+ pfn += (next - addr) >> PAGE_SHIFT;
+ } while (addr = next, addr != end);
+
+ return 0;
+}
+
static int __create_hyp_mappings(pgd_t *pgdp, unsigned long ptrs_per_pgd,
unsigned long start, unsigned long end,
unsigned long pfn, pgprot_t prot)
{
pgd_t *pgd;
- pud_t *pud;
+ p4d_t *p4d;
unsigned long addr, next;
int err = 0;
@@ -704,18 +811,18 @@ static int __create_hyp_mappings(pgd_t *pgdp, unsigned long ptrs_per_pgd,
pgd = pgdp + kvm_pgd_index(addr, ptrs_per_pgd);
if (pgd_none(*pgd)) {
- pud = pud_alloc_one(NULL, addr);
- if (!pud) {
- kvm_err("Cannot allocate Hyp pud\n");
+ p4d = p4d_alloc_one(NULL, addr);
+ if (!p4d) {
+ kvm_err("Cannot allocate Hyp p4d\n");
err = -ENOMEM;
goto out;
}
- kvm_pgd_populate(pgd, pud);
+ kvm_pgd_populate(pgd, p4d);
get_page(virt_to_page(pgd));
}
next = pgd_addr_end(addr, end);
- err = create_hyp_pud_mappings(pgd, addr, next, pfn, prot);
+ err = create_hyp_p4d_mappings(pgd, addr, next, pfn, prot);
if (err)
goto out;
pfn += (next - addr) >> PAGE_SHIFT;
@@ -1012,22 +1119,40 @@ void kvm_free_stage2_pgd(struct kvm *kvm)
free_pages_exact(pgd, stage2_pgd_size(kvm));
}
-static pud_t *stage2_get_pud(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
+static p4d_t *stage2_get_p4d(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
phys_addr_t addr)
{
pgd_t *pgd;
- pud_t *pud;
+ p4d_t *p4d;
pgd = kvm->arch.pgd + stage2_pgd_index(kvm, addr);
if (stage2_pgd_none(kvm, *pgd)) {
if (!cache)
return NULL;
- pud = mmu_memory_cache_alloc(cache);
- stage2_pgd_populate(kvm, pgd, pud);
+ p4d = mmu_memory_cache_alloc(cache);
+ stage2_pgd_populate(kvm, pgd, p4d);
get_page(virt_to_page(pgd));
}
- return stage2_pud_offset(kvm, pgd, addr);
+ return stage2_p4d_offset(kvm, pgd, addr);
+}
+
+static pud_t *stage2_get_pud(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
+ phys_addr_t addr)
+{
+ p4d_t *p4d;
+ pud_t *pud;
+
+ p4d = stage2_get_p4d(kvm, cache, addr);
+ if (stage2_p4d_none(kvm, *p4d)) {
+ if (!cache)
+ return NULL;
+ pud = mmu_memory_cache_alloc(cache);
+ stage2_p4d_populate(kvm, p4d, pud);
+ get_page(virt_to_page(p4d));
+ }
+
+ return stage2_pud_offset(kvm, p4d, addr);
}
static pmd_t *stage2_get_pmd(struct kvm *kvm, struct kvm_mmu_memory_cache *cache,
@@ -1461,18 +1586,18 @@ static void stage2_wp_pmds(struct kvm *kvm, pud_t *pud,
}
/**
- * stage2_wp_puds - write protect PGD range
+ * stage2_wp_puds - write protect P4D range
* @pgd: pointer to pgd entry
* @addr: range start address
* @end: range end address
*/
-static void stage2_wp_puds(struct kvm *kvm, pgd_t *pgd,
+static void stage2_wp_puds(struct kvm *kvm, p4d_t *p4d,
phys_addr_t addr, phys_addr_t end)
{
pud_t *pud;
phys_addr_t next;
- pud = stage2_pud_offset(kvm, pgd, addr);
+ pud = stage2_pud_offset(kvm, p4d, addr);
do {
next = stage2_pud_addr_end(kvm, addr, end);
if (!stage2_pud_none(kvm, *pud)) {
@@ -1486,6 +1611,26 @@ static void stage2_wp_puds(struct kvm *kvm, pgd_t *pgd,
} while (pud++, addr = next, addr != end);
}
+/**
+ * stage2_wp_p4ds - write protect PGD range
+ * @pgd: pointer to pgd entry
+ * @addr: range start address
+ * @end: range end address
+ */
+static void stage2_wp_p4ds(struct kvm *kvm, pgd_t *pgd,
+ phys_addr_t addr, phys_addr_t end)
+{
+ p4d_t *p4d;
+ phys_addr_t next;
+
+ p4d = stage2_p4d_offset(kvm, pgd, addr);
+ do {
+ next = stage2_p4d_addr_end(kvm, addr, end);
+ if (!stage2_p4d_none(kvm, *p4d))
+ stage2_wp_puds(kvm, p4d, addr, next);
+ } while (p4d++, addr = next, addr != end);
+}
+
/**
* stage2_wp_range() - write protect stage2 memory region range
* @kvm: The KVM pointer
@@ -1513,7 +1658,7 @@ static void stage2_wp_range(struct kvm *kvm, phys_addr_t addr, phys_addr_t end)
break;
next = stage2_pgd_addr_end(kvm, addr, end);
if (stage2_pgd_present(kvm, *pgd))
- stage2_wp_puds(kvm, pgd, addr, next);
+ stage2_wp_p4ds(kvm, pgd, addr, next);
} while (pgd++, addr = next, addr != end);
}
--
2.24.0
^ permalink raw reply related
* [PATCH v2 00/13] mm: remove __ARCH_HAS_5LEVEL_HACK
From: Mike Rapoport @ 2020-02-16 8:18 UTC (permalink / raw)
To: linux-kernel
Cc: Rich Felker, linux-ia64, Geert Uytterhoeven, linux-sh, linux-mm,
Paul Mackerras, linux-hexagon, Will Deacon, kvmarm, Jonas Bonn,
linux-arch, Brian Cain, Marc Zyngier, Russell King, Ley Foon Tan,
Mike Rapoport, Catalin Marinas, Julien Thierry, uclinux-h8-devel,
Fenghua Yu, Arnd Bergmann, Suzuki K Poulose, kvm-ppc,
Stefan Kristiansson, openrisc, Stafford Horne, Guan Xuetao,
linux-arm-kernel, Tony Luck, Yoshinori Sato, James Morse,
nios2-dev, Andrew Morton, linuxppc-dev, Mike Rapoport
From: Mike Rapoport <rppt@linux.ibm.com>
Hi,
These patches convert several architectures to use page table folding and
remove __ARCH_HAS_5LEVEL_HACK along with include/asm-generic/5level-fixup.h.
The changes are mostly about mechanical replacement of pgd accessors with p4d
ones and the addition of higher levels to page table traversals.
All the patches were sent separately to the respective arch lists and
maintainers hence the "v2" prefix.
Geert Uytterhoeven (1):
sh: fault: Modernize printing of kernel messages
Mike Rapoport (12):
arm/arm64: add support for folded p4d page tables
h8300: remove usage of __ARCH_USE_5LEVEL_HACK
hexagon: remove __ARCH_USE_5LEVEL_HACK
ia64: add support for folded p4d page tables
nios2: add support for folded p4d page tables
openrisc: add support for folded p4d page tables
powerpc: add support for folded p4d page tables
sh: drop __pXd_offset() macros that duplicate pXd_index() ones
sh: add support for folded p4d page tables
unicore32: remove __ARCH_USE_5LEVEL_HACK
asm-generic: remove pgtable-nop4d-hack.h
mm: remove __ARCH_HAS_5LEVEL_HACK and include/asm-generic/5level-fixup.h
arch/arm/include/asm/kvm_mmu.h | 5 +-
arch/arm/include/asm/pgtable.h | 1 -
arch/arm/include/asm/stage2_pgtable.h | 15 +-
arch/arm/lib/uaccess_with_memcpy.c | 9 +-
arch/arm/mach-sa1100/assabet.c | 2 +-
arch/arm/mm/dump.c | 29 ++-
arch/arm/mm/fault-armv.c | 7 +-
arch/arm/mm/fault.c | 28 ++-
arch/arm/mm/idmap.c | 3 +-
arch/arm/mm/init.c | 2 +-
arch/arm/mm/ioremap.c | 12 +-
arch/arm/mm/mm.h | 2 +-
arch/arm/mm/mmu.c | 35 ++-
arch/arm/mm/pgd.c | 40 +++-
arch/arm64/include/asm/kvm_mmu.h | 10 +-
arch/arm64/include/asm/pgalloc.h | 10 +-
arch/arm64/include/asm/pgtable-types.h | 5 +-
arch/arm64/include/asm/pgtable.h | 37 ++--
arch/arm64/include/asm/stage2_pgtable.h | 48 +++-
arch/arm64/kernel/hibernate.c | 44 +++-
arch/arm64/mm/fault.c | 9 +-
arch/arm64/mm/hugetlbpage.c | 15 +-
arch/arm64/mm/kasan_init.c | 26 ++-
arch/arm64/mm/mmu.c | 52 +++--
arch/arm64/mm/pageattr.c | 7 +-
arch/h8300/include/asm/pgtable.h | 1 -
arch/hexagon/include/asm/fixmap.h | 4 +-
arch/hexagon/include/asm/pgtable.h | 1 -
arch/ia64/include/asm/pgalloc.h | 4 +-
arch/ia64/include/asm/pgtable.h | 17 +-
arch/ia64/mm/fault.c | 7 +-
arch/ia64/mm/hugetlbpage.c | 18 +-
arch/ia64/mm/init.c | 28 ++-
arch/nios2/include/asm/pgtable.h | 3 +-
arch/nios2/mm/fault.c | 9 +-
arch/nios2/mm/ioremap.c | 6 +-
arch/openrisc/include/asm/pgtable.h | 1 -
arch/openrisc/mm/fault.c | 10 +-
arch/openrisc/mm/init.c | 4 +-
arch/powerpc/include/asm/book3s/32/pgtable.h | 1 -
arch/powerpc/include/asm/book3s/64/hash.h | 4 +-
arch/powerpc/include/asm/book3s/64/pgalloc.h | 4 +-
arch/powerpc/include/asm/book3s/64/pgtable.h | 58 +++--
arch/powerpc/include/asm/book3s/64/radix.h | 6 +-
arch/powerpc/include/asm/nohash/32/pgtable.h | 1 -
arch/powerpc/include/asm/nohash/64/pgalloc.h | 2 +-
.../include/asm/nohash/64/pgtable-4k.h | 32 +--
arch/powerpc/include/asm/nohash/64/pgtable.h | 6 +-
arch/powerpc/include/asm/pgtable.h | 8 +
arch/powerpc/kvm/book3s_64_mmu_radix.c | 59 ++++-
arch/powerpc/lib/code-patching.c | 7 +-
arch/powerpc/mm/book3s32/mmu.c | 2 +-
arch/powerpc/mm/book3s32/tlb.c | 4 +-
arch/powerpc/mm/book3s64/hash_pgtable.c | 4 +-
arch/powerpc/mm/book3s64/radix_pgtable.c | 19 +-
arch/powerpc/mm/book3s64/subpage_prot.c | 6 +-
arch/powerpc/mm/hugetlbpage.c | 28 ++-
arch/powerpc/mm/kasan/kasan_init_32.c | 8 +-
arch/powerpc/mm/mem.c | 4 +-
arch/powerpc/mm/nohash/40x.c | 4 +-
arch/powerpc/mm/nohash/book3e_pgtable.c | 15 +-
arch/powerpc/mm/pgtable.c | 25 ++-
arch/powerpc/mm/pgtable_32.c | 28 ++-
arch/powerpc/mm/pgtable_64.c | 10 +-
arch/powerpc/mm/ptdump/hashpagetable.c | 20 +-
arch/powerpc/mm/ptdump/ptdump.c | 22 +-
arch/powerpc/xmon/xmon.c | 17 +-
arch/sh/include/asm/pgtable-2level.h | 1 -
arch/sh/include/asm/pgtable-3level.h | 1 -
arch/sh/include/asm/pgtable_32.h | 5 +-
arch/sh/include/asm/pgtable_64.h | 5 +-
arch/sh/kernel/io_trapped.c | 7 +-
arch/sh/mm/cache-sh4.c | 4 +-
arch/sh/mm/cache-sh5.c | 7 +-
arch/sh/mm/fault.c | 65 ++++--
arch/sh/mm/hugetlbpage.c | 28 ++-
arch/sh/mm/init.c | 15 +-
arch/sh/mm/kmap.c | 2 +-
arch/sh/mm/tlbex_32.c | 6 +-
arch/sh/mm/tlbex_64.c | 7 +-
arch/unicore32/include/asm/pgtable.h | 1 -
arch/unicore32/kernel/hibernate.c | 4 +-
include/asm-generic/5level-fixup.h | 58 -----
include/asm-generic/pgtable-nop4d-hack.h | 64 ------
include/asm-generic/pgtable-nopud.h | 4 -
include/linux/mm.h | 6 -
mm/kasan/init.c | 11 -
mm/memory.c | 8 -
virt/kvm/arm/mmu.c | 209 +++++++++++++++---
89 files changed, 988 insertions(+), 500 deletions(-)
delete mode 100644 include/asm-generic/5level-fixup.h
delete mode 100644 include/asm-generic/pgtable-nop4d-hack.h
--
2.24.0
^ permalink raw reply
* [Bug 206525] BUG: KASAN: stack-out-of-bounds in test_bit+0x30/0x44 (kernel 5.6-rc1)
From: bugzilla-daemon @ 2020-02-15 17:52 UTC (permalink / raw)
To: linuxppc-dev
In-Reply-To: <bug-206525-206035@https.bugzilla.kernel.org/>
https://bugzilla.kernel.org/show_bug.cgi?id=206525
--- Comment #3 from Christophe Leroy (christophe.leroy@c-s.fr) ---
Bug introduced by commit ("cf5bddb95cbe net: bridge: vlan: add rtnetlink group
and notify support")
RTNLGRP_MAX is now 33.
'unsigned long groups' is 32 bits long on PPC32
Following loop in netlink_bind() overflows.
for (group = 0; group < nlk->ngroups; group++) {
if (!test_bit(group, &groups))
continue;
err = nlk->netlink_bind(net, group + 1);
if (!err)
continue;
netlink_undo_bind(group, groups, sk);
goto unlock;
}
Should 'groups' be changes to 'unsigned long long' ?
--
You are receiving this mail because:
You are watching the assignee of the bug.
^ permalink raw reply
* Re: [PATCH] powerpc/kprobes: Fix trap address when trap happened in real mode
From: Christophe Leroy @ 2020-02-15 10:28 UTC (permalink / raw)
To: Masami Hiramatsu
Cc: linux-kernel, Anil S Keshavamurthy, Paul Mackerras, stable,
Naveen N. Rao, linuxppc-dev, David S. Miller, Larry Finger
In-Reply-To: <20200214225434.464ec467ad9094961abb8ddc@kernel.org>
Hi,
Le 14/02/2020 à 14:54, Masami Hiramatsu a écrit :
> Hi,
>
> On Fri, 14 Feb 2020 12:47:49 +0000 (UTC)
> Christophe Leroy <christophe.leroy@c-s.fr> wrote:
>
>> When a program check exception happens while MMU translation is
>> disabled, following Oops happens in kprobe_handler() in the following
>> test:
>>
>> } else if (*addr != BREAKPOINT_INSTRUCTION) {
>
> Thanks for the report and patch. I'm not so sure about powerpc implementation
> but at where the MMU translation is disabled, can the handler work correctly?
> (And where did you put the probe on?)
>
> Your fix may fix this Oops, but if the handler needs special care, it is an
> option to blacklist such place (if possible).
I guess that's another story. Here we are not talking about a place
where kprobe has been illegitimately activated, but a place where there
is a valid trap, which generated a valid 'program check exception'. And
kprobe was off at that time.
As any 'program check exception' due to a trap (ie a BUG_ON, a WARN_ON,
a debugger breakpoint, a perf breakpoint, etc...) calls
kprobe_handler(), kprobe_handler() must be prepared to handle the case
where the MMU translation is disabled, even if probes are not supposed
to be set for functions running with MMU translation disabled.
Christophe
>
> Anyway, Naveen, can you review it?
>
> Thank you,
>
>>
>> [ 33.098554] BUG: Unable to handle kernel data access on read at 0x0000e268
>> [ 33.105091] Faulting instruction address: 0xc000ec34
>> [ 33.110010] Oops: Kernel access of bad area, sig: 11 [#1]
>> [ 33.115348] BE PAGE_SIZE=16K PREEMPT CMPC885
>> [ 33.119540] Modules linked in:
>> [ 33.122591] CPU: 0 PID: 429 Comm: cat Not tainted 5.6.0-rc1-s3k-dev-00824-g84195dc6c58a #3267
>> [ 33.131005] NIP: c000ec34 LR: c000ecd8 CTR: c019cab8
>> [ 33.136002] REGS: ca4d3b58 TRAP: 0300 Not tainted (5.6.0-rc1-s3k-dev-00824-g84195dc6c58a)
>> [ 33.144324] MSR: 00001032 <ME,IR,DR,RI> CR: 2a4d3c52 XER: 00000000
>> [ 33.150699] DAR: 0000e268 DSISR: c0000000
>> [ 33.150699] GPR00: c000b09c ca4d3c10 c66d0620 00000000 ca4d3c60 00000000 00009032 00000000
>> [ 33.150699] GPR08: 00020000 00000000 c087de44 c000afe0 c66d0ad0 100d3dd6 fffffff3 00000000
>> [ 33.150699] GPR16: 00000000 00000041 00000000 ca4d3d70 00000000 00000000 0000416d 00000000
>> [ 33.150699] GPR24: 00000004 c53b6128 00000000 0000e268 00000000 c07c0000 c07bb6fc ca4d3c60
>> [ 33.188015] NIP [c000ec34] kprobe_handler+0x128/0x290
>> [ 33.192989] LR [c000ecd8] kprobe_handler+0x1cc/0x290
>> [ 33.197854] Call Trace:
>> [ 33.200340] [ca4d3c30] [c000b09c] program_check_exception+0xbc/0x6fc
>> [ 33.206590] [ca4d3c50] [c000e43c] ret_from_except_full+0x0/0x4
>> [ 33.212392] --- interrupt: 700 at 0xe268
>> [ 33.270401] Instruction dump:
>> [ 33.273335] 913e0008 81220000 38600001 3929ffff 91220000 80010024 bb410008 7c0803a6
>> [ 33.280992] 38210020 4e800020 38600000 4e800020 <813b0000> 6d2a7fe0 2f8a0008 419e0154
>> [ 33.288841] ---[ end trace 5b9152d4cdadd06d ]---
>>
>> Check MSR and convert regs->nip to virtual address if the trap
>> happened with MSR_IR cleared.
>>
>> Reported-by: Larry Finger <Larry.Finger@lwfinger.net>
>> Fixes: 6cc89bad60a6 ("powerpc/kprobes: Invoke handlers directly")
>> Cc: stable@kernel.vger.org
>> Cc: Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
>> Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
>>
>> ---
>> The bug might have existed even before that commit from Naveen.
>>
>> Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
>> ---
>> arch/powerpc/kernel/kprobes.c | 4 ++++
>> 1 file changed, 4 insertions(+)
>>
>> diff --git a/arch/powerpc/kernel/kprobes.c b/arch/powerpc/kernel/kprobes.c
>> index 2d27ec4feee4..f8b848aa65bd 100644
>> --- a/arch/powerpc/kernel/kprobes.c
>> +++ b/arch/powerpc/kernel/kprobes.c
>> @@ -23,6 +23,7 @@
>> #include <asm/cacheflush.h>
>> #include <asm/sstep.h>
>> #include <asm/sections.h>
>> +#include <asm/io.h>
>> #include <linux/uaccess.h>
>>
>> DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
>> @@ -264,6 +265,9 @@ int kprobe_handler(struct pt_regs *regs)
>> if (user_mode(regs))
>> return 0;
>>
>> + if (!(regs->msr & MSR_IR))
>> + addr = phys_to_virt(regs->nip);
>> +
>> /*
>> * We don't want to be preempted for the entire
>> * duration of kprobe processing
>> --
>> 2.25.0
>>
>
>
^ permalink raw reply
* Re: [PATCH] powerpc/kprobes: Fix trap address when trap happened in real mode
From: Christophe Leroy @ 2020-02-15 10:19 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Larry Finger, Naveen N. Rao
Cc: linux-kernel, Anil S Keshavamurthy, stable, Masami Hiramatsu,
linuxppc-dev, David S. Miller
In-Reply-To: <b1451438f7148ad0e03306a1f1409f4ad1d6ec7c.1581684263.git.christophe.leroy@c-s.fr>
Le 14/02/2020 à 13:47, Christophe Leroy a écrit :
> When a program check exception happens while MMU translation is
> disabled, following Oops happens in kprobe_handler() in the following
> test:
>
> } else if (*addr != BREAKPOINT_INSTRUCTION) {
>
> [ 33.098554] BUG: Unable to handle kernel data access on read at 0x0000e268
> [ 33.105091] Faulting instruction address: 0xc000ec34
> [ 33.110010] Oops: Kernel access of bad area, sig: 11 [#1]
> [ 33.115348] BE PAGE_SIZE=16K PREEMPT CMPC885
> [ 33.119540] Modules linked in:
> [ 33.122591] CPU: 0 PID: 429 Comm: cat Not tainted 5.6.0-rc1-s3k-dev-00824-g84195dc6c58a #3267
> [ 33.131005] NIP: c000ec34 LR: c000ecd8 CTR: c019cab8
> [ 33.136002] REGS: ca4d3b58 TRAP: 0300 Not tainted (5.6.0-rc1-s3k-dev-00824-g84195dc6c58a)
> [ 33.144324] MSR: 00001032 <ME,IR,DR,RI> CR: 2a4d3c52 XER: 00000000
> [ 33.150699] DAR: 0000e268 DSISR: c0000000
> [ 33.150699] GPR00: c000b09c ca4d3c10 c66d0620 00000000 ca4d3c60 00000000 00009032 00000000
> [ 33.150699] GPR08: 00020000 00000000 c087de44 c000afe0 c66d0ad0 100d3dd6 fffffff3 00000000
> [ 33.150699] GPR16: 00000000 00000041 00000000 ca4d3d70 00000000 00000000 0000416d 00000000
> [ 33.150699] GPR24: 00000004 c53b6128 00000000 0000e268 00000000 c07c0000 c07bb6fc ca4d3c60
> [ 33.188015] NIP [c000ec34] kprobe_handler+0x128/0x290
> [ 33.192989] LR [c000ecd8] kprobe_handler+0x1cc/0x290
> [ 33.197854] Call Trace:
> [ 33.200340] [ca4d3c30] [c000b09c] program_check_exception+0xbc/0x6fc
> [ 33.206590] [ca4d3c50] [c000e43c] ret_from_except_full+0x0/0x4
> [ 33.212392] --- interrupt: 700 at 0xe268
> [ 33.270401] Instruction dump:
> [ 33.273335] 913e0008 81220000 38600001 3929ffff 91220000 80010024 bb410008 7c0803a6
> [ 33.280992] 38210020 4e800020 38600000 4e800020 <813b0000> 6d2a7fe0 2f8a0008 419e0154
> [ 33.288841] ---[ end trace 5b9152d4cdadd06d ]---
>
> Check MSR and convert regs->nip to virtual address if the trap
> happened with MSR_IR cleared.
>
> Reported-by: Larry Finger <Larry.Finger@lwfinger.net>
> Fixes: 6cc89bad60a6 ("powerpc/kprobes: Invoke handlers directly")
> Cc: stable@kernel.vger.org
Oops, I meant
Cc: stable@vger.kernel.org
> Cc: Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
> Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
>
> ---
> The bug might have existed even before that commit from Naveen.
>
> Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
> ---
> arch/powerpc/kernel/kprobes.c | 4 ++++
> 1 file changed, 4 insertions(+)
>
> diff --git a/arch/powerpc/kernel/kprobes.c b/arch/powerpc/kernel/kprobes.c
> index 2d27ec4feee4..f8b848aa65bd 100644
> --- a/arch/powerpc/kernel/kprobes.c
> +++ b/arch/powerpc/kernel/kprobes.c
> @@ -23,6 +23,7 @@
> #include <asm/cacheflush.h>
> #include <asm/sstep.h>
> #include <asm/sections.h>
> +#include <asm/io.h>
> #include <linux/uaccess.h>
>
> DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL;
> @@ -264,6 +265,9 @@ int kprobe_handler(struct pt_regs *regs)
> if (user_mode(regs))
> return 0;
>
> + if (!(regs->msr & MSR_IR))
> + addr = phys_to_virt(regs->nip);
> +
> /*
> * We don't want to be preempted for the entire
> * duration of kprobe processing
>
^ permalink raw reply
* Re: [PATCH] powerpc/8xx: Fix clearing of bits 20-23 in ITLB miss
From: Christophe Leroy @ 2020-02-15 10:17 UTC (permalink / raw)
To: Leonardo Bras, Benjamin Herrenschmidt, Paul Mackerras,
Michael Ellerman
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <546ce4737f308a4ba99a53f550de5b44abc06444.camel@linux.ibm.com>
Le 15/02/2020 à 07:28, Leonardo Bras a écrit :
> On Sun, 2020-02-09 at 18:14 +0000, Christophe Leroy wrote:
>> In ITLB miss handled the line supposed to clear bits 20-23 on the
>> L2 ITLB entry is buggy and does indeed nothing, leading to undefined
>> value which could allow execution when it shouldn't.
>>
>> Properly do the clearing with the relevant instruction.
>>
>> Fixes: 74fabcadfd43 ("powerpc/8xx: don't use r12/SPRN_SPRG_SCRATCH2 in TLB Miss handlers")
>> Cc: stable@vger.kernel.org
>> Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
>> ---
>> arch/powerpc/kernel/head_8xx.S | 2 +-
>> 1 file changed, 1 insertion(+), 1 deletion(-)
>>
>> diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
>> index 9922306ae512..073a651787df 100644
>> --- a/arch/powerpc/kernel/head_8xx.S
>> +++ b/arch/powerpc/kernel/head_8xx.S
>> @@ -256,7 +256,7 @@ InstructionTLBMiss:
>> * set. All other Linux PTE bits control the behavior
>> * of the MMU.
>> */
>> - rlwimi r10, r10, 0, 0x0f00 /* Clear bits 20-23 */
>> + rlwinm r10, r10, 0, ~0x0f00 /* Clear bits 20-23 */
>> rlwimi r10, r10, 4, 0x0400 /* Copy _PAGE_EXEC into bit 21 */
>> ori r10, r10, RPN_PATTERN | 0x200 /* Set 22 and 24-27 */
>> mtspr SPRN_MI_RPN, r10 /* Update TLB entry */
>
> Looks a valid change.
> rlwimi r10, r10, 0, 0x0f00 means:
> r10 = ((r10 << 0) & 0x0f00) | (r10 & ~0x0f00) which ends up being
> r10 = r10
>
> On ISA, rlwinm is recommended for clearing high order bits.
> rlwinm r10, r10, 0, ~0x0f00 means:
> r10 = (r10 << 0) & ~0x0f00
>
> Which does exactly what the comments suggests.
>
> FWIW:
> Reviwed-by: Leonardo Bras <leonardo@linux.ibm.com>
>
I guess you mean
Reviewed-by: Leonardo Bras <leonardo@linux.ibm.com>
^ permalink raw reply
* [PATCH v4] powerpc/32s: Fix DSI and ISI exceptions for CONFIG_VMAP_STACK
From: Christophe Leroy @ 2020-02-15 10:14 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Erhard F., Larry Finger
Cc: linuxppc-dev, linux-kernel
hash_page() needs to read page tables from kernel memory. When entire
kernel memory is mapped by BATs, which is normally the case when
CONFIG_STRICT_KERNEL_RWX is not set, it works even if the page hosting
the page table is not referenced in the MMU hash table.
However, if the page where the page table resides is not covered by
a BAT, a DSI fault can be encountered from hash_page(), and it loops
forever. This can happen when CONFIG_STRICT_KERNEL_RWX is selected
and the alignment of the different regions is too small to allow
covering the entire memory with BATs. This also happens when
CONFIG_DEBUG_PAGEALLOC is selected or when booting with 'nobats'
flag.
Also, if the page containing the kernel stack is not present in the
MMU hash table, registers cannot be saved and a recursive DSI fault
is encountered.
To allow hash_page() to properly do its job at all time and load the
MMU hash table whenever needed, it must run with data MMU disabled.
This means it must be called before re-enabling data MMU. To allow
this, registers clobbered by hash_page() and create_hpte() have to
be saved in the thread struct together with SRR0, SSR1, DAR and DSISR.
It is also necessary to ensure that DSI prolog doesn't overwrite
regs saved by prolog of the current running exception. That means:
- DSI can only use SPRN_SPRG_SCRATCH0
- Exceptions must free SPRN_SPRG_SCRATCH0 before writing to the stack.
This also fixes the Oops reported by Erhard when create_hpte() is
called by add_hash_page().
Due to prolog size increase, a few more exceptions had to get split
in two parts.
Fixes: cd08f109e262 ("powerpc/32s: Enable CONFIG_VMAP_STACK")
Reported-by: Erhard F. <erhard_f@mailbox.org>
Link: https://bugzilla.kernel.org/show_bug.cgi?id=206501
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
Tested-by: Erhard F. <erhard_f@mailbox.org>
Tested-by: Larry Finger <Larry.Finger@lwfinger.net>
---
v4: Fixed build when CONFIG_PPC_CHRP and CONFIG_KVM_BOOK3S_32 are set.
Also fixed the fact that cr1 is lost when CONFIG_VMAP_STACK is set
in Machine Check for CHRP. Made the dance with cr registers depend
on MMU_FTR_HPTE_TABLE feature, on 603 a failed access to stack
will not trigger a DSI but a data store tlb miss which doesn't use
the scratch registers.
v3: Also modify kasan_early_hash_table() accordingly.
v2: Reworked the DSI to call hash_page() even earlier and without
cloberring what ongoing exception saved in SPRG registers and
thread struct.
Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
---
arch/powerpc/include/asm/processor.h | 4 +
arch/powerpc/kernel/asm-offsets.c | 12 ++
arch/powerpc/kernel/head_32.S | 155 +++++++++++++++++++++++++-
arch/powerpc/kernel/head_32.h | 21 +++-
arch/powerpc/mm/book3s32/hash_low.S | 52 ++++-----
arch/powerpc/mm/book3s32/mmu.c | 10 +-
arch/powerpc/mm/kasan/kasan_init_32.c | 3 +-
7 files changed, 212 insertions(+), 45 deletions(-)
diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h
index 8387698bd5b6..eedcbfb9a6ff 100644
--- a/arch/powerpc/include/asm/processor.h
+++ b/arch/powerpc/include/asm/processor.h
@@ -168,6 +168,10 @@ struct thread_struct {
unsigned long srr1;
unsigned long dar;
unsigned long dsisr;
+#ifdef CONFIG_PPC_BOOK3S_32
+ unsigned long r0, r3, r4, r5, r6, r8, r9, r11;
+ unsigned long lr, ctr;
+#endif
#endif
/* Debug Registers */
struct debug_reg debug;
diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c
index c25e562f1cd9..fcf24a365fc0 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -132,6 +132,18 @@ int main(void)
OFFSET(SRR1, thread_struct, srr1);
OFFSET(DAR, thread_struct, dar);
OFFSET(DSISR, thread_struct, dsisr);
+#ifdef CONFIG_PPC_BOOK3S_32
+ OFFSET(THR0, thread_struct, r0);
+ OFFSET(THR3, thread_struct, r3);
+ OFFSET(THR4, thread_struct, r4);
+ OFFSET(THR5, thread_struct, r5);
+ OFFSET(THR6, thread_struct, r6);
+ OFFSET(THR8, thread_struct, r8);
+ OFFSET(THR9, thread_struct, r9);
+ OFFSET(THR11, thread_struct, r11);
+ OFFSET(THLR, thread_struct, lr);
+ OFFSET(THCTR, thread_struct, ctr);
+#endif
#endif
#ifdef CONFIG_SPE
OFFSET(THREAD_EVR0, thread_struct, evr[0]);
diff --git a/arch/powerpc/kernel/head_32.S b/arch/powerpc/kernel/head_32.S
index 0493fcac6409..97c887950c3c 100644
--- a/arch/powerpc/kernel/head_32.S
+++ b/arch/powerpc/kernel/head_32.S
@@ -290,17 +290,55 @@ MachineCheck:
7: EXCEPTION_PROLOG_2
addi r3,r1,STACK_FRAME_OVERHEAD
#ifdef CONFIG_PPC_CHRP
- bne cr1,1f
+#ifdef CONFIG_VMAP_STACK
+ mfspr r4, SPRN_SPRG_THREAD
+ tovirt(r4, r4)
+ lwz r4, RTAS_SP(r4)
+ cmpwi cr1, r4, 0
#endif
- EXC_XFER_STD(0x200, machine_check_exception)
-#ifdef CONFIG_PPC_CHRP
-1: b machine_check_in_rtas
+ beq cr1, machine_check_tramp
+ b machine_check_in_rtas
+#else
+ b machine_check_tramp
#endif
/* Data access exception. */
. = 0x300
DO_KVM 0x300
DataAccess:
+#ifdef CONFIG_VMAP_STACK
+ mtspr SPRN_SPRG_SCRATCH0,r10
+ mfspr r10, SPRN_SPRG_THREAD
+BEGIN_MMU_FTR_SECTION
+ stw r11, THR11(r10)
+ mfspr r10, SPRN_DSISR
+ mfcr r11
+#ifdef CONFIG_PPC_KUAP
+ andis. r10, r10, (DSISR_BAD_FAULT_32S | DSISR_DABRMATCH | DSISR_PROTFAULT)@h
+#else
+ andis. r10, r10, (DSISR_BAD_FAULT_32S | DSISR_DABRMATCH)@h
+#endif
+ mfspr r10, SPRN_SPRG_THREAD
+ beq hash_page_dsi
+.Lhash_page_dsi_cont:
+ mtcr r11
+ lwz r11, THR11(r10)
+END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
+ mtspr SPRN_SPRG_SCRATCH1,r11
+ mfspr r11, SPRN_DAR
+ stw r11, DAR(r10)
+ mfspr r11, SPRN_DSISR
+ stw r11, DSISR(r10)
+ mfspr r11, SPRN_SRR0
+ stw r11, SRR0(r10)
+ mfspr r11, SPRN_SRR1 /* check whether user or kernel */
+ stw r11, SRR1(r10)
+ mfcr r10
+ andi. r11, r11, MSR_PR
+
+ EXCEPTION_PROLOG_1
+ b handle_page_fault_tramp_1
+#else /* CONFIG_VMAP_STACK */
EXCEPTION_PROLOG handle_dar_dsisr=1
get_and_save_dar_dsisr_on_stack r4, r5, r11
BEGIN_MMU_FTR_SECTION
@@ -316,11 +354,32 @@ BEGIN_MMU_FTR_SECTION
FTR_SECTION_ELSE
b handle_page_fault_tramp_2
ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE)
+#endif /* CONFIG_VMAP_STACK */
/* Instruction access exception. */
. = 0x400
DO_KVM 0x400
InstructionAccess:
+#ifdef CONFIG_VMAP_STACK
+ mtspr SPRN_SPRG_SCRATCH0,r10
+ mtspr SPRN_SPRG_SCRATCH1,r11
+ mfspr r10, SPRN_SPRG_THREAD
+ mfspr r11, SPRN_SRR0
+ stw r11, SRR0(r10)
+ mfspr r11, SPRN_SRR1 /* check whether user or kernel */
+ stw r11, SRR1(r10)
+ mfcr r10
+BEGIN_MMU_FTR_SECTION
+ andis. r11, r11, SRR1_ISI_NOPT@h /* no pte found? */
+ bne hash_page_isi
+.Lhash_page_isi_cont:
+ mfspr r11, SPRN_SRR1 /* check whether user or kernel */
+END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
+ andi. r11, r11, MSR_PR
+
+ EXCEPTION_PROLOG_1
+ EXCEPTION_PROLOG_2
+#else /* CONFIG_VMAP_STACK */
EXCEPTION_PROLOG
andis. r0,r9,SRR1_ISI_NOPT@h /* no pte found? */
beq 1f /* if so, try to put a PTE */
@@ -329,6 +388,7 @@ InstructionAccess:
BEGIN_MMU_FTR_SECTION
bl hash_page
END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
+#endif /* CONFIG_VMAP_STACK */
1: mr r4,r12
andis. r5,r9,DSISR_SRR1_MATCH_32S@h /* Filter relevant SRR1 bits */
stw r4, _DAR(r11)
@@ -344,7 +404,7 @@ Alignment:
EXCEPTION_PROLOG handle_dar_dsisr=1
save_dar_dsisr_on_stack r4, r5, r11
addi r3,r1,STACK_FRAME_OVERHEAD
- EXC_XFER_STD(0x600, alignment_exception)
+ b alignment_exception_tramp
/* Program check exception */
EXCEPTION(0x700, ProgramCheck, program_check_exception, EXC_XFER_STD)
@@ -645,15 +705,100 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_NEED_DTLB_SW_LRU)
. = 0x3000
+machine_check_tramp:
+ EXC_XFER_STD(0x200, machine_check_exception)
+
+alignment_exception_tramp:
+ EXC_XFER_STD(0x600, alignment_exception)
+
handle_page_fault_tramp_1:
+#ifdef CONFIG_VMAP_STACK
+ EXCEPTION_PROLOG_2 handle_dar_dsisr=1
+#endif
lwz r4, _DAR(r11)
lwz r5, _DSISR(r11)
/* fall through */
handle_page_fault_tramp_2:
EXC_XFER_LITE(0x300, handle_page_fault)
+#ifdef CONFIG_VMAP_STACK
+.macro save_regs_thread thread
+ stw r0, THR0(\thread)
+ stw r3, THR3(\thread)
+ stw r4, THR4(\thread)
+ stw r5, THR5(\thread)
+ stw r6, THR6(\thread)
+ stw r8, THR8(\thread)
+ stw r9, THR9(\thread)
+ mflr r0
+ stw r0, THLR(\thread)
+ mfctr r0
+ stw r0, THCTR(\thread)
+.endm
+
+.macro restore_regs_thread thread
+ lwz r0, THLR(\thread)
+ mtlr r0
+ lwz r0, THCTR(\thread)
+ mtctr r0
+ lwz r0, THR0(\thread)
+ lwz r3, THR3(\thread)
+ lwz r4, THR4(\thread)
+ lwz r5, THR5(\thread)
+ lwz r6, THR6(\thread)
+ lwz r8, THR8(\thread)
+ lwz r9, THR9(\thread)
+.endm
+
+hash_page_dsi:
+ save_regs_thread r10
+ mfdsisr r3
+ mfdar r4
+ mfsrr0 r5
+ mfsrr1 r9
+ rlwinm r3, r3, 32 - 15, _PAGE_RW /* DSISR_STORE -> _PAGE_RW */
+ bl hash_page
+ mfspr r10, SPRN_SPRG_THREAD
+ restore_regs_thread r10
+ b .Lhash_page_dsi_cont
+
+hash_page_isi:
+ mr r11, r10
+ mfspr r10, SPRN_SPRG_THREAD
+ save_regs_thread r10
+ li r3, 0
+ lwz r4, SRR0(r10)
+ lwz r9, SRR1(r10)
+ bl hash_page
+ mfspr r10, SPRN_SPRG_THREAD
+ restore_regs_thread r10
+ mr r10, r11
+ b .Lhash_page_isi_cont
+
+ .globl fast_hash_page_return
+fast_hash_page_return:
+ andis. r10, r9, SRR1_ISI_NOPT@h /* Set on ISI, cleared on DSI */
+ mfspr r10, SPRN_SPRG_THREAD
+ restore_regs_thread r10
+ bne 1f
+
+ /* DSI */
+ mtcr r11
+ lwz r11, THR11(r10)
+ mfspr r10, SPRN_SPRG_SCRATCH0
+ SYNC
+ RFI
+
+1: /* ISI */
+ mtcr r11
+ mfspr r11, SPRN_SPRG_SCRATCH1
+ mfspr r10, SPRN_SPRG_SCRATCH0
+ SYNC
+ RFI
+
stack_overflow:
vmap_stack_overflow_exception
+#endif
AltiVecUnavailable:
EXCEPTION_PROLOG
diff --git a/arch/powerpc/kernel/head_32.h b/arch/powerpc/kernel/head_32.h
index a6a5fbbf8504..9db162f79fe6 100644
--- a/arch/powerpc/kernel/head_32.h
+++ b/arch/powerpc/kernel/head_32.h
@@ -64,11 +64,25 @@
.endm
.macro EXCEPTION_PROLOG_2 handle_dar_dsisr=0
+#if defined(CONFIG_VMAP_STACK) && defined(CONFIG_PPC_BOOK3S)
+BEGIN_MMU_FTR_SECTION
+ mtcr r10
+FTR_SECTION_ELSE
+ stw r10, _CCR(r11)
+ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_HPTE_TABLE)
+#else
stw r10,_CCR(r11) /* save registers */
+#endif
+ mfspr r10, SPRN_SPRG_SCRATCH0
stw r12,GPR12(r11)
stw r9,GPR9(r11)
- mfspr r10,SPRN_SPRG_SCRATCH0
stw r10,GPR10(r11)
+#if defined(CONFIG_VMAP_STACK) && defined(CONFIG_PPC_BOOK3S)
+BEGIN_MMU_FTR_SECTION
+ mfcr r10
+ stw r10, _CCR(r11)
+END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
+#endif
mfspr r12,SPRN_SPRG_SCRATCH1
stw r12,GPR11(r11)
mflr r10
@@ -83,6 +97,11 @@
stw r10, _DSISR(r11)
.endif
lwz r9, SRR1(r12)
+#if defined(CONFIG_VMAP_STACK) && defined(CONFIG_PPC_BOOK3S)
+BEGIN_MMU_FTR_SECTION
+ andi. r10, r9, MSR_PR
+END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
+#endif
lwz r12, SRR0(r12)
#else
mfspr r12,SPRN_SRR0
diff --git a/arch/powerpc/mm/book3s32/hash_low.S b/arch/powerpc/mm/book3s32/hash_low.S
index c11b0a005196..2015c4f96238 100644
--- a/arch/powerpc/mm/book3s32/hash_low.S
+++ b/arch/powerpc/mm/book3s32/hash_low.S
@@ -25,12 +25,6 @@
#include <asm/feature-fixups.h>
#include <asm/code-patching-asm.h>
-#ifdef CONFIG_VMAP_STACK
-#define ADDR_OFFSET 0
-#else
-#define ADDR_OFFSET PAGE_OFFSET
-#endif
-
#ifdef CONFIG_SMP
.section .bss
.align 2
@@ -53,8 +47,8 @@ mmu_hash_lock:
.text
_GLOBAL(hash_page)
#ifdef CONFIG_SMP
- lis r8, (mmu_hash_lock - ADDR_OFFSET)@h
- ori r8, r8, (mmu_hash_lock - ADDR_OFFSET)@l
+ lis r8, (mmu_hash_lock - PAGE_OFFSET)@h
+ ori r8, r8, (mmu_hash_lock - PAGE_OFFSET)@l
lis r0,0x0fff
b 10f
11: lwz r6,0(r8)
@@ -72,12 +66,9 @@ _GLOBAL(hash_page)
cmplw 0,r4,r0
ori r3,r3,_PAGE_USER|_PAGE_PRESENT /* test low addresses as user */
mfspr r5, SPRN_SPRG_PGDIR /* phys page-table root */
-#ifdef CONFIG_VMAP_STACK
- tovirt(r5, r5)
-#endif
blt+ 112f /* assume user more likely */
- lis r5, (swapper_pg_dir - ADDR_OFFSET)@ha /* if kernel address, use */
- addi r5 ,r5 ,(swapper_pg_dir - ADDR_OFFSET)@l /* kernel page table */
+ lis r5, (swapper_pg_dir - PAGE_OFFSET)@ha /* if kernel address, use */
+ addi r5 ,r5 ,(swapper_pg_dir - PAGE_OFFSET)@l /* kernel page table */
rlwimi r3,r9,32-12,29,29 /* MSR_PR -> _PAGE_USER */
112:
#ifndef CONFIG_PTE_64BIT
@@ -89,9 +80,6 @@ _GLOBAL(hash_page)
lwzx r8,r8,r5 /* Get L1 entry */
rlwinm. r8,r8,0,0,20 /* extract pt base address */
#endif
-#ifdef CONFIG_VMAP_STACK
- tovirt(r8, r8)
-#endif
#ifdef CONFIG_SMP
beq- hash_page_out /* return if no mapping */
#else
@@ -143,30 +131,36 @@ retry:
bne- retry /* retry if someone got there first */
mfsrin r3,r4 /* get segment reg for segment */
+#ifndef CONFIG_VMAP_STACK
mfctr r0
stw r0,_CTR(r11)
+#endif
bl create_hpte /* add the hash table entry */
#ifdef CONFIG_SMP
eieio
- lis r8, (mmu_hash_lock - ADDR_OFFSET)@ha
+ lis r8, (mmu_hash_lock - PAGE_OFFSET)@ha
li r0,0
- stw r0, (mmu_hash_lock - ADDR_OFFSET)@l(r8)
+ stw r0, (mmu_hash_lock - PAGE_OFFSET)@l(r8)
#endif
+#ifdef CONFIG_VMAP_STACK
+ b fast_hash_page_return
+#else
/* Return from the exception */
lwz r5,_CTR(r11)
mtctr r5
lwz r0,GPR0(r11)
lwz r8,GPR8(r11)
b fast_exception_return
+#endif
#ifdef CONFIG_SMP
hash_page_out:
eieio
- lis r8, (mmu_hash_lock - ADDR_OFFSET)@ha
+ lis r8, (mmu_hash_lock - PAGE_OFFSET)@ha
li r0,0
- stw r0, (mmu_hash_lock - ADDR_OFFSET)@l(r8)
+ stw r0, (mmu_hash_lock - PAGE_OFFSET)@l(r8)
blr
#endif /* CONFIG_SMP */
@@ -341,7 +335,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
patch_site 1f, patch__hash_page_A1
patch_site 2f, patch__hash_page_A2
/* Get the address of the primary PTE group in the hash table (r3) */
-0: lis r0, (Hash_base - ADDR_OFFSET)@h /* base address of hash table */
+0: lis r0, (Hash_base - PAGE_OFFSET)@h /* base address of hash table */
1: rlwimi r0,r3,LG_PTEG_SIZE,HASH_LEFT,HASH_RIGHT /* VSID -> hash */
2: rlwinm r3,r4,20+LG_PTEG_SIZE,HASH_LEFT,HASH_RIGHT /* PI -> hash */
xor r3,r3,r0 /* make primary hash */
@@ -355,10 +349,10 @@ END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
beq+ 10f /* no PTE: go look for an empty slot */
tlbie r4
- lis r4, (htab_hash_searches - ADDR_OFFSET)@ha
- lwz r6, (htab_hash_searches - ADDR_OFFSET)@l(r4)
+ lis r4, (htab_hash_searches - PAGE_OFFSET)@ha
+ lwz r6, (htab_hash_searches - PAGE_OFFSET)@l(r4)
addi r6,r6,1 /* count how many searches we do */
- stw r6, (htab_hash_searches - ADDR_OFFSET)@l(r4)
+ stw r6, (htab_hash_searches - PAGE_OFFSET)@l(r4)
/* Search the primary PTEG for a PTE whose 1st (d)word matches r5 */
mtctr r0
@@ -390,10 +384,10 @@ END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
beq+ found_empty
/* update counter of times that the primary PTEG is full */
- lis r4, (primary_pteg_full - ADDR_OFFSET)@ha
- lwz r6, (primary_pteg_full - ADDR_OFFSET)@l(r4)
+ lis r4, (primary_pteg_full - PAGE_OFFSET)@ha
+ lwz r6, (primary_pteg_full - PAGE_OFFSET)@l(r4)
addi r6,r6,1
- stw r6, (primary_pteg_full - ADDR_OFFSET)@l(r4)
+ stw r6, (primary_pteg_full - PAGE_OFFSET)@l(r4)
patch_site 0f, patch__hash_page_C
/* Search the secondary PTEG for an empty slot */
@@ -427,8 +421,8 @@ END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
* lockup here but that shouldn't happen
*/
-1: lis r4, (next_slot - ADDR_OFFSET)@ha /* get next evict slot */
- lwz r6, (next_slot - ADDR_OFFSET)@l(r4)
+1: lis r4, (next_slot - PAGE_OFFSET)@ha /* get next evict slot */
+ lwz r6, (next_slot - PAGE_OFFSET)@l(r4)
addi r6,r6,HPTE_SIZE /* search for candidate */
andi. r6,r6,7*HPTE_SIZE
stw r6,next_slot@l(r4)
diff --git a/arch/powerpc/mm/book3s32/mmu.c b/arch/powerpc/mm/book3s32/mmu.c
index 0a1c65a2c565..f888cbb109b9 100644
--- a/arch/powerpc/mm/book3s32/mmu.c
+++ b/arch/powerpc/mm/book3s32/mmu.c
@@ -413,7 +413,7 @@ void __init MMU_init_hw(void)
void __init MMU_init_hw_patch(void)
{
unsigned int hmask = Hash_mask >> (16 - LG_HPTEG_SIZE);
- unsigned int hash;
+ unsigned int hash = (unsigned int)Hash - PAGE_OFFSET;
if (ppc_md.progress)
ppc_md.progress("hash:patch", 0x345);
@@ -425,11 +425,6 @@ void __init MMU_init_hw_patch(void)
/*
* Patch up the instructions in hashtable.S:create_hpte
*/
- if (IS_ENABLED(CONFIG_VMAP_STACK))
- hash = (unsigned int)Hash;
- else
- hash = (unsigned int)Hash - PAGE_OFFSET;
-
modify_instruction_site(&patch__hash_page_A0, 0xffff, hash >> 16);
modify_instruction_site(&patch__hash_page_A1, 0x7c0, hash_mb << 6);
modify_instruction_site(&patch__hash_page_A2, 0x7c0, hash_mb2 << 6);
@@ -439,8 +434,7 @@ void __init MMU_init_hw_patch(void)
/*
* Patch up the instructions in hashtable.S:flush_hash_page
*/
- modify_instruction_site(&patch__flush_hash_A0, 0xffff,
- ((unsigned int)Hash - PAGE_OFFSET) >> 16);
+ modify_instruction_site(&patch__flush_hash_A0, 0xffff, hash >> 16);
modify_instruction_site(&patch__flush_hash_A1, 0x7c0, hash_mb << 6);
modify_instruction_site(&patch__flush_hash_A2, 0x7c0, hash_mb2 << 6);
modify_instruction_site(&patch__flush_hash_B, 0xffff, hmask);
diff --git a/arch/powerpc/mm/kasan/kasan_init_32.c b/arch/powerpc/mm/kasan/kasan_init_32.c
index 16dd95bd0749..db5664dde5ff 100644
--- a/arch/powerpc/mm/kasan/kasan_init_32.c
+++ b/arch/powerpc/mm/kasan/kasan_init_32.c
@@ -185,8 +185,7 @@ u8 __initdata early_hash[256 << 10] __aligned(256 << 10) = {0};
static void __init kasan_early_hash_table(void)
{
- unsigned int hash = IS_ENABLED(CONFIG_VMAP_STACK) ? (unsigned int)early_hash :
- __pa(early_hash);
+ unsigned int hash = __pa(early_hash);
modify_instruction_site(&patch__hash_page_A0, 0xffff, hash >> 16);
modify_instruction_site(&patch__flush_hash_A0, 0xffff, hash >> 16);
--
2.25.0
^ permalink raw reply related
* Re: RESEND: Re: Problem booting a PowerBook G4 Aluminum after commit cd08f109 with CONFIG_VMAP_STACK=y
From: Christophe Leroy @ 2020-02-15 7:55 UTC (permalink / raw)
To: Larry Finger; +Cc: linuxppc-dev@ozlabs.org
In-Reply-To: <04c1c725-6360-1663-c6ae-a4c7611dc385@lwfinger.net>
Le 15/02/2020 à 03:42, Larry Finger a écrit :
> Christophe,
>
> On 2/14/20 1:35 PM, Christophe Leroy wrote:
>> --- a/arch/powerpc/kernel/head_32.S
>> +++ b/arch/powerpc/kernel/head_32.S
>> @@ -270,6 +270,9 @@ __secondary_hold_acknowledge:
>> * pointer when we take an exception from supervisor mode.)
>> * -- paulus.
>> */
>> +#ifdef CONFIG_PPC_CHRP
>> +1: b machine_check_in_rtas
>> +#endif
>> . = 0x200
>> DO_KVM 0x200
>> MachineCheck:
>> @@ -290,12 +293,9 @@ MachineCheck:
>> 7: EXCEPTION_PROLOG_2
>> addi r3,r1,STACK_FRAME_OVERHEAD
>> #ifdef CONFIG_PPC_CHRP
>> - bne cr1,1f
>> + bne cr1,1b
>> #endif
>> EXC_XFER_STD(0x200, machine_check_exception)
>> -#ifdef CONFIG_PPC_CHRP
>> -1: b machine_check_in_rtas
>> -#endif
I'll need to make it a bit different because it shoehorns into your
config but won't fit if CONFIG_KVM_BOOK3S_32 is added.
>>
>> /* Data access exception. */
>> . = 0x300
>
> With the above changes and all the other patches applied, the machine
> finally boots. It is so bloody slow that it takes a long time to do
> anything, but you finally got all the places that needed patches. I
> really lost track of how many bugs were fixed in the process, but I can
> now put that old box aside until time for v5.7.0-rc1. As you can tell,
> it only gets used to verify that PPC32 is working on real G4 hardware.
> It has no real value for any other function.
Yes, I don't have a G4 myself but this is so much nested with other
stuff for the powerpc 83xx than we can't avoid the changes impacting the
G4 and other hash-MMU based PPC32 allthough the changes I'm doing are
not targetted at those platform at first. And as the 83xx is a 603 core,
it is non-hash so all hash related things can't be verified. Plus all
those small parts like power saving, RTAS, etc... which are more specific.
And checking with all possible options is also not easy.
VMAP-STACK was really a challenging functionnality, I'm happy it made
its way to mainline though.
>
> Thanks for the help,
Thanks to you for testing and for your patience.
Christophe
^ permalink raw reply
* Re: [PATCH] selftests/vm: Fix map_hugetlb length used for testing read and write
From: Leonardo Bras @ 2020-02-15 6:49 UTC (permalink / raw)
To: Christophe Leroy, Michael Ellerman, Shuah Khan
Cc: linux-mm, linuxppc-dev, linux-kernel, linux-kselftest
In-Reply-To: <9a404a13c871c4bd0ba9ede68f69a1225180dd7e.1580978385.git.christophe.leroy@c-s.fr>
[-- Attachment #1: Type: text/plain, Size: 2337 bytes --]
Hello Christophe, thank you for the patch.
On Thu, 2020-02-06 at 08:42 +0000, Christophe Leroy wrote:
> Commit fa7b9a805c79 ("tools/selftest/vm: allow choosing mem size and
> page size in map_hugetlb") added the possibility to change the size
> of memory mapped for the test, but left the read and write test using
> the default value. This is unnoticed when mapping a length greater
> than the default one, but segfaults otherwise.
>
> Fix read_bytes() and write_bytes() by giving them the real length.
>
> Also fix the call to munmap().
>
> Fixes: fa7b9a805c79 ("tools/selftest/vm: allow choosing mem size and page size in map_hugetlb")
> Cc: stable@vger.kernel.org
> Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
> ---
> tools/testing/selftests/vm/map_hugetlb.c | 14 +++++++-------
> 1 file changed, 7 insertions(+), 7 deletions(-)
>
> diff --git a/tools/testing/selftests/vm/map_hugetlb.c b/tools/testing/selftests/vm/map_hugetlb.c
> index 5a2d7b8efc40..6af951900aa3 100644
> --- a/tools/testing/selftests/vm/map_hugetlb.c
> +++ b/tools/testing/selftests/vm/map_hugetlb.c
> @@ -45,20 +45,20 @@ static void check_bytes(char *addr)
> printf("First hex is %x\n", *((unsigned int *)addr));
> }
>
> -static void write_bytes(char *addr)
> +static void write_bytes(char *addr, size_t length)
> {
> unsigned long i;
>
> - for (i = 0; i < LENGTH; i++)
> + for (i = 0; i < length; i++)
> *(addr + i) = (char)i;
> }
>
> -static int read_bytes(char *addr)
> +static int read_bytes(char *addr, size_t length)
> {
> unsigned long i;
>
> check_bytes(addr);
> - for (i = 0; i < LENGTH; i++)
> + for (i = 0; i < length; i++)
> if (*(addr + i) != (char)i) {
> printf("Mismatch at %lu\n", i);
> return 1;
> @@ -96,11 +96,11 @@ int main(int argc, char **argv)
>
> printf("Returned address is %p\n", addr);
> check_bytes(addr);
> - write_bytes(addr);
> - ret = read_bytes(addr);
> + write_bytes(addr, length);
> + ret = read_bytes(addr, length);
>
> /* munmap() length of MAP_HUGETLB memory must be hugepage aligned */
> - if (munmap(addr, LENGTH)) {
> + if (munmap(addr, length)) {
> perror("munmap");
> exit(1);
> }
I agree with you, it's a needed fix.
FWIW:
Reviwed-by: Leonardo Bras <leonardo@linux.ibm.com>
[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply
* Re: [PATCH] powerpc/8xx: Fix clearing of bits 20-23 in ITLB miss
From: Leonardo Bras @ 2020-02-15 6:28 UTC (permalink / raw)
To: Christophe Leroy, Benjamin Herrenschmidt, Paul Mackerras,
Michael Ellerman
Cc: linuxppc-dev, linux-kernel
In-Reply-To: <4f70c2778163affce8508a210f65d140e84524b4.1581272050.git.christophe.leroy@c-s.fr>
[-- Attachment #1: Type: text/plain, Size: 1617 bytes --]
On Sun, 2020-02-09 at 18:14 +0000, Christophe Leroy wrote:
> In ITLB miss handled the line supposed to clear bits 20-23 on the
> L2 ITLB entry is buggy and does indeed nothing, leading to undefined
> value which could allow execution when it shouldn't.
>
> Properly do the clearing with the relevant instruction.
>
> Fixes: 74fabcadfd43 ("powerpc/8xx: don't use r12/SPRN_SPRG_SCRATCH2 in TLB Miss handlers")
> Cc: stable@vger.kernel.org
> Signed-off-by: Christophe Leroy <christophe.leroy@c-s.fr>
> ---
> arch/powerpc/kernel/head_8xx.S | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S
> index 9922306ae512..073a651787df 100644
> --- a/arch/powerpc/kernel/head_8xx.S
> +++ b/arch/powerpc/kernel/head_8xx.S
> @@ -256,7 +256,7 @@ InstructionTLBMiss:
> * set. All other Linux PTE bits control the behavior
> * of the MMU.
> */
> - rlwimi r10, r10, 0, 0x0f00 /* Clear bits 20-23 */
> + rlwinm r10, r10, 0, ~0x0f00 /* Clear bits 20-23 */
> rlwimi r10, r10, 4, 0x0400 /* Copy _PAGE_EXEC into bit 21 */
> ori r10, r10, RPN_PATTERN | 0x200 /* Set 22 and 24-27 */
> mtspr SPRN_MI_RPN, r10 /* Update TLB entry */
Looks a valid change.
rlwimi r10, r10, 0, 0x0f00 means:
r10 = ((r10 << 0) & 0x0f00) | (r10 & ~0x0f00) which ends up being
r10 = r10
On ISA, rlwinm is recommended for clearing high order bits.
rlwinm r10, r10, 0, ~0x0f00 means:
r10 = (r10 << 0) & ~0x0f00
Which does exactly what the comments suggests.
FWIW:
Reviwed-by: Leonardo Bras <leonardo@linux.ibm.com>
[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply
* Re: [PATCH] powerpc: fix hardware PMU exception bug on PowerVM compatibility mode systems
From: Leonardo Bras @ 2020-02-15 6:26 UTC (permalink / raw)
To: Desnes A. Nunes do Rosario, linuxppc-dev; +Cc: gromero
In-Reply-To: <20200214001905.20065-1-desnesn@linux.ibm.com>
[-- Attachment #1: Type: text/plain, Size: 1509 bytes --]
Hello Desnes, thanks for the patch.
On Thu, 2020-02-13 at 21:19 -0300, Desnes A. Nunes do Rosario wrote:
> PowerVM systems running compatibility mode on a few Power8 revisions are
> still vulnerable to the hardware defect that loses PMU exceptions arriving
> prior to a context switch.
>
> The software fix for this issue is enabled through the CPU_FTR_PMAO_BUG
> cpu_feature bit, nevertheless this bit also needs to be set for PowerVM
> compatibility mode systems.
>
> Fixes: 68f2f0d431d9ea4 ("powerpc: Add a cpu feature CPU_FTR_PMAO_BUG")
> Signed-off-by: Desnes A. Nunes do Rosario <desnesn@linux.ibm.com>
> ---
> arch/powerpc/kernel/cputable.c | 2 ++
> 1 file changed, 2 insertions(+)
>
> diff --git a/arch/powerpc/kernel/cputable.c b/arch/powerpc/kernel/cputable.c
> index e745abc5457a..5bfef6263dfb 100644
> --- a/arch/powerpc/kernel/cputable.c
> +++ b/arch/powerpc/kernel/cputable.c
> @@ -2198,6 +2198,8 @@ static struct cpu_spec * __init setup_cpu_spec(unsigned long offset,
> if (old.oprofile_cpu_type != NULL) {
> t->oprofile_cpu_type = old.oprofile_cpu_type;
> t->oprofile_type = old.oprofile_type;
> + if (old.cpu_features & CPU_FTR_PMAO_BUG)
> + t->cpu_features |= CPU_FTR_PMAO_BUG;
> }
> }
>
What are your thoughts about doing:
t->cpu_features |= old.cpu_features & CPU_FTR_PMAO_BUG;
Also, I would recommend adding a short comment on top of the added
lines explaining why it is needed.
Best regards,
Leonardo Bras
[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply
* Re: [PATCH] powerpc: Fix /proc/cpuinfo revision for POWER9P processors
From: Leonardo Bras @ 2020-02-15 6:23 UTC (permalink / raw)
To: Mahesh Salgaonkar, linuxppc-dev; +Cc: Michael Neuling
In-Reply-To: <158161167949.29594.6467033073126346958.stgit@jupiter>
[-- Attachment #1: Type: text/plain, Size: 1595 bytes --]
Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com> writes:
Hello Mahesh,
> POWER9P PVR bits are same as that of POWER9. Hence mask off only the
> relevant bits for the major revision similar to POWER9.
>
> Without this patch the cpuinfo output shows 17.0 as revision:
>
> $ cat /proc/cpuinfo
> processor : 0
> cpu : POWER9P, altivec supported
> clock : 2950.000000MHz
> revision : 17.0 (pvr 004f 1100)
>
> With this change it reflects the correct revision:
>
> $ cat /proc/cpuinfo
> processor : 0
> cpu : POWER9P, altivec supported
> clock : 2633.000000MHz
> revision : 1.0 (pvr 004f 1100)
>
> Signed-off-by: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
> ---
> arch/powerpc/kernel/setup-common.c | 1 +
> 1 file changed, 1 insertion(+)
>
> diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c
> index 488f1eecc0de..37342ec42cb3 100644
> --- a/arch/powerpc/kernel/setup-common.c
> +++ b/arch/powerpc/kernel/setup-common.c
> @@ -315,6 +315,7 @@ static int show_cpuinfo(struct seq_file *m, void *v)
> min = pvr & 0xFF;
> break;
> case 0x004e: /* POWER9 bits 12-15 give chip type */
> + case 0x004f: /* POWER9P bits 12-15 give chip type */
> maj = (pvr >> 8) & 0x0F;
> min = pvr & 0xFF;
> break;
Seems fine to me.
FWIW:
Reviwed-by: Leonardo Bras <leonardo@linux.ibm.com>
[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply
* Re: [PATCH] powerpc: Fix /proc/cpuinfo revision for POWER9P processors
From: Leonardo Bras @ 2020-02-15 6:12 UTC (permalink / raw)
To: mahesh, linuxppc-dev; +Cc: mikey
[-- Attachment #1: Type: text/plain, Size: 1384 bytes --]
Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com> writes:
Hello Mahesh,
> POWER9P PVR bits are same as that of POWER9. Hence mask off only the
> relevant bits for the major revision similar to POWER9.
>
> Without this patch the cpuinfo output shows 17.0 as revision:
>
> $ cat /proc/cpuinfo
> processor : 0
> cpu : POWER9P, altivec supported
> clock : 2950.000000MHz
> revision : 17.0 (pvr 004f 1100)
>
> With this change it reflects the correct revision:
>
> $ cat /proc/cpuinfo
> processor : 0
> cpu : POWER9P, altivec supported
> clock : 2633.000000MHz
> revision : 1.0 (pvr 004f 1100)
>
> Signed-off-by: Mahesh Salgaonkar <mahesh@linux.vnet.ibm.com>
> ---
> arch/powerpc/kernel/setup-common.c | 1 +
> 1 file changed, 1 insertion(+)
>
> diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c
> index 488f1eecc0de..37342ec42cb3 100644
> --- a/arch/powerpc/kernel/setup-common.c
> +++ b/arch/powerpc/kernel/setup-common.c
> @@ -315,6 +315,7 @@ static int show_cpuinfo(struct seq_file *m, void *v)
> min = pvr & 0xFF;
> break;
> case 0x004e: /* POWER9 bits 12-15 give chip type */
> + case 0x004f: /* POWER9P bits 12-15 give chip type */
> maj = (pvr >> 8) & 0x0F;
> min = pvr & 0xFF;
> break;
Seems fine to me.
FWIW:
Reviwed-by: Leonardo Bras <leonardo@linux.ibm.com>
[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply
* Re: [PATCH] powerpc: fix hardware PMU exception bug on PowerVM compatibility mode systems
From: Leonardo Bras @ 2020-02-15 5:39 UTC (permalink / raw)
To: desnesn, linuxppc-dev; +Cc: desnesn, gromero
[-- Attachment #1: Type: text/plain, Size: 1475 bytes --]
Hello Desnes, thanks for the patch.
"Desnes A. Nunes do Rosario" <desnesn@linux.ibm.com> writes:
> PowerVM systems running compatibility mode on a few Power8 revisions are
> still vulnerable to the hardware defect that loses PMU exceptions arriving
> prior to a context switch.
>
> The software fix for this issue is enabled through the CPU_FTR_PMAO_BUG
> cpu_feature bit, nevertheless this bit also needs to be set for PowerVM
> compatibility mode systems.
>
> Fixes: 68f2f0d431d9ea4 ("powerpc: Add a cpu feature CPU_FTR_PMAO_BUG")
> Signed-off-by: Desnes A. Nunes do Rosario <desnesn@linux.ibm.com>
> ---
> arch/powerpc/kernel/cputable.c | 2 ++
> 1 file changed, 2 insertions(+)
>
> diff --git a/arch/powerpc/kernel/cputable.c b/arch/powerpc/kernel/cputable.c
> index e745abc5457a..5bfef6263dfb 100644
> --- a/arch/powerpc/kernel/cputable.c
> +++ b/arch/powerpc/kernel/cputable.c
> @@ -2198,6 +2198,8 @@ static struct cpu_spec * __init setup_cpu_spec(unsigned long offset,
> if (old.oprofile_cpu_type != NULL) {
> t->oprofile_cpu_type = old.oprofile_cpu_type;
> t->oprofile_type = old.oprofile_type;
> + if (old.cpu_features & CPU_FTR_PMAO_BUG)
> + t->cpu_features |= CPU_FTR_PMAO_BUG;
What are your thoughts about doing:
t->cpu_features |= old.cpu_features & CPU_FTR_PMAO_BUG;
Also, I would recommend adding a short comment on top of the added
lines explaining why it is needed.
Best regards,
Leonardo Bras
[-- Attachment #2: This is a digitally signed message part --]
[-- Type: application/pgp-signature, Size: 833 bytes --]
^ permalink raw reply
* [PATCH 1/1] powerpc/cputable: Remove unnecessary copy of cpu_spec->oprofile_type
From: Leonardo Bras @ 2020-02-15 5:36 UTC (permalink / raw)
To: Benjamin Herrenschmidt, Paul Mackerras, Michael Ellerman,
Christophe Leroy, Greg Kroah-Hartman, Leonardo Bras,
Thomas Gleixner, desnesn
Cc: linuxppc-dev, linux-kernel
Before checking for cpu_type == NULL, this same copy happens, so doing
it here will just write the same value to the t->oprofile_type
again.
Remove the repeated copy, as it is unnecessary.
Signed-off-by: Leonardo Bras <leonardo@linux.ibm.com>
---
arch/powerpc/kernel/cputable.c | 1 -
1 file changed, 1 deletion(-)
diff --git a/arch/powerpc/kernel/cputable.c b/arch/powerpc/kernel/cputable.c
index e745abc5457a..5a87ec96582f 100644
--- a/arch/powerpc/kernel/cputable.c
+++ b/arch/powerpc/kernel/cputable.c
@@ -2197,7 +2197,6 @@ static struct cpu_spec * __init setup_cpu_spec(unsigned long offset,
*/
if (old.oprofile_cpu_type != NULL) {
t->oprofile_cpu_type = old.oprofile_cpu_type;
- t->oprofile_type = old.oprofile_type;
}
}
--
2.24.1
^ permalink raw reply related
* Re: RESEND: Re: Problem booting a PowerBook G4 Aluminum after commit cd08f109 with CONFIG_VMAP_STACK=y
From: Larry Finger @ 2020-02-15 2:42 UTC (permalink / raw)
To: Christophe Leroy; +Cc: linuxppc-dev@ozlabs.org
In-Reply-To: <074f2468-1fbe-0714-df58-b6d2e1d2725b@c-s.fr>
Christophe,
On 2/14/20 1:35 PM, Christophe Leroy wrote:
> --- a/arch/powerpc/kernel/head_32.S
> +++ b/arch/powerpc/kernel/head_32.S
> @@ -270,6 +270,9 @@ __secondary_hold_acknowledge:
> * pointer when we take an exception from supervisor mode.)
> * -- paulus.
> */
> +#ifdef CONFIG_PPC_CHRP
> +1: b machine_check_in_rtas
> +#endif
> . = 0x200
> DO_KVM 0x200
> MachineCheck:
> @@ -290,12 +293,9 @@ MachineCheck:
> 7: EXCEPTION_PROLOG_2
> addi r3,r1,STACK_FRAME_OVERHEAD
> #ifdef CONFIG_PPC_CHRP
> - bne cr1,1f
> + bne cr1,1b
> #endif
> EXC_XFER_STD(0x200, machine_check_exception)
> -#ifdef CONFIG_PPC_CHRP
> -1: b machine_check_in_rtas
> -#endif
>
> /* Data access exception. */
> . = 0x300
With the above changes and all the other patches applied, the machine finally
boots. It is so bloody slow that it takes a long time to do anything, but you
finally got all the places that needed patches. I really lost track of how many
bugs were fixed in the process, but I can now put that old box aside until time
for v5.7.0-rc1. As you can tell, it only gets used to verify that PPC32 is
working on real G4 hardware. It has no real value for any other function.
Thanks for the help,
Larry
^ permalink raw reply
* Re: [PATCH v2 1/4] mm/memremap_pages: Introduce memremap_compat_align()
From: Dan Williams @ 2020-02-14 23:05 UTC (permalink / raw)
To: Jeff Moyer
Cc: linux-nvdimm, Aneesh Kumar K.V, Linux Kernel Mailing List,
Paul Mackerras, Vishal L Verma, linuxppc-dev
In-Reply-To: <x49k14odgwz.fsf@segfault.boston.devel.redhat.com>
On Fri, Feb 14, 2020 at 12:59 PM Jeff Moyer <jmoyer@redhat.com> wrote:
>
> Dan Williams <dan.j.williams@intel.com> writes:
>
> > On Thu, Feb 13, 2020 at 8:58 AM Jeff Moyer <jmoyer@redhat.com> wrote:
>
> >> I have just a couple of questions.
> >>
> >> First, can you please add a comment above the generic implementation of
> >> memremap_compat_align describing its purpose, and why a platform might
> >> want to override it?
> >
> > Sure, how about:
> >
> > /*
> > * The memremap() and memremap_pages() interfaces are alternately used
> > * to map persistent memory namespaces. These interfaces place different
> > * constraints on the alignment and size of the mapping (namespace).
> > * memremap() can map individual PAGE_SIZE pages. memremap_pages() can
> > * only map subsections (2MB), and at least one architecture (PowerPC)
> > * the minimum mapping granularity of memremap_pages() is 16MB.
> > *
> > * The role of memremap_compat_align() is to communicate the minimum
> > * arch supported alignment of a namespace such that it can freely
> > * switch modes without violating the arch constraint. Namely, do not
> > * allow a namespace to be PAGE_SIZE aligned since that namespace may be
> > * reconfigured into a mode that requires SUBSECTION_SIZE alignment.
> > */
>
> Well, if we modify the x86 variant to be PAGE_SIZE, I think that text
> won't work. How about:
...but I'm not looking to change it to PAGE_SIZE, I'm going to fix the
alignment check to skip if the namespace has "inner" alignment
padding, i.e. "start_pad" and/or "end_trunc" are non-zero.
^ permalink raw reply
* [PATCH 7/7] of/address: Support multiple 'dma-ranges' entries
From: Rob Herring @ 2020-02-14 22:43 UTC (permalink / raw)
To: devicetree, Frank Rowand
Cc: Michal Simek, Arnd Bergmann, Linus Walleij, linuxppc-dev,
linux-kernel, Paul Mackerras, Robin Murphy, Christoph Hellwig
In-Reply-To: <20200214224322.20030-1-robh@kernel.org>
Currently, the DMA offset and mask for a device are set based only on the
first 'dma-ranges' entry. We should really be using all the entries. The
kernel doesn't yet support multiple offsets and sizes, so the best we can
do is to find the biggest size for a single offset. The algorithm is
copied from acpi_dma_get_range().
If there's different offsets from the first entry, then we warn and
continue. It really should be an error, but this will likely break
existing DTs.
Signed-off-by: Rob Herring <robh@kernel.org>
---
drivers/of/address.c | 28 ++++++++++++++++++++++++----
1 file changed, 24 insertions(+), 4 deletions(-)
diff --git a/drivers/of/address.c b/drivers/of/address.c
index a2c45812a50e..8eea3f6e29a4 100644
--- a/drivers/of/address.c
+++ b/drivers/of/address.c
@@ -944,6 +944,7 @@ int of_dma_get_range(struct device_node *np, u64 *dma_addr, u64 *paddr, u64 *siz
bool found_dma_ranges = false;
struct of_range_parser parser;
struct of_range range;
+ u64 dma_start = U64_MAX, dma_end = 0, dma_offset = 0;
while (node) {
ranges = of_get_property(node, "dma-ranges", &len);
@@ -974,14 +975,33 @@ int of_dma_get_range(struct device_node *np, u64 *dma_addr, u64 *paddr, u64 *siz
pr_debug("dma_addr(%llx) cpu_addr(%llx) size(%llx)\n",
range.bus_addr, range.cpu_addr, range.size);
- *dma_addr = range.bus_addr;
- *paddr = range.cpu_addr;
- *size = range.size;
+ if (dma_offset && range.cpu_addr - range.bus_addr != dma_offset) {
+ pr_warn("Can't handle multiple dma-ranges with different offsets on node(%pOF)\n", node);
+ /* Don't error out as we'd break some existing DTs */
+ continue;
+ }
+ dma_offset = range.cpu_addr - range.bus_addr;
+
+ /* Take lower and upper limits */
+ if (range.bus_addr < dma_start)
+ dma_start = range.bus_addr;
+ if (range.bus_addr + range.size > dma_end)
+ dma_end = range.bus_addr + range.size;
+ }
+ if (dma_start >= dma_end) {
+ ret = -EINVAL;
+ pr_debug("Invalid DMA ranges configuration on node(%pOF)\n",
+ node);
goto out;
}
- pr_err("translation of DMA ranges failed on node(%pOF)\n", np);
+ *dma_addr = dma_start;
+ *size = dma_end - dma_start;
+ *paddr = dma_start + dma_offset;
+
+ pr_debug("final: dma_addr(%llx) cpu_addr(%llx) size(%llx)\n",
+ *dma_addr, *paddr, *size);
out:
of_node_put(node);
--
2.20.1
^ permalink raw reply related
* [PATCH 6/7] of/address: use range parser for of_dma_get_range
From: Rob Herring @ 2020-02-14 22:43 UTC (permalink / raw)
To: devicetree, Frank Rowand
Cc: Michal Simek, Arnd Bergmann, Linus Walleij, linuxppc-dev,
linux-kernel, Paul Mackerras, Robin Murphy, Christoph Hellwig
In-Reply-To: <20200214224322.20030-1-robh@kernel.org>
of_dma_get_range() does the same ranges parsing as
of_pci_range_parser_one(), so let's refactor of_dma_get_range() to use
it instead.
This commit is no functional change. Subsequent commits will parse more
than the 1st dma-ranges entry.
Signed-off-by: Rob Herring <robh@kernel.org>
---
drivers/of/address.c | 38 +++++++++++++-------------------------
1 file changed, 13 insertions(+), 25 deletions(-)
diff --git a/drivers/of/address.c b/drivers/of/address.c
index 6d33f849f114..a2c45812a50e 100644
--- a/drivers/of/address.c
+++ b/drivers/of/address.c
@@ -939,10 +939,11 @@ int of_dma_get_range(struct device_node *np, u64 *dma_addr, u64 *paddr, u64 *siz
{
struct device_node *node = of_node_get(np);
const __be32 *ranges = NULL;
- int len, naddr, nsize, pna;
+ int len;
int ret = 0;
bool found_dma_ranges = false;
- u64 dmaaddr;
+ struct of_range_parser parser;
+ struct of_range range;
while (node) {
ranges = of_get_property(node, "dma-ranges", &len);
@@ -967,33 +968,20 @@ int of_dma_get_range(struct device_node *np, u64 *dma_addr, u64 *paddr, u64 *siz
goto out;
}
- naddr = of_bus_n_addr_cells(node);
- nsize = of_bus_n_size_cells(node);
- pna = of_n_addr_cells(node);
- if ((len / sizeof(__be32)) % (pna + naddr + nsize)) {
- ret = -EINVAL;
- goto out;
- }
+ of_dma_range_parser_init(&parser, node);
+
+ for_each_of_range(&parser, &range) {
+ pr_debug("dma_addr(%llx) cpu_addr(%llx) size(%llx)\n",
+ range.bus_addr, range.cpu_addr, range.size);
+
+ *dma_addr = range.bus_addr;
+ *paddr = range.cpu_addr;
+ *size = range.size;
- /* dma-ranges format:
- * DMA addr : naddr cells
- * CPU addr : pna cells
- * size : nsize cells
- */
- dmaaddr = of_read_number(ranges, naddr);
- *paddr = of_translate_dma_address(node, ranges + naddr);
- if (*paddr == OF_BAD_ADDR) {
- pr_err("translation of DMA address(%llx) to CPU address failed node(%pOF)\n",
- dmaaddr, np);
- ret = -EINVAL;
goto out;
}
- *dma_addr = dmaaddr;
-
- *size = of_read_number(ranges + naddr + pna, nsize);
- pr_debug("dma_addr(%llx) cpu_addr(%llx) size(%llx)\n",
- *dma_addr, *paddr, *size);
+ pr_err("translation of DMA ranges failed on node(%pOF)\n", np);
out:
of_node_put(node);
--
2.20.1
^ permalink raw reply related
* [PATCH 5/7] of/address: Rework of_pci_range parsing for non-PCI buses
From: Rob Herring @ 2020-02-14 22:43 UTC (permalink / raw)
To: devicetree, Frank Rowand
Cc: Michal Simek, Arnd Bergmann, Linus Walleij, linuxppc-dev,
linux-kernel, Paul Mackerras, Robin Murphy, Christoph Hellwig
In-Reply-To: <20200214224322.20030-1-robh@kernel.org>
The only PCI specific part of of_pci_range_parser_one() is the handling
of the 3rd address cell. Rework it to work on regular 1 and 2 cell
addresses.
Use defines and a union to avoid a treewide renaming of the parsing
helpers and struct.
Signed-off-by: Rob Herring <robh@kernel.org>
---
drivers/of/address.c | 33 +++++++++++++++++++++------------
include/linux/of_address.h | 12 +++++++++---
2 files changed, 30 insertions(+), 15 deletions(-)
diff --git a/drivers/of/address.c b/drivers/of/address.c
index 5d608d7c10d6..6d33f849f114 100644
--- a/drivers/of/address.c
+++ b/drivers/of/address.c
@@ -694,12 +694,12 @@ EXPORT_SYMBOL(of_get_address);
static int parser_init(struct of_pci_range_parser *parser,
struct device_node *node, const char *name)
{
- const int na = 3, ns = 2;
int rlen;
parser->node = node;
parser->pna = of_n_addr_cells(node);
- parser->np = parser->pna + na + ns;
+ parser->na = of_bus_n_addr_cells(node);
+ parser->ns = of_bus_n_size_cells(node);
parser->dma = !strcmp(name, "dma-ranges");
parser->range = of_get_property(node, name, &rlen);
@@ -724,20 +724,28 @@ int of_pci_dma_range_parser_init(struct of_pci_range_parser *parser,
return parser_init(parser, node, "dma-ranges");
}
EXPORT_SYMBOL_GPL(of_pci_dma_range_parser_init);
+#define of_dma_range_parser_init of_pci_dma_range_parser_init
struct of_pci_range *of_pci_range_parser_one(struct of_pci_range_parser *parser,
struct of_pci_range *range)
{
- const int na = 3, ns = 2;
+ int na = parser->na;
+ int ns = parser->ns;
+ int np = parser->pna + na + ns;
if (!range)
return NULL;
- if (!parser->range || parser->range + parser->np > parser->end)
+ if (!parser->range || parser->range + np > parser->end)
return NULL;
- range->flags = of_bus_pci_get_flags(parser->range);
- range->pci_addr = of_read_number(parser->range + 1, ns);
+ if (parser->na == 3)
+ range->flags = of_bus_pci_get_flags(parser->range);
+ else
+ range->flags = 0;
+
+ range->pci_addr = of_read_number(parser->range, na);
+
if (parser->dma)
range->cpu_addr = of_translate_dma_address(parser->node,
parser->range + na);
@@ -746,15 +754,16 @@ struct of_pci_range *of_pci_range_parser_one(struct of_pci_range_parser *parser,
parser->range + na);
range->size = of_read_number(parser->range + parser->pna + na, ns);
- parser->range += parser->np;
+ parser->range += np;
/* Now consume following elements while they are contiguous */
- while (parser->range + parser->np <= parser->end) {
- u32 flags;
+ while (parser->range + np <= parser->end) {
+ u32 flags = 0;
u64 pci_addr, cpu_addr, size;
- flags = of_bus_pci_get_flags(parser->range);
- pci_addr = of_read_number(parser->range + 1, ns);
+ if (parser->na == 3)
+ flags = of_bus_pci_get_flags(parser->range);
+ pci_addr = of_read_number(parser->range, na);
if (parser->dma)
cpu_addr = of_translate_dma_address(parser->node,
parser->range + na);
@@ -770,7 +779,7 @@ struct of_pci_range *of_pci_range_parser_one(struct of_pci_range_parser *parser,
break;
range->size += size;
- parser->range += parser->np;
+ parser->range += np;
}
return range;
diff --git a/include/linux/of_address.h b/include/linux/of_address.h
index 8d12bf18e80b..763022ed3456 100644
--- a/include/linux/of_address.h
+++ b/include/linux/of_address.h
@@ -10,20 +10,27 @@ struct of_pci_range_parser {
struct device_node *node;
const __be32 *range;
const __be32 *end;
- int np;
+ int na;
+ int ns;
int pna;
bool dma;
};
+#define of_range_parser of_pci_range_parser
struct of_pci_range {
- u64 pci_addr;
+ union {
+ u64 pci_addr;
+ u64 bus_addr;
+ };
u64 cpu_addr;
u64 size;
u32 flags;
};
+#define of_range of_pci_range
#define for_each_of_pci_range(parser, range) \
for (; of_pci_range_parser_one(parser, range);)
+#define for_each_of_range for_each_of_pci_range
/* Translate a DMA address from device space to CPU space */
extern u64 of_translate_dma_address(struct device_node *dev,
@@ -142,4 +149,3 @@ static inline int of_pci_range_to_resource(struct of_pci_range *range,
#endif /* CONFIG_OF_ADDRESS && CONFIG_PCI */
#endif /* __OF_ADDRESS_H */
-
--
2.20.1
^ permalink raw reply related
* [PATCH 4/7] of: Drop struct of_pci_range.pci_space field
From: Rob Herring @ 2020-02-14 22:43 UTC (permalink / raw)
To: devicetree, Frank Rowand
Cc: Michal Simek, Arnd Bergmann, Linus Walleij, linuxppc-dev,
linux-kernel, Paul Mackerras, Robin Murphy, Christoph Hellwig
In-Reply-To: <20200214224322.20030-1-robh@kernel.org>
There's no more users of struct of_pci_range.pci_space field, so remove it.
Signed-off-by: Rob Herring <robh@kernel.org>
---
drivers/of/address.c | 1 -
include/linux/of_address.h | 1 -
2 files changed, 2 deletions(-)
diff --git a/drivers/of/address.c b/drivers/of/address.c
index 846045a48395..5d608d7c10d6 100644
--- a/drivers/of/address.c
+++ b/drivers/of/address.c
@@ -736,7 +736,6 @@ struct of_pci_range *of_pci_range_parser_one(struct of_pci_range_parser *parser,
if (!parser->range || parser->range + parser->np > parser->end)
return NULL;
- range->pci_space = be32_to_cpup(parser->range);
range->flags = of_bus_pci_get_flags(parser->range);
range->pci_addr = of_read_number(parser->range + 1, ns);
if (parser->dma)
diff --git a/include/linux/of_address.h b/include/linux/of_address.h
index eac7ab109df4..8d12bf18e80b 100644
--- a/include/linux/of_address.h
+++ b/include/linux/of_address.h
@@ -16,7 +16,6 @@ struct of_pci_range_parser {
};
struct of_pci_range {
- u32 pci_space;
u64 pci_addr;
u64 cpu_addr;
u64 size;
--
2.20.1
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox