* [PATCH v7 11/24] mm: Cache some VMA fields in the vm_fault structure
From: Laurent Dufour @ 2018-02-06 16:49 UTC (permalink / raw)
To: paulmck, peterz, akpm, kirill, ak, mhocko, dave, jack,
Matthew Wilcox, benh, mpe, paulus, Thomas Gleixner, Ingo Molnar,
hpa, Will Deacon, Sergey Senozhatsky, Andrea Arcangeli,
Alexei Starovoitov, kemi.wang, sergey.senozhatsky.work,
Daniel Jordan
Cc: linux-kernel, linux-mm, haren, khandual, npiggin, bsingharora,
Tim Chen, linuxppc-dev, x86
In-Reply-To: <1517935810-31177-1-git-send-email-ldufour@linux.vnet.ibm.com>
When handling speculative page fault, the vma->vm_flags and
vma->vm_page_prot fields are read once the page table lock is released. So
there is no more guarantee that these fields would not change in our back.
They will be saved in the vm_fault structure before the VMA is checked for
changes.
This patch also set the fields in hugetlb_no_page() and
__collapse_huge_page_swapin even if it is not need for the callee.
Signed-off-by: Laurent Dufour <ldufour@linux.vnet.ibm.com>
---
include/linux/mm.h | 6 ++++++
mm/hugetlb.c | 2 ++
mm/khugepaged.c | 2 ++
mm/memory.c | 38 ++++++++++++++++++++------------------
4 files changed, 30 insertions(+), 18 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index bb771afa59a2..c034f478b73d 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -361,6 +361,12 @@ struct vm_fault {
* page table to avoid allocation from
* atomic context.
*/
+ /*
+ * These entries are required when handling speculative page fault.
+ * This way the page handling is done using consistent field values.
+ */
+ unsigned long vma_flags;
+ pgprot_t vma_page_prot;
};
/* page entry size for vm->huge_fault() */
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 7c204e3d132b..22a818c7a6de 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3716,6 +3716,8 @@ static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma,
.vma = vma,
.address = address,
.flags = flags,
+ .vma_flags = vma->vm_flags,
+ .vma_page_prot = vma->vm_page_prot,
/*
* Hard to debug if it ends up being
* used by a callee that assumes
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 32314e9e48dd..a946d5306160 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -882,6 +882,8 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm,
.flags = FAULT_FLAG_ALLOW_RETRY,
.pmd = pmd,
.pgoff = linear_page_index(vma, address),
+ .vma_flags = vma->vm_flags,
+ .vma_page_prot = vma->vm_page_prot,
};
/* we only decide to swapin, if there is enough young ptes */
diff --git a/mm/memory.c b/mm/memory.c
index 9661b1302645..1d6b9f91f2a6 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2609,7 +2609,7 @@ static int wp_page_copy(struct vm_fault *vmf)
* Don't let another task, with possibly unlocked vma,
* keep the mlocked page.
*/
- if (page_copied && (vma->vm_flags & VM_LOCKED)) {
+ if (page_copied && (vmf->vma_flags & VM_LOCKED)) {
lock_page(old_page); /* LRU manipulation */
if (PageMlocked(old_page))
munlock_vma_page(old_page);
@@ -2643,7 +2643,7 @@ static int wp_page_copy(struct vm_fault *vmf)
*/
int finish_mkwrite_fault(struct vm_fault *vmf)
{
- WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));
+ WARN_ON_ONCE(!(vmf->vma_flags & VM_SHARED));
if (!pte_map_lock(vmf))
return VM_FAULT_RETRY;
/*
@@ -2745,7 +2745,7 @@ static int do_wp_page(struct vm_fault *vmf)
* We should not cow pages in a shared writeable mapping.
* Just mark the pages writable and/or call ops->pfn_mkwrite.
*/
- if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
+ if ((vmf->vma_flags & (VM_WRITE|VM_SHARED)) ==
(VM_WRITE|VM_SHARED))
return wp_pfn_shared(vmf);
@@ -2792,7 +2792,7 @@ static int do_wp_page(struct vm_fault *vmf)
return VM_FAULT_WRITE;
}
unlock_page(vmf->page);
- } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
+ } else if (unlikely((vmf->vma_flags & (VM_WRITE|VM_SHARED)) ==
(VM_WRITE|VM_SHARED))) {
return wp_page_shared(vmf);
}
@@ -3079,7 +3079,7 @@ int do_swap_page(struct vm_fault *vmf)
inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
dec_mm_counter_fast(vma->vm_mm, MM_SWAPENTS);
- pte = mk_pte(page, vma->vm_page_prot);
+ pte = mk_pte(page, vmf->vma_page_prot);
if ((vmf->flags & FAULT_FLAG_WRITE) && reuse_swap_page(page, NULL)) {
pte = maybe_mkwrite(pte_mkdirty(pte), vma);
vmf->flags &= ~FAULT_FLAG_WRITE;
@@ -3105,7 +3105,7 @@ int do_swap_page(struct vm_fault *vmf)
swap_free(entry);
if (mem_cgroup_swap_full(page) ||
- (vma->vm_flags & VM_LOCKED) || PageMlocked(page))
+ (vmf->vma_flags & VM_LOCKED) || PageMlocked(page))
try_to_free_swap(page);
unlock_page(page);
if (page != swapcache && swapcache) {
@@ -3162,7 +3162,7 @@ static int do_anonymous_page(struct vm_fault *vmf)
pte_t entry;
/* File mapping without ->vm_ops ? */
- if (vma->vm_flags & VM_SHARED)
+ if (vmf->vma_flags & VM_SHARED)
return VM_FAULT_SIGBUS;
/*
@@ -3186,7 +3186,7 @@ static int do_anonymous_page(struct vm_fault *vmf)
if (!(vmf->flags & FAULT_FLAG_WRITE) &&
!mm_forbids_zeropage(vma->vm_mm)) {
entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
- vma->vm_page_prot));
+ vmf->vma_page_prot));
if (!pte_map_lock(vmf))
return VM_FAULT_RETRY;
if (!pte_none(*vmf->pte))
@@ -3219,8 +3219,8 @@ static int do_anonymous_page(struct vm_fault *vmf)
*/
__SetPageUptodate(page);
- entry = mk_pte(page, vma->vm_page_prot);
- if (vma->vm_flags & VM_WRITE)
+ entry = mk_pte(page, vmf->vma_page_prot);
+ if (vmf->vma_flags & VM_WRITE)
entry = pte_mkwrite(pte_mkdirty(entry));
if (!pte_map_lock(vmf)) {
@@ -3416,7 +3416,7 @@ static int do_set_pmd(struct vm_fault *vmf, struct page *page)
for (i = 0; i < HPAGE_PMD_NR; i++)
flush_icache_page(vma, page + i);
- entry = mk_huge_pmd(page, vma->vm_page_prot);
+ entry = mk_huge_pmd(page, vmf->vma_page_prot);
if (write)
entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
@@ -3490,11 +3490,11 @@ int alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg,
return VM_FAULT_NOPAGE;
flush_icache_page(vma, page);
- entry = mk_pte(page, vma->vm_page_prot);
+ entry = mk_pte(page, vmf->vma_page_prot);
if (write)
entry = maybe_mkwrite(pte_mkdirty(entry), vma);
/* copy-on-write page */
- if (write && !(vma->vm_flags & VM_SHARED)) {
+ if (write && !(vmf->vma_flags & VM_SHARED)) {
inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
page_add_new_anon_rmap(page, vma, vmf->address, false);
mem_cgroup_commit_charge(page, memcg, false, false);
@@ -3533,7 +3533,7 @@ int finish_fault(struct vm_fault *vmf)
/* Did we COW the page? */
if ((vmf->flags & FAULT_FLAG_WRITE) &&
- !(vmf->vma->vm_flags & VM_SHARED))
+ !(vmf->vma_flags & VM_SHARED))
page = vmf->cow_page;
else
page = vmf->page;
@@ -3787,7 +3787,7 @@ static int do_fault(struct vm_fault *vmf)
ret = VM_FAULT_SIGBUS;
else if (!(vmf->flags & FAULT_FLAG_WRITE))
ret = do_read_fault(vmf);
- else if (!(vma->vm_flags & VM_SHARED))
+ else if (!(vmf->vma_flags & VM_SHARED))
ret = do_cow_fault(vmf);
else
ret = do_shared_fault(vmf);
@@ -3844,7 +3844,7 @@ static int do_numa_page(struct vm_fault *vmf)
* accessible ptes, some can allow access by kernel mode.
*/
pte = ptep_modify_prot_start(vma->vm_mm, vmf->address, vmf->pte);
- pte = pte_modify(pte, vma->vm_page_prot);
+ pte = pte_modify(pte, vmf->vma_page_prot);
pte = pte_mkyoung(pte);
if (was_writable)
pte = pte_mkwrite(pte);
@@ -3878,7 +3878,7 @@ static int do_numa_page(struct vm_fault *vmf)
* Flag if the page is shared between multiple address spaces. This
* is later used when determining whether to group tasks together
*/
- if (page_mapcount(page) > 1 && (vma->vm_flags & VM_SHARED))
+ if (page_mapcount(page) > 1 && (vmf->vma_flags & VM_SHARED))
flags |= TNF_SHARED;
last_cpupid = page_cpupid_last(page);
@@ -3923,7 +3923,7 @@ static inline int wp_huge_pmd(struct vm_fault *vmf, pmd_t orig_pmd)
return vmf->vma->vm_ops->huge_fault(vmf, PE_SIZE_PMD);
/* COW handled on pte level: split pmd */
- VM_BUG_ON_VMA(vmf->vma->vm_flags & VM_SHARED, vmf->vma);
+ VM_BUG_ON_VMA(vmf->vma_flags & VM_SHARED, vmf->vma);
__split_huge_pmd(vmf->vma, vmf->pmd, vmf->address, false, NULL);
return VM_FAULT_FALLBACK;
@@ -4070,6 +4070,8 @@ static int __handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
.flags = flags,
.pgoff = linear_page_index(vma, address),
.gfp_mask = __get_fault_gfp_mask(vma),
+ .vma_flags = vma->vm_flags,
+ .vma_page_prot = vma->vm_page_prot,
};
unsigned int dirty = flags & FAULT_FLAG_WRITE;
struct mm_struct *mm = vma->vm_mm;
--
2.7.4
^ permalink raw reply related
* [PATCH v7 10/24] mm: Protect SPF handler against anon_vma changes
From: Laurent Dufour @ 2018-02-06 16:49 UTC (permalink / raw)
To: paulmck, peterz, akpm, kirill, ak, mhocko, dave, jack,
Matthew Wilcox, benh, mpe, paulus, Thomas Gleixner, Ingo Molnar,
hpa, Will Deacon, Sergey Senozhatsky, Andrea Arcangeli,
Alexei Starovoitov, kemi.wang, sergey.senozhatsky.work,
Daniel Jordan
Cc: linux-kernel, linux-mm, haren, khandual, npiggin, bsingharora,
Tim Chen, linuxppc-dev, x86
In-Reply-To: <1517935810-31177-1-git-send-email-ldufour@linux.vnet.ibm.com>
The speculative page fault handler must be protected against anon_vma
changes. This is because page_add_new_anon_rmap() is called during the
speculative path.
In addition, don't try speculative page fault if the VMA don't have an
anon_vma structure allocated because its allocation should be
protected by the mmap_sem.
In __vma_adjust() when importer->anon_vma is set, there is no need to
protect against speculative page faults since speculative page fault
is aborted if the vma->anon_vma is not set.
When calling page_add_new_anon_rmap() vma->anon_vma is necessarily
valid since we checked for it when locking the pte and the anon_vma is
removed once the pte is unlocked. So even if the speculative page
fault handler is running concurrently with do_unmap(), as the pte is
locked in unmap_region() - through unmap_vmas() - and the anon_vma
unlinked later, because we check for the vma sequence counter which is
updated in unmap_page_range() before locking the pte, and then in
free_pgtables() so when locking the pte the change will be detected.
Signed-off-by: Laurent Dufour <ldufour@linux.vnet.ibm.com>
---
mm/memory.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/mm/memory.c b/mm/memory.c
index 39a6ea624922..9661b1302645 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -624,7 +624,9 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
* Hide vma from rmap and truncate_pagecache before freeing
* pgtables
*/
+ vm_write_begin(vma);
unlink_anon_vmas(vma);
+ vm_write_end(vma);
unlink_file_vma(vma);
if (is_vm_hugetlb_page(vma)) {
@@ -638,7 +640,9 @@ void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *vma,
&& !is_vm_hugetlb_page(next)) {
vma = next;
next = vma->vm_next;
+ vm_write_begin(vma);
unlink_anon_vmas(vma);
+ vm_write_end(vma);
unlink_file_vma(vma);
}
free_pgd_range(tlb, addr, vma->vm_end,
--
2.7.4
^ permalink raw reply related
* [PATCH v7 09/24] mm: protect mremap() against SPF hanlder
From: Laurent Dufour @ 2018-02-06 16:49 UTC (permalink / raw)
To: paulmck, peterz, akpm, kirill, ak, mhocko, dave, jack,
Matthew Wilcox, benh, mpe, paulus, Thomas Gleixner, Ingo Molnar,
hpa, Will Deacon, Sergey Senozhatsky, Andrea Arcangeli,
Alexei Starovoitov, kemi.wang, sergey.senozhatsky.work,
Daniel Jordan
Cc: linux-kernel, linux-mm, haren, khandual, npiggin, bsingharora,
Tim Chen, linuxppc-dev, x86
In-Reply-To: <1517935810-31177-1-git-send-email-ldufour@linux.vnet.ibm.com>
If a thread is remapping an area while another one is faulting on the
destination area, the SPF handler may fetch the vma from the RB tree before
the pte has been moved by the other thread. This means that the moved ptes
will overwrite those create by the page fault handler leading to page
leaked.
CPU 1 CPU2
enter mremap()
unmap the dest area
copy_vma() Enter speculative page fault handler
>> at this time the dest area is present in the RB tree
fetch the vma matching dest area
create a pte as the VMA matched
Exit the SPF handler
<data written in the new page>
move_ptes()
> it is assumed that the dest area is empty,
> the move ptes overwrite the page mapped by the CPU2.
To prevent that, when the VMA matching the dest area is extended or created
by copy_vma(), it should be marked as non available to the SPF handler.
The usual way to so is to rely on vm_write_begin()/end().
This is already in __vma_adjust() called by copy_vma() (through
vma_merge()). But __vma_adjust() is calling vm_write_end() before returning
which create a window for another thread.
This patch adds a new parameter to vma_merge() which is passed down to
vma_adjust().
The assumption is that copy_vma() is returning a vma which should be
released by calling vm_raw_write_end() by the callee once the ptes have
been moved.
Signed-off-by: Laurent Dufour <ldufour@linux.vnet.ibm.com>
---
include/linux/mm.h | 16 ++++++++++++----
mm/mmap.c | 47 ++++++++++++++++++++++++++++++++++++-----------
mm/mremap.c | 13 +++++++++++++
3 files changed, 61 insertions(+), 15 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 89e5dba22900..bb771afa59a2 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2186,16 +2186,24 @@ void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
extern int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert,
- struct vm_area_struct *expand);
+ struct vm_area_struct *expand, bool keep_locked);
static inline int vma_adjust(struct vm_area_struct *vma, unsigned long start,
unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
{
- return __vma_adjust(vma, start, end, pgoff, insert, NULL);
+ return __vma_adjust(vma, start, end, pgoff, insert, NULL, false);
}
-extern struct vm_area_struct *vma_merge(struct mm_struct *,
+extern struct vm_area_struct *__vma_merge(struct mm_struct *,
struct vm_area_struct *prev, unsigned long addr, unsigned long end,
unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
- struct mempolicy *, struct vm_userfaultfd_ctx);
+ struct mempolicy *, struct vm_userfaultfd_ctx, bool keep_locked);
+static inline struct vm_area_struct *vma_merge(struct mm_struct *vma,
+ struct vm_area_struct *prev, unsigned long addr, unsigned long end,
+ unsigned long vm_flags, struct anon_vma *anon, struct file *file,
+ pgoff_t off, struct mempolicy *pol, struct vm_userfaultfd_ctx uff)
+{
+ return __vma_merge(vma, prev, addr, end, vm_flags, anon, file, off,
+ pol, uff, false);
+}
extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
extern int __split_vma(struct mm_struct *, struct vm_area_struct *,
unsigned long addr, int new_below);
diff --git a/mm/mmap.c b/mm/mmap.c
index c4d944e1be23..13c799710a8a 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -684,7 +684,7 @@ static inline void __vma_unlink_prev(struct mm_struct *mm,
*/
int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert,
- struct vm_area_struct *expand)
+ struct vm_area_struct *expand, bool keep_locked)
{
struct mm_struct *mm = vma->vm_mm;
struct vm_area_struct *next = vma->vm_next, *orig_vma = vma;
@@ -996,7 +996,8 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
if (next && next != vma)
vm_raw_write_end(next);
- vm_raw_write_end(vma);
+ if (!keep_locked)
+ vm_raw_write_end(vma);
validate_mm(mm);
@@ -1132,12 +1133,13 @@ can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
* parameter) may establish ptes with the wrong permissions of NNNN
* instead of the right permissions of XXXX.
*/
-struct vm_area_struct *vma_merge(struct mm_struct *mm,
+struct vm_area_struct *__vma_merge(struct mm_struct *mm,
struct vm_area_struct *prev, unsigned long addr,
unsigned long end, unsigned long vm_flags,
struct anon_vma *anon_vma, struct file *file,
pgoff_t pgoff, struct mempolicy *policy,
- struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
+ struct vm_userfaultfd_ctx vm_userfaultfd_ctx,
+ bool keep_locked)
{
pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
struct vm_area_struct *area, *next;
@@ -1185,10 +1187,11 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
/* cases 1, 6 */
err = __vma_adjust(prev, prev->vm_start,
next->vm_end, prev->vm_pgoff, NULL,
- prev);
+ prev, keep_locked);
} else /* cases 2, 5, 7 */
err = __vma_adjust(prev, prev->vm_start,
- end, prev->vm_pgoff, NULL, prev);
+ end, prev->vm_pgoff, NULL, prev,
+ keep_locked);
if (err)
return NULL;
khugepaged_enter_vma_merge(prev, vm_flags);
@@ -1205,10 +1208,12 @@ struct vm_area_struct *vma_merge(struct mm_struct *mm,
vm_userfaultfd_ctx)) {
if (prev && addr < prev->vm_end) /* case 4 */
err = __vma_adjust(prev, prev->vm_start,
- addr, prev->vm_pgoff, NULL, next);
+ addr, prev->vm_pgoff, NULL, next,
+ keep_locked);
else { /* cases 3, 8 */
err = __vma_adjust(area, addr, next->vm_end,
- next->vm_pgoff - pglen, NULL, next);
+ next->vm_pgoff - pglen, NULL, next,
+ keep_locked);
/*
* In case 3 area is already equal to next and
* this is a noop, but in case 8 "area" has
@@ -3163,9 +3168,20 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent))
return NULL; /* should never get here */
- new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
- vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
- vma->vm_userfaultfd_ctx);
+
+ /* There is 3 cases to manage here in
+ * AAAA AAAA AAAA AAAA
+ * PPPP.... PPPP......NNNN PPPP....NNNN PP........NN
+ * PPPPPPPP(A) PPPP..NNNNNNNN(B) PPPPPPPPPPPP(1) NULL
+ * PPPPPPPPNNNN(2)
+ * PPPPNNNNNNNN(3)
+ *
+ * new_vma == prev in case A,1,2
+ * new_vma == next in case B,3
+ */
+ new_vma = __vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
+ vma->anon_vma, vma->vm_file, pgoff,
+ vma_policy(vma), vma->vm_userfaultfd_ctx, true);
if (new_vma) {
/*
* Source vma may have been merged into new_vma
@@ -3205,6 +3221,15 @@ struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
get_file(new_vma->vm_file);
if (new_vma->vm_ops && new_vma->vm_ops->open)
new_vma->vm_ops->open(new_vma);
+ /*
+ * As the VMA is linked right now, it may be hit by the
+ * speculative page fault handler. But we don't want it to
+ * to start mapping page in this area until the caller has
+ * potentially move the pte from the moved VMA. To prevent
+ * that we protect it right now, and let the caller unprotect
+ * it once the move is done.
+ */
+ vm_raw_write_begin(new_vma);
vma_link(mm, new_vma, prev, rb_link, rb_parent);
*need_rmap_locks = false;
}
diff --git a/mm/mremap.c b/mm/mremap.c
index 049470aa1e3e..8ed1a1d6eaed 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -302,6 +302,14 @@ static unsigned long move_vma(struct vm_area_struct *vma,
if (!new_vma)
return -ENOMEM;
+ /* new_vma is returned protected by copy_vma, to prevent speculative
+ * page fault to be done in the destination area before we move the pte.
+ * Now, we must also protect the source VMA since we don't want pages
+ * to be mapped in our back while we are copying the PTEs.
+ */
+ if (vma != new_vma)
+ vm_raw_write_begin(vma);
+
moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len,
need_rmap_locks);
if (moved_len < old_len) {
@@ -318,6 +326,8 @@ static unsigned long move_vma(struct vm_area_struct *vma,
*/
move_page_tables(new_vma, new_addr, vma, old_addr, moved_len,
true);
+ if (vma != new_vma)
+ vm_raw_write_end(vma);
vma = new_vma;
old_len = new_len;
old_addr = new_addr;
@@ -326,7 +336,10 @@ static unsigned long move_vma(struct vm_area_struct *vma,
mremap_userfaultfd_prep(new_vma, uf);
arch_remap(mm, old_addr, old_addr + old_len,
new_addr, new_addr + new_len);
+ if (vma != new_vma)
+ vm_raw_write_end(vma);
}
+ vm_raw_write_end(new_vma);
/* Conceal VM_ACCOUNT so old reservation is not undone */
if (vm_flags & VM_ACCOUNT) {
--
2.7.4
^ permalink raw reply related
* [PATCH v7 08/24] mm: Protect VMA modifications using VMA sequence count
From: Laurent Dufour @ 2018-02-06 16:49 UTC (permalink / raw)
To: paulmck, peterz, akpm, kirill, ak, mhocko, dave, jack,
Matthew Wilcox, benh, mpe, paulus, Thomas Gleixner, Ingo Molnar,
hpa, Will Deacon, Sergey Senozhatsky, Andrea Arcangeli,
Alexei Starovoitov, kemi.wang, sergey.senozhatsky.work,
Daniel Jordan
Cc: linux-kernel, linux-mm, haren, khandual, npiggin, bsingharora,
Tim Chen, linuxppc-dev, x86
In-Reply-To: <1517935810-31177-1-git-send-email-ldufour@linux.vnet.ibm.com>
The VMA sequence count has been introduced to allow fast detection of
VMA modification when running a page fault handler without holding
the mmap_sem.
This patch provides protection against the VMA modification done in :
- madvise()
- mpol_rebind_policy()
- vma_replace_policy()
- change_prot_numa()
- mlock(), munlock()
- mprotect()
- mmap_region()
- collapse_huge_page()
- userfaultd registering services
In addition, VMA fields which will be read during the speculative fault
path needs to be written using WRITE_ONCE to prevent write to be split
and intermediate values to be pushed to other CPUs.
Signed-off-by: Laurent Dufour <ldufour@linux.vnet.ibm.com>
---
fs/proc/task_mmu.c | 5 ++++-
fs/userfaultfd.c | 17 +++++++++++++----
mm/khugepaged.c | 3 +++
mm/madvise.c | 6 +++++-
mm/mempolicy.c | 51 ++++++++++++++++++++++++++++++++++-----------------
mm/mlock.c | 13 ++++++++-----
mm/mmap.c | 17 ++++++++++-------
mm/mprotect.c | 4 +++-
mm/swap_state.c | 8 ++++++--
9 files changed, 86 insertions(+), 38 deletions(-)
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index b66fc8de7d34..adb66e1989ee 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -1158,8 +1158,11 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
goto out_mm;
}
for (vma = mm->mmap; vma; vma = vma->vm_next) {
- vma->vm_flags &= ~VM_SOFTDIRTY;
+ vm_write_begin(vma);
+ WRITE_ONCE(vma->vm_flags,
+ vma->vm_flags & ~VM_SOFTDIRTY);
vma_set_page_prot(vma);
+ vm_write_end(vma);
}
downgrade_write(&mm->mmap_sem);
break;
diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c
index 87a13a7c8270..1da1ba63c7dd 100644
--- a/fs/userfaultfd.c
+++ b/fs/userfaultfd.c
@@ -659,8 +659,11 @@ int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *fcs)
octx = vma->vm_userfaultfd_ctx.ctx;
if (!octx || !(octx->features & UFFD_FEATURE_EVENT_FORK)) {
+ vm_write_begin(vma);
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
- vma->vm_flags &= ~(VM_UFFD_WP | VM_UFFD_MISSING);
+ WRITE_ONCE(vma->vm_flags,
+ vma->vm_flags & ~(VM_UFFD_WP | VM_UFFD_MISSING));
+ vm_write_end(vma);
return 0;
}
@@ -885,8 +888,10 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
vma = prev;
else
prev = vma;
- vma->vm_flags = new_flags;
+ vm_write_begin(vma);
+ WRITE_ONCE(vma->vm_flags, new_flags);
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
+ vm_write_end(vma);
}
up_write(&mm->mmap_sem);
mmput(mm);
@@ -1434,8 +1439,10 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
* the next vma was merged into the current one and
* the current one has not been updated yet.
*/
- vma->vm_flags = new_flags;
+ vm_write_begin(vma);
+ WRITE_ONCE(vma->vm_flags, new_flags);
vma->vm_userfaultfd_ctx.ctx = ctx;
+ vm_write_end(vma);
skip:
prev = vma;
@@ -1592,8 +1599,10 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
* the next vma was merged into the current one and
* the current one has not been updated yet.
*/
- vma->vm_flags = new_flags;
+ vm_write_begin(vma);
+ WRITE_ONCE(vma->vm_flags, new_flags);
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
+ vm_write_end(vma);
skip:
prev = vma;
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index b7e2268dfc9a..32314e9e48dd 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -1006,6 +1006,7 @@ static void collapse_huge_page(struct mm_struct *mm,
if (mm_find_pmd(mm, address) != pmd)
goto out;
+ vm_write_begin(vma);
anon_vma_lock_write(vma->anon_vma);
pte = pte_offset_map(pmd, address);
@@ -1041,6 +1042,7 @@ static void collapse_huge_page(struct mm_struct *mm,
pmd_populate(mm, pmd, pmd_pgtable(_pmd));
spin_unlock(pmd_ptl);
anon_vma_unlock_write(vma->anon_vma);
+ vm_write_end(vma);
result = SCAN_FAIL;
goto out;
}
@@ -1075,6 +1077,7 @@ static void collapse_huge_page(struct mm_struct *mm,
set_pmd_at(mm, address, pmd, _pmd);
update_mmu_cache_pmd(vma, address, pmd);
spin_unlock(pmd_ptl);
+ vm_write_end(vma);
*hpage = NULL;
diff --git a/mm/madvise.c b/mm/madvise.c
index 4d3c922ea1a1..e328f7ab5942 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -184,7 +184,9 @@ static long madvise_behavior(struct vm_area_struct *vma,
/*
* vm_flags is protected by the mmap_sem held in write mode.
*/
- vma->vm_flags = new_flags;
+ vm_write_begin(vma);
+ WRITE_ONCE(vma->vm_flags, new_flags);
+ vm_write_end(vma);
out:
return error;
}
@@ -450,9 +452,11 @@ static void madvise_free_page_range(struct mmu_gather *tlb,
.private = tlb,
};
+ vm_write_begin(vma);
tlb_start_vma(tlb, vma);
walk_page_range(addr, end, &free_walk);
tlb_end_vma(tlb, vma);
+ vm_write_end(vma);
}
static int madvise_free_single_vma(struct vm_area_struct *vma,
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index a8b7d59002e8..49d731c39e4f 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -380,8 +380,11 @@ void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
struct vm_area_struct *vma;
down_write(&mm->mmap_sem);
- for (vma = mm->mmap; vma; vma = vma->vm_next)
+ for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ vm_write_begin(vma);
mpol_rebind_policy(vma->vm_policy, new);
+ vm_write_end(vma);
+ }
up_write(&mm->mmap_sem);
}
@@ -554,9 +557,11 @@ unsigned long change_prot_numa(struct vm_area_struct *vma,
{
int nr_updated;
+ vm_write_begin(vma);
nr_updated = change_protection(vma, addr, end, PAGE_NONE, 0, 1);
if (nr_updated)
count_vm_numa_events(NUMA_PTE_UPDATES, nr_updated);
+ vm_write_end(vma);
return nr_updated;
}
@@ -657,6 +662,7 @@ static int vma_replace_policy(struct vm_area_struct *vma,
if (IS_ERR(new))
return PTR_ERR(new);
+ vm_write_begin(vma);
if (vma->vm_ops && vma->vm_ops->set_policy) {
err = vma->vm_ops->set_policy(vma, new);
if (err)
@@ -664,11 +670,17 @@ static int vma_replace_policy(struct vm_area_struct *vma,
}
old = vma->vm_policy;
- vma->vm_policy = new; /* protected by mmap_sem */
+ /*
+ * The speculative page fault handler access this field without
+ * hodling the mmap_sem.
+ */
+ WRITE_ONCE(vma->vm_policy, new);
+ vm_write_end(vma);
mpol_put(old);
return 0;
err_out:
+ vm_write_end(vma);
mpol_put(new);
return err;
}
@@ -1552,23 +1564,28 @@ COMPAT_SYSCALL_DEFINE6(mbind, compat_ulong_t, start, compat_ulong_t, len,
struct mempolicy *__get_vma_policy(struct vm_area_struct *vma,
unsigned long addr)
{
- struct mempolicy *pol = NULL;
+ struct mempolicy *pol;
- if (vma) {
- if (vma->vm_ops && vma->vm_ops->get_policy) {
- pol = vma->vm_ops->get_policy(vma, addr);
- } else if (vma->vm_policy) {
- pol = vma->vm_policy;
+ if (!vma)
+ return NULL;
- /*
- * shmem_alloc_page() passes MPOL_F_SHARED policy with
- * a pseudo vma whose vma->vm_ops=NULL. Take a reference
- * count on these policies which will be dropped by
- * mpol_cond_put() later
- */
- if (mpol_needs_cond_ref(pol))
- mpol_get(pol);
- }
+ if (vma->vm_ops && vma->vm_ops->get_policy)
+ return vma->vm_ops->get_policy(vma, addr);
+
+ /*
+ * This could be called without holding the mmap_sem in the
+ * speculative page fault handler's path.
+ */
+ pol = READ_ONCE(vma->vm_policy);
+ if (pol) {
+ /*
+ * shmem_alloc_page() passes MPOL_F_SHARED policy with
+ * a pseudo vma whose vma->vm_ops=NULL. Take a reference
+ * count on these policies which will be dropped by
+ * mpol_cond_put() later
+ */
+ if (mpol_needs_cond_ref(pol))
+ mpol_get(pol);
}
return pol;
diff --git a/mm/mlock.c b/mm/mlock.c
index 74e5a6547c3d..c40285c94ced 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -445,7 +445,9 @@ static unsigned long __munlock_pagevec_fill(struct pagevec *pvec,
void munlock_vma_pages_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
- vma->vm_flags &= VM_LOCKED_CLEAR_MASK;
+ vm_write_begin(vma);
+ WRITE_ONCE(vma->vm_flags, vma->vm_flags & VM_LOCKED_CLEAR_MASK);
+ vm_write_end(vma);
while (start < end) {
struct page *page;
@@ -568,10 +570,11 @@ static int mlock_fixup(struct vm_area_struct *vma, struct vm_area_struct **prev,
* It's okay if try_to_unmap_one unmaps a page just after we
* set VM_LOCKED, populate_vma_page_range will bring it back.
*/
-
- if (lock)
- vma->vm_flags = newflags;
- else
+ if (lock) {
+ vm_write_begin(vma);
+ WRITE_ONCE(vma->vm_flags, newflags);
+ vm_write_end(vma);
+ } else
munlock_vma_pages_range(vma, start, end);
out:
diff --git a/mm/mmap.c b/mm/mmap.c
index 671940b72696..c4d944e1be23 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -847,17 +847,18 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
}
if (start != vma->vm_start) {
- vma->vm_start = start;
+ WRITE_ONCE(vma->vm_start, start);
start_changed = true;
}
if (end != vma->vm_end) {
- vma->vm_end = end;
+ WRITE_ONCE(vma->vm_end, end);
end_changed = true;
}
- vma->vm_pgoff = pgoff;
+ WRITE_ONCE(vma->vm_pgoff, pgoff);
if (adjust_next) {
- next->vm_start += adjust_next << PAGE_SHIFT;
- next->vm_pgoff += adjust_next;
+ WRITE_ONCE(next->vm_start,
+ next->vm_start + (adjust_next << PAGE_SHIFT));
+ WRITE_ONCE(next->vm_pgoff, next->vm_pgoff + adjust_next);
}
if (root) {
@@ -1781,6 +1782,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
out:
perf_event_mmap(vma);
+ vm_write_begin(vma);
vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT);
if (vm_flags & VM_LOCKED) {
if (!((vm_flags & VM_SPECIAL) || is_vm_hugetlb_page(vma) ||
@@ -1803,6 +1805,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
vma->vm_flags |= VM_SOFTDIRTY;
vma_set_page_prot(vma);
+ vm_write_end(vma);
return addr;
@@ -2431,8 +2434,8 @@ int expand_downwards(struct vm_area_struct *vma,
mm->locked_vm += grow;
vm_stat_account(mm, vma->vm_flags, grow);
anon_vma_interval_tree_pre_update_vma(vma);
- vma->vm_start = address;
- vma->vm_pgoff -= grow;
+ WRITE_ONCE(vma->vm_start, address);
+ WRITE_ONCE(vma->vm_pgoff, vma->vm_pgoff - grow);
anon_vma_interval_tree_post_update_vma(vma);
vma_gap_update(vma);
spin_unlock(&mm->page_table_lock);
diff --git a/mm/mprotect.c b/mm/mprotect.c
index e3309fcf586b..9b7a71c30287 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -366,7 +366,8 @@ mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
* vm_flags and vm_page_prot are protected by the mmap_sem
* held in write mode.
*/
- vma->vm_flags = newflags;
+ vm_write_begin(vma);
+ WRITE_ONCE(vma->vm_flags, newflags);
dirty_accountable = vma_wants_writenotify(vma, vma->vm_page_prot);
vma_set_page_prot(vma);
@@ -381,6 +382,7 @@ mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
(newflags & VM_WRITE)) {
populate_vma_page_range(vma, start, end, NULL);
}
+ vm_write_end(vma);
vm_stat_account(mm, oldflags, -nrpages);
vm_stat_account(mm, newflags, nrpages);
diff --git a/mm/swap_state.c b/mm/swap_state.c
index 39ae7cfad90f..eeef59c2cd76 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -550,6 +550,10 @@ static unsigned long swapin_nr_pages(unsigned long offset)
* the readahead.
*
* Caller must hold down_read on the vma->vm_mm if vma is not NULL.
+ * This is needed to ensure the VMA will not be freed in our back. In the case
+ * of the speculative page fault handler, this cannot happen, even if we don't
+ * hold the mmap_sem. Callees are assumed to take care of reading VMA's fields
+ * using READ_ONCE() to read consistent values.
*/
struct page *swapin_readahead(swp_entry_t entry, gfp_t gfp_mask,
struct vm_area_struct *vma, unsigned long addr)
@@ -643,9 +647,9 @@ static inline void swap_ra_clamp_pfn(struct vm_area_struct *vma,
unsigned long *start,
unsigned long *end)
{
- *start = max3(lpfn, PFN_DOWN(vma->vm_start),
+ *start = max3(lpfn, PFN_DOWN(READ_ONCE(vma->vm_start)),
PFN_DOWN(faddr & PMD_MASK));
- *end = min3(rpfn, PFN_DOWN(vma->vm_end),
+ *end = min3(rpfn, PFN_DOWN(READ_ONCE(vma->vm_end)),
PFN_DOWN((faddr & PMD_MASK) + PMD_SIZE));
}
--
2.7.4
^ permalink raw reply related
* [PATCH v7 07/24] mm: VMA sequence count
From: Laurent Dufour @ 2018-02-06 16:49 UTC (permalink / raw)
To: paulmck, peterz, akpm, kirill, ak, mhocko, dave, jack,
Matthew Wilcox, benh, mpe, paulus, Thomas Gleixner, Ingo Molnar,
hpa, Will Deacon, Sergey Senozhatsky, Andrea Arcangeli,
Alexei Starovoitov, kemi.wang, sergey.senozhatsky.work,
Daniel Jordan
Cc: linux-kernel, linux-mm, haren, khandual, npiggin, bsingharora,
Tim Chen, linuxppc-dev, x86
In-Reply-To: <1517935810-31177-1-git-send-email-ldufour@linux.vnet.ibm.com>
From: Peter Zijlstra <peterz@infradead.org>
Wrap the VMA modifications (vma_adjust/unmap_page_range) with sequence
counts such that we can easily test if a VMA is changed.
The unmap_page_range() one allows us to make assumptions about
page-tables; when we find the seqcount hasn't changed we can assume
page-tables are still valid.
The flip side is that we cannot distinguish between a vma_adjust() and
the unmap_page_range() -- where with the former we could have
re-checked the vma bounds against the address.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
[Port to 4.12 kernel]
[Build depends on CONFIG_SPECULATIVE_PAGE_FAULT]
[Introduce vm_write_* inline function depending on
CONFIG_SPECULATIVE_PAGE_FAULT]
[Fix lock dependency between mapping->i_mmap_rwsem and vma->vm_sequence by
using vm_raw_write* functions]
Signed-off-by: Laurent Dufour <ldufour@linux.vnet.ibm.com>
---
include/linux/mm.h | 41 +++++++++++++++++++++++++++++++++++++++++
include/linux/mm_types.h | 3 +++
mm/memory.c | 2 ++
mm/mmap.c | 35 +++++++++++++++++++++++++++++++++++
4 files changed, 81 insertions(+)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 51d950cac772..89e5dba22900 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1369,6 +1369,47 @@ static inline void unmap_shared_mapping_range(struct address_space *mapping,
unmap_mapping_range(mapping, holebegin, holelen, 0);
}
+#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
+static inline void vm_write_begin(struct vm_area_struct *vma)
+{
+ write_seqcount_begin(&vma->vm_sequence);
+}
+static inline void vm_write_begin_nested(struct vm_area_struct *vma,
+ int subclass)
+{
+ write_seqcount_begin_nested(&vma->vm_sequence, subclass);
+}
+static inline void vm_write_end(struct vm_area_struct *vma)
+{
+ write_seqcount_end(&vma->vm_sequence);
+}
+static inline void vm_raw_write_begin(struct vm_area_struct *vma)
+{
+ raw_write_seqcount_begin(&vma->vm_sequence);
+}
+static inline void vm_raw_write_end(struct vm_area_struct *vma)
+{
+ raw_write_seqcount_end(&vma->vm_sequence);
+}
+#else
+static inline void vm_write_begin(struct vm_area_struct *vma)
+{
+}
+static inline void vm_write_begin_nested(struct vm_area_struct *vma,
+ int subclass)
+{
+}
+static inline void vm_write_end(struct vm_area_struct *vma)
+{
+}
+static inline void vm_raw_write_begin(struct vm_area_struct *vma)
+{
+}
+static inline void vm_raw_write_end(struct vm_area_struct *vma)
+{
+}
+#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
+
extern int access_process_vm(struct task_struct *tsk, unsigned long addr,
void *buf, int len, unsigned int gup_flags);
extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index fd1af6b9591d..34fde7111e88 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -333,6 +333,9 @@ struct vm_area_struct {
struct mempolicy *vm_policy; /* NUMA policy for the VMA */
#endif
struct vm_userfaultfd_ctx vm_userfaultfd_ctx;
+#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
+ seqcount_t vm_sequence;
+#endif
} __randomize_layout;
struct core_thread {
diff --git a/mm/memory.c b/mm/memory.c
index 2560d100b439..39a6ea624922 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1503,6 +1503,7 @@ void unmap_page_range(struct mmu_gather *tlb,
unsigned long next;
BUG_ON(addr >= end);
+ vm_write_begin(vma);
tlb_start_vma(tlb, vma);
pgd = pgd_offset(vma->vm_mm, addr);
do {
@@ -1512,6 +1513,7 @@ void unmap_page_range(struct mmu_gather *tlb,
next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
} while (pgd++, addr = next, addr != end);
tlb_end_vma(tlb, vma);
+ vm_write_end(vma);
}
diff --git a/mm/mmap.c b/mm/mmap.c
index 4bb038e7984b..671940b72696 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -558,6 +558,10 @@ void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
else
mm->highest_vm_end = vm_end_gap(vma);
+#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
+ seqcount_init(&vma->vm_sequence);
+#endif
+
/*
* vma->vm_prev wasn't known when we followed the rbtree to find the
* correct insertion point for that vma. As a result, we could not
@@ -692,6 +696,30 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
long adjust_next = 0;
int remove_next = 0;
+ /*
+ * Why using vm_raw_write*() functions here to avoid lockdep's warning ?
+ *
+ * Locked is complaining about a theoretical lock dependency, involving
+ * 3 locks:
+ * mapping->i_mmap_rwsem --> vma->vm_sequence --> fs_reclaim
+ *
+ * Here are the major path leading to this dependency :
+ * 1. __vma_adjust() mmap_sem -> vm_sequence -> i_mmap_rwsem
+ * 2. move_vmap() mmap_sem -> vm_sequence -> fs_reclaim
+ * 3. __alloc_pages_nodemask() fs_reclaim -> i_mmap_rwsem
+ * 4. unmap_mapping_range() i_mmap_rwsem -> vm_sequence
+ *
+ * So there is no way to solve this easily, especially because in
+ * unmap_mapping_range() the i_mmap_rwsem is grab while the impacted
+ * VMAs are not yet known.
+ * However, the way the vm_seq is used is guarantying that we will
+ * never block on it since we just check for its value and never wait
+ * for it to move, see vma_has_changed() and handle_speculative_fault().
+ */
+ vm_raw_write_begin(vma);
+ if (next)
+ vm_raw_write_begin(next);
+
if (next && !insert) {
struct vm_area_struct *exporter = NULL, *importer = NULL;
@@ -902,6 +930,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
anon_vma_merge(vma, next);
mm->map_count--;
mpol_put(vma_policy(next));
+ vm_raw_write_end(next);
kmem_cache_free(vm_area_cachep, next);
/*
* In mprotect's case 6 (see comments on vma_merge),
@@ -916,6 +945,8 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
* "vma->vm_next" gap must be updated.
*/
next = vma->vm_next;
+ if (next)
+ vm_raw_write_begin(next);
} else {
/*
* For the scope of the comment "next" and
@@ -962,6 +993,10 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
if (insert && file)
uprobe_mmap(insert);
+ if (next && next != vma)
+ vm_raw_write_end(next);
+ vm_raw_write_end(vma);
+
validate_mm(mm);
return 0;
--
2.7.4
^ permalink raw reply related
* [PATCH v7 06/24] mm: Introduce pte_spinlock for FAULT_FLAG_SPECULATIVE
From: Laurent Dufour @ 2018-02-06 16:49 UTC (permalink / raw)
To: paulmck, peterz, akpm, kirill, ak, mhocko, dave, jack,
Matthew Wilcox, benh, mpe, paulus, Thomas Gleixner, Ingo Molnar,
hpa, Will Deacon, Sergey Senozhatsky, Andrea Arcangeli,
Alexei Starovoitov, kemi.wang, sergey.senozhatsky.work,
Daniel Jordan
Cc: linux-kernel, linux-mm, haren, khandual, npiggin, bsingharora,
Tim Chen, linuxppc-dev, x86
In-Reply-To: <1517935810-31177-1-git-send-email-ldufour@linux.vnet.ibm.com>
When handling page fault without holding the mmap_sem the fetch of the
pte lock pointer and the locking will have to be done while ensuring
that the VMA is not touched in our back.
So move the fetch and locking operations in a dedicated function.
Signed-off-by: Laurent Dufour <ldufour@linux.vnet.ibm.com>
---
mm/memory.c | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
diff --git a/mm/memory.c b/mm/memory.c
index bb058527525a..2560d100b439 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2452,6 +2452,13 @@ static inline void wp_page_reuse(struct vm_fault *vmf)
pte_unmap_unlock(vmf->pte, vmf->ptl);
}
+static bool pte_spinlock(struct vm_fault *vmf)
+{
+ vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
+ spin_lock(vmf->ptl);
+ return true;
+}
+
static bool pte_map_lock(struct vm_fault *vmf)
{
vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
@@ -3819,8 +3826,8 @@ static int do_numa_page(struct vm_fault *vmf)
* validation through pte_unmap_same(). It's of NUMA type but
* the pfn may be screwed if the read is non atomic.
*/
- vmf->ptl = pte_lockptr(vma->vm_mm, vmf->pmd);
- spin_lock(vmf->ptl);
+ if (!pte_spinlock(vmf))
+ return VM_FAULT_RETRY;
if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) {
pte_unmap_unlock(vmf->pte, vmf->ptl);
goto out;
@@ -4013,8 +4020,8 @@ static int handle_pte_fault(struct vm_fault *vmf)
if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
return do_numa_page(vmf);
- vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
- spin_lock(vmf->ptl);
+ if (!pte_spinlock(vmf))
+ return VM_FAULT_RETRY;
entry = vmf->orig_pte;
if (unlikely(!pte_same(*vmf->pte, entry)))
goto unlock;
--
2.7.4
^ permalink raw reply related
* [PATCH v7 05/24] mm: Prepare for FAULT_FLAG_SPECULATIVE
From: Laurent Dufour @ 2018-02-06 16:49 UTC (permalink / raw)
To: paulmck, peterz, akpm, kirill, ak, mhocko, dave, jack,
Matthew Wilcox, benh, mpe, paulus, Thomas Gleixner, Ingo Molnar,
hpa, Will Deacon, Sergey Senozhatsky, Andrea Arcangeli,
Alexei Starovoitov, kemi.wang, sergey.senozhatsky.work,
Daniel Jordan
Cc: linux-kernel, linux-mm, haren, khandual, npiggin, bsingharora,
Tim Chen, linuxppc-dev, x86
In-Reply-To: <1517935810-31177-1-git-send-email-ldufour@linux.vnet.ibm.com>
From: Peter Zijlstra <peterz@infradead.org>
When speculating faults (without holding mmap_sem) we need to validate
that the vma against which we loaded pages is still valid when we're
ready to install the new PTE.
Therefore, replace the pte_offset_map_lock() calls that (re)take the
PTL with pte_map_lock() which can fail in case we find the VMA changed
since we started the fault.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
[Port to 4.12 kernel]
[Remove the comment about the fault_env structure which has been
implemented as the vm_fault structure in the kernel]
Signed-off-by: Laurent Dufour <ldufour@linux.vnet.ibm.com>
---
include/linux/mm.h | 1 +
mm/memory.c | 56 ++++++++++++++++++++++++++++++++++++++----------------
2 files changed, 41 insertions(+), 16 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 47c06fd20f6a..51d950cac772 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -302,6 +302,7 @@ extern pgprot_t protection_map[16];
#define FAULT_FLAG_USER 0x40 /* The fault originated in userspace */
#define FAULT_FLAG_REMOTE 0x80 /* faulting for non current tsk/mm */
#define FAULT_FLAG_INSTRUCTION 0x100 /* The fault was during an instruction fetch */
+#define FAULT_FLAG_SPECULATIVE 0x200 /* Speculative fault, not holding mmap_sem */
#define FAULT_FLAG_TRACE \
{ FAULT_FLAG_WRITE, "WRITE" }, \
diff --git a/mm/memory.c b/mm/memory.c
index 32b9eb77d95c..bb058527525a 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2452,6 +2452,13 @@ static inline void wp_page_reuse(struct vm_fault *vmf)
pte_unmap_unlock(vmf->pte, vmf->ptl);
}
+static bool pte_map_lock(struct vm_fault *vmf)
+{
+ vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd,
+ vmf->address, &vmf->ptl);
+ return true;
+}
+
/*
* Handle the case of a page which we actually need to copy to a new page.
*
@@ -2479,6 +2486,7 @@ static int wp_page_copy(struct vm_fault *vmf)
const unsigned long mmun_start = vmf->address & PAGE_MASK;
const unsigned long mmun_end = mmun_start + PAGE_SIZE;
struct mem_cgroup *memcg;
+ int ret = VM_FAULT_OOM;
if (unlikely(anon_vma_prepare(vma)))
goto oom;
@@ -2506,7 +2514,11 @@ static int wp_page_copy(struct vm_fault *vmf)
/*
* Re-check the pte - we dropped the lock
*/
- vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
+ if (!pte_map_lock(vmf)) {
+ mem_cgroup_cancel_charge(new_page, memcg, false);
+ ret = VM_FAULT_RETRY;
+ goto oom_free_new;
+ }
if (likely(pte_same(*vmf->pte, vmf->orig_pte))) {
if (old_page) {
if (!PageAnon(old_page)) {
@@ -2598,7 +2610,7 @@ static int wp_page_copy(struct vm_fault *vmf)
oom:
if (old_page)
put_page(old_page);
- return VM_FAULT_OOM;
+ return ret;
}
/**
@@ -2619,8 +2631,8 @@ static int wp_page_copy(struct vm_fault *vmf)
int finish_mkwrite_fault(struct vm_fault *vmf)
{
WARN_ON_ONCE(!(vmf->vma->vm_flags & VM_SHARED));
- vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, vmf->address,
- &vmf->ptl);
+ if (!pte_map_lock(vmf))
+ return VM_FAULT_RETRY;
/*
* We might have raced with another page fault while we released the
* pte_offset_map_lock.
@@ -2738,8 +2750,11 @@ static int do_wp_page(struct vm_fault *vmf)
get_page(vmf->page);
pte_unmap_unlock(vmf->pte, vmf->ptl);
lock_page(vmf->page);
- vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
- vmf->address, &vmf->ptl);
+ if (!pte_map_lock(vmf)) {
+ unlock_page(vmf->page);
+ put_page(vmf->page);
+ return VM_FAULT_RETRY;
+ }
if (!pte_same(*vmf->pte, vmf->orig_pte)) {
unlock_page(vmf->page);
pte_unmap_unlock(vmf->pte, vmf->ptl);
@@ -2967,8 +2982,10 @@ int do_swap_page(struct vm_fault *vmf)
* Back out if somebody else faulted in this pte
* while we released the pte lock.
*/
- vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
- vmf->address, &vmf->ptl);
+ if (!pte_map_lock(vmf)) {
+ delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
+ return VM_FAULT_RETRY;
+ }
if (likely(pte_same(*vmf->pte, vmf->orig_pte)))
ret = VM_FAULT_OOM;
delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
@@ -3024,8 +3041,11 @@ int do_swap_page(struct vm_fault *vmf)
/*
* Back out if somebody else already faulted in this pte.
*/
- vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
- &vmf->ptl);
+ if (!pte_map_lock(vmf)) {
+ ret = VM_FAULT_RETRY;
+ mem_cgroup_cancel_charge(page, memcg, false);
+ goto out_page;
+ }
if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte)))
goto out_nomap;
@@ -3154,8 +3174,8 @@ static int do_anonymous_page(struct vm_fault *vmf)
!mm_forbids_zeropage(vma->vm_mm)) {
entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
vma->vm_page_prot));
- vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
- vmf->address, &vmf->ptl);
+ if (!pte_map_lock(vmf))
+ return VM_FAULT_RETRY;
if (!pte_none(*vmf->pte))
goto unlock;
ret = check_stable_address_space(vma->vm_mm);
@@ -3190,8 +3210,11 @@ static int do_anonymous_page(struct vm_fault *vmf)
if (vma->vm_flags & VM_WRITE)
entry = pte_mkwrite(pte_mkdirty(entry));
- vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
- &vmf->ptl);
+ if (!pte_map_lock(vmf)) {
+ mem_cgroup_cancel_charge(page, memcg, false);
+ put_page(page);
+ return VM_FAULT_RETRY;
+ }
if (!pte_none(*vmf->pte))
goto release;
@@ -3315,8 +3338,9 @@ static int pte_alloc_one_map(struct vm_fault *vmf)
* pte_none() under vmf->ptl protection when we return to
* alloc_set_pte().
*/
- vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
- &vmf->ptl);
+ if (!pte_map_lock(vmf))
+ return VM_FAULT_RETRY;
+
return 0;
}
--
2.7.4
^ permalink raw reply related
* [PATCH v7 01/24] mm: Introduce CONFIG_SPECULATIVE_PAGE_FAULT
From: Laurent Dufour @ 2018-02-06 16:49 UTC (permalink / raw)
To: paulmck, peterz, akpm, kirill, ak, mhocko, dave, jack,
Matthew Wilcox, benh, mpe, paulus, Thomas Gleixner, Ingo Molnar,
hpa, Will Deacon, Sergey Senozhatsky, Andrea Arcangeli,
Alexei Starovoitov, kemi.wang, sergey.senozhatsky.work,
Daniel Jordan
Cc: linux-kernel, linux-mm, haren, khandual, npiggin, bsingharora,
Tim Chen, linuxppc-dev, x86
In-Reply-To: <1517935810-31177-1-git-send-email-ldufour@linux.vnet.ibm.com>
This configuration variable will be used to build the code needed to
handle speculative page fault.
By default it is turned off, and activated depending on architecture
support.
Suggested-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Laurent Dufour <ldufour@linux.vnet.ibm.com>
---
mm/Kconfig | 3 +++
1 file changed, 3 insertions(+)
diff --git a/mm/Kconfig b/mm/Kconfig
index c782e8fb7235..42be0d92336e 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -760,3 +760,6 @@ config GUP_BENCHMARK
performance of get_user_pages_fast().
See tools/testing/selftests/vm/gup_benchmark.c
+
+config SPECULATIVE_PAGE_FAULT
+ bool
--
2.7.4
^ permalink raw reply related
* [PATCH v7 04/24] mm: Dont assume page-table invariance during faults
From: Laurent Dufour @ 2018-02-06 16:49 UTC (permalink / raw)
To: paulmck, peterz, akpm, kirill, ak, mhocko, dave, jack,
Matthew Wilcox, benh, mpe, paulus, Thomas Gleixner, Ingo Molnar,
hpa, Will Deacon, Sergey Senozhatsky, Andrea Arcangeli,
Alexei Starovoitov, kemi.wang, sergey.senozhatsky.work,
Daniel Jordan
Cc: linux-kernel, linux-mm, haren, khandual, npiggin, bsingharora,
Tim Chen, linuxppc-dev, x86
In-Reply-To: <1517935810-31177-1-git-send-email-ldufour@linux.vnet.ibm.com>
From: Peter Zijlstra <peterz@infradead.org>
One of the side effects of speculating on faults (without holding
mmap_sem) is that we can race with free_pgtables() and therefore we
cannot assume the page-tables will stick around.
Remove the reliance on the pte pointer.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
In most of the case pte_unmap_same() was returning 1, which meaning that
do_swap_page() should do its processing. So in most of the case there will
be no impact.
Now regarding the case where pte_unmap_safe() was returning 0, and thus
do_swap_page return 0 too, this happens when the page has already been
swapped back. This may happen before do_swap_page() get called or while in
the call to do_swap_page(). In that later case, the check done when
swapin_readahead() returns will detect that case.
The worst case would be that a page fault is occuring on 2 threads at the
same time on the same swapped out page. In that case one thread will take
much time looping in __read_swap_cache_async(). But in the regular page
fault path, this is even worse since the thread would wait for semaphore to
be released before starting anything.
[Remove only if !CONFIG_SPECULATIVE_PAGE_FAULT]
Signed-off-by: Laurent Dufour <ldufour@linux.vnet.ibm.com>
---
mm/memory.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/mm/memory.c b/mm/memory.c
index 5ec6433d6a5c..32b9eb77d95c 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2288,6 +2288,7 @@ int apply_to_page_range(struct mm_struct *mm, unsigned long addr,
}
EXPORT_SYMBOL_GPL(apply_to_page_range);
+#ifndef CONFIG_SPECULATIVE_PAGE_FAULT
/*
* handle_pte_fault chooses page fault handler according to an entry which was
* read non-atomically. Before making any commitment, on those architectures
@@ -2311,6 +2312,7 @@ static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd,
pte_unmap(page_table);
return same;
}
+#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
{
@@ -2898,11 +2900,13 @@ int do_swap_page(struct vm_fault *vmf)
swapcache = page;
}
+#ifndef CONFIG_SPECULATIVE_PAGE_FAULT
if (!pte_unmap_same(vma->vm_mm, vmf->pmd, vmf->pte, vmf->orig_pte)) {
if (page)
put_page(page);
goto out;
}
+#endif
entry = pte_to_swp_entry(vmf->orig_pte);
if (unlikely(non_swap_entry(entry))) {
--
2.7.4
^ permalink raw reply related
* [PATCH v7 03/24] powerpc/mm: Define CONFIG_SPECULATIVE_PAGE_FAULT
From: Laurent Dufour @ 2018-02-06 16:49 UTC (permalink / raw)
To: paulmck, peterz, akpm, kirill, ak, mhocko, dave, jack,
Matthew Wilcox, benh, mpe, paulus, Thomas Gleixner, Ingo Molnar,
hpa, Will Deacon, Sergey Senozhatsky, Andrea Arcangeli,
Alexei Starovoitov, kemi.wang, sergey.senozhatsky.work,
Daniel Jordan
Cc: linux-kernel, linux-mm, haren, khandual, npiggin, bsingharora,
Tim Chen, linuxppc-dev, x86
In-Reply-To: <1517935810-31177-1-git-send-email-ldufour@linux.vnet.ibm.com>
Define CONFIG_SPECULATIVE_PAGE_FAULT for BOOK3S_64 and SMP. This enables
the Speculative Page Fault handler.
Support is only provide for BOOK3S_64 currently because:
- require CONFIG_PPC_STD_MMU because checks done in
set_access_flags_filter()
- require BOOK3S because we can't support for book3e_hugetlb_preload()
called by update_mmu_cache()
Signed-off-by: Laurent Dufour <ldufour@linux.vnet.ibm.com>
---
arch/powerpc/Kconfig | 1 +
1 file changed, 1 insertion(+)
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 9d3329811cc1..57c19ee79c00 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -232,6 +232,7 @@ config PPC
select OLD_SIGACTION if PPC32
select OLD_SIGSUSPEND
select SPARSE_IRQ
+ select SPECULATIVE_PAGE_FAULT if PPC_BOOK3S_64 && SMP
select SYSCTL_EXCEPTION_TRACE
select VIRT_TO_BUS if !PPC64
#
--
2.7.4
^ permalink raw reply related
* [PATCH v7 02/24] x86/mm: Define CONFIG_SPECULATIVE_PAGE_FAULT
From: Laurent Dufour @ 2018-02-06 16:49 UTC (permalink / raw)
To: paulmck, peterz, akpm, kirill, ak, mhocko, dave, jack,
Matthew Wilcox, benh, mpe, paulus, Thomas Gleixner, Ingo Molnar,
hpa, Will Deacon, Sergey Senozhatsky, Andrea Arcangeli,
Alexei Starovoitov, kemi.wang, sergey.senozhatsky.work,
Daniel Jordan
Cc: linux-kernel, linux-mm, haren, khandual, npiggin, bsingharora,
Tim Chen, linuxppc-dev, x86
In-Reply-To: <1517935810-31177-1-git-send-email-ldufour@linux.vnet.ibm.com>
Introduce CONFIG_SPECULATIVE_PAGE_FAULT which turns on the Speculative Page
Fault handler when building for 64bits with SMP.
Cc: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Laurent Dufour <ldufour@linux.vnet.ibm.com>
---
arch/x86/Kconfig | 1 +
1 file changed, 1 insertion(+)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index d9878f1f3bac..cb3a0e8dbf27 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -29,6 +29,7 @@ config X86_64
select HAVE_ARCH_SOFT_DIRTY
select MODULES_USE_ELF_RELA
select X86_DEV_DMA_OPS
+ select SPECULATIVE_PAGE_FAULT if SMP
#
# Arch settings
--
2.7.4
^ permalink raw reply related
* [PATCH v7 00/24] Speculative page faults
From: Laurent Dufour @ 2018-02-06 16:49 UTC (permalink / raw)
To: paulmck, peterz, akpm, kirill, ak, mhocko, dave, jack,
Matthew Wilcox, benh, mpe, paulus, Thomas Gleixner, Ingo Molnar,
hpa, Will Deacon, Sergey Senozhatsky, Andrea Arcangeli,
Alexei Starovoitov, kemi.wang, sergey.senozhatsky.work,
Daniel Jordan
Cc: linux-kernel, linux-mm, haren, khandual, npiggin, bsingharora,
Tim Chen, linuxppc-dev, x86
This is a port on kernel 4.15 of the work done by Peter Zijlstra to
handle page fault without holding the mm semaphore [1].
The idea is to try to handle user space page faults without holding the
mmap_sem. This should allow better concurrency for massively threaded
process since the page fault handler will not wait for other threads memory
layout change to be done, assuming that this change is done in another part
of the process's memory space. This type page fault is named speculative
page fault. If the speculative page fault fails because of a concurrency is
detected or because underlying PMD or PTE tables are not yet allocating, it
is failing its processing and a classic page fault is then tried.
The speculative page fault (SPF) has to look for the VMA matching the fault
address without holding the mmap_sem, this is done by introducing a rwlock
which protects the access to the mm_rb tree. Previously this was done using
SRCU but it was introducing a lot of scheduling to process the VMA's
freeing
operation which was hitting the performance by 20% as reported by Kemi Wang
[2].Using a rwlock to protect access to the mm_rb tree is limiting the
locking contention to these operations which are expected to be in a O(log
n)
order. In addition to ensure that the VMA is not freed in our back a
reference count is added and 2 services (get_vma() and put_vma()) are
introduced to handle the reference count. When a VMA is fetch from the RB
tree using get_vma() is must be later freeed using put_vma(). Furthermore,
to allow the VMA to be used again by the classic page fault handler a
service is introduced can_reuse_spf_vma(). This service is expected to be
called with the mmap_sem hold. It checked that the VMA is still matching
the specified address and is releasing its reference count as the mmap_sem
is hold it is ensure that it will not be freed in our back. In general, the
VMA's reference count could be decremented when holding the mmap_sem but it
should not be increased as holding the mmap_sem is ensuring that the VMA is
stable. I can't see anymore the overhead I got while will-it-scale
benchmark anymore.
The VMA's attributes checked during the speculative page fault processing
have to be protected against parallel changes. This is done by using a per
VMA sequence lock. This sequence lock allows the speculative page fault
handler to fast check for parallel changes in progress and to abort the
speculative page fault in that case.
Once the VMA is found, the speculative page fault handler would check for
the VMA's attributes to verify that the page fault has to be handled
correctly or not. Thus the VMA is protected through a sequence lock which
allows fast detection of concurrent VMA changes. If such a change is
detected, the speculative page fault is aborted and a *classic* page fault
is tried. VMA sequence lockings are added when VMA attributes which are
checked during the page fault are modified.
When the PTE is fetched, the VMA is checked to see if it has been changed,
so once the page table is locked, the VMA is valid, so any other changes
leading to touching this PTE will need to lock the page table, so no
parallel change is possible at this time.
The locking of the PTE is done with interrupts disabled, this allows to
check for the PMD to ensure that there is not an ongoing collapsing
operation. Since khugepaged is firstly set the PMD to pmd_none and then is
waiting for the other CPU to have catch the IPI interrupt, if the pmd is
valid at the time the PTE is locked, we have the guarantee that the
collapsing opertion will have to wait on the PTE lock to move foward. This
allows the SPF handler to map the PTE safely. If the PMD value is different
than the one recorded at the beginning of the SPF operation, the classic
page fault handler will be called to handle the operation while holding the
mmap_sem. As the PTE lock is done with the interrupts disabled, the lock is
done using spin_trylock() to avoid dead lock when handling a page fault
while a TLB invalidate is requested by an other CPU holding the PTE.
Support for THP is not done because when checking for the PMD, we can be
confused by an in progress collapsing operation done by khugepaged. The
issue is that pmd_none() could be true either if the PMD is not already
populate or if the underlying PTE are in the way to be collapsed. So we
cannot safely allocate a PMD if pmd_none() is true.
This series builds on top of v4.15-mmotm-2018-01-31-16-51 and is
functional on x86 and PowerPC.
------------------
Benchmarks results
There is no functional change compared to the v6 so benchmark results are
the same.
Please see https://lkml.org/lkml/2018/1/12/515 for details.
------------------
Changes since v6:
- Rename config variable to CONFIG_SPECULATIVE_PAGE_FAULT (patch 1)
- Review the way the config variable is set (patch 1 to 3)
- Introduce mm_rb_write_*lock() in mm/mmap.c (patch 18)
- Merge patch introducing pte try locking in the patch 18.
Changes since v5:
- use rwlock agains the mm RB tree in place of SRCU
- add a VMA's reference count to protect VMA while using it without
holding the mmap_sem.
- check PMD value to detect collapsing operation
- don't try speculative page fault for mono threaded processes
- try to reuse the fetched VMA if VM_RETRY is returned
- go directly to the error path if an error is detected during the SPF
path
- fix race window when moving VMA in move_vma()
Changes since v4:
- As requested by Andrew Morton, use CONFIG_SPF and define it earlier in
the series to ease bisection.
Changes since v3:
- Don't build when CONFIG_SMP is not set
- Fixed a lock dependency warning in __vma_adjust()
- Use READ_ONCE to access p*d values in handle_speculative_fault()
- Call memcp_oom() service in handle_speculative_fault()
Changes since v2:
- Perf event is renamed in PERF_COUNT_SW_SPF
- On Power handle do_page_fault()'s cleaning
- On Power if the VM_FAULT_ERROR is returned by
handle_speculative_fault(), do not retry but jump to the error path
- If VMA's flags are not matching the fault, directly returns
VM_FAULT_SIGSEGV and not VM_FAULT_RETRY
- Check for pud_trans_huge() to avoid speculative path
- Handles _vm_normal_page()'s introduced by 6f16211df3bf
("mm/device-public-memory: device memory cache coherent with CPU")
- add and review few comments in the code
Changes since v1:
- Remove PERF_COUNT_SW_SPF_FAILED perf event.
- Add tracing events to details speculative page fault failures.
- Cache VMA fields values which are used once the PTE is unlocked at the
end of the page fault events.
- Ensure that fields read during the speculative path are written and read
using WRITE_ONCE and READ_ONCE.
- Add checks at the beginning of the speculative path to abort it if the
VMA is known to not be supported.
Changes since RFC V5 [5]
- Port to 4.13 kernel
- Merging patch fixing lock dependency into the original patch
- Replace the 2 parameters of vma_has_changed() with the vmf pointer
- In patch 7, don't call __do_fault() in the speculative path as it may
want to unlock the mmap_sem.
- In patch 11-12, don't check for vma boundaries when
page_add_new_anon_rmap() is called during the spf path and protect against
anon_vma pointer's update.
- In patch 13-16, add performance events to report number of successful
and failed speculative events.
[1] http://linux-kernel.2935.n7.nabble.com/RFC-PATCH-0-6-Another-go-at-speculative-page-faults-tt965642.html#none
[2] https://patchwork.kernel.org/patch/9999687/
Laurent Dufour (19):
mm: Introduce CONFIG_SPECULATIVE_PAGE_FAULT
x86/mm: Define CONFIG_SPECULATIVE_PAGE_FAULT
powerpc/mm: Define CONFIG_SPECULATIVE_PAGE_FAULT
mm: Introduce pte_spinlock for FAULT_FLAG_SPECULATIVE
mm: Protect VMA modifications using VMA sequence count
mm: protect mremap() against SPF hanlder
mm: Protect SPF handler against anon_vma changes
mm: Cache some VMA fields in the vm_fault structure
mm/migrate: Pass vm_fault pointer to migrate_misplaced_page()
mm: Introduce __lru_cache_add_active_or_unevictable
mm: Introduce __maybe_mkwrite()
mm: Introduce __vm_normal_page()
mm: Introduce __page_add_new_anon_rmap()
mm: Protect mm_rb tree with a rwlock
mm: Adding speculative page fault failure trace events
perf: Add a speculative page fault sw event
perf tools: Add support for the SPF perf event
mm: Speculative page fault handler return VMA
powerpc/mm: Add speculative page fault
Peter Zijlstra (5):
mm: Dont assume page-table invariance during faults
mm: Prepare for FAULT_FLAG_SPECULATIVE
mm: VMA sequence count
mm: Provide speculative fault infrastructure
x86/mm: Add speculative pagefault handling
arch/powerpc/Kconfig | 1 +
arch/powerpc/mm/fault.c | 31 +-
arch/x86/Kconfig | 1 +
arch/x86/mm/fault.c | 38 ++-
fs/proc/task_mmu.c | 5 +-
fs/userfaultfd.c | 17 +-
include/linux/hugetlb_inline.h | 2 +-
include/linux/migrate.h | 4 +-
include/linux/mm.h | 91 +++++-
include/linux/mm_types.h | 7 +
include/linux/pagemap.h | 4 +-
include/linux/rmap.h | 12 +-
include/linux/swap.h | 10 +-
include/trace/events/pagefault.h | 87 ++++++
include/uapi/linux/perf_event.h | 1 +
kernel/fork.c | 3 +
mm/Kconfig | 3 +
mm/hugetlb.c | 2 +
mm/init-mm.c | 3 +
mm/internal.h | 20 ++
mm/khugepaged.c | 5 +
mm/madvise.c | 6 +-
mm/memory.c | 563 ++++++++++++++++++++++++++++++----
mm/mempolicy.c | 51 ++-
mm/migrate.c | 4 +-
mm/mlock.c | 13 +-
mm/mmap.c | 211 ++++++++++---
mm/mprotect.c | 4 +-
mm/mremap.c | 13 +
mm/rmap.c | 5 +-
mm/swap.c | 6 +-
mm/swap_state.c | 8 +-
tools/include/uapi/linux/perf_event.h | 1 +
tools/perf/util/evsel.c | 1 +
tools/perf/util/parse-events.c | 4 +
tools/perf/util/parse-events.l | 1 +
tools/perf/util/python.c | 1 +
37 files changed, 1075 insertions(+), 164 deletions(-)
create mode 100644 include/trace/events/pagefault.h
--
2.7.4
^ permalink raw reply
* Re: rtc-opal: Fix handling of firmware error codes, prevent busy loops
From: Michael Ellerman @ 2018-02-06 14:06 UTC (permalink / raw)
To: Alexandre Belloni; +Cc: Stewart Smith, linuxppc-dev, neelegup, stable
In-Reply-To: <20180206124212.GQ3404@piout.net>
Alexandre Belloni <alexandre.belloni@bootlin.com> writes:
> On 06/02/2018 at 16:22:47 +1100, Michael Ellerman wrote:
>> > Just a note to let you know that this patch should have gone through my
>> > tree but it was not sent to linux-rtc or me.
>>
>> Sorry, I saw it had been languishing for a long time and assumed you'd
>> missed it.
>>
>> Happy to revert/rework it if you're not happy with it.
>>
>
> No, that's fine. It's just that the commit title stands out when using
> git log --oneline and that triggered my OCD ;)
Oh no! Now I'm *really* sorry, that's the worst! :)
cheers
^ permalink raw reply
* Re: [GIT PULL] Please pull powerpc/linux.git powerpc-4.16-1 tag
From: Michael Ellerman @ 2018-02-06 13:39 UTC (permalink / raw)
To: Linus Torvalds
Cc: aik, Alan Modra, Andrew Donnellan, Aneesh Kumar K. V, anju,
Anton Blanchard, Arnd Bergmann, Thiago Jung Bauermann,
Benjamin Herrenschmidt, bhaktipriya96, Bjorn Helgaas, bryantly,
Balbir Singh, Christophe Leroy, clombard, Cyril Bur,
Daniel Thompson, david, Dmitry Torokhov, fbarrat,
Geert Uytterhoeven, geoff, gomonovych, gpiccoli, gromero,
Gustavo A. R. Silva, ivan, Jeremy Kerr, joakim.tjernlund,
Joe Perches, Josh Poimboeuf, jsperbeck, julia, kamalesh, khandual,
leoyang.li, Linux Kernel Mailing List, ppc-dev, Ram Pai, maddy,
mahesh, Mathieu Malaterre, mwb, naveen.n.rao, nfont, Nick Piggin,
oss, Paul Mackerras, Rob Herring, ruscur, santosh, Andreas Schwab,
Stewart Smith, sukadev, Tejun Heo, wei.guo.simon,
Greg Kroah-Hartman, Dan Williams
In-Reply-To: <CA+55aFypU331cQy-6ZJ6szF=2KVLqcbwCUGd+gTwPViRqRWN+g@mail.gmail.com>
Linus Torvalds <torvalds@linux-foundation.org> writes:
> Hmm. This adds a
>
> static inline void pci_uevent_ers(struct pci_dev *pdev, ..
>
> to include/linux/pci.h.
>
> Why?
>
> You do realize that that header file is included by almost every
> driver out there. Why is that magical function *so* important that it
> needs to be an inline function, and those strings etc duplicated in
> every user?
>
> Yes, that header file is already full of random inline functions, but
> they are generally wrapper functions that don't really do anything,
> and in many cases ithey are also basically configurable to really do
> nothing at all so that the compiler can remove all trace of something
> that isn't enabled.
>
> This function, in contrast, is simply not that at all. I see literally
> _zero_ reason why it should be an inline in a really core file.
>
> It's not important.
>
> It's not performance critical.
>
> It's not even used by any core drivers.
All true.
> I pulled this, but honestly, this part was *GARBAGE*. I'm used to
> people always thinking that *their* code is so important that it needs
> to be front and center, but this is just ridiculous shit.
I disagree. This was just a relatively new contributor looking for somewhere to
put a function so that it could be shared between powerpc code and the PCI AER
code, and choosing the easy but wrong option.
None of us think this code is important, we're not trying to put it "front and
center", there just isn't an obvious home for this function in an existing .c
file that doesn't cause it to be built in when it's not needed.
And yeah I should have caught it in review, so I have no problem with you being
annoyed about *that*.
Anyway I'll talk to Bjorn and find a better place for it.
cheers
^ permalink raw reply
* Re: Kernel 4.15 lost set_robust_list support on POWER 9
From: Mauricio Faria de Oliveira @ 2018-02-06 13:17 UTC (permalink / raw)
To: Nicholas Piggin, Benjamin Herrenschmidt
Cc: Florian Weimer, linuxppc-dev, Aneesh Kumar K.V, Jiri Slaby
In-Reply-To: <20180206110616.5d1d5881@roar.ozlabs.ibm.com>
On 02/05/2018 11:06 PM, Nicholas Piggin wrote:
> Does this help?
>
> powerpc/64s/radix: allocate guard-PID for kernel contexts at boot
Yes, the test-case passes:
# strace -e set_robust_list -f ./test
set_robust_list(0x7fff8d453910, 24) = 0
+++ exited with 0 +++
# uname -r
4.15.0.guardpid
Thanks!
cheers,
Mauricio
^ permalink raw reply
* Re: [PATCH] powerpc/64s: Fix MASKABLE_RELON_EXCEPTION_HV_OOL macro
From: Alexey Kardashevskiy @ 2018-02-06 12:42 UTC (permalink / raw)
To: Madhavan Srinivasan, mpe; +Cc: linuxppc-dev
In-Reply-To: <1517920597-14588-1-git-send-email-maddy@linux.vnet.ibm.com>
On 06/02/18 23:36, Madhavan Srinivasan wrote:
> Commit f14e953b191f ("powerpc/64s: Add support to take additional parameter in MASKABLE_* macro")
> messed up MASKABLE_RELON_EXCEPTION_HV_OOL macro by adding the wrong
> __SOFTEN__ test which caused guest kernel trash at boot. Patch to fix
> the macro to use SOFTEN_TEST_HV instead of SOFTEN_NOTEST_HV.
>
> Reported-by: Alexey Kardashevskiy <aik@ozlabs.ru>
> Fix-Suggested-by: Michael Ellerman <mpe@ellerman.id.au>
> Signed-off-by: Madhavan Srinivasan <maddy@linux.vnet.ibm.com>
> ---
> arch/powerpc/include/asm/exception-64s.h | 2 +-
> 1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h
> index 176dfb73d42c..471b2274fbeb 100644
> --- a/arch/powerpc/include/asm/exception-64s.h
> +++ b/arch/powerpc/include/asm/exception-64s.h
> @@ -645,7 +645,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
> EXC_HV, SOFTEN_TEST_HV, bitmask)
>
> #define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label, bitmask) \
> - MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_NOTEST_HV, vec, bitmask);\
> + MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec, bitmask);\
> EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_HV)
>
> /*
>
Tested-by: Alexey Kardashevskiy <aik@ozlabs.ru>
--
Alexey
^ permalink raw reply
* Re: rtc-opal: Fix handling of firmware error codes, prevent busy loops
From: Alexandre Belloni @ 2018-02-06 12:42 UTC (permalink / raw)
To: Michael Ellerman; +Cc: Stewart Smith, linuxppc-dev, neelegup, stable
In-Reply-To: <87bmh24rq0.fsf@concordia.ellerman.id.au>
On 06/02/2018 at 16:22:47 +1100, Michael Ellerman wrote:
> > Just a note to let you know that this patch should have gone through my
> > tree but it was not sent to linux-rtc or me.
>
> Sorry, I saw it had been languishing for a long time and assumed you'd
> missed it.
>
> Happy to revert/rework it if you're not happy with it.
>
No, that's fine. It's just that the commit title stands out when using
git log --oneline and that triggered my OCD ;)
> > I guess what happened is that Michael cleaned up the Linux PPC patchwork
> > queue.
>
> Yeah I did.
>
> In future I'll ping you if there's something that seems to have fallen
> through the cracks.
>
Great thanks!
--
Alexandre Belloni, Bootlin (formerly Free Electrons)
Embedded Linux and Kernel engineering
http://bootlin.com
^ permalink raw reply
* [PATCH] powerpc: wii: Probe the whole devicetree
From: Jonathan Neuschäfer @ 2018-02-06 12:37 UTC (permalink / raw)
To: linuxppc-dev
Cc: Joel Stanley, Jonathan Neuschäfer, Benjamin Herrenschmidt,
Paul Mackerras, Michael Ellerman, linux-kernel
Previously, wii_device_probe would only initialize devices under the
/hollywood node. After this patch, platform devices placed outside of
/hollywood will also be initialized.
The intended usecase for this are devices located outside of the
Hollywood chip, such as GPIO LEDs and GPIO buttons.
Signed-off-by: Jonathan Neuschäfer <j.neuschaefer@gmx.net>
---
arch/powerpc/platforms/embedded6xx/wii.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/powerpc/platforms/embedded6xx/wii.c b/arch/powerpc/platforms/embedded6xx/wii.c
index f598d5fe1d72..4de26c3c4f6d 100644
--- a/arch/powerpc/platforms/embedded6xx/wii.c
+++ b/arch/powerpc/platforms/embedded6xx/wii.c
@@ -248,7 +248,7 @@ static int __init wii_device_probe(void)
if (!machine_is(wii))
return 0;
- of_platform_bus_probe(NULL, wii_of_bus, NULL);
+ of_platform_populate(NULL, wii_of_bus, NULL, NULL);
return 0;
}
device_initcall(wii_device_probe);
--
2.15.1
^ permalink raw reply related
* [PATCH] powerpc/64s: Fix MASKABLE_RELON_EXCEPTION_HV_OOL macro
From: Madhavan Srinivasan @ 2018-02-06 12:36 UTC (permalink / raw)
To: mpe; +Cc: linuxppc-dev, Madhavan Srinivasan
Commit f14e953b191f ("powerpc/64s: Add support to take additional parameter in MASKABLE_* macro")
messed up MASKABLE_RELON_EXCEPTION_HV_OOL macro by adding the wrong
__SOFTEN__ test which caused guest kernel trash at boot. Patch to fix
the macro to use SOFTEN_TEST_HV instead of SOFTEN_NOTEST_HV.
Reported-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Fix-Suggested-by: Michael Ellerman <mpe@ellerman.id.au>
Signed-off-by: Madhavan Srinivasan <maddy@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/exception-64s.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h
index 176dfb73d42c..471b2274fbeb 100644
--- a/arch/powerpc/include/asm/exception-64s.h
+++ b/arch/powerpc/include/asm/exception-64s.h
@@ -645,7 +645,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
EXC_HV, SOFTEN_TEST_HV, bitmask)
#define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label, bitmask) \
- MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_NOTEST_HV, vec, bitmask);\
+ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec, bitmask);\
EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_HV)
/*
--
2.7.4
^ permalink raw reply related
* Re: [PATCH] powerpc/64s: fix may_hard_irq_enable for PMI soft masking
From: Nicholas Piggin @ 2018-02-06 12:28 UTC (permalink / raw)
To: Madhavan Srinivasan; +Cc: linuxppc-dev
In-Reply-To: <764f5253-eaab-4bae-24f6-6ca3bd5d4700@linux.vnet.ibm.com>
On Tue, 6 Feb 2018 15:30:43 +0530
Madhavan Srinivasan <maddy@linux.vnet.ibm.com> wrote:
> On Saturday 03 February 2018 12:47 PM, Nicholas Piggin wrote:
> > vThe soft IRQ masking code has to hard-disable interrupts in cases
> > where the exception is not cleared by the masked handler. External
> > interrupts used this approach for soft masking. Now recently PMU
> > interrupts do the same thing.
> >
> > The soft IRQ masking code additionally allowed for interrupt handlers
> > to hard-enable interrupts after soft-disabling them. The idea is to
> > allow PMU interrupts through to profile interrupt handlers.
> >
> > So when interrupts are being replayed when there is a pending
> > interrupt that requires hard-disabling, there is a test to prevent
> > those handlers from hard-enabling them if there is a pending external
> > interrupt. may_hard_irq_enable() handles this.
> >
> > After f442d00480 ("powerpc/64s: Add support to mask perf interrupts
> > and replay them"), may_hard_irq_enable() could prematurely enable
> > MSR[EE] when a PMU exception exists, which would result in the
> > interrupt firing again while masked, and MSR[EE] being disabled again.
> >
> > I haven't seen that this could cause a serious problem, but it's
> > more consistent to handle these soft-masked interrupts in the same
> > way. So introduce a define for all types of interrupts that require
> > MSR[EE] masking in their soft-disable handlers, and use that in
> > may_hard_irq_enable().
> >
> > Cc: Madhavan Srinivasan <maddy@linux.vnet.ibm.com>
> > Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
>
> Yes nice catch. We could get into a messy state.
> May be if I run my perf+kernbench longer, could get lucky.
I think we just get away with it, because I think the worst that
happens is a PMU exception will fire an interrupt again while it
is still soft-masked, during the interrupt replay code. It would
run its masked handler and set MSR[EE]=0 again and later be
replayed.
But this tidies things up and makes them consistent, and won't
cause any surprises if code changes in future. Thanks for the
review.
Thanks,
Nick
^ permalink raw reply
* Re: DPAA Ethernet traffice troubles with Linux kernel
From: Christian Zigotzky @ 2018-02-06 11:20 UTC (permalink / raw)
To: mad skateman, Madalin-cristian Bucur
Cc: Andrew Lunn, linuxppc-dev@lists.ozlabs.org, Jamie Krueger
In-Reply-To: <CAHkrCmGxFUbfEyufdYhFYA4jyOkh+W2z=1yZsZwv+O4t+hE_1w@mail.gmail.com>
Hello,
I have tried to figure out why there is a problem with the buffer space
but unfortunately without any success. Any ideas? Could you please watch
Skateman's video? [1]
Thanks,
Christian
[1] https://drive.google.com/file/d/18RhksfcavRJPr86asQDTzrmsN20D0Xim/view
On 03 February 2018 at 12:54PM, mad skateman wrote:
>
> For those interested... i have recorded a video of my X5000 DPAA
Ethernet, and the weird problems..
> In this Video i am also transfering hundereds of megabytes from my
NAS to the X5000.
> You will also see pings die... mostly after the 12th packet and
giving the no buffer space error..
> Hopefully someone might have a clue about what is happening.
>
> https://drive.google.com/file/d/18RhksfcavRJPr86asQDTzrmsN20D0Xim/view
>
> On Wed, Jan 17, 2018 at 3:43 PM, Madalin-cristian Bucur
<madalin.bucur@nxp.com> wrote:
>
> > -----Original Message-----
> > From: Madalin-cristian Bucur
> > Sent: Wednesday, January 17, 2018 4:25 PM
> > To: David S . Miller <davem@davemloft.net>
> > Cc: linuxppc-dev@lists.ozlabs.org; netdev@vger.kernel.org;
> > madskateman@gmail.com; 'Madalin-cristian Bucur'
<madalin.bucur@nxp.com>;
> > Andrew Lunn <andrew@lunn.ch>; Joakim Tjernlund
> > <Joakim.Tjernlund@infinera.com>
> > Subject: RE: DPAA Ethernet traffice troubles with Linux kernel
> >
> > > -----Original Message-----
> > > From: netdev-owner@vger.kernel.org
[mailto:netdev-owner@vger.kernel.org]
> > > On Behalf Of Madalin-cristian Bucur
> > > Sent: Wednesday, January 17, 2018 4:16 PM
> > > To: Andrew Lunn <andrew@lunn.ch>; Joakim Tjernlund
> > > <Joakim.Tjernlund@infinera.com>
> > > Cc: linuxppc-dev@lists.ozlabs.org; netdev@vger.kernel.org;
> > > madskateman@gmail.com; David S . Miller <davem@davemloft.net>
> > > Subject: RE: DPAA Ethernet traffice troubles with Linux kernel
> > >
> > > > -----Original Message-----
> > > > From: Andrew Lunn [mailto:andrew@lunn.ch]
> > > > Sent: Wednesday, January 17, 2018 3:44 PM
> > > > To: Joakim Tjernlund <Joakim.Tjernlund@infinera.com>
> > > > Subject: Re: DPAA Ethernet traffice troubles with Linux kernel
> > > >
> > > > > That doesn't work really, having users to hit the bug,
debug it, fix
> > > it
> > > > and then
> > > > > find it fixed already in upstream, then specifically
request it to
> > be
> > > > backported to stable.
> > > > > I don't need this fix to be backported, already got it.
Someone else
> > > > might though.
> > > >
> > > > The "someone else might though" is a big point of asking
for it to
> > > > added to stable. The other reason is it means one less
patch you need
> > > > to maintain in your build.
> > >
> > > I've sent that patch [1] for net but I guess the timing was
wrong and
> > > it was merged to net-next.
> > >
> > > > > I would be interested in bug fixes upstream which fixes:
> > > >
> > > > Did you try upstream? Does it give the same errors?
> > > >
> > > > Andrew
> > >
> > > [1] https://patchwork.kernel.org/patch/10146119/
> > >
> > > Madalin
> >
> > Hi Dave,
> >
> > Can you please add the fix [1] to stable?
> >
> > Thank you,
> > Madalin
>
> Sorry,
>
> I've provided the wrong link towards the patch (v1 instead of v3),
> here's the correct one:
>
> https://patchwork.kernel.org/patch/10151969/
>
> Madalin
>
>
^ permalink raw reply
* Re: [PATCH] powerpc/64s: fix may_hard_irq_enable for PMI soft masking
From: Madhavan Srinivasan @ 2018-02-06 10:00 UTC (permalink / raw)
To: Nicholas Piggin, linuxppc-dev
In-Reply-To: <20180203071750.6469-1-npiggin@gmail.com>
On Saturday 03 February 2018 12:47 PM, Nicholas Piggin wrote:
> vThe soft IRQ masking code has to hard-disable interrupts in cases
> where the exception is not cleared by the masked handler. External
> interrupts used this approach for soft masking. Now recently PMU
> interrupts do the same thing.
>
> The soft IRQ masking code additionally allowed for interrupt handlers
> to hard-enable interrupts after soft-disabling them. The idea is to
> allow PMU interrupts through to profile interrupt handlers.
>
> So when interrupts are being replayed when there is a pending
> interrupt that requires hard-disabling, there is a test to prevent
> those handlers from hard-enabling them if there is a pending external
> interrupt. may_hard_irq_enable() handles this.
>
> After f442d00480 ("powerpc/64s: Add support to mask perf interrupts
> and replay them"), may_hard_irq_enable() could prematurely enable
> MSR[EE] when a PMU exception exists, which would result in the
> interrupt firing again while masked, and MSR[EE] being disabled again.
>
> I haven't seen that this could cause a serious problem, but it's
> more consistent to handle these soft-masked interrupts in the same
> way. So introduce a define for all types of interrupts that require
> MSR[EE] masking in their soft-disable handlers, and use that in
> may_hard_irq_enable().
>
> Cc: Madhavan Srinivasan <maddy@linux.vnet.ibm.com>
> Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Yes nice catch. We could get into a messy state.
May be if I run my perf+kernbench longer, could get lucky.
Reviewed-by: Madhavan Srinivasan <maddy@linux.vnet.ibm.com>
> ---
> Hi,
>
> A review of this would be helpful. I think it probably might as well
> go into 4.16 unless I misunderstood the code. My changelog got a bit
> long-winded in the end, I could try improve it if it's hard to
> understand.
>
> Thanks,
> Nick
>
> arch/powerpc/include/asm/hw_irq.h | 12 +++++++++++-
> arch/powerpc/kernel/exceptions-64e.S | 2 ++
> arch/powerpc/kernel/exceptions-64s.S | 6 +++---
> 3 files changed, 16 insertions(+), 4 deletions(-)
>
> diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h
> index 88e5e8f17e98..855e17d158b1 100644
> --- a/arch/powerpc/include/asm/hw_irq.h
> +++ b/arch/powerpc/include/asm/hw_irq.h
> @@ -29,6 +29,16 @@
> #define PACA_IRQ_HMI 0x20
> #define PACA_IRQ_PMI 0x40
>
> +/*
> + * Some soft-masked interrupts must be hard masked until they are replayed
> + * (e.g., because the soft-masked handler does not clear the exception).
> + */
> +#ifdef CONFIG_PPC_BOOK3S
> +#define PACA_IRQ_MUST_HARD_MASK (PACA_IRQ_EE|PACA_IRQ_PMI)
> +#else
> +#define PACA_IRQ_MUST_HARD_MASK (PACA_IRQ_EE)
> +#endif
> +
> /*
> * flags for paca->irq_soft_mask
> */
> @@ -244,7 +254,7 @@ static inline bool lazy_irq_pending(void)
> static inline void may_hard_irq_enable(void)
> {
> get_paca()->irq_happened &= ~PACA_IRQ_HARD_DIS;
> - if (!(get_paca()->irq_happened & PACA_IRQ_EE))
> + if (!(get_paca()->irq_happened & PACA_IRQ_MUST_HARD_MASK))
> __hard_irq_enable();
> }
>
> diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S
> index ee832d344a5a..9b6e653e501a 100644
> --- a/arch/powerpc/kernel/exceptions-64e.S
> +++ b/arch/powerpc/kernel/exceptions-64e.S
> @@ -943,6 +943,8 @@ kernel_dbg_exc:
> /*
> * An interrupt came in while soft-disabled; We mark paca->irq_happened
> * accordingly and if the interrupt is level sensitive, we hard disable
> + * hard disable (full_mask) corresponds to PACA_IRQ_MUST_HARD_MASK, so
> + * keep these in synch.
> */
>
> .macro masked_interrupt_book3e paca_irq full_mask
> diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
> index 243d072a225a..3ac87e53b3da 100644
> --- a/arch/powerpc/kernel/exceptions-64s.S
> +++ b/arch/powerpc/kernel/exceptions-64s.S
> @@ -1426,7 +1426,7 @@ EXC_COMMON_BEGIN(soft_nmi_common)
> * triggered and won't automatically refire.
> * - If it was a HMI we return immediately since we handled it in realmode
> * and it won't refire.
> - * - else we hard disable and return.
> + * - Else it is one of PACA_IRQ_MUST_HARD_MASK, so hard disable and return.
> * This is called with r10 containing the value to OR to the paca field.
> */
> #define MASKED_INTERRUPT(_H) \
> @@ -1441,8 +1441,8 @@ masked_##_H##interrupt: \
> ori r10,r10,0xffff; \
> mtspr SPRN_DEC,r10; \
> b MASKED_DEC_HANDLER_LABEL; \
> -1: andi. r10,r10,(PACA_IRQ_DBELL|PACA_IRQ_HMI); \
> - bne 2f; \
> +1: andi. r10,r10,PACA_IRQ_MUST_HARD_MASK; \
> + beq 2f; \
> mfspr r10,SPRN_##_H##SRR1; \
> xori r10,r10,MSR_EE; /* clear MSR_EE */ \
> mtspr SPRN_##_H##SRR1,r10; \
^ permalink raw reply
* Re: rtc-opal: Fix handling of firmware error codes, prevent busy loops
From: Stewart Smith @ 2018-02-06 7:57 UTC (permalink / raw)
To: Alexandre Belloni, Michael Ellerman; +Cc: linuxppc-dev, neelegup, stable
In-Reply-To: <20180206022646.GP3404@piout.net>
Alexandre Belloni <alexandre.belloni@bootlin.com> writes:
> On 02/08/2016 at 11:50:16 +1000, Stewart Smith wrote:
>> According to the OPAL docs:
>> https://github.com/open-power/skiboot/blob/skiboot-5.2.5/doc/opal-api/opal-rtc-read-3.txt
>> https://github.com/open-power/skiboot/blob/skiboot-5.2.5/doc/opal-api/opal-rtc-write-4.txt
>> OPAL_HARDWARE may be returned from OPAL_RTC_READ or OPAL_RTC_WRITE and this
>> indicates either a transient or permanent error.
>>
>> Prior to this patch, Linux was not dealing with OPAL_HARDWARE being a
>> permanent error particularly well, in that you could end up in a busy
>> loop.
>>
>> This was not too hard to trigger on an AMI BMC based OpenPOWER machine
>> doing a continuous "ipmitool mc reset cold" to the BMC, the result of
>> that being that we'd get stuck in an infinite loop in opal_get_rtc_time.
>>
>> We now retry a few times before returning the error higher up the stack.
>>
>> Cc: stable@vger.kernel.org
>> Signed-off-by: Stewart Smith <stewart@linux.vnet.ibm.com>
>> ---
>> drivers/rtc/rtc-opal.c | 12 ++++++++++--
>> 1 file changed, 10 insertions(+), 2 deletions(-)
>>
>
> Just a note to let you know that this patch should have gone through my
> tree but it was not sent to linux-rtc or me.
>
> I guess what happened is that Michael cleaned up the Linux PPC patchwork
> queue.
Apologies for not sending there. My (18 month ago self) bad.
--
Stewart Smith
OPAL Architect, IBM.
^ permalink raw reply
* Re: [PATCH v11 3/3] mm, x86: display pkey in smaps only if arch supports pkeys
From: Michael Ellerman @ 2018-02-06 5:51 UTC (permalink / raw)
To: Ram Pai, kbuild test robot
Cc: kbuild-all, mingo, akpm, linuxppc-dev, linux-mm, x86, linux-arch,
linux-kernel, dave.hansen, benh, paulus, khandual, aneesh.kumar,
bsingharora, hbabu, mhocko, bauerman, ebiederm, corbet, arnd,
fweimer, msuchanek
In-Reply-To: <20180202072139.GD5411@ram.oc3035372033.ibm.com>
Ram Pai <linuxram@us.ibm.com> writes:
> On Fri, Feb 02, 2018 at 12:27:27PM +0800, kbuild test robot wrote:
>> Hi Ram,
>>=20
>> Thank you for the patch! Yet something to improve:
>>=20
>> [auto build test ERROR on linus/master]
>> [also build test ERROR on v4.15 next-20180201]
>> [if your patch is applied to the wrong git tree, please drop us a note t=
o help improve the system]
>>=20
>> url: https://urldefense.proofpoint.com/v2/url?u=3Dhttps-3A__github.co=
m_0day-2Dci_linux_commits_Ram-2DPai_mm-2Dx86-2Dpowerpc-2DEnhancements-2Dto-=
2DMemory-2DProtection-2DKeys_20180202-2D120004&d=3DDwIBAg&c=3Djf_iaSHvJObTb=
x-siA1ZOg&r=3Dm-UrKChQVkZtnPpjbF6YY99NbT8FBByQ-E-ygV8luxw&m=3DFv3tEHet1bTUr=
DjOnzEhXvGM_4tGlkYhJHPBnWNWgVA&s=3DZ1W6CV2tfPmLYU8lVv1oDRl2cAyQA76KE2P064A2=
CQY&e=3D
>> config: x86_64-randconfig-x005-201804 (attached as .config)
>> compiler: gcc-7 (Debian 7.2.0-12) 7.2.1 20171025
>> reproduce:
>> # save the attached .config to linux build tree
>> make ARCH=3Dx86_64=20
>>=20
>> All error/warnings (new ones prefixed by >>):
>>=20
>> In file included from arch/x86/include/asm/mmu_context.h:8:0,
>> from arch/x86/events/core.c:36:
>> >> include/linux/pkeys.h:16:23: error: expected identifier or '(' before=
numeric constant
>> #define vma_pkey(vma) 0
>> ^
>> >> arch/x86/include/asm/mmu_context.h:298:19: note: in expansion of macr=
o 'vma_pkey'
>> static inline int vma_pkey(struct vm_area_struct *vma)
>> ^~~~~~~~
>>=20
>> vim +16 include/linux/pkeys.h
>>=20
>> 7=09
>> 8 #ifdef CONFIG_ARCH_HAS_PKEYS
>> 9 #include <asm/pkeys.h>
>> 10 #else /* ! CONFIG_ARCH_HAS_PKEYS */
>> 11 #define arch_max_pkey() (1)
>> 12 #define execute_only_pkey(mm) (0)
>> 13 #define arch_override_mprotect_pkey(vma, prot, pkey) (0)
>> 14 #define PKEY_DEDICATED_EXECUTE_ONLY 0
>> 15 #define ARCH_VM_PKEY_FLAGS 0
>> > 16 #define vma_pkey(vma) 0
>
> Oops. Thanks for catching the issue. The following fix will resolve the e=
rror.
>
> diff --git a/arch/x86/include/asm/mmu_context.h
> b/arch/x86/include/asm/mmu_context.h
> index 6d16d15..c1aeb19 100644
> --- a/arch/x86/include/asm/mmu_context.h
> +++ b/arch/x86/include/asm/mmu_context.h
> @@ -238,11 +238,6 @@ static inline int vma_pkey(struct vm_area_struct
> *vma)
>=20=20
> return (vma->vm_flags & vma_pkey_mask) >> VM_PKEY_SHIFT;
> }
> -#else
> -static inline int vma_pkey(struct vm_area_struct *vma)
> -{
> - return 0;
> -}
> #endif
That's not working for me (i386_defconfig):
In file included from ../include/linux/pkeys.h:6:0,
from ../arch/x86/kernel/fpu/xstate.c:9:
../arch/x86/include/asm/mmu_context.h: In function =E2=80=98arch_vma_acce=
ss_permitted=E2=80=99:
../arch/x86/include/asm/mmu_context.h:276:28: error: implicit declaration=
of function =E2=80=98vma_pkey=E2=80=99 [-Werror=3Dimplicit-function-declar=
ation]
return __pkru_allows_pkey(vma_pkey(vma), write);
^~~~~~~~
In file included from ../include/linux/pkeys.h:6:0,
from ../fs/proc/task_mmu.c:21:
../arch/x86/include/asm/mmu_context.h: In function =E2=80=98arch_vma_acce=
ss_permitted=E2=80=99:
../arch/x86/include/asm/mmu_context.h:276:28: error: implicit declaration=
of function =E2=80=98vma_pkey=E2=80=99 [-Werror=3Dimplicit-function-declar=
ation]
return __pkru_allows_pkey(vma_pkey(vma), write);
^~~~~~~~
In file included from ../include/linux/pkeys.h:6:0,
from ../mm/mmap.c:46:
../arch/x86/include/asm/mmu_context.h: In function =E2=80=98arch_vma_acce=
ss_permitted=E2=80=99:
../arch/x86/include/asm/mmu_context.h:276:28: error: implicit declaration=
of function =E2=80=98vma_pkey=E2=80=99 [-Werror=3Dimplicit-function-declar=
ation]
return __pkru_allows_pkey(vma_pkey(vma), write);
^~~~~~~~
In file included from ../include/linux/pkeys.h:6:0,
from ../mm/mprotect.c:27:
../arch/x86/include/asm/mmu_context.h: In function =E2=80=98arch_vma_acce=
ss_permitted=E2=80=99:
../arch/x86/include/asm/mmu_context.h:276:28: error: implicit declaration=
of function =E2=80=98vma_pkey=E2=80=99 [-Werror=3Dimplicit-function-declar=
ation]
return __pkru_allows_pkey(vma_pkey(vma), write);
^~~~~~~~
In file included from ../include/linux/pkeys.h:6:0,
from ../arch/x86/kernel/fpu/core.c:15:
../arch/x86/include/asm/mmu_context.h: In function =E2=80=98arch_vma_acce=
ss_permitted=E2=80=99:
../arch/x86/include/asm/mmu_context.h:276:28: error: implicit declaration=
of function =E2=80=98vma_pkey=E2=80=99 [-Werror=3Dimplicit-function-declar=
ation]
return __pkru_allows_pkey(vma_pkey(vma), write);
^~~~~~~~
cheers
^ permalink raw reply
* Re: rtc-opal: Fix handling of firmware error codes, prevent busy loops
From: Michael Ellerman @ 2018-02-06 5:22 UTC (permalink / raw)
To: Alexandre Belloni, Stewart Smith; +Cc: linuxppc-dev, neelegup, stable
In-Reply-To: <20180206022646.GP3404@piout.net>
Alexandre Belloni <alexandre.belloni@bootlin.com> writes:
> Hi,
>
> On 02/08/2016 at 11:50:16 +1000, Stewart Smith wrote:
>> According to the OPAL docs:
>> https://github.com/open-power/skiboot/blob/skiboot-5.2.5/doc/opal-api/opal-rtc-read-3.txt
>> https://github.com/open-power/skiboot/blob/skiboot-5.2.5/doc/opal-api/opal-rtc-write-4.txt
>> OPAL_HARDWARE may be returned from OPAL_RTC_READ or OPAL_RTC_WRITE and this
>> indicates either a transient or permanent error.
>>
>> Prior to this patch, Linux was not dealing with OPAL_HARDWARE being a
>> permanent error particularly well, in that you could end up in a busy
>> loop.
>>
>> This was not too hard to trigger on an AMI BMC based OpenPOWER machine
>> doing a continuous "ipmitool mc reset cold" to the BMC, the result of
>> that being that we'd get stuck in an infinite loop in opal_get_rtc_time.
>>
>> We now retry a few times before returning the error higher up the stack.
>>
>> Cc: stable@vger.kernel.org
>> Signed-off-by: Stewart Smith <stewart@linux.vnet.ibm.com>
>> ---
>> drivers/rtc/rtc-opal.c | 12 ++++++++++--
>> 1 file changed, 10 insertions(+), 2 deletions(-)
>>
>
> Just a note to let you know that this patch should have gone through my
> tree but it was not sent to linux-rtc or me.
Sorry, I saw it had been languishing for a long time and assumed you'd
missed it.
Happy to revert/rework it if you're not happy with it.
> I guess what happened is that Michael cleaned up the Linux PPC patchwork
> queue.
Yeah I did.
In future I'll ping you if there's something that seems to have fallen
through the cracks.
cheers
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox