From: Jan Kara <jack@suse.cz>
To: linux-mm@kvack.org
Cc: linux-fsdevel@vger.kernel.org, linux-nvdimm@lists.01.org,
Dan Williams <dan.j.williams@intel.com>,
Ross Zwisler <ross.zwisler@linux.intel.com>,
"Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>,
Jan Kara <jack@suse.cz>
Subject: [PATCH 07/20] mm: Add orig_pte field into vm_fault
Date: Tue, 27 Sep 2016 18:08:11 +0200 [thread overview]
Message-ID: <1474992504-20133-8-git-send-email-jack@suse.cz> (raw)
In-Reply-To: <1474992504-20133-1-git-send-email-jack@suse.cz>
Add orig_pte field to vm_fault structure to allow ->page_mkwrite
handlers to fully handle the fault. This also allows us to save some
passing of extra arguments around.
Signed-off-by: Jan Kara <jack@suse.cz>
---
include/linux/mm.h | 4 +--
mm/internal.h | 2 +-
mm/khugepaged.c | 5 ++--
mm/memory.c | 76 +++++++++++++++++++++++++++---------------------------
4 files changed, 44 insertions(+), 43 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 5fc6daf5242c..c908fd7243ea 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -300,8 +300,8 @@ struct vm_fault {
unsigned long virtual_address; /* Faulting virtual address masked by
* PAGE_MASK */
pmd_t *pmd; /* Pointer to pmd entry matching
- * the 'address'
- */
+ * the 'address' */
+ pte_t orig_pte; /* Value of PTE at the time of fault */
struct page *cow_page; /* Handler may choose to COW */
struct page *page; /* ->fault handlers should return a
diff --git a/mm/internal.h b/mm/internal.h
index cc80060914f6..7c7421da5d63 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -36,7 +36,7 @@
/* Do not use these with a slab allocator */
#define GFP_SLAB_BUG_MASK (__GFP_DMA32|__GFP_HIGHMEM|~__GFP_BITS_MASK)
-int do_swap_page(struct vm_fault *vmf, pte_t orig_pte);
+int do_swap_page(struct vm_fault *vmf);
void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
unsigned long floor, unsigned long ceiling);
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index f88b2d3810a7..66bc77f2d1d2 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -890,11 +890,12 @@ static bool __collapse_huge_page_swapin(struct mm_struct *mm,
vmf.pte = pte_offset_map(pmd, address);
for (; vmf.address < address + HPAGE_PMD_NR*PAGE_SIZE;
vmf.pte++, vmf.address += PAGE_SIZE) {
- pteval = *vmf.pte;
+ vmf.orig_pte = *vmf.pte;
+ pteval = vmf.orig_pte;
if (!is_swap_pte(pteval))
continue;
swapped_in++;
- ret = do_swap_page(&vmf, pteval);
+ ret = do_swap_page(&vmf);
/* do_swap_page returns VM_FAULT_RETRY with released mmap_sem */
if (ret & VM_FAULT_RETRY) {
diff --git a/mm/memory.c b/mm/memory.c
index 48de8187d7b2..0c8779c23925 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2070,8 +2070,8 @@ static int do_page_mkwrite(struct vm_area_struct *vma, struct page *page,
* case, all we need to do here is to mark the page as writable and update
* any related book-keeping.
*/
-static inline int wp_page_reuse(struct vm_fault *vmf, pte_t orig_pte,
- struct page *page, int page_mkwrite, int dirty_shared)
+static inline int wp_page_reuse(struct vm_fault *vmf, struct page *page,
+ int page_mkwrite, int dirty_shared)
__releases(vmf->ptl)
{
struct vm_area_struct *vma = vmf->vma;
@@ -2084,8 +2084,8 @@ static inline int wp_page_reuse(struct vm_fault *vmf, pte_t orig_pte,
if (page)
page_cpupid_xchg_last(page, (1 << LAST_CPUPID_SHIFT) - 1);
- flush_cache_page(vma, vmf->address, pte_pfn(orig_pte));
- entry = pte_mkyoung(orig_pte);
+ flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
+ entry = pte_mkyoung(vmf->orig_pte);
entry = maybe_mkwrite(pte_mkdirty(entry), vma);
if (ptep_set_access_flags(vma, vmf->address, vmf->pte, entry, 1))
update_mmu_cache(vma, vmf->address, vmf->pte);
@@ -2135,8 +2135,7 @@ static inline int wp_page_reuse(struct vm_fault *vmf, pte_t orig_pte,
* held to the old page, as well as updating the rmap.
* - In any case, unlock the PTL and drop the reference we took to the old page.
*/
-static int wp_page_copy(struct vm_fault *vmf, pte_t orig_pte,
- struct page *old_page)
+static int wp_page_copy(struct vm_fault *vmf, struct page *old_page)
{
struct vm_area_struct *vma = vmf->vma;
struct mm_struct *mm = vma->vm_mm;
@@ -2150,7 +2149,7 @@ static int wp_page_copy(struct vm_fault *vmf, pte_t orig_pte,
if (unlikely(anon_vma_prepare(vma)))
goto oom;
- if (is_zero_pfn(pte_pfn(orig_pte))) {
+ if (is_zero_pfn(pte_pfn(vmf->orig_pte))) {
new_page = alloc_zeroed_user_highpage_movable(vma,
vmf->address);
if (!new_page)
@@ -2174,7 +2173,7 @@ static int wp_page_copy(struct vm_fault *vmf, pte_t orig_pte,
* Re-check the pte - we dropped the lock
*/
vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
- if (likely(pte_same(*vmf->pte, orig_pte))) {
+ if (likely(pte_same(*vmf->pte, vmf->orig_pte))) {
if (old_page) {
if (!PageAnon(old_page)) {
dec_mm_counter_fast(mm,
@@ -2184,7 +2183,7 @@ static int wp_page_copy(struct vm_fault *vmf, pte_t orig_pte,
} else {
inc_mm_counter_fast(mm, MM_ANONPAGES);
}
- flush_cache_page(vma, vmf->address, pte_pfn(orig_pte));
+ flush_cache_page(vma, vmf->address, pte_pfn(vmf->orig_pte));
entry = mk_pte(new_page, vma->vm_page_prot);
entry = maybe_mkwrite(pte_mkdirty(entry), vma);
/*
@@ -2268,7 +2267,7 @@ oom:
* Handle write page faults for VM_MIXEDMAP or VM_PFNMAP for a VM_SHARED
* mapping
*/
-static int wp_pfn_shared(struct vm_fault *vmf, pte_t orig_pte)
+static int wp_pfn_shared(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
@@ -2286,16 +2285,15 @@ static int wp_pfn_shared(struct vm_fault *vmf, pte_t orig_pte)
* We might have raced with another page fault while we
* released the pte_offset_map_lock.
*/
- if (!pte_same(*vmf->pte, orig_pte)) {
+ if (!pte_same(*vmf->pte, vmf->orig_pte)) {
pte_unmap_unlock(vmf->pte, vmf->ptl);
return 0;
}
}
- return wp_page_reuse(vmf, orig_pte, NULL, 0, 0);
+ return wp_page_reuse(vmf, NULL, 0, 0);
}
-static int wp_page_shared(struct vm_fault *vmf, pte_t orig_pte,
- struct page *old_page)
+static int wp_page_shared(struct vm_fault *vmf, struct page *old_page)
__releases(vmf->ptl)
{
struct vm_area_struct *vma = vmf->vma;
@@ -2321,7 +2319,7 @@ static int wp_page_shared(struct vm_fault *vmf, pte_t orig_pte,
*/
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
vmf->address, &vmf->ptl);
- if (!pte_same(*vmf->pte, orig_pte)) {
+ if (!pte_same(*vmf->pte, vmf->orig_pte)) {
unlock_page(old_page);
pte_unmap_unlock(vmf->pte, vmf->ptl);
put_page(old_page);
@@ -2330,7 +2328,7 @@ static int wp_page_shared(struct vm_fault *vmf, pte_t orig_pte,
page_mkwrite = 1;
}
- return wp_page_reuse(vmf, orig_pte, old_page, page_mkwrite, 1);
+ return wp_page_reuse(vmf, old_page, page_mkwrite, 1);
}
/*
@@ -2351,13 +2349,13 @@ static int wp_page_shared(struct vm_fault *vmf, pte_t orig_pte,
* but allow concurrent faults), with pte both mapped and locked.
* We return with mmap_sem still held, but pte unmapped and unlocked.
*/
-static int do_wp_page(struct vm_fault *vmf, pte_t orig_pte)
+static int do_wp_page(struct vm_fault *vmf)
__releases(vmf->ptl)
{
struct vm_area_struct *vma = vmf->vma;
struct page *old_page;
- old_page = vm_normal_page(vma, vmf->address, orig_pte);
+ old_page = vm_normal_page(vma, vmf->address, vmf->orig_pte);
if (!old_page) {
/*
* VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
@@ -2368,10 +2366,10 @@ static int do_wp_page(struct vm_fault *vmf, pte_t orig_pte)
*/
if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
(VM_WRITE|VM_SHARED))
- return wp_pfn_shared(vmf, orig_pte);
+ return wp_pfn_shared(vmf);
pte_unmap_unlock(vmf->pte, vmf->ptl);
- return wp_page_copy(vmf, orig_pte, old_page);
+ return wp_page_copy(vmf, old_page);
}
/*
@@ -2386,7 +2384,7 @@ static int do_wp_page(struct vm_fault *vmf, pte_t orig_pte)
lock_page(old_page);
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
vmf->address, &vmf->ptl);
- if (!pte_same(*vmf->pte, orig_pte)) {
+ if (!pte_same(*vmf->pte, vmf->orig_pte)) {
unlock_page(old_page);
pte_unmap_unlock(vmf->pte, vmf->ptl);
put_page(old_page);
@@ -2406,12 +2404,12 @@ static int do_wp_page(struct vm_fault *vmf, pte_t orig_pte)
page_move_anon_rmap(old_page, vma);
}
unlock_page(old_page);
- return wp_page_reuse(vmf, orig_pte, old_page, 0, 0);
+ return wp_page_reuse(vmf, old_page, 0, 0);
}
unlock_page(old_page);
} else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
(VM_WRITE|VM_SHARED))) {
- return wp_page_shared(vmf, orig_pte, old_page);
+ return wp_page_shared(vmf, old_page);
}
/*
@@ -2420,7 +2418,7 @@ static int do_wp_page(struct vm_fault *vmf, pte_t orig_pte)
get_page(old_page);
pte_unmap_unlock(vmf->pte, vmf->ptl);
- return wp_page_copy(vmf, orig_pte, old_page);
+ return wp_page_copy(vmf, old_page);
}
static void unmap_mapping_range_vma(struct vm_area_struct *vma,
@@ -2508,7 +2506,7 @@ EXPORT_SYMBOL(unmap_mapping_range);
* We return with the mmap_sem locked or unlocked in the same cases
* as does filemap_fault().
*/
-int do_swap_page(struct vm_fault *vmf, pte_t orig_pte)
+int do_swap_page(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
struct page *page, *swapcache;
@@ -2519,10 +2517,10 @@ int do_swap_page(struct vm_fault *vmf, pte_t orig_pte)
int exclusive = 0;
int ret = 0;
- if (!pte_unmap_same(vma->vm_mm, vmf->pmd, vmf->pte, orig_pte))
+ if (!pte_unmap_same(vma->vm_mm, vmf->pmd, vmf->pte, vmf->orig_pte))
goto out;
- entry = pte_to_swp_entry(orig_pte);
+ entry = pte_to_swp_entry(vmf->orig_pte);
if (unlikely(non_swap_entry(entry))) {
if (is_migration_entry(entry)) {
migration_entry_wait(vma->vm_mm, vmf->pmd,
@@ -2530,7 +2528,7 @@ int do_swap_page(struct vm_fault *vmf, pte_t orig_pte)
} else if (is_hwpoison_entry(entry)) {
ret = VM_FAULT_HWPOISON;
} else {
- print_bad_pte(vma, vmf->address, orig_pte, NULL);
+ print_bad_pte(vma, vmf->address, vmf->orig_pte, NULL);
ret = VM_FAULT_SIGBUS;
}
goto out;
@@ -2547,7 +2545,7 @@ int do_swap_page(struct vm_fault *vmf, pte_t orig_pte)
*/
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
vmf->address, &vmf->ptl);
- if (likely(pte_same(*vmf->pte, orig_pte)))
+ if (likely(pte_same(*vmf->pte, vmf->orig_pte)))
ret = VM_FAULT_OOM;
delayacct_clear_flag(DELAYACCT_PF_SWAPIN);
goto unlock;
@@ -2604,7 +2602,7 @@ int do_swap_page(struct vm_fault *vmf, pte_t orig_pte)
*/
vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
&vmf->ptl);
- if (unlikely(!pte_same(*vmf->pte, orig_pte)))
+ if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte)))
goto out_nomap;
if (unlikely(!PageUptodate(page))) {
@@ -2632,9 +2630,10 @@ int do_swap_page(struct vm_fault *vmf, pte_t orig_pte)
exclusive = RMAP_EXCLUSIVE;
}
flush_icache_page(vma, page);
- if (pte_swp_soft_dirty(orig_pte))
+ if (pte_swp_soft_dirty(vmf->orig_pte))
pte = pte_mksoft_dirty(pte);
set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
+ vmf->orig_pte = pte;
if (page == swapcache) {
do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
mem_cgroup_commit_charge(page, memcg, true, false);
@@ -2664,7 +2663,7 @@ int do_swap_page(struct vm_fault *vmf, pte_t orig_pte)
}
if (vmf->flags & FAULT_FLAG_WRITE) {
- ret |= do_wp_page(vmf, pte);
+ ret |= do_wp_page(vmf);
if (ret & VM_FAULT_ERROR)
ret &= VM_FAULT_ERROR;
goto out;
@@ -3326,7 +3325,7 @@ static int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
return mpol_misplaced(page, vma, addr);
}
-static int do_numa_page(struct vm_fault *vmf, pte_t pte)
+static int do_numa_page(struct vm_fault *vmf)
{
struct vm_area_struct *vma = vmf->vma;
struct page *page = NULL;
@@ -3334,6 +3333,7 @@ static int do_numa_page(struct vm_fault *vmf, pte_t pte)
int last_cpupid;
int target_nid;
bool migrated = false;
+ pte_t pte = vmf->orig_pte;
bool was_writable = pte_write(pte);
int flags = 0;
@@ -3484,8 +3484,7 @@ static int handle_pte_fault(struct vm_fault *vmf)
* So now it's safe to run pte_offset_map().
*/
vmf->pte = pte_offset_map(vmf->pmd, vmf->address);
-
- entry = *vmf->pte;
+ vmf->orig_pte = *vmf->pte;
/*
* some architectures can have larger ptes than wordsize,
@@ -3496,6 +3495,7 @@ static int handle_pte_fault(struct vm_fault *vmf)
* ptl lock held. So here a barrier will do.
*/
barrier();
+ entry = vmf->orig_pte;
if (pte_none(entry)) {
pte_unmap(vmf->pte);
vmf->pte = NULL;
@@ -3510,10 +3510,10 @@ static int handle_pte_fault(struct vm_fault *vmf)
}
if (!pte_present(entry))
- return do_swap_page(vmf, entry);
+ return do_swap_page(vmf);
if (pte_protnone(entry) && vma_is_accessible(vmf->vma))
- return do_numa_page(vmf, entry);
+ return do_numa_page(vmf);
vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
spin_lock(vmf->ptl);
@@ -3521,7 +3521,7 @@ static int handle_pte_fault(struct vm_fault *vmf)
goto unlock;
if (vmf->flags & FAULT_FLAG_WRITE) {
if (!pte_write(entry))
- return do_wp_page(vmf, entry);
+ return do_wp_page(vmf);
entry = pte_mkdirty(entry);
}
entry = pte_mkyoung(entry);
--
2.6.6
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
next prev parent reply other threads:[~2016-09-27 16:08 UTC|newest]
Thread overview: 72+ messages / expand[flat|nested] mbox.gz Atom feed top
2016-09-27 16:08 [PATCH 0/20 v3] dax: Clear dirty bits after flushing caches Jan Kara
2016-09-27 16:08 ` [PATCH 01/20] mm: Change type of vmf->virtual_address Jan Kara
2016-09-30 9:07 ` Christoph Hellwig
2016-10-14 18:02 ` Ross Zwisler
2016-09-27 16:08 ` [PATCH 02/20] mm: Join struct fault_env and vm_fault Jan Kara
2016-09-30 9:10 ` Christoph Hellwig
2016-10-03 7:43 ` Jan Kara
2016-09-27 16:08 ` [PATCH 03/20] mm: Use pgoff in struct vm_fault instead of passing it separately Jan Kara
2016-10-14 18:42 ` Ross Zwisler
2016-10-17 9:01 ` Jan Kara
2016-09-27 16:08 ` [PATCH 04/20] mm: Use passed vm_fault structure in __do_fault() Jan Kara
2016-10-14 19:05 ` Ross Zwisler
2016-09-27 16:08 ` [PATCH 05/20] mm: Trim __do_fault() arguments Jan Kara
2016-10-14 20:31 ` Ross Zwisler
2016-10-17 9:04 ` Jan Kara
2016-09-27 16:08 ` [PATCH 06/20] mm: Use pass vm_fault structure for in wp_pfn_shared() Jan Kara
2016-10-14 21:04 ` Ross Zwisler
2016-09-27 16:08 ` Jan Kara [this message]
2016-10-17 16:45 ` [PATCH 07/20] mm: Add orig_pte field into vm_fault Ross Zwisler
2016-10-18 10:13 ` Jan Kara
2016-09-27 16:08 ` [PATCH 08/20] mm: Allow full handling of COW faults in ->fault handlers Jan Kara
2016-10-17 16:50 ` Ross Zwisler
2016-09-27 16:08 ` [PATCH 09/20] mm: Factor out functionality to finish page faults Jan Kara
2016-10-17 17:38 ` Ross Zwisler
2016-10-17 17:40 ` Ross Zwisler
2016-10-18 9:44 ` Jan Kara
2016-09-27 16:08 ` [PATCH 10/20] mm: Move handling of COW faults into DAX code Jan Kara
2016-10-17 19:29 ` Ross Zwisler
2016-10-18 10:32 ` Jan Kara
2016-09-27 16:08 ` [PATCH 11/20] mm: Remove unnecessary vma->vm_ops check Jan Kara
2016-10-17 19:40 ` Ross Zwisler
2016-10-18 10:37 ` Jan Kara
2016-09-27 16:08 ` [PATCH 12/20] mm: Factor out common parts of write fault handling Jan Kara
2016-10-17 22:08 ` Ross Zwisler
2016-10-18 10:50 ` Jan Kara
2016-10-18 17:32 ` Ross Zwisler
2016-09-27 16:08 ` [PATCH 13/20] mm: Pass vm_fault structure into do_page_mkwrite() Jan Kara
2016-10-17 22:29 ` Ross Zwisler
2016-09-27 16:08 ` [PATCH 14/20] mm: Use vmf->page during WP faults Jan Kara
2016-10-18 17:56 ` Ross Zwisler
2016-09-27 16:08 ` [PATCH 15/20] mm: Move part of wp_page_reuse() into the single call site Jan Kara
2016-10-18 17:59 ` Ross Zwisler
2016-09-27 16:08 ` [PATCH 16/20] mm: Provide helper for finishing mkwrite faults Jan Kara
2016-10-18 18:35 ` Ross Zwisler
2016-10-19 7:16 ` Jan Kara
2016-10-19 17:21 ` Ross Zwisler
2016-10-20 8:48 ` Jan Kara
2016-09-27 16:08 ` [PATCH 17/20] mm: Export follow_pte() Jan Kara
2016-10-18 18:37 ` Ross Zwisler
2016-09-27 16:08 ` [PATCH 18/20] dax: Make cache flushing protected by entry lock Jan Kara
2016-10-18 19:20 ` Ross Zwisler
2016-10-19 7:19 ` Jan Kara
2016-10-19 18:25 ` Ross Zwisler
2016-09-27 16:08 ` [PATCH 19/20] dax: Protect PTE modification on WP fault by radix tree " Jan Kara
2016-10-18 19:53 ` Ross Zwisler
2016-10-19 7:25 ` Jan Kara
2016-10-19 17:25 ` Ross Zwisler
2016-09-27 16:08 ` [PATCH 20/20] dax: Clear dirty entry tags on cache flush Jan Kara
2016-10-18 22:12 ` Ross Zwisler
2016-10-19 7:30 ` Jan Kara
2016-10-19 16:38 ` Ross Zwisler
2016-09-30 9:14 ` [PATCH 0/20 v3] dax: Clear dirty bits after flushing caches Christoph Hellwig
2016-10-03 7:59 ` Jan Kara
2016-10-03 8:03 ` Christoph Hellwig
2016-10-03 8:15 ` Jan Kara
2016-10-03 9:32 ` Christoph Hellwig
2016-10-03 11:13 ` Jan Kara
2016-10-13 20:34 ` Ross Zwisler
2016-10-17 8:47 ` Jan Kara
2016-10-17 18:59 ` Ross Zwisler
2016-10-18 9:49 ` Jan Kara
-- strict thread matches above, loose matches on Subject: below --
2016-11-18 9:17 [PATCH 0/20 v5] " Jan Kara
2016-11-18 9:17 ` [PATCH 07/20] mm: Add orig_pte field into vm_fault Jan Kara
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1474992504-20133-8-git-send-email-jack@suse.cz \
--to=jack@suse.cz \
--cc=dan.j.williams@intel.com \
--cc=kirill.shutemov@linux.intel.com \
--cc=linux-fsdevel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=linux-nvdimm@lists.01.org \
--cc=ross.zwisler@linux.intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).