From: jglisse@redhat.com
To: linux-mm@kvack.org
Cc: "Andrew Morton" <akpm@linux-foundation.org>,
linux-kernel@vger.kernel.org,
"Jérôme Glisse" <jglisse@redhat.com>,
"Evgeny Baskakov" <ebaskakov@nvidia.com>,
"Ralph Campbell" <rcampbell@nvidia.com>,
"Mark Hairgrove" <mhairgrove@nvidia.com>,
"John Hubbard" <jhubbard@nvidia.com>
Subject: [PATCH 13/15] mm/hmm: factor out pte and pmd handling to simplify hmm_vma_walk_pmd()
Date: Mon, 19 Mar 2018 22:00:35 -0400 [thread overview]
Message-ID: <20180320020038.3360-14-jglisse@redhat.com> (raw)
In-Reply-To: <20180320020038.3360-1-jglisse@redhat.com>
From: Jérôme Glisse <jglisse@redhat.com>
No functional change, just create one function to handle pmd and one
to handle pte (hmm_vma_handle_pmd() and hmm_vma_handle_pte()).
Signed-off-by: Jérôme Glisse <jglisse@redhat.com>
Cc: Evgeny Baskakov <ebaskakov@nvidia.com>
Cc: Ralph Campbell <rcampbell@nvidia.com>
Cc: Mark Hairgrove <mhairgrove@nvidia.com>
Cc: John Hubbard <jhubbard@nvidia.com>
---
mm/hmm.c | 174 +++++++++++++++++++++++++++++++++++++--------------------------
1 file changed, 102 insertions(+), 72 deletions(-)
diff --git a/mm/hmm.c b/mm/hmm.c
index 52cdceb35733..dc703e9c3a95 100644
--- a/mm/hmm.c
+++ b/mm/hmm.c
@@ -351,6 +351,99 @@ static int hmm_vma_walk_hole(unsigned long addr,
return hmm_vma_walk->fault ? -EAGAIN : 0;
}
+static int hmm_vma_handle_pmd(struct mm_walk *walk,
+ unsigned long addr,
+ unsigned long end,
+ uint64_t *pfns,
+ pmd_t pmd)
+{
+ struct hmm_vma_walk *hmm_vma_walk = walk->private;
+ unsigned long pfn, i;
+ uint64_t flag = 0;
+
+ if (pmd_protnone(pmd))
+ return hmm_vma_walk_hole(addr, end, walk);
+
+ if ((hmm_vma_walk->fault & hmm_vma_walk->write) && !pmd_write(pmd))
+ return hmm_vma_walk_hole(addr, end, walk);
+
+ pfn = pmd_pfn(pmd) + pte_index(addr);
+ flag |= pmd_write(pmd) ? HMM_PFN_WRITE : 0;
+ for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++)
+ pfns[i] = hmm_pfn_from_pfn(pfn) | flag;
+ hmm_vma_walk->last = end;
+ return 0;
+}
+
+static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
+ unsigned long end, pmd_t *pmdp, pte_t *ptep,
+ uint64_t *pfns)
+{
+ struct hmm_vma_walk *hmm_vma_walk = walk->private;
+ struct vm_area_struct *vma = walk->vma;
+ pte_t pte = *ptep;
+
+ *pfns = 0;
+
+ if (pte_none(pte)) {
+ *pfns = 0;
+ if (hmm_vma_walk->fault)
+ goto fault;
+ return 0;
+ }
+
+ if (!pte_present(pte)) {
+ swp_entry_t entry = pte_to_swp_entry(pte);
+
+ if (!non_swap_entry(entry)) {
+ if (hmm_vma_walk->fault)
+ goto fault;
+ return 0;
+ }
+
+ /*
+ * This is a special swap entry, ignore migration, use
+ * device and report anything else as error.
+ */
+ if (is_device_private_entry(entry)) {
+ *pfns = hmm_pfn_from_pfn(swp_offset(entry));
+ if (is_write_device_private_entry(entry)) {
+ *pfns |= HMM_PFN_WRITE;
+ } else if ((hmm_vma_walk->fault & hmm_vma_walk->write))
+ goto fault;
+ *pfns |= HMM_PFN_DEVICE_PRIVATE;
+ return 0;
+ }
+
+ if (is_migration_entry(entry)) {
+ if (hmm_vma_walk->fault) {
+ pte_unmap(ptep);
+ hmm_vma_walk->last = addr;
+ migration_entry_wait(vma->vm_mm,
+ pmdp, addr);
+ return -EAGAIN;
+ }
+ return 0;
+ }
+
+ /* Report error for everything else */
+ *pfns = HMM_PFN_ERROR;
+ return -EFAULT;
+ }
+
+ if ((hmm_vma_walk->fault & hmm_vma_walk->write) && !pte_write(pte))
+ goto fault;
+
+ *pfns = hmm_pfn_from_pfn(pte_pfn(pte));
+ *pfns |= pte_write(pte) ? HMM_PFN_WRITE : 0;
+ return 0;
+
+fault:
+ pte_unmap(ptep);
+ /* Fault any virtual address we were ask to fault */
+ return hmm_vma_walk_hole(addr, end, walk);
+}
+
static int hmm_vma_walk_pmd(pmd_t *pmdp,
unsigned long start,
unsigned long end,
@@ -358,25 +451,20 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,
{
struct hmm_vma_walk *hmm_vma_walk = walk->private;
struct hmm_range *range = hmm_vma_walk->range;
- struct vm_area_struct *vma = walk->vma;
uint64_t *pfns = range->pfns;
unsigned long addr = start, i;
- bool write_fault;
pte_t *ptep;
i = (addr - range->start) >> PAGE_SHIFT;
- write_fault = hmm_vma_walk->fault & hmm_vma_walk->write;
again:
if (pmd_none(*pmdp))
return hmm_vma_walk_hole(start, end, walk);
- if (pmd_huge(*pmdp) && vma->vm_flags & VM_HUGETLB)
+ if (pmd_huge(*pmdp) && (range->vma->vm_flags & VM_HUGETLB))
return hmm_pfns_bad(start, end, walk);
if (pmd_devmap(*pmdp) || pmd_trans_huge(*pmdp)) {
- unsigned long pfn;
- uint64_t flag = 0;
pmd_t pmd;
/*
@@ -392,17 +480,8 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,
barrier();
if (!pmd_devmap(pmd) && !pmd_trans_huge(pmd))
goto again;
- if (pmd_protnone(pmd))
- return hmm_vma_walk_hole(start, end, walk);
- if (write_fault && !pmd_write(pmd))
- return hmm_vma_walk_hole(start, end, walk);
-
- pfn = pmd_pfn(pmd) + pte_index(addr);
- flag |= pmd_write(pmd) ? HMM_PFN_WRITE : 0;
- for (; addr < end; addr += PAGE_SIZE, i++, pfn++)
- pfns[i] = hmm_pfn_from_pfn(pfn) | flag;
- return 0;
+ return hmm_vma_handle_pmd(walk, addr, end, &pfns[i], pmd);
}
if (pmd_bad(*pmdp))
@@ -410,67 +489,18 @@ static int hmm_vma_walk_pmd(pmd_t *pmdp,
ptep = pte_offset_map(pmdp, addr);
for (; addr < end; addr += PAGE_SIZE, ptep++, i++) {
- pte_t pte = *ptep;
+ int r;
- pfns[i] = 0;
-
- if (pte_none(pte)) {
- pfns[i] = 0;
- if (hmm_vma_walk->fault)
- goto fault;
- continue;
- }
-
- if (!pte_present(pte)) {
- swp_entry_t entry = pte_to_swp_entry(pte);
-
- if (!non_swap_entry(entry)) {
- if (hmm_vma_walk->fault)
- goto fault;
- continue;
- }
-
- /*
- * This is a special swap entry, ignore migration, use
- * device and report anything else as error.
- */
- if (is_device_private_entry(entry)) {
- pfns[i] = hmm_pfn_from_pfn(swp_offset(entry));
- if (is_write_device_private_entry(entry)) {
- pfns[i] |= HMM_PFN_WRITE;
- } else if (write_fault)
- goto fault;
- pfns[i] |= HMM_PFN_DEVICE_PRIVATE;
- } else if (is_migration_entry(entry)) {
- if (hmm_vma_walk->fault) {
- pte_unmap(ptep);
- hmm_vma_walk->last = addr;
- migration_entry_wait(vma->vm_mm,
- pmdp, addr);
- return -EAGAIN;
- }
- continue;
- } else {
- /* Report error for everything else */
- pfns[i] = HMM_PFN_ERROR;
- }
- continue;
+ r = hmm_vma_handle_pte(walk, addr, end, pmdp, ptep, &pfns[i]);
+ if (r) {
+ /* hmm_vma_handle_pte() did unmap pte directory */
+ hmm_vma_walk->last = addr;
+ return r;
}
-
- if (write_fault && !pte_write(pte))
- goto fault;
-
- pfns[i] = hmm_pfn_from_pfn(pte_pfn(pte));
- pfns[i] |= pte_write(pte) ? HMM_PFN_WRITE : 0;
- continue;
-
-fault:
- pte_unmap(ptep);
- /* Fault any virtual address we were ask to fault */
- return hmm_vma_walk_hole(start, end, walk);
}
pte_unmap(ptep - 1);
+ hmm_vma_walk->last = addr;
return 0;
}
--
2.14.3
next prev parent reply other threads:[~2018-03-20 2:01 UTC|newest]
Thread overview: 47+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-03-20 2:00 [PATCH 00/15] hmm: fixes and documentations v3 jglisse
2018-03-20 2:00 ` [PATCH 01/15] mm/hmm: documentation editorial update to HMM documentation jglisse
2018-03-20 2:00 ` [PATCH 02/15] mm/hmm: fix header file if/else/endif maze v2 jglisse
2018-03-20 2:00 ` [PATCH 03/15] mm/hmm: HMM should have a callback before MM is destroyed v2 jglisse
2018-03-21 4:14 ` John Hubbard
2018-03-21 18:03 ` Jerome Glisse
2018-03-21 22:16 ` John Hubbard
2018-03-21 22:46 ` Jerome Glisse
2018-03-21 23:10 ` John Hubbard
2018-03-21 23:37 ` Jerome Glisse
2018-03-22 0:11 ` John Hubbard
2018-03-22 1:32 ` Jerome Glisse
2018-03-22 1:28 ` [PATCH 03/15] mm/hmm: HMM should have a callback before MM is destroyed v3 jglisse
2018-03-22 6:58 ` John Hubbard
2018-03-20 2:00 ` [PATCH 04/15] mm/hmm: unregister mmu_notifier when last HMM client quit jglisse
2018-03-21 4:24 ` John Hubbard
2018-03-21 18:12 ` Jerome Glisse
2018-03-21 18:16 ` [PATCH 04/15] mm/hmm: unregister mmu_notifier when last HMM client quit v2 jglisse
2018-03-21 23:22 ` John Hubbard
2018-03-21 23:41 ` Jerome Glisse
2018-03-22 22:47 ` John Hubbard
2018-03-22 23:37 ` Jerome Glisse
2018-03-23 0:13 ` John Hubbard
2018-03-23 0:50 ` Jerome Glisse
2018-03-23 0:56 ` John Hubbard
2018-03-22 1:30 ` [PATCH 04/15] mm/hmm: unregister mmu_notifier when last HMM client quit v3 jglisse
2018-03-22 22:36 ` Andrew Morton
2018-03-20 2:00 ` [PATCH 05/15] mm/hmm: hmm_pfns_bad() was accessing wrong struct jglisse
2018-03-20 2:00 ` [PATCH 06/15] mm/hmm: use struct for hmm_vma_fault(), hmm_vma_get_pfns() parameters v2 jglisse
2018-03-20 2:00 ` [PATCH 07/15] mm/hmm: remove HMM_PFN_READ flag and ignore peculiar architecture v2 jglisse
2018-03-20 2:00 ` [PATCH 08/15] mm/hmm: use uint64_t for HMM pfn instead of defining hmm_pfn_t to ulong v2 jglisse
2018-03-20 2:00 ` [PATCH 09/15] mm/hmm: cleanup special vma handling (VM_SPECIAL) jglisse
2018-03-20 2:00 ` [PATCH 10/15] mm/hmm: do not differentiate between empty entry or missing directory v2 jglisse
2018-03-21 5:24 ` John Hubbard
2018-03-21 14:48 ` Jerome Glisse
2018-03-21 23:16 ` John Hubbard
2018-03-20 2:00 ` [PATCH 11/15] mm/hmm: rename HMM_PFN_DEVICE_UNADDRESSABLE to HMM_PFN_DEVICE_PRIVATE jglisse
2018-03-20 2:00 ` [PATCH 12/15] mm/hmm: move hmm_pfns_clear() closer to where it is use jglisse
2018-03-20 2:00 ` jglisse [this message]
2018-03-21 5:07 ` [PATCH 13/15] mm/hmm: factor out pte and pmd handling to simplify hmm_vma_walk_pmd() John Hubbard
2018-03-21 15:08 ` Jerome Glisse
2018-03-21 22:36 ` John Hubbard
2018-03-20 2:00 ` [PATCH 14/15] mm/hmm: change hmm_vma_fault() to allow write fault on page basis jglisse
2018-03-20 2:00 ` [PATCH 15/15] mm/hmm: use device driver encoding for HMM pfn v2 jglisse
2018-03-21 4:39 ` John Hubbard
2018-03-21 15:52 ` Jerome Glisse
2018-03-21 23:19 ` John Hubbard
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20180320020038.3360-14-jglisse@redhat.com \
--to=jglisse@redhat.com \
--cc=akpm@linux-foundation.org \
--cc=ebaskakov@nvidia.com \
--cc=jhubbard@nvidia.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
--cc=mhairgrove@nvidia.com \
--cc=rcampbell@nvidia.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).