From: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
To: benh@kernel.crashing.org, paulus@samba.org, mpe@ellerman.id.au
Cc: linuxppc-dev@lists.ozlabs.org,
"Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Subject: [PATCH 04/16] powerpc/mm: Add hash invalidate callback
Date: Fri, 27 Oct 2017 09:38:21 +0530 [thread overview]
Message-ID: <20171027040833.3644-5-aneesh.kumar@linux.vnet.ibm.com> (raw)
In-Reply-To: <20171027040833.3644-1-aneesh.kumar@linux.vnet.ibm.com>
Add hash based invalidate callback and use that in flush_hash_page.
Note: In a later patch, we will drop the slot tracking completely. At that point
we will also loose the __rpte_sub_valid() check in
pte_iterate_hashed_subpages(). That means we call the invalidate for all
subpages irrespective of whether we took a hash fault on that or not.
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/book3s/64/mmu-hash.h | 4 ++
arch/powerpc/mm/hash_native_64.c | 27 ++++++++++++
arch/powerpc/mm/hash_utils_64.c | 11 ++---
arch/powerpc/platforms/ps3/htab.c | 59 +++++++++++++++++++++++++++
arch/powerpc/platforms/pseries/lpar.c | 26 ++++++++++++
5 files changed, 119 insertions(+), 8 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
index 508275bb05d5..79f141e721ee 100644
--- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h
@@ -136,6 +136,10 @@ struct mmu_hash_ops {
unsigned long vpn,
int bpsize, int apsize,
int ssize, int local);
+ void (*hash_invalidate)(unsigned long hash,
+ unsigned long vpn,
+ int bpsize, int apsize,
+ int ssize, int local);
long (*hpte_updatepp)(unsigned long slot,
unsigned long newpp,
unsigned long vpn,
diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index 496b1680ba24..f473a78baab7 100644
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -497,6 +497,32 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
local_irq_restore(flags);
}
+static void native_hash_invalidate(unsigned long hash, unsigned long vpn,
+ int bpsize, int apsize, int ssize, int local)
+{
+ unsigned long flags;
+ struct hash_pte *hptep;
+
+ DBG_LOW(" invalidate(vpn=%016lx, hash: %lx)\n", vpn, hash);
+ local_irq_save(flags);
+ hptep = native_hpte_find(hash, vpn, bpsize, ssize);
+ if (hptep) {
+ /*
+ * Invalidate the hpte. NOTE: this also unlocks it
+ */
+ hptep->v = 0;
+ }
+ /*
+ * We need to invalidate the TLB always because hpte_remove doesn't do
+ * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
+ * random entry from it. When we do that we don't invalidate the TLB
+ * (hpte_remove) because we assume the old translation is still
+ * technically "valid".
+ */
+ tlbie(vpn, bpsize, apsize, ssize, local);
+ local_irq_restore(flags);
+}
+
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static void native_hugepage_invalidate(unsigned long vsid,
unsigned long addr,
@@ -776,6 +802,7 @@ static int native_register_proc_table(unsigned long base, unsigned long page_siz
void __init hpte_init_native(void)
{
mmu_hash_ops.hpte_invalidate = native_hpte_invalidate;
+ mmu_hash_ops.hash_invalidate = native_hash_invalidate;
mmu_hash_ops.hpte_updatepp = native_hpte_updatepp;
mmu_hash_ops.hpte_updateboltedpp = native_hpte_updateboltedpp;
mmu_hash_ops.hpte_removebolted = native_hpte_removebolted;
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 4d4662a77c14..b197fe57547e 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -1598,23 +1598,18 @@ static inline void tm_flush_hash_page(int local)
void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize,
unsigned long flags)
{
- unsigned long hash, index, shift, hidx, slot;
+ unsigned long hash, index, shift;
int local = flags & HPTE_LOCAL_UPDATE;
DBG_LOW("flush_hash_page(vpn=%016lx)\n", vpn);
pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) {
hash = hpt_hash(vpn, shift, ssize);
- hidx = __rpte_to_hidx(pte, index);
- if (hidx & _PTEIDX_SECONDARY)
- hash = ~hash;
- slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
- slot += hidx & _PTEIDX_GROUP_IX;
- DBG_LOW(" sub %ld: hash=%lx, hidx=%lx\n", index, slot, hidx);
+ DBG_LOW(" sub %ld: hash=%lx\n", index, hash);
/*
* We use same base page size and actual psize, because we don't
* use these functions for hugepage
*/
- mmu_hash_ops.hpte_invalidate(slot, vpn, psize, psize,
+ mmu_hash_ops.hash_invalidate(hash, vpn, psize, psize,
ssize, local);
} pte_iterate_hashed_end();
diff --git a/arch/powerpc/platforms/ps3/htab.c b/arch/powerpc/platforms/ps3/htab.c
index cc2b281a3766..813c2f77f75d 100644
--- a/arch/powerpc/platforms/ps3/htab.c
+++ b/arch/powerpc/platforms/ps3/htab.c
@@ -193,9 +193,68 @@ static void ps3_hpte_clear(void)
ps3_mm_vas_destroy();
}
+static long ps3_hpte_find(unsigned long hash, unsigned long want_v)
+{
+ unsigned long i, j, result;
+ unsigned long hpte_group;
+ bool secondary_search = false;
+ u64 hpte_v_array[4], hpte_rs;
+
+
+ /* first check primary */
+ hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP;
+
+search_again:
+ for (i = 0; i < HPTES_PER_GROUP; i += 4, hpte_group += 4) {
+
+ result = lv1_read_htab_entries(PS3_LPAR_VAS_ID_CURRENT,
+ hpte_group & ~0x3UL, &hpte_v_array[0],
+ &hpte_v_array[1], &hpte_v_array[2],
+ &hpte_v_array[3], &hpte_rs);
+ /* ignore failures ? */
+ if (result)
+ continue;
+
+ for (j = 0; j < 4; j++) {
+ if (HPTE_V_COMPARE(hpte_v_array[j], want_v) &&
+ (hpte_v_array[j] & HPTE_V_VALID)) {
+ return hpte_group + j;
+ }
+ }
+ }
+ if (!secondary_search) {
+ hpte_group = (~hash & htab_hash_mask) * HPTES_PER_GROUP;
+ secondary_search = true;
+ goto search_again;
+ }
+ return -1;
+}
+
+static void ps3_hash_invalidate(unsigned long hash, unsigned long vpn,
+ int psize, int apsize, int ssize, int local)
+{
+ long slot;
+ unsigned long flags;
+ unsigned long want_v;
+
+ want_v = hpte_encode_avpn(vpn, psize, ssize);
+
+ spin_lock_irqsave(&ps3_htab_lock, flags);
+ slot = ps3_hpte_find(hash, want_v);
+ if (slot < 0)
+ /* HPTE not found */
+ goto err_out;
+ /* invalidate the entry */
+ lv1_write_htab_entry(PS3_LPAR_VAS_ID_CURRENT, slot, 0, 0);
+err_out:
+ spin_unlock_irqrestore(&ps3_htab_lock, flags);
+ return;
+}
+
void __init ps3_hpte_init(unsigned long htab_size)
{
mmu_hash_ops.hpte_invalidate = ps3_hpte_invalidate;
+ mmu_hash_ops.hash_invalidate = ps3_hash_invalidate;
mmu_hash_ops.hpte_updatepp = ps3_hpte_updatepp;
mmu_hash_ops.hpte_updateboltedpp = ps3_hpte_updateboltedpp;
mmu_hash_ops.hpte_insert = ps3_hpte_insert;
diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c
index edab68d9f9f3..e366252e0e93 100644
--- a/arch/powerpc/platforms/pseries/lpar.c
+++ b/arch/powerpc/platforms/pseries/lpar.c
@@ -419,6 +419,31 @@ static void pSeries_lpar_hpte_invalidate(unsigned long slot, unsigned long vpn,
BUG_ON(lpar_rc != H_SUCCESS);
}
+static void pSeries_lpar_hash_invalidate(unsigned long hash, unsigned long vpn,
+ int psize, int apsize,
+ int ssize, int local)
+{
+ long slot;
+ unsigned long want_v;
+ unsigned long lpar_rc;
+ unsigned long dummy1, dummy2;
+
+ pr_devel(" inval : hash=%lx, vpn=%016lx, psize: %d, local: %d\n",
+ hash, vpn, psize, local);
+
+ want_v = hpte_encode_avpn(vpn, psize, ssize);
+ slot = __pSeries_lpar_hpte_find(hash, want_v);
+ if (slot < 0)
+ /* HPTE not found */
+ return;
+ lpar_rc = plpar_pte_remove(H_AVPN, slot, want_v, &dummy1, &dummy2);
+ if (lpar_rc == H_NOT_FOUND)
+ return;
+
+ BUG_ON(lpar_rc != H_SUCCESS);
+}
+
+
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
/*
* Limit iterations holding pSeries_lpar_tlbie_lock to 3. We also need
@@ -758,6 +783,7 @@ static int pseries_lpar_register_process_table(unsigned long base,
void __init hpte_init_pseries(void)
{
mmu_hash_ops.hpte_invalidate = pSeries_lpar_hpte_invalidate;
+ mmu_hash_ops.hash_invalidate = pSeries_lpar_hash_invalidate;
mmu_hash_ops.hpte_updatepp = pSeries_lpar_hpte_updatepp;
mmu_hash_ops.hpte_updateboltedpp = pSeries_lpar_hpte_updateboltedpp;
mmu_hash_ops.hpte_insert = pSeries_lpar_hpte_insert;
--
2.13.6
next prev parent reply other threads:[~2017-10-27 4:09 UTC|newest]
Thread overview: 31+ messages / expand[flat|nested] mbox.gz Atom feed top
2017-10-27 4:08 [PATCH 00/16] Remove hash page table slot tracking from linux PTE Aneesh Kumar K.V
2017-10-27 4:08 ` [PATCH 01/16] powerpc/mm/hash: Remove the superfluous bitwise operation when find hpte group Aneesh Kumar K.V
2017-10-27 4:08 ` [PATCH 02/16] powerpc/mm: Update native_hpte_find to return hash pte Aneesh Kumar K.V
2017-10-27 4:08 ` [PATCH 03/16] powerpc/pseries: Update hpte find helper to take hash value Aneesh Kumar K.V
2017-10-27 4:08 ` Aneesh Kumar K.V [this message]
2017-10-27 4:08 ` [PATCH 05/16] powerpc/mm: use hash_invalidate for __kernel_map_pages() Aneesh Kumar K.V
2017-10-27 4:08 ` [PATCH 06/16] powerpc/mm: Switch flush_hash_range to not use slot Aneesh Kumar K.V
2017-10-27 4:08 ` [PATCH 07/16] powerpc/mm: Add hash updatepp callback Aneesh Kumar K.V
2017-10-27 4:08 ` [PATCH 08/16] powerpc/mm/hash: Don't track hash pte slot number in linux page table Aneesh Kumar K.V
2017-10-27 4:08 ` [PATCH 09/16] powerpc/mm: Add new firmware feature HASH API Aneesh Kumar K.V
2017-10-27 4:08 ` [PATCH 10/16] powerpc/kvm/hash: Implement HASH_REMOVE hcall Aneesh Kumar K.V
2017-10-27 4:08 ` [PATCH 11/16] powerpc/kvm/hash: Implement HASH_PROTECT hcall Aneesh Kumar K.V
2017-10-27 4:08 ` [PATCH 12/16] powerpc/kvm/hash: Implement HASH_BULK_REMOVE hcall Aneesh Kumar K.V
2017-10-27 4:08 ` [PATCH 13/16] powerpc/mm/pseries: Use HASH_PROTECT hcall in guest Aneesh Kumar K.V
2017-10-27 4:08 ` [PATCH 14/16] powerpc/mm/pseries: Use HASH_REMOVE " Aneesh Kumar K.V
2017-10-27 4:08 ` [PATCH 15/16] powerpc/mm/pseries: Move slot based bulk remove to helper Aneesh Kumar K.V
2017-10-27 4:08 ` [PATCH 16/16] powerpc/mm/pseries: Use HASH_BULK_REMOVE hcall in guest Aneesh Kumar K.V
2017-10-27 4:34 ` [PATCH 00/16] Remove hash page table slot tracking from linux PTE Paul Mackerras
2017-10-27 5:27 ` Aneesh Kumar K.V
2017-10-27 5:41 ` Paul Mackerras
2017-10-30 7:57 ` Aneesh Kumar K.V
2017-10-30 11:49 ` Aneesh Kumar K.V
2017-10-30 13:14 ` Aneesh Kumar K.V
2017-10-30 13:49 ` Aneesh Kumar K.V
2017-11-21 8:41 ` Aneesh Kumar K.V
2017-10-28 22:35 ` Ram Pai
2017-10-29 14:05 ` Aneesh Kumar K.V
2017-10-29 22:04 ` Paul Mackerras
2017-10-30 0:51 ` Ram Pai
2017-11-01 4:46 ` Michael Ellerman
2017-11-01 11:02 ` Paul Mackerras
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20171027040833.3644-5-aneesh.kumar@linux.vnet.ibm.com \
--to=aneesh.kumar@linux.vnet.ibm.com \
--cc=benh@kernel.crashing.org \
--cc=linuxppc-dev@lists.ozlabs.org \
--cc=mpe@ellerman.id.au \
--cc=paulus@samba.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.