From: "Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
To: benh@kernel.crashing.org, paulus@samba.org, mpe@ellerman.id.au,
Scott Wood <scottwood@freescale.com>,
Denis Kirjanov <kda@linux-powerpc.org>
Cc: linuxppc-dev@lists.ozlabs.org,
"Aneesh Kumar K.V" <aneesh.kumar@linux.vnet.ibm.com>
Subject: [PATCH V2 10/10] powerpc/mm: Optmize the hashed subpage iteration
Date: Mon, 23 Nov 2015 16:03:45 +0530 [thread overview]
Message-ID: <1448274825-30289-11-git-send-email-aneesh.kumar@linux.vnet.ibm.com> (raw)
In-Reply-To: <1448274825-30289-1-git-send-email-aneesh.kumar@linux.vnet.ibm.com>
If we have _PAGE_COMBO set, we override the _PAGE_F_GIX_SHIFT
and _PAGE_F_SECOND. Together we have 4 bits, each of them
used to indicate whether any of the 4 4k subpage in that group
is valid. ie,
[ group 1 bit ] [ group 2 bit ] ..... [ group 4 ]
[ subpage 1 - 4] [ subpage 5- 8] ..... [ subpage 13 - 16]
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/book3s/64/hash-64k.h | 9 ++++-
arch/powerpc/include/asm/book3s/64/pgtable.h | 6 ++--
arch/powerpc/mm/hash64_64k.c | 51 ++++++++++++++++++++-------
arch/powerpc/mm/hash_native_64.c | 12 ++-----
arch/powerpc/mm/hash_utils_64.c | 2 +-
arch/powerpc/platforms/pseries/lpar.c | 2 +-
6 files changed, 55 insertions(+), 27 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h
index 5f18801ae722..9ae5eb82fb85 100644
--- a/arch/powerpc/include/asm/book3s/64/hash-64k.h
+++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h
@@ -27,6 +27,11 @@
#define _PAGE_COMBO 0x00020000 /* this is a combo 4k page */
#define _PAGE_4K_PFN 0x00040000 /* PFN is for a single 4k page */
+/*
+ * Used to track subpage group valid if _PAGE_COMBO is set
+ * This overloads _PAGE_F_GIX and _PAGE_F_SECOND
+ */
+#define _PAGE_COMBO_VALID (_PAGE_F_GIX | _PAGE_F_SECOND)
/* PTE flags to conserve for HPTE identification */
#define _PAGE_HPTEFLAGS (_PAGE_BUSY | _PAGE_F_SECOND | \
@@ -66,17 +71,19 @@
#define pte_to_hidx pte_to_hidx
extern unsigned long pte_to_hidx(pte_t pte, unsigned long hash,
unsigned long vpn, int ssize, bool *valid);
+extern bool pte_or_subptegroup_valid(pte_t pte, unsigned long index);
/*
* Trick: we set __end to va + 64k, which happens works for
* a 16M page as well as we want only one iteration
*/
-#define pte_iterate_hashed_subpages(vpn, psize, shift) \
+#define pte_iterate_hashed_subpages(pte, vpn, psize, shift) \
do { \
unsigned long index; \
unsigned long __end = vpn + (1UL << (PAGE_SHIFT - VPN_SHIFT)); \
shift = mmu_psize_defs[psize].shift; \
for (index = 0; vpn < __end; index++, \
vpn += (1L << (shift - VPN_SHIFT))) { \
+ if (pte_or_subptegroup_valid(pte, index)) \
do {
#define pte_iterate_hashed_end() } while(0); } } while(0)
diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h
index 74be69c8e5de..9000884cf715 100644
--- a/arch/powerpc/include/asm/book3s/64/pgtable.h
+++ b/arch/powerpc/include/asm/book3s/64/pgtable.h
@@ -55,9 +55,9 @@ static inline unsigned long pte_to_hidx(pte_t pte, unsigned long hash,
#endif
#ifndef pte_iterate_hashed_subpages
-#define pte_iterate_hashed_subpages(vpn, psize, shift) \
- do { \
- shift = mmu_psize_defs[psize].shift; \
+#define pte_iterate_hashed_subpages(pte, vpn, psize, shift) \
+ do { \
+ shift = mmu_psize_defs[psize].shift; \
#define pte_iterate_hashed_end() } while(0)
#endif
diff --git a/arch/powerpc/mm/hash64_64k.c b/arch/powerpc/mm/hash64_64k.c
index c0bed3d01c1c..4c38ccd1e52f 100644
--- a/arch/powerpc/mm/hash64_64k.c
+++ b/arch/powerpc/mm/hash64_64k.c
@@ -16,6 +16,43 @@
#include <asm/machdep.h>
#include <asm/mmu.h>
+/*
+ * index from 0 - 15
+ */
+bool pte_or_subptegroup_valid(pte_t pte, unsigned long index)
+{
+ unsigned long ptev = pte_val(pte);
+
+ if (!(ptev & _PAGE_HASHPTE))
+ return false;
+ if (ptev & _PAGE_COMBO) {
+ unsigned long g_idx;
+
+ g_idx = (ptev & _PAGE_COMBO_VALID) >> _PAGE_F_GIX_SHIFT;
+ index = index >> 2;
+ if (g_idx & (0x1 << index))
+ return true;
+ else
+ return false;
+ }
+ return true;
+}
+
+/*
+ * index from 0 - 15
+ */
+static unsigned long mark_subptegroup_valid(unsigned long ptev, unsigned long index)
+{
+ unsigned long g_idx;
+
+ if (!(ptev & _PAGE_COMBO))
+ return ptev;
+ index = index >> 2;
+ g_idx = 0x1 << index;
+
+ return ptev | (g_idx << _PAGE_F_GIX_SHIFT);
+}
+
unsigned long pte_to_hidx(pte_t pte, unsigned long hash,
unsigned long vpn, int ssize, bool *valid)
{
@@ -182,18 +219,8 @@ repeat:
MMU_PAGE_4K, MMU_PAGE_4K, old_pte);
return -1;
}
- /*
- * Insert slot number & secondary bit in PTE second half,
- * clear _PAGE_BUSY and set appropriate HPTE slot bit
- * Since we have _PAGE_BUSY set on ptep, we can be sure
- * nobody is undating hidx.
- */
- new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HASHPTE | _PAGE_COMBO;
- /*
- * check __real_pte for details on matching smp_rmb()
- * FIXME!! We can possibly get rid of this ?
- */
- smp_wmb();
+ new_pte = mark_subptegroup_valid(new_pte, subpg_index);
+ new_pte |= _PAGE_HASHPTE;
*ptep = __pte(new_pte & ~_PAGE_BUSY);
return 0;
}
diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index 9f7f6673e726..9bd0c6f505f0 100644
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -664,7 +664,7 @@ static void native_flush_hash_range(unsigned long number, int local)
vpn = batch->vpn[i];
pte = batch->pte[i];
- pte_iterate_hashed_subpages(vpn, psize, shift) {
+ pte_iterate_hashed_subpages(pte, vpn, psize, shift) {
hash = hpt_hash(vpn, shift, ssize);
hidx = pte_to_hidx(pte, hash, vpn, ssize, &valid_slot);
if (!valid_slot)
@@ -692,10 +692,7 @@ static void native_flush_hash_range(unsigned long number, int local)
vpn = batch->vpn[i];
pte = batch->pte[i];
- pte_iterate_hashed_subpages(vpn, psize, shift) {
- /*
- * We are not looking at subpage valid here
- */
+ pte_iterate_hashed_subpages(pte, vpn, psize, shift) {
__tlbiel(vpn, psize, psize, ssize);
} pte_iterate_hashed_end();
}
@@ -711,10 +708,7 @@ static void native_flush_hash_range(unsigned long number, int local)
vpn = batch->vpn[i];
pte = batch->pte[i];
- pte_iterate_hashed_subpages(vpn, psize, shift) {
- /*
- * We are not looking at subpage valid here
- */
+ pte_iterate_hashed_subpages(pte, vpn, psize, shift) {
__tlbie(vpn, psize, psize, ssize);
} pte_iterate_hashed_end();
}
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 80e71ccc9474..d2fa41effa23 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -1302,7 +1302,7 @@ void flush_hash_page(unsigned long vpn, pte_t pte, int psize, int ssize,
int local = flags & HPTE_LOCAL_UPDATE;
DBG_LOW("flush_hash_page(vpn=%016lx)\n", vpn);
- pte_iterate_hashed_subpages(vpn, psize, shift) {
+ pte_iterate_hashed_subpages(pte, vpn, psize, shift) {
hash = hpt_hash(vpn, shift, ssize);
hidx = pte_to_hidx(pte, hash, vpn, ssize, &valid_slot);
if (!valid_slot)
diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c
index 1708cab20fc8..06af06420a35 100644
--- a/arch/powerpc/platforms/pseries/lpar.c
+++ b/arch/powerpc/platforms/pseries/lpar.c
@@ -549,7 +549,7 @@ static void pSeries_lpar_flush_hash_range(unsigned long number, int local)
vpn = batch->vpn[i];
pte = batch->pte[i];
- pte_iterate_hashed_subpages(vpn, psize, shift) {
+ pte_iterate_hashed_subpages(pte, vpn, psize, shift) {
hash = hpt_hash(vpn, shift, ssize);
hidx = pte_to_hidx(pte, hash, vpn, ssize, &valid_slot);
if (!valid_slot)
--
2.5.0
prev parent reply other threads:[~2015-11-23 10:35 UTC|newest]
Thread overview: 13+ messages / expand[flat|nested] mbox.gz Atom feed top
2015-11-23 10:33 [PATCH V2 00/10] Reduce the pte framgment size Aneesh Kumar K.V
2015-11-23 10:33 ` [PATCH V2 01/10] powerpc/mm: Don't hardcode page table size Aneesh Kumar K.V
2015-11-23 10:33 ` [PATCH V2 02/10] powerpc/mm: Don't hardcode the hash pte slot shift Aneesh Kumar K.V
2015-11-23 10:33 ` [PATCH V2 03/10] powerpc/nohash: Update 64K nohash config to have 32 pte fragement Aneesh Kumar K.V
2015-11-23 10:33 ` [PATCH V2 04/10] powerpc/nohash: we don't use real_pte_t for nohash Aneesh Kumar K.V
2015-11-23 10:33 ` [PATCH V2 05/10] powerpc/mm: Use H_READ with H_READ_4 Aneesh Kumar K.V
2015-11-23 10:33 ` [PATCH V2 06/10] powerpc/mm: Don't track 4k subpage information with 64k linux page size Aneesh Kumar K.V
2015-11-23 10:33 ` [PATCH V2 07/10] powerpc/mm: update PTE frag size Aneesh Kumar K.V
2015-11-27 7:27 ` Aneesh Kumar K.V
2015-11-27 11:56 ` Aneesh Kumar K.V
2015-11-23 10:33 ` [PATCH V2 08/10] powerpc/mm: Update pte_iterate_hashed_subpages args Aneesh Kumar K.V
2015-11-23 10:33 ` [PATCH V2 09/10] powerpc/mm: Drop real_pte_t usage Aneesh Kumar K.V
2015-11-23 10:33 ` Aneesh Kumar K.V [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1448274825-30289-11-git-send-email-aneesh.kumar@linux.vnet.ibm.com \
--to=aneesh.kumar@linux.vnet.ibm.com \
--cc=benh@kernel.crashing.org \
--cc=kda@linux-powerpc.org \
--cc=linuxppc-dev@lists.ozlabs.org \
--cc=mpe@ellerman.id.au \
--cc=paulus@samba.org \
--cc=scottwood@freescale.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).