From: Nicholas Piggin <npiggin@gmail.com>
To: linuxppc-dev@lists.ozlabs.org
Cc: Nicholas Piggin <npiggin@gmail.com>,
"Aneesh Kumar K . V" <aneesh.kumar@linux.vnet.ibm.com>,
Christophe Leroy <christophe.leroy@c-s.fr>
Subject: [PATCH v2 10/10] powerpc/mm/slice: use the dynamic high slice size to limit bitmap operations
Date: Wed, 7 Mar 2018 11:37:18 +1000 [thread overview]
Message-ID: <20180307013718.13749-11-npiggin@gmail.com> (raw)
In-Reply-To: <20180307013718.13749-1-npiggin@gmail.com>
The number of high slices a process might use now depends on its
address space size, and what allocation address it has requested.
This patch uses that limit throughout call chains where possible,
rather than use the fixed SLICE_NUM_HIGH for bitmap operations.
This saves some cost for processes that don't use very large address
spaces.
Perormance numbers aren't changed significantly, this may change
with larger address spaces or different mmap access patterns that
require more slice mask building.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/mm/slice.c | 75 +++++++++++++++++++++++++++++--------------------
1 file changed, 45 insertions(+), 30 deletions(-)
diff --git a/arch/powerpc/mm/slice.c b/arch/powerpc/mm/slice.c
index 1297b3ad7dd2..09b95e976de9 100644
--- a/arch/powerpc/mm/slice.c
+++ b/arch/powerpc/mm/slice.c
@@ -61,14 +61,12 @@ static void slice_print_mask(const char *label, const struct slice_mask *mask) {
#endif
static void slice_range_to_mask(unsigned long start, unsigned long len,
- struct slice_mask *ret)
+ struct slice_mask *ret,
+ unsigned long high_slices)
{
unsigned long end = start + len - 1;
ret->low_slices = 0;
- if (SLICE_NUM_HIGH)
- bitmap_zero(ret->high_slices, SLICE_NUM_HIGH);
-
if (start < SLICE_LOW_TOP) {
unsigned long mend = min(end,
(unsigned long)(SLICE_LOW_TOP - 1));
@@ -77,6 +75,11 @@ static void slice_range_to_mask(unsigned long start, unsigned long len,
- (1u << GET_LOW_SLICE_INDEX(start));
}
+ if (!SLICE_NUM_HIGH)
+ return;
+
+#error XXX: bitmap_zero with non-const size? Good code?
+ bitmap_zero(ret->high_slices, high_slices);
if ((start + len) > SLICE_LOW_TOP) {
unsigned long start_index = GET_HIGH_SLICE_INDEX(start);
unsigned long align_end = ALIGN(end, (1UL << SLICE_HIGH_SHIFT));
@@ -120,22 +123,20 @@ static int slice_high_has_vma(struct mm_struct *mm, unsigned long slice)
}
static void slice_mask_for_free(struct mm_struct *mm, struct slice_mask *ret,
- unsigned long high_limit)
+ unsigned long high_slices)
{
unsigned long i;
ret->low_slices = 0;
- if (SLICE_NUM_HIGH)
- bitmap_zero(ret->high_slices, SLICE_NUM_HIGH);
-
for (i = 0; i < SLICE_NUM_LOW; i++)
if (!slice_low_has_vma(mm, i))
ret->low_slices |= 1u << i;
- if (high_limit <= SLICE_LOW_TOP)
+ if (!SLICE_NUM_HIGH || !high_slices)
return;
- for (i = 0; i < GET_HIGH_SLICE_INDEX(high_limit); i++)
+ bitmap_zero(ret->high_slices, high_slices);
+ for (i = 0; i < high_slices; i++)
if (!slice_high_has_vma(mm, i))
__set_bit(i, ret->high_slices);
}
@@ -228,6 +229,7 @@ static void slice_convert(struct mm_struct *mm,
{
int index, mask_index;
/* Write the new slice psize bits */
+ unsigned long high_slices;
unsigned char *hpsizes, *lpsizes;
struct slice_mask *psize_mask, *old_mask;
unsigned long i, flags;
@@ -263,7 +265,8 @@ static void slice_convert(struct mm_struct *mm,
}
hpsizes = mm->context.high_slices_psize;
- for (i = 0; i < GET_HIGH_SLICE_INDEX(mm->context.slb_addr_limit); i++) {
+ high_slices = GET_HIGH_SLICE_INDEX(mm->context.slb_addr_limit);
+ for (i = 0; SLICE_NUM_HIGH && i < high_slices; i++) {
if (!test_bit(i, mask->high_slices))
continue;
@@ -430,32 +433,35 @@ static unsigned long slice_find_area(struct mm_struct *mm, unsigned long len,
}
static inline void slice_copy_mask(struct slice_mask *dst,
- const struct slice_mask *src)
+ const struct slice_mask *src,
+ unsigned long high_slices)
{
dst->low_slices = src->low_slices;
if (!SLICE_NUM_HIGH)
return;
- bitmap_copy(dst->high_slices, src->high_slices, SLICE_NUM_HIGH);
+ bitmap_copy(dst->high_slices, src->high_slices, high_slices);
}
static inline void slice_or_mask(struct slice_mask *dst,
const struct slice_mask *src1,
- const struct slice_mask *src2)
+ const struct slice_mask *src2,
+ unsigned long high_slices)
{
dst->low_slices = src1->low_slices | src2->low_slices;
if (!SLICE_NUM_HIGH)
return;
- bitmap_or(dst->high_slices, src1->high_slices, src2->high_slices, SLICE_NUM_HIGH);
+ bitmap_or(dst->high_slices, src1->high_slices, src2->high_slices, high_slices);
}
static inline void slice_andnot_mask(struct slice_mask *dst,
const struct slice_mask *src1,
- const struct slice_mask *src2)
+ const struct slice_mask *src2,
+ unsigned long high_slices)
{
dst->low_slices = src1->low_slices & ~src2->low_slices;
if (!SLICE_NUM_HIGH)
return;
- bitmap_andnot(dst->high_slices, src1->high_slices, src2->high_slices, SLICE_NUM_HIGH);
+ bitmap_andnot(dst->high_slices, src1->high_slices, src2->high_slices, high_slices);
}
#ifdef CONFIG_PPC_64K_PAGES
@@ -478,6 +484,7 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
struct mm_struct *mm = current->mm;
unsigned long newaddr;
unsigned long high_limit;
+ unsigned long high_slices;
high_limit = DEFAULT_MAP_WINDOW;
if (addr >= high_limit || (fixed && (addr + len > high_limit)))
@@ -494,6 +501,7 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
return -ENOMEM;
}
+ high_slices = GET_HIGH_SLICE_INDEX(high_limit);
if (high_limit > mm->context.slb_addr_limit) {
/*
* Increasing the slb_addr_limit does not require
@@ -556,11 +564,11 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
if (IS_ENABLED(CONFIG_PPC_64K_PAGES) && psize == MMU_PAGE_64K) {
compat_maskp = slice_mask_for_size(mm, MMU_PAGE_4K);
if (fixed)
- slice_or_mask(&good_mask, maskp, compat_maskp);
+ slice_or_mask(&good_mask, maskp, compat_maskp, high_slices);
else
- slice_copy_mask(&good_mask, maskp);
+ slice_copy_mask(&good_mask, maskp, high_slices);
} else {
- slice_copy_mask(&good_mask, maskp);
+ slice_copy_mask(&good_mask, maskp, high_slices);
}
slice_print_mask(" good_mask", &good_mask);
@@ -594,8 +602,8 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
* We don't fit in the good mask, check what other slices are
* empty and thus can be converted
*/
- slice_mask_for_free(mm, &potential_mask, high_limit);
- slice_or_mask(&potential_mask, &potential_mask, &good_mask);
+ slice_mask_for_free(mm, &potential_mask, high_slices);
+ slice_or_mask(&potential_mask, &potential_mask, &good_mask, high_slices);
slice_print_mask(" potential", &potential_mask);
if (addr != 0 || fixed) {
@@ -632,7 +640,7 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
#ifdef CONFIG_PPC_64K_PAGES
if (addr == -ENOMEM && psize == MMU_PAGE_64K) {
/* retry the search with 4k-page slices included */
- slice_or_mask(&potential_mask, &potential_mask, compat_maskp);
+ slice_or_mask(&potential_mask, &potential_mask, compat_maskp, high_slices);
addr = slice_find_area(mm, len, &potential_mask,
psize, topdown, high_limit);
}
@@ -641,17 +649,18 @@ unsigned long slice_get_unmapped_area(unsigned long addr, unsigned long len,
if (addr == -ENOMEM)
return -ENOMEM;
- slice_range_to_mask(addr, len, &potential_mask);
+ slice_range_to_mask(addr, len, &potential_mask, high_slices);
slice_dbg(" found potential area at 0x%lx\n", addr);
slice_print_mask(" mask", &potential_mask);
convert:
- slice_andnot_mask(&potential_mask, &potential_mask, &good_mask);
+ slice_andnot_mask(&potential_mask, &potential_mask, &good_mask, high_slices);
if (compat_maskp && !fixed)
- slice_andnot_mask(&potential_mask, &potential_mask, compat_maskp);
+ slice_andnot_mask(&potential_mask, &potential_mask, compat_maskp, high_slices);
+#error XXX: are we sure high_slices is > 0 here when SLICE_NUM_HIGH? Perhaps not for 32-bit tasks on 64?
if (potential_mask.low_slices ||
(SLICE_NUM_HIGH &&
- !bitmap_empty(potential_mask.high_slices, SLICE_NUM_HIGH))) {
+ !bitmap_empty(potential_mask.high_slices, high_slices))) {
slice_convert(mm, &potential_mask, psize);
if (psize > MMU_PAGE_BASE)
on_each_cpu(slice_flush_segments, mm, 1);
@@ -722,7 +731,9 @@ void slice_init_new_context_exec(struct mm_struct *mm)
mm->context.user_psize = psize;
/*
- * Set all slice psizes to the default.
+ * Set all slice psizes to the default. High slices could
+ * be initialised up to slb_addr_limit if we ensure to
+ * initialise the rest of them as slb_addr_limit is expanded.
*/
lpsizes = mm->context.low_slices_psize;
memset(lpsizes, (psize << 4) | psize, SLICE_NUM_LOW >> 1);
@@ -743,10 +754,12 @@ void slice_set_range_psize(struct mm_struct *mm, unsigned long start,
unsigned long len, unsigned int psize)
{
struct slice_mask mask;
+ unsigned long high_slices;
VM_BUG_ON(radix_enabled());
- slice_range_to_mask(start, len, &mask);
+ high_slices = GET_HIGH_SLICE_INDEX(mm->context.slb_addr_limit);
+ slice_range_to_mask(start, len, &mask, high_slices);
slice_convert(mm, &mask, psize);
}
@@ -784,9 +797,11 @@ int slice_is_hugepage_only_range(struct mm_struct *mm, unsigned long addr,
if (psize == MMU_PAGE_64K) {
const struct slice_mask *compat_maskp;
struct slice_mask available;
+ unsigned long high_slices;
compat_maskp = slice_mask_for_size(mm, MMU_PAGE_4K);
- slice_or_mask(&available, maskp, compat_maskp);
+ high_slices = GET_HIGH_SLICE_INDEX(mm->context.slb_addr_limit);
+ slice_or_mask(&available, maskp, compat_maskp, high_slices);
return !slice_check_range_fits(mm, &available, addr, len);
}
#endif
--
2.16.1
next prev parent reply other threads:[~2018-03-07 1:38 UTC|newest]
Thread overview: 14+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-03-07 1:37 [PATCH v2 00/10] powerpc/mm/slice: improve slice speed and stack use Nicholas Piggin
2018-03-07 1:37 ` [PATCH v2 01/10] powerpc/mm/slice: Simplify and optimise slice context initialisation Nicholas Piggin
2018-03-14 9:28 ` [v2, " Michael Ellerman
2018-03-07 1:37 ` [PATCH v2 02/10] powerpc/mm/slice: tidy lpsizes and hpsizes update loops Nicholas Piggin
2018-03-07 1:37 ` [PATCH v2 03/10] powerpc/mm/slice: pass pointers to struct slice_mask where possible Nicholas Piggin
2018-03-07 1:37 ` [PATCH v2 04/10] powerpc/mm/slice: implement a slice mask cache Nicholas Piggin
2018-03-07 1:37 ` [PATCH v2 05/10] powerpc/mm/slice: implement slice_check_range_fits Nicholas Piggin
2018-03-07 1:37 ` [PATCH v2 06/10] powerpc/mm/slice: Switch to 3-operand slice bitops helpers Nicholas Piggin
2018-03-07 1:37 ` [PATCH v2 07/10] powerpc/mm/slice: remove dead code Nicholas Piggin
2018-03-07 1:37 ` [PATCH v2 08/10] powerpc/mm/slice: Use const pointers to cached slice masks where possible Nicholas Piggin
2018-03-07 1:37 ` [PATCH v2 09/10] powerpc/mm/slice: remove radix calls to the slice code Nicholas Piggin
2018-03-07 1:37 ` Nicholas Piggin [this message]
2018-03-07 3:22 ` [PATCH v2 10/10] powerpc/mm/slice: use the dynamic high slice size to limit bitmap operations Nicholas Piggin
2018-03-07 10:45 ` Michael Ellerman
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20180307013718.13749-11-npiggin@gmail.com \
--to=npiggin@gmail.com \
--cc=aneesh.kumar@linux.vnet.ibm.com \
--cc=christophe.leroy@c-s.fr \
--cc=linuxppc-dev@lists.ozlabs.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).