* [PATCH 1/4] sparc64: Kill page table quicklists.
2011-07-12 12:26 [PATCH 0/4] mm, sparc64: Implement gup_fast() Peter Zijlstra
@ 2011-07-12 12:26 ` Peter Zijlstra
2011-07-13 22:31 ` Andrew Morton
2011-07-12 12:26 ` [PATCH 2/4] sparc64: Use RCU page table freeing Peter Zijlstra
` (3 subsequent siblings)
4 siblings, 1 reply; 10+ messages in thread
From: Peter Zijlstra @ 2011-07-12 12:26 UTC (permalink / raw)
To: Andrew Morton; +Cc: Linux-MM, LKML, David Miller, Peter Zijlstra
[-- Attachment #1: davem-sparc64-Kill_page_table_quicklists.patch --]
[-- Type: text/plain, Size: 4444 bytes --]
They are pointless and make it harder to use RCU page
table freeing and share code with other architectures.
BTW, this is the second time this has happened, see
commit 3c936465249f863f322154ff1aaa628b84ee5750
("[SPARC64]: Kill pgtable quicklists and use SLAB.")
:-)
Signed-off-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/r/20100408.180005.76674367.davem@davemloft.net
---
arch/sparc/Kconfig | 4 ----
arch/sparc/include/asm/pgalloc_64.h | 32 +++++++++++++++-----------------
arch/sparc/mm/tsb.c | 11 +++++++++++
3 files changed, 26 insertions(+), 21 deletions(-)
Index: linux-2.6/arch/sparc/Kconfig
===================================================================
--- linux-2.6.orig/arch/sparc/Kconfig
+++ linux-2.6/arch/sparc/Kconfig
@@ -81,10 +81,6 @@ config IOMMU_HELPER
bool
default y if SPARC64
-config QUICKLIST
- bool
- default y if SPARC64
-
config STACKTRACE_SUPPORT
bool
default y if SPARC64
Index: linux-2.6/arch/sparc/include/asm/pgalloc_64.h
===================================================================
--- linux-2.6.orig/arch/sparc/include/asm/pgalloc_64.h
+++ linux-2.6/arch/sparc/include/asm/pgalloc_64.h
@@ -5,7 +5,6 @@
#include <linux/sched.h>
#include <linux/mm.h>
#include <linux/slab.h>
-#include <linux/quicklist.h>
#include <asm/spitfire.h>
#include <asm/cpudata.h>
@@ -14,69 +13,68 @@
/* Page table allocation/freeing. */
+extern struct kmem_cache *pgtable_cache;
+
static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{
- return quicklist_alloc(0, GFP_KERNEL, NULL);
+ return kmem_cache_alloc(pgtable_cache, GFP_KERNEL);
}
static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
{
- quicklist_free(0, NULL, pgd);
+ kmem_cache_free(pgtable_cache, pgd);
}
#define pud_populate(MM, PUD, PMD) pud_set(PUD, PMD)
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
{
- return quicklist_alloc(0, GFP_KERNEL, NULL);
+ return kmem_cache_alloc(pgtable_cache,
+ GFP_KERNEL|__GFP_REPEAT);
}
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
{
- quicklist_free(0, NULL, pmd);
+ kmem_cache_free(pgtable_cache, pmd);
}
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
unsigned long address)
{
- return quicklist_alloc(0, GFP_KERNEL, NULL);
+ return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO);
}
static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
unsigned long address)
{
struct page *page;
- void *pg;
+ pte_t *pte;
- pg = quicklist_alloc(0, GFP_KERNEL, NULL);
- if (!pg)
+ pte = pte_alloc_one_kernel(mm, address);
+ if (!pte)
return NULL;
- page = virt_to_page(pg);
+ page = virt_to_page(pte);
pgtable_page_ctor(page);
return page;
}
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
{
- quicklist_free(0, NULL, pte);
+ free_page((unsigned long)pte);
}
static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
{
pgtable_page_dtor(ptepage);
- quicklist_free_page(0, NULL, ptepage);
+ __free_page(ptepage);
}
-
#define pmd_populate_kernel(MM, PMD, PTE) pmd_set(PMD, PTE)
#define pmd_populate(MM,PMD,PTE_PAGE) \
pmd_populate_kernel(MM,PMD,page_address(PTE_PAGE))
#define pmd_pgtable(pmd) pmd_page(pmd)
-static inline void check_pgt_cache(void)
-{
- quicklist_trim(0, NULL, 25, 16);
-}
+#define check_pgt_cache() do { } while (0)
#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte)
#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd)
Index: linux-2.6/arch/sparc/mm/tsb.c
===================================================================
--- linux-2.6.orig/arch/sparc/mm/tsb.c
+++ linux-2.6/arch/sparc/mm/tsb.c
@@ -236,6 +236,8 @@ static void setup_tsb_params(struct mm_s
}
}
+struct kmem_cache *pgtable_cache __read_mostly;
+
static struct kmem_cache *tsb_caches[8] __read_mostly;
static const char *tsb_cache_names[8] = {
@@ -253,6 +255,15 @@ void __init pgtable_cache_init(void)
{
unsigned long i;
+ pgtable_cache = kmem_cache_create("pgtable_cache",
+ PAGE_SIZE, PAGE_SIZE,
+ 0,
+ _clear_page);
+ if (!pgtable_cache) {
+ prom_printf("pgtable_cache_init(): Could not create!\n");
+ prom_halt();
+ }
+
for (i = 0; i < 8; i++) {
unsigned long size = 8192 << i;
const char *name = tsb_cache_names[i];
^ permalink raw reply [flat|nested] 10+ messages in thread* Re: [PATCH 1/4] sparc64: Kill page table quicklists.
2011-07-12 12:26 ` [PATCH 1/4] sparc64: Kill page table quicklists Peter Zijlstra
@ 2011-07-13 22:31 ` Andrew Morton
0 siblings, 0 replies; 10+ messages in thread
From: Andrew Morton @ 2011-07-13 22:31 UTC (permalink / raw)
To: Peter Zijlstra; +Cc: Linux-MM, LKML, David Miller
On Tue, 12 Jul 2011 14:26:09 +0200
Peter Zijlstra <a.p.zijlstra@chello.nl> wrote:
> --- linux-2.6.orig/arch/sparc/mm/tsb.c
> +++ linux-2.6/arch/sparc/mm/tsb.c
> @@ -236,6 +236,8 @@ static void setup_tsb_params(struct mm_s
> }
> }
>
> +struct kmem_cache *pgtable_cache __read_mostly;
> +
> static struct kmem_cache *tsb_caches[8] __read_mostly;
>
> static const char *tsb_cache_names[8] = {
> @@ -253,6 +255,15 @@ void __init pgtable_cache_init(void)
> {
> unsigned long i;
>
> + pgtable_cache = kmem_cache_create("pgtable_cache",
> + PAGE_SIZE, PAGE_SIZE,
> + 0,
> + _clear_page);
The use of slab constructors is often dubious from a cache usage POV.
But the lifecycle of a page-table page might well be that it slowly
gets non-zeroes written into it and then slowly gets zeroes written
into it until it is all-zeroes and then we free it up. And often only
a subset of the page will ever be written.
So it could be that the slab constructor behaviour is a good match
here. And not just for sparc!
Did such thinking and/or any testing go into this decision?
> + if (!pgtable_cache) {
> + prom_printf("pgtable_cache_init(): Could not create!\n");
> + prom_halt();
> + }
> +
^ permalink raw reply [flat|nested] 10+ messages in thread
* [PATCH 2/4] sparc64: Use RCU page table freeing.
2011-07-12 12:26 [PATCH 0/4] mm, sparc64: Implement gup_fast() Peter Zijlstra
2011-07-12 12:26 ` [PATCH 1/4] sparc64: Kill page table quicklists Peter Zijlstra
@ 2011-07-12 12:26 ` Peter Zijlstra
2011-07-12 12:26 ` [PATCH 3/4] sparc64: Add support for _PAGE_SPECIAL Peter Zijlstra
` (2 subsequent siblings)
4 siblings, 0 replies; 10+ messages in thread
From: Peter Zijlstra @ 2011-07-12 12:26 UTC (permalink / raw)
To: Andrew Morton; +Cc: Linux-MM, LKML, David Miller, Peter Zijlstra
[-- Attachment #1: davem-sparc64-Use_RCU_page_table_freeing.patch --]
[-- Type: text/plain, Size: 2540 bytes --]
Make use of the generic RCU page table freeing on Sparc64, doing so
allows for race-free software page-table walkers like gup_fast().
Signed-off-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/r/20100408.180008.180206930.davem@davemloft.net
---
arch/sparc/Kconfig | 1
arch/sparc/include/asm/pgalloc_64.h | 48 ++++++++++++++++++++++++++++++++++--
2 files changed, 47 insertions(+), 2 deletions(-)
Index: linux-2.6/arch/sparc/include/asm/pgalloc_64.h
===================================================================
--- linux-2.6.orig/arch/sparc/include/asm/pgalloc_64.h
+++ linux-2.6/arch/sparc/include/asm/pgalloc_64.h
@@ -76,7 +76,51 @@ static inline void pte_free(struct mm_st
#define check_pgt_cache() do { } while (0)
-#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte)
-#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd)
+static inline void pgtable_free(void *table, bool is_page)
+{
+ if (is_page)
+ free_page((unsigned long)table);
+ else
+ kmem_cache_free(pgtable_cache, table);
+}
+
+#ifdef CONFIG_SMP
+
+struct mmu_gather;
+extern void tlb_remove_table(struct mmu_gather *, void *);
+
+static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page)
+{
+ unsigned long pgf = (unsigned long)table;
+ if (is_page)
+ pgf |= 0x1UL;
+ tlb_remove_table(tlb, (void *)pgf);
+}
+
+static inline void __tlb_remove_table(void *_table)
+{
+ void *table = (void *)((unsigned long)_table & ~0x1UL);
+ bool is_page = false;
+
+ if ((unsigned long)_table & 0x1UL)
+ is_page = true;
+ pgtable_free(table, is_page);
+}
+#else /* CONFIG_SMP */
+static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page)
+{
+ pgtable_free(table, is_page);
+}
+#endif /* !CONFIG_SMP */
+
+static inline void __pte_free_tlb(struct mmu_gather *tlb, struct page *ptepage,
+ unsigned long address)
+{
+ pgtable_page_dtor(ptepage);
+ pgtable_free_tlb(tlb, page_address(ptepage), true);
+}
+
+#define __pmd_free_tlb(tlb, pmd, addr) \
+ pgtable_free_tlb(tlb, pmd, false)
#endif /* _SPARC64_PGALLOC_H */
Index: linux-2.6/arch/sparc/Kconfig
===================================================================
--- linux-2.6.orig/arch/sparc/Kconfig
+++ linux-2.6/arch/sparc/Kconfig
@@ -53,6 +53,7 @@ config SPARC64
select HAVE_PERF_EVENTS
select PERF_USE_VMALLOC
select IRQ_PREFLOW_FASTEOI
+ select HAVE_RCU_TABLE_FREE if SMP
config ARCH_DEFCONFIG
string
^ permalink raw reply [flat|nested] 10+ messages in thread* [PATCH 3/4] sparc64: Add support for _PAGE_SPECIAL
2011-07-12 12:26 [PATCH 0/4] mm, sparc64: Implement gup_fast() Peter Zijlstra
2011-07-12 12:26 ` [PATCH 1/4] sparc64: Kill page table quicklists Peter Zijlstra
2011-07-12 12:26 ` [PATCH 2/4] sparc64: Use RCU page table freeing Peter Zijlstra
@ 2011-07-12 12:26 ` Peter Zijlstra
2011-07-12 12:26 ` [PATCH 4/4] sparc64: Implement get_user_pages_fast() Peter Zijlstra
2011-07-12 12:42 ` [PATCH 0/4] mm, sparc64: Implement gup_fast() Peter Zijlstra
4 siblings, 0 replies; 10+ messages in thread
From: Peter Zijlstra @ 2011-07-12 12:26 UTC (permalink / raw)
To: Andrew Morton; +Cc: Linux-MM, LKML, David Miller, Peter Zijlstra
[-- Attachment #1: davem-sparc64-Add_support_for__PAGE_SPECIA.patch --]
[-- Type: text/plain, Size: 3306 bytes --]
Luckily there are still a few software PTE bits remaining
and they even match up in both the sun4u and sun4v pte
layouts.
Signed-off-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/r/20100408.180010.76058269.davem@davemloft.net
---
arch/sparc/include/asm/pgtable_64.h | 15 +++++++++++----
1 file changed, 11 insertions(+), 4 deletions(-)
Index: linux-2.6/arch/sparc/include/asm/pgtable_64.h
===================================================================
--- linux-2.6.orig/arch/sparc/include/asm/pgtable_64.h
+++ linux-2.6/arch/sparc/include/asm/pgtable_64.h
@@ -95,6 +95,10 @@
/* PTE bits which are the same in SUN4U and SUN4V format. */
#define _PAGE_VALID _AC(0x8000000000000000,UL) /* Valid TTE */
#define _PAGE_R _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/
+#define _PAGE_SPECIAL _AC(0x0200000000000000,UL) /* Special page */
+
+/* Advertise support for _PAGE_SPECIAL */
+#define __HAVE_ARCH_PTE_SPECIAL
/* SUN4U pte bits... */
#define _PAGE_SZ4MB_4U _AC(0x6000000000000000,UL) /* 4MB Page */
@@ -104,6 +108,7 @@
#define _PAGE_NFO_4U _AC(0x1000000000000000,UL) /* No Fault Only */
#define _PAGE_IE_4U _AC(0x0800000000000000,UL) /* Invert Endianness */
#define _PAGE_SOFT2_4U _AC(0x07FC000000000000,UL) /* Software bits, set 2 */
+#define _PAGE_SPECIAL_4U _AC(0x0200000000000000,UL) /* Special page */
#define _PAGE_RES1_4U _AC(0x0002000000000000,UL) /* Reserved */
#define _PAGE_SZ32MB_4U _AC(0x0001000000000000,UL) /* (Panther) 32MB page */
#define _PAGE_SZ256MB_4U _AC(0x2001000000000000,UL) /* (Panther) 256MB page */
@@ -133,6 +138,7 @@
#define _PAGE_ACCESSED_4V _AC(0x1000000000000000,UL) /* Accessed (ref'd) */
#define _PAGE_READ_4V _AC(0x0800000000000000,UL) /* Readable SW Bit */
#define _PAGE_WRITE_4V _AC(0x0400000000000000,UL) /* Writable SW Bit */
+#define _PAGE_SPECIAL_4V _AC(0x0200000000000000,UL) /* Special page */
#define _PAGE_PADDR_4V _AC(0x00FFFFFFFFFFE000,UL) /* paddr[55:13] */
#define _PAGE_IE_4V _AC(0x0000000000001000,UL) /* Invert Endianness */
#define _PAGE_E_4V _AC(0x0000000000000800,UL) /* side-Effect */
@@ -302,10 +308,10 @@ static inline pte_t pte_modify(pte_t pte
: "=r" (mask), "=r" (tmp)
: "i" (_PAGE_PADDR_4U | _PAGE_MODIFIED_4U | _PAGE_ACCESSED_4U |
_PAGE_CP_4U | _PAGE_CV_4U | _PAGE_E_4U | _PAGE_PRESENT_4U |
- _PAGE_SZBITS_4U),
+ _PAGE_SZBITS_4U | _PAGE_SPECIAL),
"i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V |
_PAGE_CP_4V | _PAGE_CV_4V | _PAGE_E_4V | _PAGE_PRESENT_4V |
- _PAGE_SZBITS_4V));
+ _PAGE_SZBITS_4V | _PAGE_SPECIAL));
return __pte((pte_val(pte) & mask) | (pgprot_val(prot) & ~mask));
}
@@ -502,6 +508,7 @@ static inline pte_t pte_mkyoung(pte_t pt
static inline pte_t pte_mkspecial(pte_t pte)
{
+ pte_val(pte) |= _PAGE_SPECIAL;
return pte;
}
@@ -607,9 +614,9 @@ static inline unsigned long pte_present(
return val;
}
-static inline int pte_special(pte_t pte)
+static inline unsigned long pte_special(pte_t pte)
{
- return 0;
+ return pte_val(pte) & _PAGE_SPECIAL;
}
#define pmd_set(pmdp, ptep) \
^ permalink raw reply [flat|nested] 10+ messages in thread* [PATCH 4/4] sparc64: Implement get_user_pages_fast().
2011-07-12 12:26 [PATCH 0/4] mm, sparc64: Implement gup_fast() Peter Zijlstra
` (2 preceding siblings ...)
2011-07-12 12:26 ` [PATCH 3/4] sparc64: Add support for _PAGE_SPECIAL Peter Zijlstra
@ 2011-07-12 12:26 ` Peter Zijlstra
2011-07-12 12:42 ` [PATCH 0/4] mm, sparc64: Implement gup_fast() Peter Zijlstra
4 siblings, 0 replies; 10+ messages in thread
From: Peter Zijlstra @ 2011-07-12 12:26 UTC (permalink / raw)
To: Andrew Morton; +Cc: Linux-MM, LKML, David Miller, Peter Zijlstra
[-- Attachment #1: davem-sparc64-Implement_get_user_pages_fast.patch --]
[-- Type: text/plain, Size: 5454 bytes --]
Signed-off-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/r/20100408.180015.30977694.davem@davemloft.net
---
arch/sparc/mm/Makefile | 2
arch/sparc/mm/gup.c | 181 +++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 182 insertions(+), 1 deletion(-)
create mode 100644 arch/sparc/mm/gup.c
Index: linux-2.6/arch/sparc/mm/Makefile
===================================================================
--- linux-2.6.orig/arch/sparc/mm/Makefile
+++ linux-2.6/arch/sparc/mm/Makefile
@@ -4,7 +4,7 @@
asflags-y := -ansi
ccflags-y := -Werror
-obj-$(CONFIG_SPARC64) += ultra.o tlb.o tsb.o
+obj-$(CONFIG_SPARC64) += ultra.o tlb.o tsb.o gup.o
obj-y += fault_$(BITS).o
obj-y += init_$(BITS).o
obj-$(CONFIG_SPARC32) += loadmmu.o
Index: linux-2.6/arch/sparc/mm/gup.c
===================================================================
--- /dev/null
+++ linux-2.6/arch/sparc/mm/gup.c
@@ -0,0 +1,181 @@
+/*
+ * Lockless get_user_pages_fast for sparc, cribbed from powerpc
+ *
+ * Copyright (C) 2008 Nick Piggin
+ * Copyright (C) 2008 Novell Inc.
+ */
+
+#include <linux/sched.h>
+#include <linux/mm.h>
+#include <linux/vmstat.h>
+#include <linux/pagemap.h>
+#include <linux/rwsem.h>
+#include <asm/pgtable.h>
+
+/*
+ * The performance critical leaf functions are made noinline otherwise gcc
+ * inlines everything into a single function which results in too much
+ * register pressure.
+ */
+static noinline int gup_pte_range(pmd_t pmd, unsigned long addr,
+ unsigned long end, int write, struct page **pages, int *nr)
+{
+ unsigned long mask, result;
+ pte_t *ptep;
+
+ if (tlb_type == hypervisor) {
+ result = _PAGE_PRESENT_4V|_PAGE_P_4V;
+ if (write)
+ result |= _PAGE_WRITE_4V;
+ } else {
+ result = _PAGE_PRESENT_4U|_PAGE_P_4U;
+ if (write)
+ result |= _PAGE_WRITE_4U;
+ }
+ mask = result | _PAGE_SPECIAL;
+
+ ptep = pte_offset_kernel(&pmd, addr);
+ do {
+ struct page *page, *head;
+ pte_t pte = *ptep;
+
+ if ((pte_val(pte) & mask) != result)
+ return 0;
+ VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
+
+ /* The hugepage case is simplified on sparc64 because
+ * we encode the sub-page pfn offsets into the
+ * hugepage PTEs. We could optimize this in the future
+ * use page_cache_add_speculative() for the hugepage case.
+ */
+ page = pte_page(pte);
+ head = compound_head(page);
+ if (!page_cache_get_speculative(head))
+ return 0;
+ if (unlikely(pte_val(pte) != pte_val(*ptep))) {
+ put_page(head);
+ return 0;
+ }
+
+ pages[*nr] = page;
+ (*nr)++;
+ } while (ptep++, addr += PAGE_SIZE, addr != end);
+
+ return 1;
+}
+
+static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
+ int write, struct page **pages, int *nr)
+{
+ unsigned long next;
+ pmd_t *pmdp;
+
+ pmdp = pmd_offset(&pud, addr);
+ do {
+ pmd_t pmd = *pmdp;
+
+ next = pmd_addr_end(addr, end);
+ if (pmd_none(pmd))
+ return 0;
+ if (!gup_pte_range(pmd, addr, next, write, pages, nr))
+ return 0;
+ } while (pmdp++, addr = next, addr != end);
+
+ return 1;
+}
+
+static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
+ int write, struct page **pages, int *nr)
+{
+ unsigned long next;
+ pud_t *pudp;
+
+ pudp = pud_offset(&pgd, addr);
+ do {
+ pud_t pud = *pudp;
+
+ next = pud_addr_end(addr, end);
+ if (pud_none(pud))
+ return 0;
+ if (!gup_pmd_range(pud, addr, next, write, pages, nr))
+ return 0;
+ } while (pudp++, addr = next, addr != end);
+
+ return 1;
+}
+
+int get_user_pages_fast(unsigned long start, int nr_pages, int write,
+ struct page **pages)
+{
+ struct mm_struct *mm = current->mm;
+ unsigned long addr, len, end;
+ unsigned long next;
+ pgd_t *pgdp;
+ int nr = 0;
+
+ start &= PAGE_MASK;
+ addr = start;
+ len = (unsigned long) nr_pages << PAGE_SHIFT;
+ end = start + len;
+
+ /*
+ * XXX: batch / limit 'nr', to avoid large irq off latency
+ * needs some instrumenting to determine the common sizes used by
+ * important workloads (eg. DB2), and whether limiting the batch size
+ * will decrease performance.
+ *
+ * It seems like we're in the clear for the moment. Direct-IO is
+ * the main guy that batches up lots of get_user_pages, and even
+ * they are limited to 64-at-a-time which is not so many.
+ */
+ /*
+ * This doesn't prevent pagetable teardown, but does prevent
+ * the pagetables from being freed on sparc.
+ *
+ * So long as we atomically load page table pointers versus teardown,
+ * we can follow the address down to the the page and take a ref on it.
+ */
+ local_irq_disable();
+
+ pgdp = pgd_offset(mm, addr);
+ do {
+ pgd_t pgd = *pgdp;
+
+ next = pgd_addr_end(addr, end);
+ if (pgd_none(pgd))
+ goto slow;
+ if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
+ goto slow;
+ } while (pgdp++, addr = next, addr != end);
+
+ local_irq_enable();
+
+ VM_BUG_ON(nr != (end - start) >> PAGE_SHIFT);
+ return nr;
+
+ {
+ int ret;
+
+slow:
+ local_irq_enable();
+
+ /* Try to get the remaining pages with get_user_pages */
+ start += nr << PAGE_SHIFT;
+ pages += nr;
+
+ down_read(&mm->mmap_sem);
+ ret = get_user_pages(current, mm, start,
+ (end - start) >> PAGE_SHIFT, write, 0, pages, NULL);
+ up_read(&mm->mmap_sem);
+
+ /* Have to be a bit careful with return values */
+ if (nr > 0) {
+ if (ret < 0)
+ ret = nr;
+ else
+ ret += nr;
+ }
+
+ return ret;
+ }
+}
^ permalink raw reply [flat|nested] 10+ messages in thread* Re: [PATCH 0/4] mm, sparc64: Implement gup_fast()
2011-07-12 12:26 [PATCH 0/4] mm, sparc64: Implement gup_fast() Peter Zijlstra
` (3 preceding siblings ...)
2011-07-12 12:26 ` [PATCH 4/4] sparc64: Implement get_user_pages_fast() Peter Zijlstra
@ 2011-07-12 12:42 ` Peter Zijlstra
2011-07-13 22:33 ` Andrew Morton
4 siblings, 1 reply; 10+ messages in thread
From: Peter Zijlstra @ 2011-07-12 12:42 UTC (permalink / raw)
To: Andrew Morton; +Cc: Linux-MM, LKML, David Miller
On Tue, 2011-07-12 at 14:26 +0200, Peter Zijlstra wrote:
> With the recent mmu_gather changes that included generic RCU freeing of
> page-tables, it is now quite straight forward to implement gup_fast() on
> sparc64.
>
> Andrew, please consider merging these patches.
Gah, quilt-mail ate all the From: headers again.. all 4 patches are in
fact written by davem. Do you want a resend?
^ permalink raw reply [flat|nested] 10+ messages in thread* Re: [PATCH 0/4] mm, sparc64: Implement gup_fast()
2011-07-12 12:42 ` [PATCH 0/4] mm, sparc64: Implement gup_fast() Peter Zijlstra
@ 2011-07-13 22:33 ` Andrew Morton
2011-07-15 16:21 ` David Miller
0 siblings, 1 reply; 10+ messages in thread
From: Andrew Morton @ 2011-07-13 22:33 UTC (permalink / raw)
To: Peter Zijlstra; +Cc: Linux-MM, LKML, David Miller
On Tue, 12 Jul 2011 14:42:11 +0200
Peter Zijlstra <a.p.zijlstra@chello.nl> wrote:
> On Tue, 2011-07-12 at 14:26 +0200, Peter Zijlstra wrote:
> > With the recent mmu_gather changes that included generic RCU freeing of
> > page-tables, it is now quite straight forward to implement gup_fast() on
> > sparc64.
> >
> > Andrew, please consider merging these patches.
>
> Gah, quilt-mail ate all the From: headers again.. all 4 patches are in
> fact written by davem. Do you want a resend?
I have an editor ;)
I expect these would get more (ie: non-zero) testing if they were
merged via Dave's tree?
^ permalink raw reply [flat|nested] 10+ messages in thread
* Re: [PATCH 0/4] mm, sparc64: Implement gup_fast()
2011-07-13 22:33 ` Andrew Morton
@ 2011-07-15 16:21 ` David Miller
0 siblings, 0 replies; 10+ messages in thread
From: David Miller @ 2011-07-15 16:21 UTC (permalink / raw)
To: akpm; +Cc: a.p.zijlstra, linux-mm, linux-kernel
From: Andrew Morton <akpm@linux-foundation.org>
Date: Wed, 13 Jul 2011 15:33:48 -0700
> On Tue, 12 Jul 2011 14:42:11 +0200
> Peter Zijlstra <a.p.zijlstra@chello.nl> wrote:
>
>> On Tue, 2011-07-12 at 14:26 +0200, Peter Zijlstra wrote:
>> > With the recent mmu_gather changes that included generic RCU freeing of
>> > page-tables, it is now quite straight forward to implement gup_fast() on
>> > sparc64.
>> >
>> > Andrew, please consider merging these patches.
>>
>> Gah, quilt-mail ate all the From: headers again.. all 4 patches are in
>> fact written by davem. Do you want a resend?
>
> I have an editor ;)
>
> I expect these would get more (ie: non-zero) testing if they were
> merged via Dave's tree?
It's actually easier if you merge these in for me Andrew, thanks!
^ permalink raw reply [flat|nested] 10+ messages in thread