[patch 01/13] x86/mm: get_user_pages_fast_atomic

Marcelo Tosatti
2008-09-06T19:26:55Z

[-- Attachment #1: gupfast-atomic --]
[-- Type: text/plain, Size: 3035 bytes --]

From: Nick Piggin <nickpiggin@yahoo.com.au>

Provide a lockless pagetable walk function without fallback to mmap_sem
on error.

Index: kvm/arch/x86/mm/gup.c
===================================================================
--- kvm.orig/arch/x86/mm/gup.c
+++ kvm/arch/x86/mm/gup.c
@@ -8,6 +8,7 @@
#include <linux/mm.h>
#include <linux/vmstat.h>
#include <linux/highmem.h>
+#include <linux/module.h>

#include <asm/pgtable.h>

@@ -219,7 +220,7 @@ static int gup_pud_range(pgd_t pgd, unsi
return 1;
}

-int get_user_pages_fast(unsigned long start, int nr_pages, int write,
+int get_user_pages_fast_atomic(unsigned long start, int nr_pages, int write,
struct page **pages)
{
struct mm_struct *mm = current->mm;
@@ -234,7 +235,7 @@ int get_user_pages_fast(unsigned long st
end = start + len;
if (unlikely(!access_ok(write ? VERIFY_WRITE : VERIFY_READ,
start, len)))
- goto slow_irqon;
+ return -EFAULT;

/*
* XXX: batch / limit 'nr', to avoid large irq off latency
@@ -261,38 +262,49 @@ int get_user_pages_fast(unsigned long st

next = pgd_addr_end(addr, end);
if (pgd_none(pgd))
- goto slow;
+ goto out_short;
if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
- goto slow;
+ goto out_short;
} while (pgdp++, addr = next, addr != end);
- local_irq_enable();

VM_BUG_ON(nr != (end - start) >> PAGE_SHIFT);
+out_short:
+ local_irq_enable();
return nr;
+}

- {
- int ret;
-
-slow:
- local_irq_enable();
-slow_irqon:
- /* Try to get the remaining pages with get_user_pages */
- start += nr << PAGE_SHIFT;
- pages += nr;
-
- down_read(&mm->mmap_sem);
- ret = get_user_pages(current, mm, start,
- (end - start) >> PAGE_SHIFT, write, 0, pages, NULL);
- up_read(&mm->mmap_sem);
-
- /* Have to be a bit careful with return values */
- if (nr > 0) {
- if (ret < 0)
- ret = nr;
- else
- ret += nr;
- }
+int get_user_pages_fast(unsigned long start, int nr_pages, int write,
+ struct page **pages)
+{
+ struct mm_struct *mm = current->mm;
+ int nr = 0;
+ int ret;

- return ret;
+ nr = get_user_pages_fast_atomic(start, nr_pages, write, pages);
+ if (likely(nr == nr_pages))
+ return nr;
+
+ if (unlikely(nr < 0))
+ return nr;
+
+ start += nr << PAGE_SHIFT;
+ pages += nr;
+ nr_pages -= nr;
+
+ down_read(&mm->mmap_sem);
+ ret = get_user_pages(current, mm, start,
+ nr_pages, write, 0, pages, NULL);
+ up_read(&mm->mmap_sem);
+
+ /* Have to be a bit careful with return values */
+ if (nr > 0) {
+ if (ret < 0)
+ ret = nr;
+ else
+ ret += nr;
}
+
+ return ret;
}
+
+EXPORT_SYMBOL_GPL(get_user_pages_fast_atomic);
Index: kvm/include/asm-x86/uaccess.h
===================================================================
--- kvm.orig/include/asm-x86/uaccess.h
+++ kvm/include/asm-x86/uaccess.h
@@ -443,6 +443,9 @@ extern struct movsl_mask {

#define ARCH_HAS_NOCACHE_UACCESS 1

+int get_user_pages_fast_atomic(unsigned long start, int nr_pages, int write,
+ struct page **pages);
+
#ifdef CONFIG_X86_32
# include "uaccess_32.h"
#else

--