[patch 01/13] x86/mm: get_user_pages_fast_atomic

[Marcelo Tosatti <mtosatti@redhat.com>]

[-- Attachment #1: gupfast-atomic --]
[-- Type: text/plain, Size: 3035 bytes --]

From: Nick Piggin <nickpiggin@yahoo.com.au>

Provide a lockless pagetable walk function without fallback to mmap_sem
on error.

Index: kvm/arch/x86/mm/gup.c
===================================================================
--- kvm.orig/arch/x86/mm/gup.c
+++ kvm/arch/x86/mm/gup.c
@@ -8,6 +8,7 @@
 #include <linux/mm.h>
 #include <linux/vmstat.h>
 #include <linux/highmem.h>
+#include <linux/module.h>
 
 #include <asm/pgtable.h>
 
@@ -219,7 +220,7 @@ static int gup_pud_range(pgd_t pgd, unsi
 	return 1;
 }
 
-int get_user_pages_fast(unsigned long start, int nr_pages, int write,
+int get_user_pages_fast_atomic(unsigned long start, int nr_pages, int write,
 			struct page **pages)
 {
 	struct mm_struct *mm = current->mm;
@@ -234,7 +235,7 @@ int get_user_pages_fast(unsigned long st
 	end = start + len;
 	if (unlikely(!access_ok(write ? VERIFY_WRITE : VERIFY_READ,
 					start, len)))
-		goto slow_irqon;
+		return -EFAULT;
 
 	/*
 	 * XXX: batch / limit 'nr', to avoid large irq off latency
@@ -261,38 +262,49 @@ int get_user_pages_fast(unsigned long st
 
 		next = pgd_addr_end(addr, end);
 		if (pgd_none(pgd))
-			goto slow;
+			goto out_short;
 		if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
-			goto slow;
+			goto out_short;
 	} while (pgdp++, addr = next, addr != end);
-	local_irq_enable();
 
 	VM_BUG_ON(nr != (end - start) >> PAGE_SHIFT);
+out_short:
+	local_irq_enable();
 	return nr;
+}
 
-	{
-		int ret;
-
-slow:
-		local_irq_enable();
-slow_irqon:
-		/* Try to get the remaining pages with get_user_pages */
-		start += nr << PAGE_SHIFT;
-		pages += nr;
-
-		down_read(&mm->mmap_sem);
-		ret = get_user_pages(current, mm, start,
-			(end - start) >> PAGE_SHIFT, write, 0, pages, NULL);
-		up_read(&mm->mmap_sem);
-
-		/* Have to be a bit careful with return values */
-		if (nr > 0) {
-			if (ret < 0)
-				ret = nr;
-			else
-				ret += nr;
-		}
+int get_user_pages_fast(unsigned long start, int nr_pages, int write,
+			struct page **pages)
+{
+	struct mm_struct *mm = current->mm;
+	int nr = 0;
+	int ret;
 
-		return ret;
+	nr = get_user_pages_fast_atomic(start, nr_pages, write, pages);
+	if (likely(nr == nr_pages))
+		return nr;
+
+	if (unlikely(nr < 0))
+		return nr;
+
+	start += nr << PAGE_SHIFT;
+	pages += nr;
+	nr_pages -= nr;
+
+	down_read(&mm->mmap_sem);
+	ret = get_user_pages(current, mm, start,
+		nr_pages, write, 0, pages, NULL);
+	up_read(&mm->mmap_sem);
+
+	/* Have to be a bit careful with return values */
+	if (nr > 0) {
+		if (ret < 0)
+			ret = nr;
+		else
+			ret += nr;
 	}
+
+	return ret;
 }
+
+EXPORT_SYMBOL_GPL(get_user_pages_fast_atomic);
Index: kvm/include/asm-x86/uaccess.h
===================================================================
--- kvm.orig/include/asm-x86/uaccess.h
+++ kvm/include/asm-x86/uaccess.h
@@ -443,6 +443,9 @@ extern struct movsl_mask {
 
 #define ARCH_HAS_NOCACHE_UACCESS 1
 
+int get_user_pages_fast_atomic(unsigned long start, int nr_pages, int write,
+			struct page **pages);
+
 #ifdef CONFIG_X86_32
 # include "uaccess_32.h"
 #else

--