From mboxrd@z Thu Jan  1 00:00:00 1970
From: Marcelo Tosatti <mtosatti@redhat.com>
Subject: [patch 01/13] x86/mm: get_user_pages_fast_atomic
Date: Sat, 06 Sep 2008 15:48:23 -0300
Message-ID: <20080906192430.514756352@localhost.localdomain>
References: <20080906184822.560099087@localhost.localdomain>
Cc: kvm@vger.kernel.org
To: Avi Kivity <avi@qumranet.com>
Return-path: <kvm-owner@vger.kernel.org>
Received: from mx1.redhat.com ([66.187.233.31]:60541 "EHLO mx1.redhat.com"
	rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP
	id S1752235AbYIFT0z (ORCPT <rfc822;kvm@vger.kernel.org>);
	Sat, 6 Sep 2008 15:26:55 -0400
Content-Disposition: inline; filename=gupfast-atomic
Sender: kvm-owner@vger.kernel.org
List-ID: <kvm.vger.kernel.org>

From: Nick Piggin <nickpiggin@yahoo.com.au>

Provide a lockless pagetable walk function without fallback to mmap_sem
on error.

Index: kvm/arch/x86/mm/gup.c
===================================================================
--- kvm.orig/arch/x86/mm/gup.c
+++ kvm/arch/x86/mm/gup.c
@@ -8,6 +8,7 @@
 #include <linux/mm.h>
 #include <linux/vmstat.h>
 #include <linux/highmem.h>
+#include <linux/module.h>
 
 #include <asm/pgtable.h>
 
@@ -219,7 +220,7 @@ static int gup_pud_range(pgd_t pgd, unsi
 	return 1;
 }
 
-int get_user_pages_fast(unsigned long start, int nr_pages, int write,
+int get_user_pages_fast_atomic(unsigned long start, int nr_pages, int write,
 			struct page **pages)
 {
 	struct mm_struct *mm = current->mm;
@@ -234,7 +235,7 @@ int get_user_pages_fast(unsigned long st
 	end = start + len;
 	if (unlikely(!access_ok(write ? VERIFY_WRITE : VERIFY_READ,
 					start, len)))
-		goto slow_irqon;
+		return -EFAULT;
 
 	/*
 	 * XXX: batch / limit 'nr', to avoid large irq off latency
@@ -261,38 +262,49 @@ int get_user_pages_fast(unsigned long st
 
 		next = pgd_addr_end(addr, end);
 		if (pgd_none(pgd))
-			goto slow;
+			goto out_short;
 		if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
-			goto slow;
+			goto out_short;
 	} while (pgdp++, addr = next, addr != end);
-	local_irq_enable();
 
 	VM_BUG_ON(nr != (end - start) >> PAGE_SHIFT);
+out_short:
+	local_irq_enable();
 	return nr;
+}
 
-	{
-		int ret;
-
-slow:
-		local_irq_enable();
-slow_irqon:
-		/* Try to get the remaining pages with get_user_pages */
-		start += nr << PAGE_SHIFT;
-		pages += nr;
-
-		down_read(&mm->mmap_sem);
-		ret = get_user_pages(current, mm, start,
-			(end - start) >> PAGE_SHIFT, write, 0, pages, NULL);
-		up_read(&mm->mmap_sem);
-
-		/* Have to be a bit careful with return values */
-		if (nr > 0) {
-			if (ret < 0)
-				ret = nr;
-			else
-				ret += nr;
-		}
+int get_user_pages_fast(unsigned long start, int nr_pages, int write,
+			struct page **pages)
+{
+	struct mm_struct *mm = current->mm;
+	int nr = 0;
+	int ret;
 
-		return ret;
+	nr = get_user_pages_fast_atomic(start, nr_pages, write, pages);
+	if (likely(nr == nr_pages))
+		return nr;
+
+	if (unlikely(nr < 0))
+		return nr;
+
+	start += nr << PAGE_SHIFT;
+	pages += nr;
+	nr_pages -= nr;
+
+	down_read(&mm->mmap_sem);
+	ret = get_user_pages(current, mm, start,
+		nr_pages, write, 0, pages, NULL);
+	up_read(&mm->mmap_sem);
+
+	/* Have to be a bit careful with return values */
+	if (nr > 0) {
+		if (ret < 0)
+			ret = nr;
+		else
+			ret += nr;
 	}
+
+	return ret;
 }
+
+EXPORT_SYMBOL_GPL(get_user_pages_fast_atomic);
Index: kvm/include/asm-x86/uaccess.h
===================================================================
--- kvm.orig/include/asm-x86/uaccess.h
+++ kvm/include/asm-x86/uaccess.h
@@ -443,6 +443,9 @@ extern struct movsl_mask {
 
 #define ARCH_HAS_NOCACHE_UACCESS 1
 
+int get_user_pages_fast_atomic(unsigned long start, int nr_pages, int write,
+			struct page **pages);
+
 #ifdef CONFIG_X86_32
 # include "uaccess_32.h"
 #else

--