From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from eggs.gnu.org ([2001:4830:134:3::10]:35980) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1YswzC-0003HU-3w for qemu-devel@nongnu.org; Thu, 14 May 2015 13:31:55 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1YswzB-0005sD-6J for qemu-devel@nongnu.org; Thu, 14 May 2015 13:31:54 -0400 Received: from mx1.redhat.com ([209.132.183.28]:61047) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1YswzA-0005ry-WB for qemu-devel@nongnu.org; Thu, 14 May 2015 13:31:53 -0400 From: Andrea Arcangeli Date: Thu, 14 May 2015 19:31:13 +0200 Message-Id: <1431624680-20153-17-git-send-email-aarcange@redhat.com> In-Reply-To: <1431624680-20153-1-git-send-email-aarcange@redhat.com> References: <1431624680-20153-1-git-send-email-aarcange@redhat.com> Subject: [Qemu-devel] [PATCH 16/23] userfaultfd: allocate the userfaultfd_ctx cacheline aligned List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , To: Andrew Morton , linux-kernel@vger.kernel.org, linux-mm@kvack.org, qemu-devel@nongnu.org, kvm@vger.kernel.org, linux-api@vger.kernel.org Cc: zhang.zhanghailiang@huawei.com, Pavel Emelyanov , Johannes Weiner , Hugh Dickins , "Dr. David Alan Gilbert" , Sanidhya Kashyap , Dave Hansen , Andres Lagar-Cavilla , Mel Gorman , Paolo Bonzini , "Kirill A. Shutemov" , "Huangpeng (Peter)" , Andy Lutomirski , Linus Torvalds , Peter Feiner Use proper slab to guarantee alignment. Signed-off-by: Andrea Arcangeli --- fs/userfaultfd.c | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c index 3d26f41..5542fe7 100644 --- a/fs/userfaultfd.c +++ b/fs/userfaultfd.c @@ -27,20 +27,26 @@ #include #include +static struct kmem_cache *userfaultfd_ctx_cachep __read_mostly; + enum userfaultfd_state { UFFD_STATE_WAIT_API, UFFD_STATE_RUNNING, }; +/* + * Start with fault_pending_wqh and fault_wqh so they're more likely + * to be in the same cacheline. + */ struct userfaultfd_ctx { - /* pseudo fd refcounting */ - atomic_t refcount; /* waitqueue head for the pending (i.e. not read) userfaults */ wait_queue_head_t fault_pending_wqh; /* waitqueue head for the userfaults */ wait_queue_head_t fault_wqh; /* waitqueue head for the pseudo fd to wakeup poll/read */ wait_queue_head_t fd_wqh; + /* pseudo fd refcounting */ + atomic_t refcount; /* userfaultfd syscall flags */ unsigned int flags; /* state machine */ @@ -117,7 +123,7 @@ static void userfaultfd_ctx_put(struct userfaultfd_ctx *ctx) VM_BUG_ON(spin_is_locked(&ctx->fd_wqh.lock)); VM_BUG_ON(waitqueue_active(&ctx->fd_wqh)); mmput(ctx->mm); - kfree(ctx); + kmem_cache_free(userfaultfd_ctx_cachep, ctx); } } @@ -987,6 +993,15 @@ static const struct file_operations userfaultfd_fops = { .llseek = noop_llseek, }; +static void init_once_userfaultfd_ctx(void *mem) +{ + struct userfaultfd_ctx *ctx = (struct userfaultfd_ctx *) mem; + + init_waitqueue_head(&ctx->fault_pending_wqh); + init_waitqueue_head(&ctx->fault_wqh); + init_waitqueue_head(&ctx->fd_wqh); +} + /** * userfaultfd_file_create - Creates an userfaultfd file pointer. * @flags: Flags for the userfaultfd file. @@ -1017,14 +1032,11 @@ static struct file *userfaultfd_file_create(int flags) goto out; file = ERR_PTR(-ENOMEM); - ctx = kmalloc(sizeof(*ctx), GFP_KERNEL); + ctx = kmem_cache_alloc(userfaultfd_ctx_cachep, GFP_KERNEL); if (!ctx) goto out; atomic_set(&ctx->refcount, 1); - init_waitqueue_head(&ctx->fault_pending_wqh); - init_waitqueue_head(&ctx->fault_wqh); - init_waitqueue_head(&ctx->fd_wqh); ctx->flags = flags; ctx->state = UFFD_STATE_WAIT_API; ctx->released = false; @@ -1035,7 +1047,7 @@ static struct file *userfaultfd_file_create(int flags) file = anon_inode_getfile("[userfaultfd]", &userfaultfd_fops, ctx, O_RDWR | (flags & UFFD_SHARED_FCNTL_FLAGS)); if (IS_ERR(file)) - kfree(ctx); + kmem_cache_free(userfaultfd_ctx_cachep, ctx); out: return file; } @@ -1064,3 +1076,14 @@ err_put_unused_fd: return error; } + +static int __init userfaultfd_init(void) +{ + userfaultfd_ctx_cachep = kmem_cache_create("userfaultfd_ctx_cache", + sizeof(struct userfaultfd_ctx), + 0, + SLAB_HWCACHE_ALIGN|SLAB_PANIC, + init_once_userfaultfd_ctx); + return 0; +} +__initcall(userfaultfd_init);