From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1755481AbbBUEM0 (ORCPT ); Fri, 20 Feb 2015 23:12:26 -0500 Received: from mail-pa0-f45.google.com ([209.85.220.45]:46461 "EHLO mail-pa0-f45.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1755394AbbBUEMY (ORCPT ); Fri, 20 Feb 2015 23:12:24 -0500 Date: Fri, 20 Feb 2015 20:12:22 -0800 (PST) From: Hugh Dickins X-X-Sender: hugh@eggly.anvils To: "Kirill A. Shutemov" cc: Andrea Arcangeli , Ning Qu , Andrew Morton , linux-kernel@vger.kernel.org, linux-mm@kvack.org Subject: [PATCH 13/24] huge tmpfs: extend get_user_pages_fast to shmem pmd In-Reply-To: Message-ID: References: User-Agent: Alpine 2.11 (LSU 23 2013-08-11) MIME-Version: 1.0 Content-Type: TEXT/PLAIN; charset=US-ASCII Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Factor out one small part of the shmem pmd handling: the arch-specific get_user_pages_fast() has special code to cope with the peculiar refcounting on anonymous THP tail pages (and on hugetlbfs tail pages): which must be avoided in the straightforward shmem pmd case. Signed-off-by: Hugh Dickins --- arch/mips/mm/gup.c | 17 ++++++++++++----- arch/s390/mm/gup.c | 22 +++++++++++++++++++++- arch/sparc/mm/gup.c | 22 +++++++++++++++++++++- arch/x86/mm/gup.c | 17 ++++++++++++----- mm/gup.c | 22 +++++++++++++++++++++- 5 files changed, 87 insertions(+), 13 deletions(-) --- thpfs.orig/arch/mips/mm/gup.c 2015-02-08 18:54:22.000000000 -0800 +++ thpfs/arch/mips/mm/gup.c 2015-02-20 19:34:26.971957306 -0800 @@ -64,7 +64,8 @@ static inline void get_head_page_multipl { VM_BUG_ON(page != compound_head(page)); VM_BUG_ON(page_count(page) == 0); - atomic_add(nr, &page->_count); + if (nr) + atomic_add(nr, &page->_count); SetPageReferenced(page); } @@ -85,13 +86,19 @@ static int gup_huge_pmd(pmd_t pmd, unsig head = pte_page(pte); page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT); do { - VM_BUG_ON(compound_head(page) != head); - pages[*nr] = page; - if (PageTail(page)) + if (PageTail(page)) { + VM_BUG_ON(compound_head(page) != head); get_huge_page_tail(page); + refs++; + } else { + /* + * Handle head or huge tmpfs with normal refcounting. + */ + get_page(page); + } + pages[*nr] = page; (*nr)++; page++; - refs++; } while (addr += PAGE_SIZE, addr != end); get_head_page_multiple(head, refs); --- thpfs.orig/arch/s390/mm/gup.c 2014-01-19 18:40:07.000000000 -0800 +++ thpfs/arch/s390/mm/gup.c 2015-02-20 19:34:26.971957306 -0800 @@ -61,10 +61,30 @@ static inline int gup_huge_pmd(pmd_t *pm return 0; VM_BUG_ON(!pfn_valid(pmd_val(pmd) >> PAGE_SHIFT)); - refs = 0; head = pmd_page(pmd); page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT); + + if (!PageHead(head)) { + /* + * Handle a huge tmpfs team with normal refcounting. + */ + do { + if (!page_cache_get_speculative(page)) + return 0; + if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) { + put_page(page); + return 0; + } + pages[*nr] = page; + (*nr)++; + page++; + } while (addr += PAGE_SIZE, addr != end); + return 1; + } + tail = page; + refs = 0; + do { VM_BUG_ON(compound_head(page) != head); pages[*nr] = page; --- thpfs.orig/arch/sparc/mm/gup.c 2014-12-07 14:21:05.000000000 -0800 +++ thpfs/arch/sparc/mm/gup.c 2015-02-20 19:34:26.975957297 -0800 @@ -79,10 +79,30 @@ static int gup_huge_pmd(pmd_t *pmdp, pmd if (write && !pmd_write(pmd)) return 0; - refs = 0; head = pmd_page(pmd); page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT); + + if (!PageHead(head)) { + /* + * Handle a huge tmpfs team with normal refcounting. + */ + do { + if (!page_cache_get_speculative(page)) + return 0; + if (unlikely(pmd_val(pmd) != pmd_val(*pmdp))) { + put_page(page); + return 0; + } + pages[*nr] = page; + (*nr)++; + page++; + } while (addr += PAGE_SIZE, addr != end); + return 1; + } + tail = page; + refs = 0; + do { VM_BUG_ON(compound_head(page) != head); pages[*nr] = page; --- thpfs.orig/arch/x86/mm/gup.c 2015-02-08 18:54:22.000000000 -0800 +++ thpfs/arch/x86/mm/gup.c 2015-02-20 19:34:26.975957297 -0800 @@ -110,7 +110,8 @@ static inline void get_head_page_multipl { VM_BUG_ON_PAGE(page != compound_head(page), page); VM_BUG_ON_PAGE(page_count(page) == 0, page); - atomic_add(nr, &page->_count); + if (nr) + atomic_add(nr, &page->_count); SetPageReferenced(page); } @@ -135,13 +136,19 @@ static noinline int gup_huge_pmd(pmd_t p head = pte_page(pte); page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT); do { - VM_BUG_ON_PAGE(compound_head(page) != head, page); - pages[*nr] = page; - if (PageTail(page)) + if (PageTail(page)) { + VM_BUG_ON_PAGE(compound_head(page) != head, page); get_huge_page_tail(page); + refs++; + } else { + /* + * Handle head or huge tmpfs with normal refcounting. + */ + get_page(page); + } + pages[*nr] = page; (*nr)++; page++; - refs++; } while (addr += PAGE_SIZE, addr != end); get_head_page_multiple(head, refs); --- thpfs.orig/mm/gup.c 2015-02-08 18:54:22.000000000 -0800 +++ thpfs/mm/gup.c 2015-02-20 19:34:26.975957297 -0800 @@ -795,10 +795,30 @@ static int gup_huge_pmd(pmd_t orig, pmd_ if (write && !pmd_write(orig)) return 0; - refs = 0; head = pmd_page(orig); page = head + ((addr & ~PMD_MASK) >> PAGE_SHIFT); + + if (!PageHead(head)) { + /* + * Handle a huge tmpfs team with normal refcounting. + */ + do { + if (!page_cache_get_speculative(page)) + return 0; + if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) { + put_page(page); + return 0; + } + pages[*nr] = page; + (*nr)++; + page++; + } while (addr += PAGE_SIZE, addr != end); + return 1; + } + tail = page; + refs = 0; + do { VM_BUG_ON_PAGE(compound_head(page) != head, page); pages[*nr] = page;