From mboxrd@z Thu Jan 1 00:00:00 1970 From: John Hubbard Subject: [PATCH v2 02/18] mm/gup: factor out duplicate code from four routines Date: Sun, 3 Nov 2019 13:17:57 -0800 Message-ID: <20191103211813.213227-3-jhubbard@nvidia.com> References: <20191103211813.213227-1-jhubbard@nvidia.com> Mime-Version: 1.0 Content-Type: text/plain Content-Transfer-Encoding: quoted-printable Return-path: In-Reply-To: <20191103211813.213227-1-jhubbard@nvidia.com> Sender: netdev-owner@vger.kernel.org To: Andrew Morton Cc: Al Viro , Alex Williamson , Benjamin Herrenschmidt , =?UTF-8?q?Bj=C3=B6rn=20T=C3=B6pel?= , Christoph Hellwig , Dan Williams , Daniel Vetter , Dave Chinner , David Airlie , "David S . Miller" , Ira Weiny , Jan Kara , Jason Gunthorpe , Jens Axboe , Jonathan Corbet , =?UTF-8?q?J=C3=A9r=C3=B4me=20Glisse?= , Magnus Karlsson , Mauro Carvalho Chehab , Michael Ellerman M List-Id: dri-devel@lists.freedesktop.org There are four locations in gup.c that have a fair amount of code duplication. This means that changing one requires making the same changes in four places, not to mention reading the same code four times, and wondering if there are subtle differences. Factor out the common code into static functions, thus reducing the overall line count and the code's complexity. Also, take the opportunity to slightly improve the efficiency of the error cases, by doing a mass subtraction of the refcount, surrounded by get_page()/put_page(). Also, further simplify (slightly), by waiting until the the successful end of each routine, to increment *nr. Cc: Ira Weiny Cc: Christoph Hellwig Cc: Aneesh Kumar K.V Signed-off-by: John Hubbard --- mm/gup.c | 104 ++++++++++++++++++++++++------------------------------- 1 file changed, 45 insertions(+), 59 deletions(-) diff --git a/mm/gup.c b/mm/gup.c index 85caf76b3012..199da99e8ffc 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -1969,6 +1969,34 @@ static int __gup_device_huge_pud(pud_t pud, pud_t *p= udp, unsigned long addr, } #endif =20 +static int __record_subpages(struct page *page, unsigned long addr, + unsigned long end, struct page **pages, int nr) +{ + int nr_recorded_pages =3D 0; + + do { + pages[nr] =3D page; + nr++; + page++; + nr_recorded_pages++; + } while (addr +=3D PAGE_SIZE, addr !=3D end); + return nr_recorded_pages; +} + +static void put_compound_head(struct page *page, int refs) +{ + /* Do a get_page() first, in case refs =3D=3D page->_refcount */ + get_page(page); + page_ref_sub(page, refs); + put_page(page); +} + +static void __huge_pt_done(struct page *head, int nr_recorded_pages, int *= nr) +{ + *nr +=3D nr_recorded_pages; + SetPageReferenced(head); +} + #ifdef CONFIG_ARCH_HAS_HUGEPD static unsigned long hugepte_addr_end(unsigned long addr, unsigned long en= d, unsigned long sz) @@ -1998,33 +2026,20 @@ static int gup_hugepte(pte_t *ptep, unsigned long s= z, unsigned long addr, /* hugepages are never "special" */ VM_BUG_ON(!pfn_valid(pte_pfn(pte))); =20 - refs =3D 0; head =3D pte_page(pte); - page =3D head + ((addr & (sz-1)) >> PAGE_SHIFT); - do { - VM_BUG_ON(compound_head(page) !=3D head); - pages[*nr] =3D page; - (*nr)++; - page++; - refs++; - } while (addr +=3D PAGE_SIZE, addr !=3D end); + refs =3D __record_subpages(page, addr, end, pages, *nr); =20 head =3D try_get_compound_head(head, refs); - if (!head) { - *nr -=3D refs; + if (!head) return 0; - } =20 if (unlikely(pte_val(pte) !=3D pte_val(*ptep))) { - /* Could be optimized better */ - *nr -=3D refs; - while (refs--) - put_page(head); + put_compound_head(head, refs); return 0; } =20 - SetPageReferenced(head); + __huge_pt_done(head, refs, nr); return 1; } =20 @@ -2071,29 +2086,19 @@ static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, un= signed long addr, pages, nr); } =20 - refs =3D 0; page =3D pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); - do { - pages[*nr] =3D page; - (*nr)++; - page++; - refs++; - } while (addr +=3D PAGE_SIZE, addr !=3D end); + refs =3D __record_subpages(page, addr, end, pages, *nr); =20 head =3D try_get_compound_head(pmd_page(orig), refs); - if (!head) { - *nr -=3D refs; + if (!head) return 0; - } =20 if (unlikely(pmd_val(orig) !=3D pmd_val(*pmdp))) { - *nr -=3D refs; - while (refs--) - put_page(head); + put_compound_head(head, refs); return 0; } =20 - SetPageReferenced(head); + __huge_pt_done(head, refs, nr); return 1; } =20 @@ -2114,29 +2119,19 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, un= signed long addr, pages, nr); } =20 - refs =3D 0; page =3D pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); - do { - pages[*nr] =3D page; - (*nr)++; - page++; - refs++; - } while (addr +=3D PAGE_SIZE, addr !=3D end); + refs =3D __record_subpages(page, addr, end, pages, *nr); =20 head =3D try_get_compound_head(pud_page(orig), refs); - if (!head) { - *nr -=3D refs; + if (!head) return 0; - } =20 if (unlikely(pud_val(orig) !=3D pud_val(*pudp))) { - *nr -=3D refs; - while (refs--) - put_page(head); + put_compound_head(head, refs); return 0; } =20 - SetPageReferenced(head); + __huge_pt_done(head, refs, nr); return 1; } =20 @@ -2151,29 +2146,20 @@ static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, un= signed long addr, return 0; =20 BUILD_BUG_ON(pgd_devmap(orig)); - refs =3D 0; + page =3D pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT); - do { - pages[*nr] =3D page; - (*nr)++; - page++; - refs++; - } while (addr +=3D PAGE_SIZE, addr !=3D end); + refs =3D __record_subpages(page, addr, end, pages, *nr); =20 head =3D try_get_compound_head(pgd_page(orig), refs); - if (!head) { - *nr -=3D refs; + if (!head) return 0; - } =20 if (unlikely(pgd_val(orig) !=3D pgd_val(*pgdp))) { - *nr -=3D refs; - while (refs--) - put_page(head); + put_compound_head(head, refs); return 0; } =20 - SetPageReferenced(head); + __huge_pt_done(head, refs, nr); return 1; } =20 --=20 2.23.0