From mboxrd@z Thu Jan 1 00:00:00 1970 From: Jeremy Fitzhardinge Subject: Re: [RFC] linux: add p[mug]d_clear_full() accessors Date: Tue, 20 May 2008 16:17:35 +0100 Message-ID: <4832EB8F.8010405@goop.org> References: <4832FF91.76E4.0078.0@novell.com> Mime-Version: 1.0 Content-Type: text/plain; charset=ISO-8859-1; format=flowed Content-Transfer-Encoding: 7bit Return-path: In-Reply-To: <4832FF91.76E4.0078.0@novell.com> List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Sender: xen-devel-bounces@lists.xensource.com Errors-To: xen-devel-bounces@lists.xensource.com To: Jan Beulich Cc: xen-devel@lists.xensource.com List-Id: xen-devel@lists.xenproject.org Jan Beulich wrote: > .. to avoid using hypercalls for clearing dead (un-pinned) page tables. > This eliminates well over a quarter of a million hypercalls during > kernel builds (up to about a million depending on the configuration). > Does it have much effect on performance? > This is against 2.6.25.4, it does not apply to the 2.6.18 tree (I'd > make this effort only if the patch is desired into that tree). The > primary goal of sending this is to find out whether the required change > to include/asm-generic/pgtable.h and mm/memory.c is considered > worthwhile (in which case I'd make an attempt at getting this accepted > upstream). > Is this based on your forward-port of the Xen patches? Stock 2.6.25.4 doesn't have mm_is_pinned(). In fact, in pvops-Xen, I set PG_pinned on all pinned pagetable pages (not just the top level), so you can tell whether you're updating a pinned pgd/pud/pmd without needing an mm on hand. So I think this optimisation can be implemented in current Linux entirely within the Xen-specific code. J > Signed-off-by: Jan Beulich > > Index: head-2008-05-19/include/asm-generic/pgtable.h > =================================================================== > --- head-2008-05-19.orig/include/asm-generic/pgtable.h 2008-05-19 10:48:15.000000000 +0200 > +++ head-2008-05-19/include/asm-generic/pgtable.h 2008-05-19 10:57:25.000000000 +0200 > @@ -133,6 +133,18 @@ static inline void ptep_set_wrprotect(st > #define move_pte(pte, prot, old_addr, new_addr) (pte) > #endif > > +#ifndef __HAVE_ARCH_PMD_CLEAR_FULL > +#define pmd_clear_full(mm, addr, pmd, full) pmd_clear(pmd) > +#endif > + > +#ifndef __HAVE_ARCH_PUD_CLEAR_FULL > +#define pud_clear_full(mm, addr, pud, full) pud_clear(pud) > +#endif > + > +#ifndef __HAVE_ARCH_PGD_CLEAR_FULL > +#define pgd_clear_full(mm, addr, pgd, full) pgd_clear(pgd) > +#endif > + > /* > * When walking page tables, get the address of the next boundary, > * or the end address of the range if that comes earlier. Although no > Index: head-2008-05-19/include/asm-x86/mach-xen/asm/pgtable.h > =================================================================== > --- head-2008-05-19.orig/include/asm-x86/mach-xen/asm/pgtable.h 2008-05-19 10:57:24.000000000 +0200 > +++ head-2008-05-19/include/asm-x86/mach-xen/asm/pgtable.h 2008-05-19 10:57:25.000000000 +0200 > @@ -397,6 +397,22 @@ static inline pte_t ptep_get_and_clear(s > > pte_t xen_ptep_get_and_clear_full(struct vm_area_struct *, unsigned long, pte_t *, int); > > +#define __HAVE_ARCH_PMD_CLEAR_FULL > +#define pmd_clear_full(mm, addr, pmd, full) \ > + (!mm_is_pinned(mm) ? __xen_pmd_clear(pmd) : xen_pmd_clear(pmd)) > + > +#ifndef __PAGETABLE_PMD_FOLDED > +#define __HAVE_ARCH_PUD_CLEAR_FULL > +#define pud_clear_full(mm, addr, pud, full) \ > + (!mm_is_pinned(mm) ? __xen_pud_clear(pud) : xen_pud_clear(pud)) > +#endif > + > +#ifndef __PAGETABLE_PUD_FOLDED > +#define __HAVE_ARCH_PGD_CLEAR_FULL > +#define pgd_clear_full(mm, addr, pgd, full) \ > + (!mm_is_pinned(mm) ? __xen_pgd_clear(pgd) : xen_pgd_clear(pgd)) > +#endif > + > #define __HAVE_ARCH_PTEP_SET_WRPROTECT > static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) > { > Index: head-2008-05-19/include/asm-x86/mach-xen/asm/pgtable-2level.h > =================================================================== > --- head-2008-05-19.orig/include/asm-x86/mach-xen/asm/pgtable-2level.h 2008-05-19 10:49:58.000000000 +0200 > +++ head-2008-05-19/include/asm-x86/mach-xen/asm/pgtable-2level.h 2008-05-19 10:57:25.000000000 +0200 > @@ -35,6 +35,11 @@ static inline void xen_pmd_clear(pmd_t * > > #define __xen_pte_clear(ptep) xen_set_pte(ptep, __pte(0)) > > +static inline void __xen_pmd_clear(pmd_t *pmdp) > +{ > + *pmdp = __pmd(0); > +} > + > #ifdef CONFIG_SMP > static inline pte_t xen_ptep_get_and_clear(pte_t *xp, pte_t res) > { > Index: head-2008-05-19/include/asm-x86/mach-xen/asm/pgtable-3level.h > =================================================================== > --- head-2008-05-19.orig/include/asm-x86/mach-xen/asm/pgtable-3level.h 2008-05-19 10:49:58.000000000 +0200 > +++ head-2008-05-19/include/asm-x86/mach-xen/asm/pgtable-3level.h 2008-05-19 10:57:25.000000000 +0200 > @@ -76,6 +76,11 @@ static inline void xen_pmd_clear(pmd_t * > xen_l2_entry_update(pmd, __pmd(0)); > } > > +static inline void __xen_pmd_clear(pmd_t *pmd) > +{ > + *pmd = __pmd(0); > +} > + > static inline void pud_clear(pud_t *pudp) > { > pgdval_t pgd; > @@ -96,6 +101,11 @@ static inline void pud_clear(pud_t *pudp > xen_tlb_flush(); > } > > +static inline void __xen_pud_clear(pud_t *pudp) > +{ > + *pudp = __pud(0); > +} > + > #define pud_page(pud) \ > ((struct page *) __va(pud_val(pud) & PAGE_MASK)) > > Index: head-2008-05-19/include/asm-x86/mach-xen/asm/pgtable_64.h > =================================================================== > --- head-2008-05-19.orig/include/asm-x86/mach-xen/asm/pgtable_64.h 2008-05-19 10:57:24.000000000 +0200 > +++ head-2008-05-19/include/asm-x86/mach-xen/asm/pgtable_64.h 2008-05-19 10:57:25.000000000 +0200 > @@ -109,6 +109,11 @@ static inline void xen_pmd_clear(pmd_t * > xen_set_pmd(pmd, xen_make_pmd(0)); > } > > +static inline void __xen_pmd_clear(pmd_t *pmd) > +{ > + *pmd = xen_make_pmd(0); > +} > + > static inline void xen_set_pud(pud_t *pudp, pud_t pud) > { > xen_l3_entry_update(pudp, pud); > @@ -119,6 +124,11 @@ static inline void xen_pud_clear(pud_t * > xen_set_pud(pud, xen_make_pud(0)); > } > > +static inline void __xen_pud_clear(pud_t *pud) > +{ > + *pud = xen_make_pud(0); > +} > + > #define __user_pgd(pgd) ((pgd) + PTRS_PER_PGD) > > static inline void xen_set_pgd(pgd_t *pgdp, pgd_t pgd) > @@ -132,6 +142,12 @@ static inline void xen_pgd_clear(pgd_t * > xen_set_pgd(__user_pgd(pgd), xen_make_pgd(0)); > } > > +static inline void __xen_pgd_clear(pgd_t *pgd) > +{ > + *pgd = xen_make_pgd(0); > + *__user_pgd(pgd) = xen_make_pgd(0); > +} > + > #define pte_same(a, b) ((a).pte == (b).pte) > > #endif /* !__ASSEMBLY__ */ > Index: head-2008-05-19/mm/memory.c > =================================================================== > --- head-2008-05-19.orig/mm/memory.c 2008-05-19 10:49:46.000000000 +0200 > +++ head-2008-05-19/mm/memory.c 2008-05-19 10:57:25.000000000 +0200 > @@ -132,10 +132,11 @@ void pmd_clear_bad(pmd_t *pmd) > * Note: this doesn't free the actual pages themselves. That > * has been handled earlier when unmapping all the memory regions. > */ > -static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd) > +static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, > + unsigned long addr) > { > pgtable_t token = pmd_pgtable(*pmd); > - pmd_clear(pmd); > + pmd_clear_full(tlb->mm, addr, pmd, tlb->fullmm); > pte_free_tlb(tlb, token); > tlb->mm->nr_ptes--; > } > @@ -154,7 +155,7 @@ static inline void free_pmd_range(struct > next = pmd_addr_end(addr, end); > if (pmd_none_or_clear_bad(pmd)) > continue; > - free_pte_range(tlb, pmd); > + free_pte_range(tlb, pmd, addr); > } while (pmd++, addr = next, addr != end); > > start &= PUD_MASK; > @@ -169,7 +170,7 @@ static inline void free_pmd_range(struct > return; > > pmd = pmd_offset(pud, start); > - pud_clear(pud); > + pud_clear_full(tlb->mm, start, pud, tlb->fullmm); > pmd_free_tlb(tlb, pmd); > } > > @@ -202,7 +203,7 @@ static inline void free_pud_range(struct > return; > > pud = pud_offset(pgd, start); > - pgd_clear(pgd); > + pgd_clear_full(tlb->mm, start, pgd, tlb->fullmm); > pud_free_tlb(tlb, pud); > } > > > > > _______________________________________________ > Xen-devel mailing list > Xen-devel@lists.xensource.com > http://lists.xensource.com/xen-devel >