From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1753487Ab0C1FaQ (ORCPT ); Sun, 28 Mar 2010 01:30:16 -0400 Received: from mail-pw0-f46.google.com ([209.85.160.46]:50794 "EHLO mail-pw0-f46.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1753093Ab0C1FaN convert rfc822-to-8bit (ORCPT ); Sun, 28 Mar 2010 01:30:13 -0400 DomainKey-Signature: a=rsa-sha1; c=nofws; d=gmail.com; s=gamma; h=mime-version:in-reply-to:references:date:message-id:subject:from:to :cc:content-type:content-transfer-encoding; b=a1GUcxYzoFurWmIEHSzQlIBeTDndbv1mdgcvZxhHU1psnE23qh0Xw4nQD7FPytv2Ch BfG8FRHX+vvxWIziuPaU1pGbezg3YHLpaLF6nYD36fun3P2AZ3b/yVURfvm5LFyiB2Nm 65S5n4cUGYu7YNDwQrk6lFxVIblgt3ut5aEnU= MIME-Version: 1.0 In-Reply-To: <4BAB6646.7040302@cn.fujitsu.com> References: <4B94CD2D.8070401@cn.fujitsu.com> <4B95F802.9020308@cn.fujitsu.com> <20100311081548.GJ5812@laptop> <4B98C6DE.3060602@cn.fujitsu.com> <20100311110317.GL5812@laptop> <4BAB6646.7040302@cn.fujitsu.com> Date: Sun, 28 Mar 2010 13:30:13 +0800 Message-ID: Subject: Re: [PATCH] [PATCH -mmotm] cpuset,mm: use seqlock to protect task->mempolicy and mems_allowed (v2) (was: Re: [PATCH V2 4/4] cpuset,mm: update task's mems_allowed lazily) From: Bob Liu To: miaox@cn.fujitsu.com Cc: Nick Piggin , David Rientjes , Lee Schermerhorn , Paul Menage , Linux-Kernel , Linux-MM , Andrew Morton Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8BIT Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org On Thu, Mar 25, 2010 at 9:33 PM, Miao Xie wrote: > on 2010-3-11 19:03, Nick Piggin wrote: >>> Ok, I try to make a new patch by using seqlock. >> >> Well... I do think seqlocks would be a bit simpler because they don't >> require this checking and synchronizing of this patch. > Hi, Nick Piggin > > I have made a new patch which uses seqlock to protect mems_allowed and mempolicy. > please review it. > > Subject: [PATCH] [PATCH -mmotm] cpuset,mm: use seqlock to protect task->mempolicy and mems_allowed (v2) > > Before applying this patch, cpuset updates task->mems_allowed by setting all > new bits in the nodemask first, and clearing all old unallowed bits later. > But in the way, the allocator can see an empty nodemask, though it is infrequent. > > The problem is following: > The size of nodemask_t is greater than the size of long integer, so loading > and storing of nodemask_t are not atomic operations. If task->mems_allowed > don't intersect with new_mask, such as the first word of the mask is empty > and only the first word of new_mask is not empty. When the allocator > loads a word of the mask before > >        current->mems_allowed |= new_mask; > > and then loads another word of the mask after > >        current->mems_allowed = new_mask; > > the allocator gets an empty nodemask. > > Besides that, if the size of nodemask_t is less than the size of long integer, > there is another problem. when the kernel allocater invokes the following function, > >        struct zoneref *next_zones_zonelist(struct zoneref *z, >                                                enum zone_type highest_zoneidx, >                                                nodemask_t *nodes, >                                                struct zone **zone) >        { >                /* >                 * Find the next suitable zone to use for the allocation. >                 * Only filter based on nodemask if it's set >                 */ >                if (likely(nodes == NULL)) >                        ...... >               else >                        while (zonelist_zone_idx(z) > highest_zoneidx || >                                        (z->zone && !zref_in_nodemask(z, nodes))) >                                z++; > >                *zone = zonelist_zone(z); >                return z; >        } > > if we change nodemask between two calls of zref_in_nodemask(), such as >        Task1                                           Task2 >        zref_in_nodemask(z = node0's z, nodes = 1-2) >        zref_in_nodemask return 0 >                                                        nodes = 0 >        zref_in_nodemask(z = node1's z, nodes = 0) >        zref_in_nodemask return 0 > z will overflow. > > when the kernel allocater accesses task->mempolicy, there is the same problem. > > The following method is used to fix these two problem. > A seqlock is used to protect task's mempolicy and mems_allowed for configs where > MAX_NUMNODES > BITS_PER_LONG, and when the kernel allocater accesses nodemask, > it locks the seqlock and gets the copy of nodemask, then it passes the copy of > nodemask to the memory allocating function. > > Signed-off-by: Miao Xie > --- >  include/linux/cpuset.h    |   79 +++++++++++++++++++++++- >  include/linux/init_task.h |    8 +++ >  include/linux/sched.h     |   17 ++++- >  kernel/cpuset.c           |   97 +++++++++++++++++++++++------- >  kernel/exit.c             |    4 + >  kernel/fork.c             |    4 + >  mm/hugetlb.c              |   22 ++++++- >  mm/mempolicy.c            |  144 ++++++++++++++++++++++++++++++++++----------- >  mm/slab.c                 |   26 +++++++- >  mm/slub.c                 |   12 ++++- >  10 files changed, 341 insertions(+), 72 deletions(-) > > diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h > index a5740fc..e307f89 100644 > --- a/include/linux/cpuset.h > +++ b/include/linux/cpuset.h > @@ -53,8 +53,8 @@ static inline int cpuset_zone_allowed_hardwall(struct zone *z, gfp_t gfp_mask) >        return cpuset_node_allowed_hardwall(zone_to_nid(z), gfp_mask); >  } > > -extern int cpuset_mems_allowed_intersects(const struct task_struct *tsk1, > -                                         const struct task_struct *tsk2); > +extern int cpuset_mems_allowed_intersects(struct task_struct *tsk1, > +                                         struct task_struct *tsk2); > >  #define cpuset_memory_pressure_bump()                          \ >        do {                                                    \ > @@ -90,9 +90,68 @@ extern void rebuild_sched_domains(void); > >  extern void cpuset_print_task_mems_allowed(struct task_struct *p); > > +# if MAX_NUMNODES > BITS_PER_LONG > +/* > + * Be used to protect task->mempolicy and mems_allowed when reading them for > + * page allocation. > + * > + * we don't care that the kernel page allocator allocate a page on a node in > + * the old mems_allowed, which isn't a big deal, especially since it was > + * previously allowed. > + * > + * We just worry whether the kernel page allocator gets an empty mems_allowed > + * or not. But > + *   if MAX_NUMNODES <= BITS_PER_LONG, loading/storing task->mems_allowed are > + *   atomic operations. So we needn't do anything to protect the loading of > + *   task->mems_allowed in fastpaths. > + * > + *   if MAX_NUMNODES > BITS_PER_LONG, loading/storing task->mems_allowed are > + *   not atomic operations. So we use a seqlock to protect the loading of > + *   task->mems_allowed in fastpaths. > + */ > +#define mems_fastpath_lock_irqsave(p, flags)                           \ > +       ({                                                              \ > +               read_seqbegin_irqsave(&p->mems_seqlock, flags);         \ > +       }) > + > +#define mems_fastpath_unlock_irqrestore(p, seq, flags)                 \ > +       ({                                                              \ > +               read_seqretry_irqrestore(&p->mems_seqlock, seq, flags); \ > +       }) > + > +#define mems_slowpath_lock_irqsave(p, flags)                           \ > +       do {                                                            \ > +               write_seqlock_irqsave(&p->mems_seqlock, flags);         \ > +       } while (0) > + > +#define mems_slowpath_unlock_irqrestore(p, flags)                      \ > +       do {                                                            \ > +               write_sequnlock_irqrestore(&p->mems_seqlock, flags);    \ > +       } while (0) > +# else > +#define mems_fastpath_lock_irqsave(p, flags)           ({ (void)(flags); 0; }) > + > +#define mems_fastpath_unlock_irqrestore(p, flags)      ({ (void)(flags); 0; }) > + > +#define mems_slowpath_lock_irqsave(p, flags)                   \ > +       do {                                                    \ > +               task_lock(p);                                   \ > +               (void)(flags);                                  \ > +       } while (0) > + > +#define mems_slowpath_unlock_irqrestore(p, flags)              \ > +       do {                                                    \ > +               task_unlock(p);                                 \ > +               (void)(flags);                                  \ > +       } while (0) > +# endif > + >  static inline void set_mems_allowed(nodemask_t nodemask) >  { > +       unsigned long flags; > +       mems_slowpath_lock_irqsave(current, flags); >        current->mems_allowed = nodemask; > +       mems_slowpath_unlock_irqrestore(current, flags); >  } > >  #else /* !CONFIG_CPUSETS */ > @@ -144,8 +203,8 @@ static inline int cpuset_zone_allowed_hardwall(struct zone *z, gfp_t gfp_mask) >        return 1; >  } > > -static inline int cpuset_mems_allowed_intersects(const struct task_struct *tsk1, > -                                                const struct task_struct *tsk2) > +static inline int cpuset_mems_allowed_intersects(struct task_struct *tsk1, > +                                                struct task_struct *tsk2) >  { >        return 1; >  } > @@ -193,6 +252,18 @@ static inline void set_mems_allowed(nodemask_t nodemask) >  { >  } > > +#define mems_fastpath_lock_irqsave(p, flags)                           \ > +       ({ (void)(flags); 0; }) > + > +#define mems_fastpath_unlock_irqrestore(p, seq, flags)                 \ > +       ({ (void)(flags); 0; }) > + > +#define mems_slowpath_lock_irqsave(p, flags)                           \ > +       do { (void)(flags); } while (0) > + > +#define mems_slowpath_unlock_irqrestore(p, flags)                      \ > +       do { (void)(flags); } while (0) > + >  #endif /* !CONFIG_CPUSETS */ > >  #endif /* _LINUX_CPUSET_H */ > diff --git a/include/linux/init_task.h b/include/linux/init_task.h > index 1ed6797..0394e20 100644 > --- a/include/linux/init_task.h > +++ b/include/linux/init_task.h > @@ -102,6 +102,13 @@ extern struct cred init_cred; >  # define INIT_PERF_EVENTS(tsk) >  #endif > > +#if defined(CONFIG_CPUSETS) && MAX_NUMNODES > BITS_PER_LONG > +# define INIT_MEM_SEQLOCK(tsk)                                         \ > +       .mems_seqlock   = __SEQLOCK_UNLOCKED(tsk.mems_seqlock), > +#else > +# define INIT_MEM_SEQLOCK(tsk) > +#endif > + >  /* >  *  INIT_TASK is used to set up the first task table, touch at >  * your own risk!. Base=0, limit=0x1fffff (=2MB) > @@ -171,6 +178,7 @@ extern struct cred init_cred; >        INIT_FTRACE_GRAPH                                               \ >        INIT_TRACE_RECURSION                                            \ >        INIT_TASK_RCU_PREEMPT(tsk)                                      \ > +       INIT_MEM_SEQLOCK(tsk)                                           \ >  } > > > diff --git a/include/linux/sched.h b/include/linux/sched.h > index 84b8c22..1cf5fd3 100644 > --- a/include/linux/sched.h > +++ b/include/linux/sched.h > @@ -1356,8 +1356,9 @@ struct task_struct { >  /* Thread group tracking */ >        u32 parent_exec_id; >        u32 self_exec_id; > -/* Protection of (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, > - * mempolicy */ > +/* Protection of (de-)allocation: mm, files, fs, tty, keyrings. > + * if MAX_NUMNODES <= BITS_PER_LONG,it will protect mems_allowed and mempolicy. > + * Or we use other seqlock - mems_seqlock to protect them. */ >        spinlock_t alloc_lock; > >  #ifdef CONFIG_GENERIC_HARDIRQS > @@ -1425,7 +1426,13 @@ struct task_struct { >        cputime_t acct_timexpd; /* stime + utime since last update */ >  #endif >  #ifdef CONFIG_CPUSETS > -       nodemask_t mems_allowed;        /* Protected by alloc_lock */ > +# if MAX_NUMNODES > BITS_PER_LONG > +       /* Protection of mems_allowed, and mempolicy */ > +       seqlock_t mems_seqlock; > +# endif > +       /* if MAX_NUMNODES <= BITS_PER_LONG, Protected by alloc_lock; > +        * else Protected by mems_seqlock */ > +       nodemask_t mems_allowed; >        int cpuset_mem_spread_rotor; >  #endif >  #ifdef CONFIG_CGROUPS > @@ -1448,7 +1455,9 @@ struct task_struct { >        struct list_head perf_event_list; >  #endif >  #ifdef CONFIG_NUMA > -       struct mempolicy *mempolicy;    /* Protected by alloc_lock */ > +       /* if MAX_NUMNODES <= BITS_PER_LONG, Protected by alloc_lock; > +        * else Protected by mems_seqlock */ > +       struct mempolicy *mempolicy; >        short il_next; >  #endif >        atomic_t fs_excl;       /* holding fs exclusive resources */ > diff --git a/kernel/cpuset.c b/kernel/cpuset.c > index d109467..52e6f51 100644 > --- a/kernel/cpuset.c > +++ b/kernel/cpuset.c > @@ -198,12 +198,13 @@ static struct cpuset top_cpuset = { >  * from one of the callbacks into the cpuset code from within >  * __alloc_pages(). >  * > - * If a task is only holding callback_mutex, then it has read-only > - * access to cpusets. > + * If a task is only holding callback_mutex or cgroup_mutext, then it has > + * read-only access to cpusets. >  * >  * Now, the task_struct fields mems_allowed and mempolicy may be changed > - * by other task, we use alloc_lock in the task_struct fields to protect > - * them. > + * by other task, we use alloc_lock(if MAX_NUMNODES <= BITS_PER_LONG) or > + * mems_seqlock(if MAX_NUMNODES > BITS_PER_LONG) in the task_struct fields > + * to protect them. >  * >  * The cpuset_common_file_read() handlers only hold callback_mutex across >  * small pieces of code, such as when reading out possibly multi-word > @@ -920,6 +921,10 @@ static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs, >  *    call to guarantee_online_mems(), as we know no one is changing >  *    our task's cpuset. >  * > + *    As the above comment said, no one can change current task's mems_allowed > + *    except itself. so we needn't hold lock to protect task's mems_allowed > + *    during this call. > + * >  *    While the mm_struct we are migrating is typically from some >  *    other task, the task_struct mems_allowed that we are hacking >  *    is for our current task, which must allocate new pages for that > @@ -947,15 +952,13 @@ static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from, >  * we structure updates as setting all new allowed nodes, then clearing newly >  * disallowed ones. >  * > - * Called with task's alloc_lock held > + * Called with mems_slowpath_lock held >  */ >  static void cpuset_change_task_nodemask(struct task_struct *tsk, >                                        nodemask_t *newmems) >  { > -       nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems); > -       mpol_rebind_task(tsk, &tsk->mems_allowed); > -       mpol_rebind_task(tsk, newmems); >        tsk->mems_allowed = *newmems; > +       mpol_rebind_task(tsk, newmems); >  } > >  /* > @@ -970,6 +973,7 @@ static void cpuset_change_nodemask(struct task_struct *p, >        struct cpuset *cs; >        int migrate; >        const nodemask_t *oldmem = scan->data; > +       unsigned long flags; >        NODEMASK_ALLOC(nodemask_t, newmems, GFP_KERNEL); > >        if (!newmems) > @@ -978,9 +982,9 @@ static void cpuset_change_nodemask(struct task_struct *p, >        cs = cgroup_cs(scan->cg); >        guarantee_online_mems(cs, newmems); > > -       task_lock(p); > +       mems_slowpath_lock_irqsave(p, flags); >        cpuset_change_task_nodemask(p, newmems); > -       task_unlock(p); > +       mems_slowpath_unlock_irqrestore(p, flags); > >        NODEMASK_FREE(newmems); > > @@ -1375,6 +1379,7 @@ static int cpuset_can_attach(struct cgroup_subsys *ss, struct cgroup *cont, >  static void cpuset_attach_task(struct task_struct *tsk, nodemask_t *to, >                               struct cpuset *cs) >  { > +       unsigned long flags; >        int err; >        /* >         * can_attach beforehand should guarantee that this doesn't fail. > @@ -1383,9 +1388,10 @@ static void cpuset_attach_task(struct task_struct *tsk, nodemask_t *to, >        err = set_cpus_allowed_ptr(tsk, cpus_attach); >        WARN_ON_ONCE(err); > > -       task_lock(tsk); > +       mems_slowpath_lock_irqsave(tsk, flags); >        cpuset_change_task_nodemask(tsk, to); > -       task_unlock(tsk); > +       mems_slowpath_unlock_irqrestore(tsk, flags); > + >        cpuset_update_task_spread_flag(cs, tsk); > >  } > @@ -2233,7 +2239,15 @@ nodemask_t cpuset_mems_allowed(struct task_struct *tsk) >  */ >  int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask) >  { > -       return nodes_intersects(*nodemask, current->mems_allowed); > +       unsigned long flags, seq; > +       int retval; > + > +       do { > +               seq = mems_fastpath_lock_irqsave(current, flags); > +               retval = nodes_intersects(*nodemask, current->mems_allowed); > +       } while (mems_fastpath_unlock_irqrestore(current, seq, flags)); > + > +       return retval; >  } > >  /* > @@ -2314,11 +2328,18 @@ int __cpuset_node_allowed_softwall(int node, gfp_t gfp_mask) >  { >        const struct cpuset *cs;        /* current cpuset ancestors */ >        int allowed;                    /* is allocation in zone z allowed? */ > +       unsigned long flags, seq; > >        if (in_interrupt() || (gfp_mask & __GFP_THISNODE)) >                return 1; >        might_sleep_if(!(gfp_mask & __GFP_HARDWALL)); > -       if (node_isset(node, current->mems_allowed)) > + > +       do { > +               seq = mems_fastpath_lock_irqsave(current, flags); > +               allowed = node_isset(node, current->mems_allowed); > +       } while (mems_fastpath_unlock_irqrestore(current, seq, flags)); > + > +       if (allowed) >                return 1; >        /* >         * Allow tasks that have access to memory reserves because they have > @@ -2369,9 +2390,18 @@ int __cpuset_node_allowed_softwall(int node, gfp_t gfp_mask) >  */ >  int __cpuset_node_allowed_hardwall(int node, gfp_t gfp_mask) >  { > +       int allowed; > +       unsigned long flags, seq; > + >        if (in_interrupt() || (gfp_mask & __GFP_THISNODE)) >                return 1; > -       if (node_isset(node, current->mems_allowed)) > + > +       do { > +               seq = mems_fastpath_lock_irqsave(current, flags); > +               allowed = node_isset(node, current->mems_allowed); > +       } while (mems_fastpath_unlock_irqrestore(current, seq, flags)); > + > +       if (allowed) >                return 1; >        /* >         * Allow tasks that have access to memory reserves because they have > @@ -2438,11 +2468,16 @@ void cpuset_unlock(void) >  int cpuset_mem_spread_node(void) >  { >        int node; > - > -       node = next_node(current->cpuset_mem_spread_rotor, current->mems_allowed); > -       if (node == MAX_NUMNODES) > -               node = first_node(current->mems_allowed); > -       current->cpuset_mem_spread_rotor = node; > +       unsigned long flags, seq; > + > +       do { > +               seq = mems_fastpath_lock_irqsave(current, flags); > +               node = next_node(current->cpuset_mem_spread_rotor, > +                                       current->mems_allowed); > +               if (node == MAX_NUMNODES) > +                       node = first_node(current->mems_allowed); > +               current->cpuset_mem_spread_rotor = node; > +       } while (mems_fastpath_unlock_irqrestore(current, seq, flags)); >        return node; >  } >  EXPORT_SYMBOL_GPL(cpuset_mem_spread_node); > @@ -2458,10 +2493,26 @@ EXPORT_SYMBOL_GPL(cpuset_mem_spread_node); >  * to the other. >  **/ > > -int cpuset_mems_allowed_intersects(const struct task_struct *tsk1, > -                                  const struct task_struct *tsk2) > +int cpuset_mems_allowed_intersects(struct task_struct *tsk1, > +                                  struct task_struct *tsk2) >  { > -       return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed); > +       unsigned long flags1, flags2; > +       int retval; > +       struct task_struct *tsk; > + > +       if (tsk1 > tsk2) { > +               tsk = tsk1; > +               tsk1 = tsk2; > +               tsk2 = tsk; > +       } > + > +       mems_slowpath_lock_irqsave(tsk1, flags1); > +       mems_slowpath_lock_irqsave(tsk2, flags2); > +       retval = nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed); > +       mems_slowpath_unlock_irqrestore(tsk2, flags2); > +       mems_slowpath_unlock_irqrestore(tsk1, flags1); > + > +       return retval; >  } > >  /** > diff --git a/kernel/exit.c b/kernel/exit.c > index 7b012a0..cbf045d 100644 > --- a/kernel/exit.c > +++ b/kernel/exit.c > @@ -16,6 +16,7 @@ >  #include >  #include >  #include > +#include >  #include >  #include >  #include > @@ -649,6 +650,7 @@ static void exit_mm(struct task_struct * tsk) >  { >        struct mm_struct *mm = tsk->mm; >        struct core_state *core_state; > +       unsigned long flags; > >        mm_release(tsk, mm); >        if (!mm) > @@ -694,8 +696,10 @@ static void exit_mm(struct task_struct * tsk) >        /* We don't want this task to be frozen prematurely */ >        clear_freeze_flag(tsk); >  #ifdef CONFIG_NUMA > +       mems_slowpath_lock_irqsave(tsk, flags); >        mpol_put(tsk->mempolicy); >        tsk->mempolicy = NULL; > +       mems_slowpath_unlock_irqrestore(tsk, flags); >  #endif >        task_unlock(tsk); >        mm_update_next_owner(mm); > diff --git a/kernel/fork.c b/kernel/fork.c > index fe73f8d..591346a 100644 > --- a/kernel/fork.c > +++ b/kernel/fork.c > @@ -32,6 +32,7 @@ >  #include >  #include >  #include > +#include >  #include >  #include >  #include > @@ -1075,6 +1076,9 @@ static struct task_struct *copy_process(unsigned long clone_flags, >        p->io_context = NULL; >        p->audit_context = NULL; >        cgroup_fork(p); > +#if defined(CONFIG_CPUSETS) && MAX_NUMNODES > BITS_PER_LONG > +       seqlock_init(&p->mems_seqlock); > +#endif >  #ifdef CONFIG_NUMA >        p->mempolicy = mpol_dup(p->mempolicy); >        if (IS_ERR(p->mempolicy)) { > diff --git a/mm/hugetlb.c b/mm/hugetlb.c > index 3a5aeb3..b40cc52 100644 > --- a/mm/hugetlb.c > +++ b/mm/hugetlb.c > @@ -465,6 +465,8 @@ static struct page *dequeue_huge_page_vma(struct hstate *h, >        struct page *page = NULL; >        struct mempolicy *mpol; >        nodemask_t *nodemask; > +       nodemask_t tmp_mask; > +       unsigned long seq, irqflag; >        struct zonelist *zonelist = huge_zonelist(vma, address, >                                        htlb_alloc_mask, &mpol, &nodemask); >        struct zone *zone; > @@ -483,6 +485,15 @@ static struct page *dequeue_huge_page_vma(struct hstate *h, >        if (avoid_reserve && h->free_huge_pages - h->resv_huge_pages == 0) >                return NULL; > > +       if (mpol == current->mempolicy && nodemask) { > +               do { > +                       seq = mems_fastpath_lock_irqsave(current, irqflag); > +                       tmp_mask = *nodemask; > +               } while (mems_fastpath_unlock_irqrestore(current, > +                                                               seq, irqflag)); > +               nodemask = &tmp_mask; > +       } > + Maybe you can define these to a macro or inline function, I saw serveral similar places :-) >        for_each_zone_zonelist_nodemask(zone, z, zonelist, >                                                MAX_NR_ZONES - 1, nodemask) { >                nid = zone_to_nid(zone); > @@ -1835,10 +1846,15 @@ __setup("default_hugepagesz=", hugetlb_default_setup); >  static unsigned int cpuset_mems_nr(unsigned int *array) >  { >        int node; > -       unsigned int nr = 0; > +       unsigned int nr; > +       unsigned long flags, seq; > > -       for_each_node_mask(node, cpuset_current_mems_allowed) > -               nr += array[node]; > +       do { > +               nr = 0; > +               seq = mems_fastpath_lock_irqsave(current, flags); > +               for_each_node_mask(node, cpuset_current_mems_allowed) > +                       nr += array[node]; > +       } while (mems_fastpath_unlock_irqrestore(current, seq, flags)); > >        return nr; >  } > diff --git a/mm/mempolicy.c b/mm/mempolicy.c > index dd3f5c5..49abf11 100644 > --- a/mm/mempolicy.c > +++ b/mm/mempolicy.c > @@ -187,8 +187,10 @@ static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes) >  * parameter with respect to the policy mode and flags.  But, we need to >  * handle an empty nodemask with MPOL_PREFERRED here. >  * > - * Must be called holding task's alloc_lock to protect task's mems_allowed > - * and mempolicy.  May also be called holding the mmap_semaphore for write. > + * Must be called using > + *     mems_slowpath_lock_irqsave()/mems_slowpath_unlock_irqrestore() > + * to protect task's mems_allowed and mempolicy.  May also be called holding > + * the mmap_semaphore for write. >  */ >  static int mpol_set_nodemask(struct mempolicy *pol, >                     const nodemask_t *nodes, struct nodemask_scratch *nsc) > @@ -344,9 +346,10 @@ static void mpol_rebind_policy(struct mempolicy *pol, >  * Wrapper for mpol_rebind_policy() that just requires task >  * pointer, and updates task mempolicy. >  * > - * Called with task's alloc_lock held. > + * Using > + *     mems_slowpath_lock_irqsave()/mems_slowpath_unlock_irqrestore() > + * to protect it. >  */ > - >  void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new) >  { >        mpol_rebind_policy(tsk->mempolicy, new); > @@ -644,6 +647,7 @@ static long do_set_mempolicy(unsigned short mode, unsigned short flags, >        struct mempolicy *new, *old; >        struct mm_struct *mm = current->mm; >        NODEMASK_SCRATCH(scratch); > +       unsigned long irqflags; >        int ret; > >        if (!scratch) > @@ -662,10 +666,10 @@ static long do_set_mempolicy(unsigned short mode, unsigned short flags, >         */ >        if (mm) >                down_write(&mm->mmap_sem); > -       task_lock(current); > +       mems_slowpath_lock_irqsave(current, irqflags); >        ret = mpol_set_nodemask(new, nodes, scratch); >        if (ret) { > -               task_unlock(current); > +               mems_slowpath_unlock_irqrestore(current, irqflags); >                if (mm) >                        up_write(&mm->mmap_sem); >                mpol_put(new); > @@ -677,7 +681,7 @@ static long do_set_mempolicy(unsigned short mode, unsigned short flags, >        if (new && new->mode == MPOL_INTERLEAVE && >            nodes_weight(new->v.nodes)) >                current->il_next = first_node(new->v.nodes); > -       task_unlock(current); > +       mems_slowpath_unlock_irqrestore(current, irqflags); >        if (mm) >                up_write(&mm->mmap_sem); > > @@ -691,7 +695,9 @@ out: >  /* >  * Return nodemask for policy for get_mempolicy() query >  * > - * Called with task's alloc_lock held > + * Must be called using mems_slowpath_lock_irqsave()/ > + * mems_slowpath_unlock_irqrestore() to > + * protect it. >  */ >  static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes) >  { > @@ -736,6 +742,7 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask, >        struct mm_struct *mm = current->mm; >        struct vm_area_struct *vma = NULL; >        struct mempolicy *pol = current->mempolicy; > +       unsigned long irqflags; > >        if (flags & >                ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED)) > @@ -745,9 +752,10 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask, >                if (flags & (MPOL_F_NODE|MPOL_F_ADDR)) >                        return -EINVAL; >                *policy = 0;    /* just so it's initialized */ > -               task_lock(current); > + > +               mems_slowpath_lock_irqsave(current, irqflags); >                *nmask  = cpuset_current_mems_allowed; > -               task_unlock(current); > +               mems_slowpath_unlock_irqrestore(current, irqflags); >                return 0; >        } > > @@ -803,13 +811,13 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask, > >        err = 0; >        if (nmask) { > +               mems_slowpath_lock_irqsave(current, irqflags); >                if (mpol_store_user_nodemask(pol)) { >                        *nmask = pol->w.user_nodemask; >                } else { > -                       task_lock(current); >                        get_policy_nodemask(pol, nmask); > -                       task_unlock(current); >                } > +               mems_slowpath_unlock_irqrestore(current, irqflags); >        } > >  out: > @@ -1008,6 +1016,7 @@ static long do_mbind(unsigned long start, unsigned long len, >        struct mempolicy *new; >        unsigned long end; >        int err; > +       unsigned long irqflags; >        LIST_HEAD(pagelist); > >        if (flags & ~(unsigned long)(MPOL_MF_STRICT | > @@ -1055,9 +1064,9 @@ static long do_mbind(unsigned long start, unsigned long len, >                NODEMASK_SCRATCH(scratch); >                if (scratch) { >                        down_write(&mm->mmap_sem); > -                       task_lock(current); > +                       mems_slowpath_lock_irqsave(current, irqflags); >                        err = mpol_set_nodemask(new, nmask, scratch); > -                       task_unlock(current); > +                       mems_slowpath_unlock_irqrestore(current, irqflags); >                        if (err) >                                up_write(&mm->mmap_sem); >                } else > @@ -1408,8 +1417,10 @@ static struct mempolicy *get_vma_policy(struct task_struct *task, >                } else if (vma->vm_policy) >                        pol = vma->vm_policy; >        } > + >        if (!pol) >                pol = &default_policy; > + >        return pol; >  } > > @@ -1475,7 +1486,7 @@ static unsigned interleave_nodes(struct mempolicy *policy) >  * next slab entry. >  * @policy must be protected by freeing by the caller.  If @policy is >  * the current task's mempolicy, this protection is implicit, as only the > - * task can change it's policy.  The system default policy requires no > + * task can free it's policy.  The system default policy requires no >  * such protection. >  */ >  unsigned slab_node(struct mempolicy *policy) > @@ -1574,16 +1585,33 @@ struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr, >                                nodemask_t **nodemask) >  { >        struct zonelist *zl; > +       struct mempolicy policy; > +       struct mempolicy *pol; > +       unsigned long seq, irqflag; > >        *mpol = get_vma_policy(current, vma, addr); >        *nodemask = NULL;       /* assume !MPOL_BIND */ > > -       if (unlikely((*mpol)->mode == MPOL_INTERLEAVE)) { > -               zl = node_zonelist(interleave_nid(*mpol, vma, addr, > +       pol = *mpol; > +       if (pol == current->mempolicy) { > +               /* > +                * get_vma_policy() doesn't return NULL, so we needn't worry > +                * whether pol is NULL or not. > +                */ > +               do { > +                       seq = mems_fastpath_lock_irqsave(current, irqflag); > +                       policy = *pol; > +               } while (mems_fastpath_unlock_irqrestore(current, > +                                                               seq, irqflag)); > +               pol = &policy; > +       } > + > +       if (unlikely(pol->mode == MPOL_INTERLEAVE)) { > +               zl = node_zonelist(interleave_nid(pol, vma, addr, >                                huge_page_shift(hstate_vma(vma))), gfp_flags); >        } else { > -               zl = policy_zonelist(gfp_flags, *mpol); > -               if ((*mpol)->mode == MPOL_BIND) > +               zl = policy_zonelist(gfp_flags, pol); > +               if (pol->mode == MPOL_BIND) >                        *nodemask = &(*mpol)->v.nodes; >        } >        return zl; > @@ -1609,11 +1637,14 @@ bool init_nodemask_of_mempolicy(nodemask_t *mask) >  { >        struct mempolicy *mempolicy; >        int nid; > +       unsigned long irqflags; > >        if (!(mask && current->mempolicy)) >                return false; > > +       mems_slowpath_lock_irqsave(current, irqflags); >        mempolicy = current->mempolicy; > + >        switch (mempolicy->mode) { >        case MPOL_PREFERRED: >                if (mempolicy->flags & MPOL_F_LOCAL) > @@ -1633,6 +1664,8 @@ bool init_nodemask_of_mempolicy(nodemask_t *mask) >                BUG(); >        } > > +       mems_slowpath_unlock_irqrestore(current, irqflags); > + >        return true; >  } >  #endif > @@ -1722,7 +1755,22 @@ struct page * >  alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr) >  { >        struct mempolicy *pol = get_vma_policy(current, vma, addr); > +       struct mempolicy policy; >        struct zonelist *zl; > +       struct page *page; > +       unsigned long seq, iflags; > + > +       if (pol == current->mempolicy) { > +               /* > +                * get_vma_policy() doesn't return NULL, so we needn't worry > +                * whether pol is NULL or not. > +                */ > +               do { > +                       seq = mems_fastpath_lock_irqsave(current, iflags); > +                       policy = *pol; > +               } while (mems_fastpath_unlock_irqrestore(current, seq, iflags)); > +               pol = &policy; > +       } > >        if (unlikely(pol->mode == MPOL_INTERLEAVE)) { >                unsigned nid; > @@ -1736,15 +1784,16 @@ alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr) >                /* >                 * slow path: ref counted shared policy >                 */ > -               struct page *page =  __alloc_pages_nodemask(gfp, 0, > -                                               zl, policy_nodemask(gfp, pol)); > +               page =  __alloc_pages_nodemask(gfp, 0, zl, > +                                       policy_nodemask(gfp, pol)); >                __mpol_put(pol); >                return page; >        } >        /* >         * fast path:  default or task policy >         */ > -       return __alloc_pages_nodemask(gfp, 0, zl, policy_nodemask(gfp, pol)); > +       page = __alloc_pages_nodemask(gfp, 0, zl, policy_nodemask(gfp, pol)); > +       return page; >  } > >  /** > @@ -1761,26 +1810,37 @@ alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr) >  *     Allocate a page from the kernel page pool.  When not in >  *     interrupt context and apply the current process NUMA policy. >  *     Returns NULL when no page can be allocated. > - * > - *     Don't call cpuset_update_task_memory_state() unless > - *     1) it's ok to take cpuset_sem (can WAIT), and > - *     2) allocating for current task (not interrupt). >  */ >  struct page *alloc_pages_current(gfp_t gfp, unsigned order) >  { >        struct mempolicy *pol = current->mempolicy; > +       struct mempolicy policy; > +       struct page *page; > +       unsigned long seq, irqflags; > + > >        if (!pol || in_interrupt() || (gfp & __GFP_THISNODE)) >                pol = &default_policy; > - > +       else { > +               do { > +                       seq = mems_fastpath_lock_irqsave(current, irqflags); > +                       policy = *pol; > +               } while (mems_fastpath_unlock_irqrestore(current, > +                                                               seq, irqflags)); > +               pol = &policy; > +       } >        /* >         * No reference counting needed for current->mempolicy >         * nor system default_policy >         */ >        if (pol->mode == MPOL_INTERLEAVE) > -               return alloc_page_interleave(gfp, order, interleave_nodes(pol)); > -       return __alloc_pages_nodemask(gfp, order, > -                       policy_zonelist(gfp, pol), policy_nodemask(gfp, pol)); > +               page = alloc_page_interleave(gfp, order, interleave_nodes(pol)); > +       else > +               page =  __alloc_pages_nodemask(gfp, order, > +                                       policy_zonelist(gfp, pol), > +                                       policy_nodemask(gfp, pol)); > + > +       return page; >  } >  EXPORT_SYMBOL(alloc_pages_current); > > @@ -2026,6 +2086,7 @@ restart: >  */ >  void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol) >  { > +       unsigned long irqflags; >        int ret; > >        sp->root = RB_ROOT;             /* empty tree == default mempolicy */ > @@ -2043,9 +2104,9 @@ void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol) >                if (IS_ERR(new)) >                        goto put_free; /* no valid nodemask intersection */ > > -               task_lock(current); > +               mems_slowpath_lock_irqsave(current, irqflags); >                ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch); > -               task_unlock(current); > +               mems_slowpath_unlock_irqrestore(current, irqflags); >                mpol_put(mpol); /* drop our ref on sb mpol */ >                if (ret) >                        goto put_free; > @@ -2200,6 +2261,7 @@ int mpol_parse_str(char *str, struct mempolicy **mpol, int no_context) >        nodemask_t nodes; >        char *nodelist = strchr(str, ':'); >        char *flags = strchr(str, '='); > +       unsigned long irqflags; >        int err = 1; > >        if (nodelist) { > @@ -2291,9 +2353,9 @@ int mpol_parse_str(char *str, struct mempolicy **mpol, int no_context) >                int ret; >                NODEMASK_SCRATCH(scratch); >                if (scratch) { > -                       task_lock(current); > +                       mems_slowpath_lock_irqsave(current, irqflags); >                        ret = mpol_set_nodemask(new, &nodes, scratch); > -                       task_unlock(current); > +                       mems_slowpath_unlock_irqrestore(current, irqflags); >                } else >                        ret = -ENOMEM; >                NODEMASK_SCRATCH_FREE(scratch); > @@ -2487,8 +2549,10 @@ int show_numa_map(struct seq_file *m, void *v) >        struct file *file = vma->vm_file; >        struct mm_struct *mm = vma->vm_mm; >        struct mempolicy *pol; > +       struct mempolicy policy; >        int n; >        char buffer[50]; > +       unsigned long iflags, seq; > >        if (!mm) >                return 0; > @@ -2498,6 +2562,18 @@ int show_numa_map(struct seq_file *m, void *v) >                return 0; > >        pol = get_vma_policy(priv->task, vma, vma->vm_start); > +       if (pol == current->mempolicy) { > +               /* > +                * get_vma_policy() doesn't return NULL, so we needn't worry > +                * whether pol is NULL or not. > +                */ > +               do { > +                       seq = mems_fastpath_lock_irqsave(current, iflags); > +                       policy = *pol; > +               } while (mems_fastpath_unlock_irqrestore(current, seq, iflags)); > +               pol = &policy; > +       } > + >        mpol_to_str(buffer, sizeof(buffer), pol, 0); >        mpol_cond_put(pol); > > diff --git a/mm/slab.c b/mm/slab.c > index 09f1572..b8f5acb 100644 > --- a/mm/slab.c > +++ b/mm/slab.c > @@ -3282,14 +3282,24 @@ static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags) >  static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags) >  { >        int nid_alloc, nid_here; > +       unsigned long lflags, seq; > +       struct mempolicy mpol; > >        if (in_interrupt() || (flags & __GFP_THISNODE)) >                return NULL; > + >        nid_alloc = nid_here = numa_node_id(); >        if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD)) >                nid_alloc = cpuset_mem_spread_node(); > -       else if (current->mempolicy) > -               nid_alloc = slab_node(current->mempolicy); > +       else if (current->mempolicy) { > +               do { > +                       seq = mems_fastpath_lock_irqsave(current, lflags); > +                       mpol = *(current->mempolicy); > +               } while (mems_fastpath_unlock_irqrestore(current, seq, lflags)); > + > +               nid_alloc = slab_node(&mpol); > +       } > + >        if (nid_alloc != nid_here) >                return ____cache_alloc_node(cachep, flags, nid_alloc); >        return NULL; > @@ -3312,11 +3322,21 @@ static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags) >        enum zone_type high_zoneidx = gfp_zone(flags); >        void *obj = NULL; >        int nid; > +       unsigned long lflags, seq; > +       struct mempolicy mpol; > >        if (flags & __GFP_THISNODE) >                return NULL; > > -       zonelist = node_zonelist(slab_node(current->mempolicy), flags); > +       if (current->mempolicy) { > +               do { > +                       seq = mems_fastpath_lock_irqsave(current, lflags); > +                       mpol = *(current->mempolicy); > +               } while (mems_fastpath_unlock_irqrestore(current, seq, lflags)); > +               zonelist = node_zonelist(slab_node(&mpol), flags); > +       } else > +               zonelist = node_zonelist(slab_node(NULL), flags); > + >        local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK); > >  retry: > diff --git a/mm/slub.c b/mm/slub.c > index b364844..436c521 100644 > --- a/mm/slub.c > +++ b/mm/slub.c > @@ -1345,6 +1345,8 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags) >        struct zone *zone; >        enum zone_type high_zoneidx = gfp_zone(flags); >        struct page *page; > +       unsigned long lflags, seq; > +       struct mempolicy mpol; > >        /* >         * The defrag ratio allows a configuration of the tradeoffs between > @@ -1368,7 +1370,15 @@ static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags) >                        get_cycles() % 1024 > s->remote_node_defrag_ratio) >                return NULL; > > -       zonelist = node_zonelist(slab_node(current->mempolicy), flags); > +       if (current->mempolicy) { > +               do { > +                       seq = mems_fastpath_lock_irqsave(current, lflags); > +                       mpol = *(current->mempolicy); > +               } while (mems_fastpath_unlock_irqrestore(current, seq, lflags)); > +               zonelist = node_zonelist(slab_node(&mpol), flags); > +       } else > +               zonelist = node_zonelist(slab_node(NULL), flags); > + >        for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { >                struct kmem_cache_node *n; > > -- > 1.6.5.2 > > > > -- > To unsubscribe, send a message with 'unsubscribe linux-mm' in > the body to majordomo@kvack.org.  For more info on Linux MM, > see: http://www.linux-mm.org/ . > Don't email: email@kvack.org > -- Regards, -Bob Liu