diff for duplicates of <20140702212004.GF1369@cmpxchg.org> diff --git a/a/1.txt b/N1/1.txt index ea17a62..2c5c169 100644 --- a/a/1.txt +++ b/N1/1.txt @@ -54,3 +54,330 @@ stack on top of mm-memcontrol-rewrite-charge-api-fix-shmem_unuse-fix. Thanks! --- +>From b13bbe7774296388ca28afc1ce5776d6d6b371fb Mon Sep 17 00:00:00 2001 +From: Johannes Weiner <hannes@cmpxchg.org> +Date: Wed, 2 Jul 2014 08:50:23 -0400 +Subject: [patch] mm: memcontrol: rewrite uncharge API fix + +Hugh reports: + +====================================================== +[ INFO: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detected ] +3.16.0-rc2-mm1 #3 Not tainted +------------------------------------------------------ +cc1/2771 [HC0[0]:SC0[0]:HE0:SE1] is trying to acquire: + (&(&rtpz->lock)->rlock){+.+.-.}, at: [<ffffffff811518b5>] memcg_check_events+0x17e/0x206 +dd +and this task is already holding: + (&(&zone->lru_lock)->rlock){..-.-.}, at: [<ffffffff8110da3f>] release_pages+0xe7/0x239 +which would create a new lock dependency: + (&(&zone->lru_lock)->rlock){..-.-.} -> (&(&rtpz->lock)->rlock){+.+.-.} + +but this new dependency connects a SOFTIRQ-irq-safe lock: + (&(&zone->lru_lock)->rlock){..-.-.} +... which became SOFTIRQ-irq-safe at: + [<ffffffff810c201e>] __lock_acquire+0x59f/0x17e8 + [<ffffffff810c38a6>] lock_acquire+0x61/0x78 + [<ffffffff815bdfbd>] _raw_spin_lock_irqsave+0x3f/0x51 + [<ffffffff8110dc0e>] pagevec_lru_move_fn+0x7d/0xf6 + [<ffffffff8110dca4>] pagevec_move_tail+0x1d/0x2c + [<ffffffff8110e298>] rotate_reclaimable_page+0xb2/0xd4 + [<ffffffff811018bf>] end_page_writeback+0x1c/0x45 + [<ffffffff81134400>] end_swap_bio_write+0x5c/0x69 + [<ffffffff8123473e>] bio_endio+0x50/0x6e + [<ffffffff81238dee>] blk_update_request+0x163/0x255 + [<ffffffff81238ef7>] blk_update_bidi_request+0x17/0x65 + [<ffffffff81239242>] blk_end_bidi_request+0x1a/0x56 + [<ffffffff81239289>] blk_end_request+0xb/0xd + [<ffffffff813a075a>] scsi_io_completion+0x16d/0x553 + [<ffffffff81399c0f>] scsi_finish_command+0xb6/0xbf + [<ffffffff813a0564>] scsi_softirq_done+0xe9/0xf0 + [<ffffffff8123e8e5>] blk_done_softirq+0x79/0x8b + [<ffffffff81088675>] __do_softirq+0xfc/0x21f + [<ffffffff8108898f>] irq_exit+0x3d/0x92 + [<ffffffff81032379>] do_IRQ+0xcc/0xe5 + [<ffffffff815bf5ac>] ret_from_intr+0x0/0x13 + [<ffffffff81443ac0>] cpuidle_enter+0x12/0x14 + [<ffffffff810bb4e4>] cpu_startup_entry+0x187/0x243 + [<ffffffff815a90ab>] rest_init+0x12f/0x133 + [<ffffffff81970e7c>] start_kernel+0x396/0x3a3 + [<ffffffff81970489>] x86_64_start_reservations+0x2a/0x2c + [<ffffffff81970552>] x86_64_start_kernel+0xc7/0xca + +to a SOFTIRQ-irq-unsafe lock: + (&(&rtpz->lock)->rlock){+.+.-.} +... which became SOFTIRQ-irq-unsafe at: +... [<ffffffff810c2095>] __lock_acquire+0x616/0x17e8 + [<ffffffff810c38a6>] lock_acquire+0x61/0x78 + [<ffffffff815bde9f>] _raw_spin_lock+0x34/0x41 + [<ffffffff811518b5>] memcg_check_events+0x17e/0x206 + [<ffffffff811535bb>] commit_charge+0x260/0x26f + [<ffffffff81157004>] mem_cgroup_commit_charge+0xb1/0xdb + [<ffffffff81115b51>] shmem_getpage_gfp+0x400/0x6c2 + [<ffffffff81115ecc>] shmem_write_begin+0x33/0x35 + [<ffffffff81102a24>] generic_perform_write+0xb7/0x1a4 + [<ffffffff8110391e>] __generic_file_write_iter+0x25b/0x29b + [<ffffffff81103999>] generic_file_write_iter+0x3b/0xa5 + [<ffffffff8115a115>] new_sync_write+0x7b/0x9f + [<ffffffff8115a56c>] vfs_write+0xb5/0x169 + [<ffffffff8115ae1f>] SyS_write+0x45/0x8c + [<ffffffff815bead2>] system_call_fastpath+0x16/0x1b + +The soft limit tree lock needs to be IRQ-safe as it's acquired while +holding the IRQ-safe zone->lru_lock. + +But more importantly, with uncharge happening in release_pages() now, +this path is executed from interrupt context. + +Make the soft limit tree lock, uncharge batching, and charge +statistics IRQ-safe. + +Reported-by: Hugh Dickins <hughd@google.com> +Signed-off-by: Johannes Weiner <hannes@cmpxchg.org> +--- + mm/memcontrol.c | 113 +++++++++++++++++++++++++++++++++----------------------- + 1 file changed, 67 insertions(+), 46 deletions(-) + +diff --git a/mm/memcontrol.c b/mm/memcontrol.c +index 6c3ffb02651e..91b621846e10 100644 +--- a/mm/memcontrol.c ++++ b/mm/memcontrol.c +@@ -754,9 +754,11 @@ static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_zone *mz, + static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_zone *mz, + struct mem_cgroup_tree_per_zone *mctz) + { +- spin_lock(&mctz->lock); ++ unsigned long flags; ++ ++ spin_lock_irqsave(&mctz->lock, flags); + __mem_cgroup_remove_exceeded(mz, mctz); +- spin_unlock(&mctz->lock); ++ spin_unlock_irqrestore(&mctz->lock, flags); + } + + +@@ -779,7 +781,9 @@ static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page) + * mem is over its softlimit. + */ + if (excess || mz->on_tree) { +- spin_lock(&mctz->lock); ++ unsigned long flags; ++ ++ spin_lock_irqsave(&mctz->lock, flags); + /* if on-tree, remove it */ + if (mz->on_tree) + __mem_cgroup_remove_exceeded(mz, mctz); +@@ -788,7 +792,7 @@ static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page) + * If excess is 0, no tree ops. + */ + __mem_cgroup_insert_exceeded(mz, mctz, excess); +- spin_unlock(&mctz->lock); ++ spin_unlock_irqrestore(&mctz->lock, flags); + } + } + } +@@ -839,9 +843,9 @@ mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) + { + struct mem_cgroup_per_zone *mz; + +- spin_lock(&mctz->lock); ++ spin_lock_irq(&mctz->lock); + mz = __mem_cgroup_largest_soft_limit_node(mctz); +- spin_unlock(&mctz->lock); ++ spin_unlock_irq(&mctz->lock); + return mz; + } + +@@ -904,8 +908,9 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, + struct page *page, + int nr_pages) + { +- preempt_disable(); ++ unsigned long flags; + ++ local_irq_save(flags); + /* + * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is + * counted as CACHE even if it's on ANON LRU. +@@ -930,7 +935,7 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg, + } + + __this_cpu_add(memcg->stat->nr_page_events, nr_pages); +- preempt_enable(); ++ local_irq_restore(flags); + } + + unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru) +@@ -1009,7 +1014,9 @@ static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg, + */ + static void memcg_check_events(struct mem_cgroup *memcg, struct page *page) + { +- preempt_disable(); ++ unsigned long flags; ++ ++ local_irq_save(flags); + /* threshold event is triggered in finer grain than soft limit */ + if (unlikely(mem_cgroup_event_ratelimit(memcg, + MEM_CGROUP_TARGET_THRESH))) { +@@ -1022,7 +1029,7 @@ static void memcg_check_events(struct mem_cgroup *memcg, struct page *page) + do_numainfo = mem_cgroup_event_ratelimit(memcg, + MEM_CGROUP_TARGET_NUMAINFO); + #endif +- preempt_enable(); ++ local_irq_restore(flags); + + mem_cgroup_threshold(memcg); + if (unlikely(do_softlimit)) +@@ -1032,7 +1039,7 @@ static void memcg_check_events(struct mem_cgroup *memcg, struct page *page) + atomic_inc(&memcg->numainfo_events); + #endif + } else +- preempt_enable(); ++ local_irq_restore(flags); + } + + struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) +@@ -3620,6 +3627,9 @@ out: + + void mem_cgroup_uncharge_start(void) + { ++ unsigned long flags; ++ ++ local_irq_save(flags); + current->memcg_batch.do_batch++; + /* We can do nest. */ + if (current->memcg_batch.do_batch == 1) { +@@ -3627,21 +3637,18 @@ void mem_cgroup_uncharge_start(void) + current->memcg_batch.nr_pages = 0; + current->memcg_batch.memsw_nr_pages = 0; + } ++ local_irq_restore(flags); + } + + void mem_cgroup_uncharge_end(void) + { + struct memcg_batch_info *batch = ¤t->memcg_batch; ++ unsigned long flags; + +- if (!batch->do_batch) +- return; +- +- batch->do_batch--; +- if (batch->do_batch) /* If stacked, do nothing. */ +- return; +- +- if (!batch->memcg) +- return; ++ local_irq_save(flags); ++ VM_BUG_ON(!batch->do_batch); ++ if (--batch->do_batch) /* If stacked, do nothing */ ++ goto out; + /* + * This "batch->memcg" is valid without any css_get/put etc... + * bacause we hide charges behind us. +@@ -3653,8 +3660,8 @@ void mem_cgroup_uncharge_end(void) + res_counter_uncharge(&batch->memcg->memsw, + batch->memsw_nr_pages * PAGE_SIZE); + memcg_oom_recover(batch->memcg); +- /* forget this pointer (for sanity check) */ +- batch->memcg = NULL; ++out: ++ local_irq_restore(flags); + } + + #ifdef CONFIG_MEMCG_SWAP +@@ -6554,6 +6561,36 @@ void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg) + cancel_charge(memcg, nr_pages); + } + ++static bool uncharge_batched(struct mem_cgroup *memcg, ++ unsigned long pc_flags, ++ unsigned int nr_pages) ++{ ++ struct memcg_batch_info *batch = ¤t->memcg_batch; ++ bool uncharged = false; ++ unsigned long flags; ++ ++ if (nr_pages > 1) ++ return false; ++ if (test_thread_flag(TIF_MEMDIE)) ++ return false; ++ ++ local_irq_save(flags); ++ if (!batch->do_batch) ++ goto out; ++ if (batch->memcg && batch->memcg != memcg) ++ goto out; ++ if (!batch->memcg) ++ batch->memcg = memcg; ++ if (pc_flags & PCG_MEM) ++ batch->nr_pages++; ++ if (pc_flags & PCG_MEMSW) ++ batch->memsw_nr_pages++; ++ uncharged = true; ++out: ++ local_irq_restore(flags); ++ return uncharged; ++} ++ + /** + * mem_cgroup_uncharge - uncharge a page + * @page: page to uncharge +@@ -6563,11 +6600,10 @@ void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg) + */ + void mem_cgroup_uncharge(struct page *page) + { +- struct memcg_batch_info *batch; + unsigned int nr_pages = 1; + struct mem_cgroup *memcg; + struct page_cgroup *pc; +- unsigned long flags; ++ unsigned long pc_flags; + + VM_BUG_ON_PAGE(PageLRU(page), page); + VM_BUG_ON_PAGE(page_count(page), page); +@@ -6591,35 +6627,20 @@ void mem_cgroup_uncharge(struct page *page) + * exclusive access to the page. + */ + memcg = pc->mem_cgroup; +- flags = pc->flags; ++ pc_flags = pc->flags; + pc->flags = 0; + + mem_cgroup_charge_statistics(memcg, page, -nr_pages); + memcg_check_events(memcg, page); + +- batch = ¤t->memcg_batch; +- if (!batch->memcg) +- batch->memcg = memcg; +- else if (batch->memcg != memcg) +- goto uncharge; +- if (nr_pages > 1) +- goto uncharge; +- if (!batch->do_batch) +- goto uncharge; +- if (test_thread_flag(TIF_MEMDIE)) +- goto uncharge; +- if (flags & PCG_MEM) +- batch->nr_pages++; +- if (flags & PCG_MEMSW) +- batch->memsw_nr_pages++; +- return; +-uncharge: +- if (flags & PCG_MEM) ++ if (uncharge_batched(memcg, pc_flags, nr_pages)) ++ return; ++ ++ if (pc_flags & PCG_MEM) + res_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE); +- if (flags & PCG_MEMSW) ++ if (pc_flags & PCG_MEMSW) + res_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE); +- if (batch->memcg != memcg) +- memcg_oom_recover(memcg); ++ memcg_oom_recover(memcg); + } + + /** +-- +2.0.0 diff --git a/a/content_digest b/N1/content_digest index 8592b54..d3c06df 100644 --- a/a/content_digest +++ b/N1/content_digest @@ -65,6 +65,333 @@ "\n" "Thanks!\n" "\n" - --- + "---\n" + ">From b13bbe7774296388ca28afc1ce5776d6d6b371fb Mon Sep 17 00:00:00 2001\n" + "From: Johannes Weiner <hannes@cmpxchg.org>\n" + "Date: Wed, 2 Jul 2014 08:50:23 -0400\n" + "Subject: [patch] mm: memcontrol: rewrite uncharge API fix\n" + "\n" + "Hugh reports:\n" + "\n" + "======================================================\n" + "[ INFO: SOFTIRQ-safe -> SOFTIRQ-unsafe lock order detected ]\n" + "3.16.0-rc2-mm1 #3 Not tainted\n" + "------------------------------------------------------\n" + "cc1/2771 [HC0[0]:SC0[0]:HE0:SE1] is trying to acquire:\n" + " (&(&rtpz->lock)->rlock){+.+.-.}, at: [<ffffffff811518b5>] memcg_check_events+0x17e/0x206\n" + "dd\n" + "and this task is already holding:\n" + " (&(&zone->lru_lock)->rlock){..-.-.}, at: [<ffffffff8110da3f>] release_pages+0xe7/0x239\n" + "which would create a new lock dependency:\n" + " (&(&zone->lru_lock)->rlock){..-.-.} -> (&(&rtpz->lock)->rlock){+.+.-.}\n" + "\n" + "but this new dependency connects a SOFTIRQ-irq-safe lock:\n" + " (&(&zone->lru_lock)->rlock){..-.-.}\n" + "... which became SOFTIRQ-irq-safe at:\n" + " [<ffffffff810c201e>] __lock_acquire+0x59f/0x17e8\n" + " [<ffffffff810c38a6>] lock_acquire+0x61/0x78\n" + " [<ffffffff815bdfbd>] _raw_spin_lock_irqsave+0x3f/0x51\n" + " [<ffffffff8110dc0e>] pagevec_lru_move_fn+0x7d/0xf6\n" + " [<ffffffff8110dca4>] pagevec_move_tail+0x1d/0x2c\n" + " [<ffffffff8110e298>] rotate_reclaimable_page+0xb2/0xd4\n" + " [<ffffffff811018bf>] end_page_writeback+0x1c/0x45\n" + " [<ffffffff81134400>] end_swap_bio_write+0x5c/0x69\n" + " [<ffffffff8123473e>] bio_endio+0x50/0x6e\n" + " [<ffffffff81238dee>] blk_update_request+0x163/0x255\n" + " [<ffffffff81238ef7>] blk_update_bidi_request+0x17/0x65\n" + " [<ffffffff81239242>] blk_end_bidi_request+0x1a/0x56\n" + " [<ffffffff81239289>] blk_end_request+0xb/0xd\n" + " [<ffffffff813a075a>] scsi_io_completion+0x16d/0x553\n" + " [<ffffffff81399c0f>] scsi_finish_command+0xb6/0xbf\n" + " [<ffffffff813a0564>] scsi_softirq_done+0xe9/0xf0\n" + " [<ffffffff8123e8e5>] blk_done_softirq+0x79/0x8b\n" + " [<ffffffff81088675>] __do_softirq+0xfc/0x21f\n" + " [<ffffffff8108898f>] irq_exit+0x3d/0x92\n" + " [<ffffffff81032379>] do_IRQ+0xcc/0xe5\n" + " [<ffffffff815bf5ac>] ret_from_intr+0x0/0x13\n" + " [<ffffffff81443ac0>] cpuidle_enter+0x12/0x14\n" + " [<ffffffff810bb4e4>] cpu_startup_entry+0x187/0x243\n" + " [<ffffffff815a90ab>] rest_init+0x12f/0x133\n" + " [<ffffffff81970e7c>] start_kernel+0x396/0x3a3\n" + " [<ffffffff81970489>] x86_64_start_reservations+0x2a/0x2c\n" + " [<ffffffff81970552>] x86_64_start_kernel+0xc7/0xca\n" + "\n" + "to a SOFTIRQ-irq-unsafe lock:\n" + " (&(&rtpz->lock)->rlock){+.+.-.}\n" + "... which became SOFTIRQ-irq-unsafe at:\n" + "... [<ffffffff810c2095>] __lock_acquire+0x616/0x17e8\n" + " [<ffffffff810c38a6>] lock_acquire+0x61/0x78\n" + " [<ffffffff815bde9f>] _raw_spin_lock+0x34/0x41\n" + " [<ffffffff811518b5>] memcg_check_events+0x17e/0x206\n" + " [<ffffffff811535bb>] commit_charge+0x260/0x26f\n" + " [<ffffffff81157004>] mem_cgroup_commit_charge+0xb1/0xdb\n" + " [<ffffffff81115b51>] shmem_getpage_gfp+0x400/0x6c2\n" + " [<ffffffff81115ecc>] shmem_write_begin+0x33/0x35\n" + " [<ffffffff81102a24>] generic_perform_write+0xb7/0x1a4\n" + " [<ffffffff8110391e>] __generic_file_write_iter+0x25b/0x29b\n" + " [<ffffffff81103999>] generic_file_write_iter+0x3b/0xa5\n" + " [<ffffffff8115a115>] new_sync_write+0x7b/0x9f\n" + " [<ffffffff8115a56c>] vfs_write+0xb5/0x169\n" + " [<ffffffff8115ae1f>] SyS_write+0x45/0x8c\n" + " [<ffffffff815bead2>] system_call_fastpath+0x16/0x1b\n" + "\n" + "The soft limit tree lock needs to be IRQ-safe as it's acquired while\n" + "holding the IRQ-safe zone->lru_lock.\n" + "\n" + "But more importantly, with uncharge happening in release_pages() now,\n" + "this path is executed from interrupt context.\n" + "\n" + "Make the soft limit tree lock, uncharge batching, and charge\n" + "statistics IRQ-safe.\n" + "\n" + "Reported-by: Hugh Dickins <hughd@google.com>\n" + "Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>\n" + "---\n" + " mm/memcontrol.c | 113 +++++++++++++++++++++++++++++++++-----------------------\n" + " 1 file changed, 67 insertions(+), 46 deletions(-)\n" + "\n" + "diff --git a/mm/memcontrol.c b/mm/memcontrol.c\n" + "index 6c3ffb02651e..91b621846e10 100644\n" + "--- a/mm/memcontrol.c\n" + "+++ b/mm/memcontrol.c\n" + "@@ -754,9 +754,11 @@ static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_zone *mz,\n" + " static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_zone *mz,\n" + " \t\t\t\t struct mem_cgroup_tree_per_zone *mctz)\n" + " {\n" + "-\tspin_lock(&mctz->lock);\n" + "+\tunsigned long flags;\n" + "+\n" + "+\tspin_lock_irqsave(&mctz->lock, flags);\n" + " \t__mem_cgroup_remove_exceeded(mz, mctz);\n" + "-\tspin_unlock(&mctz->lock);\n" + "+\tspin_unlock_irqrestore(&mctz->lock, flags);\n" + " }\n" + " \n" + " \n" + "@@ -779,7 +781,9 @@ static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)\n" + " \t\t * mem is over its softlimit.\n" + " \t\t */\n" + " \t\tif (excess || mz->on_tree) {\n" + "-\t\t\tspin_lock(&mctz->lock);\n" + "+\t\t\tunsigned long flags;\n" + "+\n" + "+\t\t\tspin_lock_irqsave(&mctz->lock, flags);\n" + " \t\t\t/* if on-tree, remove it */\n" + " \t\t\tif (mz->on_tree)\n" + " \t\t\t\t__mem_cgroup_remove_exceeded(mz, mctz);\n" + "@@ -788,7 +792,7 @@ static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)\n" + " \t\t\t * If excess is 0, no tree ops.\n" + " \t\t\t */\n" + " \t\t\t__mem_cgroup_insert_exceeded(mz, mctz, excess);\n" + "-\t\t\tspin_unlock(&mctz->lock);\n" + "+\t\t\tspin_unlock_irqrestore(&mctz->lock, flags);\n" + " \t\t}\n" + " \t}\n" + " }\n" + "@@ -839,9 +843,9 @@ mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)\n" + " {\n" + " \tstruct mem_cgroup_per_zone *mz;\n" + " \n" + "-\tspin_lock(&mctz->lock);\n" + "+\tspin_lock_irq(&mctz->lock);\n" + " \tmz = __mem_cgroup_largest_soft_limit_node(mctz);\n" + "-\tspin_unlock(&mctz->lock);\n" + "+\tspin_unlock_irq(&mctz->lock);\n" + " \treturn mz;\n" + " }\n" + " \n" + "@@ -904,8 +908,9 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,\n" + " \t\t\t\t\t struct page *page,\n" + " \t\t\t\t\t int nr_pages)\n" + " {\n" + "-\tpreempt_disable();\n" + "+\tunsigned long flags;\n" + " \n" + "+\tlocal_irq_save(flags);\n" + " \t/*\n" + " \t * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is\n" + " \t * counted as CACHE even if it's on ANON LRU.\n" + "@@ -930,7 +935,7 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,\n" + " \t}\n" + " \n" + " \t__this_cpu_add(memcg->stat->nr_page_events, nr_pages);\n" + "-\tpreempt_enable();\n" + "+\tlocal_irq_restore(flags);\n" + " }\n" + " \n" + " unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)\n" + "@@ -1009,7 +1014,9 @@ static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,\n" + " */\n" + " static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)\n" + " {\n" + "-\tpreempt_disable();\n" + "+\tunsigned long flags;\n" + "+\n" + "+\tlocal_irq_save(flags);\n" + " \t/* threshold event is triggered in finer grain than soft limit */\n" + " \tif (unlikely(mem_cgroup_event_ratelimit(memcg,\n" + " \t\t\t\t\t\tMEM_CGROUP_TARGET_THRESH))) {\n" + "@@ -1022,7 +1029,7 @@ static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)\n" + " \t\tdo_numainfo = mem_cgroup_event_ratelimit(memcg,\n" + " \t\t\t\t\t\tMEM_CGROUP_TARGET_NUMAINFO);\n" + " #endif\n" + "-\t\tpreempt_enable();\n" + "+\t\tlocal_irq_restore(flags);\n" + " \n" + " \t\tmem_cgroup_threshold(memcg);\n" + " \t\tif (unlikely(do_softlimit))\n" + "@@ -1032,7 +1039,7 @@ static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)\n" + " \t\t\tatomic_inc(&memcg->numainfo_events);\n" + " #endif\n" + " \t} else\n" + "-\t\tpreempt_enable();\n" + "+\t\tlocal_irq_restore(flags);\n" + " }\n" + " \n" + " struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)\n" + "@@ -3620,6 +3627,9 @@ out:\n" + " \n" + " void mem_cgroup_uncharge_start(void)\n" + " {\n" + "+\tunsigned long flags;\n" + "+\n" + "+\tlocal_irq_save(flags);\n" + " \tcurrent->memcg_batch.do_batch++;\n" + " \t/* We can do nest. */\n" + " \tif (current->memcg_batch.do_batch == 1) {\n" + "@@ -3627,21 +3637,18 @@ void mem_cgroup_uncharge_start(void)\n" + " \t\tcurrent->memcg_batch.nr_pages = 0;\n" + " \t\tcurrent->memcg_batch.memsw_nr_pages = 0;\n" + " \t}\n" + "+\tlocal_irq_restore(flags);\n" + " }\n" + " \n" + " void mem_cgroup_uncharge_end(void)\n" + " {\n" + " \tstruct memcg_batch_info *batch = ¤t->memcg_batch;\n" + "+\tunsigned long flags;\n" + " \n" + "-\tif (!batch->do_batch)\n" + "-\t\treturn;\n" + "-\n" + "-\tbatch->do_batch--;\n" + "-\tif (batch->do_batch) /* If stacked, do nothing. */\n" + "-\t\treturn;\n" + "-\n" + "-\tif (!batch->memcg)\n" + "-\t\treturn;\n" + "+\tlocal_irq_save(flags);\n" + "+\tVM_BUG_ON(!batch->do_batch);\n" + "+\tif (--batch->do_batch) /* If stacked, do nothing */\n" + "+\t\tgoto out;\n" + " \t/*\n" + " \t * This \"batch->memcg\" is valid without any css_get/put etc...\n" + " \t * bacause we hide charges behind us.\n" + "@@ -3653,8 +3660,8 @@ void mem_cgroup_uncharge_end(void)\n" + " \t\tres_counter_uncharge(&batch->memcg->memsw,\n" + " \t\t\t\t batch->memsw_nr_pages * PAGE_SIZE);\n" + " \tmemcg_oom_recover(batch->memcg);\n" + "-\t/* forget this pointer (for sanity check) */\n" + "-\tbatch->memcg = NULL;\n" + "+out:\n" + "+\tlocal_irq_restore(flags);\n" + " }\n" + " \n" + " #ifdef CONFIG_MEMCG_SWAP\n" + "@@ -6554,6 +6561,36 @@ void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg)\n" + " \tcancel_charge(memcg, nr_pages);\n" + " }\n" + " \n" + "+static bool uncharge_batched(struct mem_cgroup *memcg,\n" + "+\t\t\t unsigned long pc_flags,\n" + "+\t\t\t unsigned int nr_pages)\n" + "+{\n" + "+\tstruct memcg_batch_info *batch = ¤t->memcg_batch;\n" + "+\tbool uncharged = false;\n" + "+\tunsigned long flags;\n" + "+\n" + "+\tif (nr_pages > 1)\n" + "+\t\treturn false;\n" + "+\tif (test_thread_flag(TIF_MEMDIE))\n" + "+\t\treturn false;\n" + "+\n" + "+\tlocal_irq_save(flags);\n" + "+\tif (!batch->do_batch)\n" + "+\t\tgoto out;\n" + "+\tif (batch->memcg && batch->memcg != memcg)\n" + "+\t\tgoto out;\n" + "+\tif (!batch->memcg)\n" + "+\t\tbatch->memcg = memcg;\n" + "+\tif (pc_flags & PCG_MEM)\n" + "+\t\tbatch->nr_pages++;\n" + "+\tif (pc_flags & PCG_MEMSW)\n" + "+\t\tbatch->memsw_nr_pages++;\n" + "+\tuncharged = true;\n" + "+out:\n" + "+\tlocal_irq_restore(flags);\n" + "+\treturn uncharged;\n" + "+}\n" + "+\n" + " /**\n" + " * mem_cgroup_uncharge - uncharge a page\n" + " * @page: page to uncharge\n" + "@@ -6563,11 +6600,10 @@ void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg)\n" + " */\n" + " void mem_cgroup_uncharge(struct page *page)\n" + " {\n" + "-\tstruct memcg_batch_info *batch;\n" + " \tunsigned int nr_pages = 1;\n" + " \tstruct mem_cgroup *memcg;\n" + " \tstruct page_cgroup *pc;\n" + "-\tunsigned long flags;\n" + "+\tunsigned long pc_flags;\n" + " \n" + " \tVM_BUG_ON_PAGE(PageLRU(page), page);\n" + " \tVM_BUG_ON_PAGE(page_count(page), page);\n" + "@@ -6591,35 +6627,20 @@ void mem_cgroup_uncharge(struct page *page)\n" + " \t * exclusive access to the page.\n" + " \t */\n" + " \tmemcg = pc->mem_cgroup;\n" + "-\tflags = pc->flags;\n" + "+\tpc_flags = pc->flags;\n" + " \tpc->flags = 0;\n" + " \n" + " \tmem_cgroup_charge_statistics(memcg, page, -nr_pages);\n" + " \tmemcg_check_events(memcg, page);\n" + " \n" + "-\tbatch = ¤t->memcg_batch;\n" + "-\tif (!batch->memcg)\n" + "-\t\tbatch->memcg = memcg;\n" + "-\telse if (batch->memcg != memcg)\n" + "-\t\tgoto uncharge;\n" + "-\tif (nr_pages > 1)\n" + "-\t\tgoto uncharge;\n" + "-\tif (!batch->do_batch)\n" + "-\t\tgoto uncharge;\n" + "-\tif (test_thread_flag(TIF_MEMDIE))\n" + "-\t\tgoto uncharge;\n" + "-\tif (flags & PCG_MEM)\n" + "-\t\tbatch->nr_pages++;\n" + "-\tif (flags & PCG_MEMSW)\n" + "-\t\tbatch->memsw_nr_pages++;\n" + "-\treturn;\n" + "-uncharge:\n" + "-\tif (flags & PCG_MEM)\n" + "+\tif (uncharge_batched(memcg, pc_flags, nr_pages))\n" + "+\t\treturn;\n" + "+\n" + "+\tif (pc_flags & PCG_MEM)\n" + " \t\tres_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE);\n" + "-\tif (flags & PCG_MEMSW)\n" + "+\tif (pc_flags & PCG_MEMSW)\n" + " \t\tres_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE);\n" + "-\tif (batch->memcg != memcg)\n" + "-\t\tmemcg_oom_recover(memcg);\n" + "+\tmemcg_oom_recover(memcg);\n" + " }\n" + " \n" + " /**\n" + "-- \n" + 2.0.0 -da5c41a7462fb3fc391e9b454c69d870d53bb52cc76ba3f607868965e437b596 +bfa7f08fdcb4016328de6e9ce06acaf3a1e315644bd908c6e4fb286f2de2b5ad
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.