From: aarcange@redhat.com
To: linux-mm@kvack.org
Cc: Marcelo Tosatti <mtosatti@redhat.com>,
Adam Litke <agl@us.ibm.com>, Avi Kivity <avi@redhat.com>,
Izik Eidus <ieidus@redhat.com>,
Hugh Dickins <hugh.dickins@tiscali.co.uk>,
Nick Piggin <npiggin@suse.de>, Rik van Riel <riel@redhat.com>,
Mel Gorman <mel@csn.ul.ie>, Dave Hansen <dave@linux.vnet.ibm.com>,
Benjamin Herrenschmidt <benh@kernel.crashing.org>,
Ingo Molnar <mingo@elte.hu>, Mike Travis <travis@sgi.com>,
KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>,
Christoph Lameter <cl@linux-foundation.org>,
Chris Wright <chrisw@sous-sol.org>,
Andrew Morton <akpm@linux-foundation.org>,
bpicco@redhat.com,
KOSAKI Motohiro <kosaki.motohiro@jp.fujitsu.com>,
Balbir Singh <balbir@linux.vnet.ibm.com>,
Arnd Bergmann <arnd@arndb.de>,
Andrea Arcangeli <aarcange@redhat.com>
Subject: [patch 33/36] memcg compound
Date: Sun, 21 Feb 2010 15:10:42 +0100 [thread overview]
Message-ID: <20100221141758.070493777@redhat.com> (raw)
In-Reply-To: 20100221141009.581909647@redhat.com
[-- Attachment #1: memcg_compound --]
[-- Type: text/plain, Size: 9315 bytes --]
From: Andrea Arcangeli <aarcange@redhat.com>
Teach memcg to charge/uncharge compound pages.
Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Acked-by: Rik van Riel <riel@redhat.com>
---
---
Documentation/cgroups/memory.txt | 4 +
mm/memcontrol.c | 80 +++++++++++++++++++++++----------------
2 files changed, 52 insertions(+), 32 deletions(-)
--- a/Documentation/cgroups/memory.txt
+++ b/Documentation/cgroups/memory.txt
@@ -4,6 +4,10 @@ NOTE: The Memory Resource Controller has
to as the memory controller in this document. Do not confuse memory controller
used here with the memory controller that is used in hardware.
+NOTE: When in this documentation we refer to PAGE_SIZE, we actually
+mean the real page size of the page being accounted which is bigger than
+PAGE_SIZE for compound pages.
+
Salient features
a. Enable control of Anonymous, Page Cache (mapped and unmapped) and
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -1424,8 +1424,8 @@ static int __cpuinit memcg_stock_cpu_cal
* oom-killer can be invoked.
*/
static int __mem_cgroup_try_charge(struct mm_struct *mm,
- gfp_t gfp_mask, struct mem_cgroup **memcg,
- bool oom, struct page *page)
+ gfp_t gfp_mask, struct mem_cgroup **memcg,
+ bool oom, struct page *page, int page_size)
{
struct mem_cgroup *mem, *mem_over_limit;
int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
@@ -1438,6 +1438,9 @@ static int __mem_cgroup_try_charge(struc
return 0;
}
+ if (PageTransHuge(page))
+ csize = page_size;
+
/*
* We always charge the cgroup the mm_struct belongs to.
* The mm_struct's mem_cgroup changes on task migration if the
@@ -1462,8 +1465,9 @@ static int __mem_cgroup_try_charge(struc
int ret = 0;
unsigned long flags = 0;
- if (consume_stock(mem))
- goto charged;
+ if (!PageTransHuge(page))
+ if (consume_stock(mem))
+ goto charged;
ret = res_counter_charge(&mem->res, csize, &fail_res);
if (likely(!ret)) {
@@ -1483,7 +1487,7 @@ static int __mem_cgroup_try_charge(struc
res);
/* reduce request size and retry */
- if (csize > PAGE_SIZE) {
+ if (csize > page_size) {
csize = PAGE_SIZE;
continue;
}
@@ -1556,7 +1560,7 @@ static int __mem_cgroup_try_charge(struc
goto nomem;
}
}
- if (csize > PAGE_SIZE)
+ if (csize > page_size)
refill_stock(mem, csize - PAGE_SIZE);
charged:
/*
@@ -1593,9 +1597,10 @@ static void __mem_cgroup_cancel_charge(s
/* we don't need css_put for root */
}
-static void mem_cgroup_cancel_charge(struct mem_cgroup *mem)
+static void mem_cgroup_cancel_charge(struct mem_cgroup *mem,
+ int page_size)
{
- __mem_cgroup_cancel_charge(mem, 1);
+ __mem_cgroup_cancel_charge(mem, page_size >> PAGE_SHIFT);
}
/*
@@ -1651,8 +1656,9 @@ struct mem_cgroup *try_get_mem_cgroup_fr
*/
static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
- struct page_cgroup *pc,
- enum charge_type ctype)
+ struct page_cgroup *pc,
+ enum charge_type ctype,
+ int page_size)
{
/* try_charge() can return NULL to *memcg, taking care of it. */
if (!mem)
@@ -1661,7 +1667,7 @@ static void __mem_cgroup_commit_charge(s
lock_page_cgroup(pc);
if (unlikely(PageCgroupUsed(pc))) {
unlock_page_cgroup(pc);
- mem_cgroup_cancel_charge(mem);
+ mem_cgroup_cancel_charge(mem, page_size);
return;
}
@@ -1732,7 +1738,7 @@ static void __mem_cgroup_move_account(st
mem_cgroup_charge_statistics(from, pc, false);
if (uncharge)
/* This is not "cancel", but cancel_charge does all we need. */
- mem_cgroup_cancel_charge(from);
+ mem_cgroup_cancel_charge(from, PAGE_SIZE);
/* caller should have done css_get */
pc->mem_cgroup = to;
@@ -1788,13 +1794,14 @@ static int mem_cgroup_move_parent(struct
goto put;
parent = mem_cgroup_from_cont(pcg);
- ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false, page);
+ ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false, page,
+ PAGE_SIZE);
if (ret || !parent)
goto put_back;
ret = mem_cgroup_move_account(pc, child, parent, true);
if (ret)
- mem_cgroup_cancel_charge(parent);
+ mem_cgroup_cancel_charge(parent, PAGE_SIZE);
put_back:
putback_lru_page(page);
put:
@@ -1816,6 +1823,10 @@ static int mem_cgroup_charge_common(stru
struct mem_cgroup *mem;
struct page_cgroup *pc;
int ret;
+ int page_size = PAGE_SIZE;
+
+ if (PageTransHuge(page))
+ page_size <<= compound_order(page);
pc = lookup_page_cgroup(page);
/* can happen at boot */
@@ -1824,11 +1835,12 @@ static int mem_cgroup_charge_common(stru
prefetchw(pc);
mem = memcg;
- ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true, page);
+ ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true, page,
+ page_size);
if (ret || !mem)
return ret;
- __mem_cgroup_commit_charge(mem, pc, ctype);
+ __mem_cgroup_commit_charge(mem, pc, ctype, page_size);
return 0;
}
@@ -1837,8 +1849,6 @@ int mem_cgroup_newpage_charge(struct pag
{
if (mem_cgroup_disabled())
return 0;
- if (PageCompound(page))
- return 0;
/*
* If already mapped, we don't have to account.
* If page cache, page->mapping has address_space.
@@ -1851,7 +1861,7 @@ int mem_cgroup_newpage_charge(struct pag
if (unlikely(!mm))
mm = &init_mm;
return mem_cgroup_charge_common(page, mm, gfp_mask,
- MEM_CGROUP_CHARGE_TYPE_MAPPED, NULL);
+ MEM_CGROUP_CHARGE_TYPE_MAPPED, NULL);
}
static void
@@ -1944,14 +1954,14 @@ int mem_cgroup_try_charge_swapin(struct
if (!mem)
goto charge_cur_mm;
*ptr = mem;
- ret = __mem_cgroup_try_charge(NULL, mask, ptr, true, page);
+ ret = __mem_cgroup_try_charge(NULL, mask, ptr, true, page, PAGE_SIZE);
/* drop extra refcnt from tryget */
css_put(&mem->css);
return ret;
charge_cur_mm:
if (unlikely(!mm))
mm = &init_mm;
- return __mem_cgroup_try_charge(mm, mask, ptr, true, page);
+ return __mem_cgroup_try_charge(mm, mask, ptr, true, page, PAGE_SIZE);
}
static void
@@ -1967,7 +1977,7 @@ __mem_cgroup_commit_charge_swapin(struct
cgroup_exclude_rmdir(&ptr->css);
pc = lookup_page_cgroup(page);
mem_cgroup_lru_del_before_commit_swapcache(page);
- __mem_cgroup_commit_charge(ptr, pc, ctype);
+ __mem_cgroup_commit_charge(ptr, pc, ctype, PAGE_SIZE);
mem_cgroup_lru_add_after_commit_swapcache(page);
/*
* Now swap is on-memory. This means this page may be
@@ -2016,11 +2026,12 @@ void mem_cgroup_cancel_charge_swapin(str
return;
if (!mem)
return;
- mem_cgroup_cancel_charge(mem);
+ mem_cgroup_cancel_charge(mem, PAGE_SIZE);
}
static void
-__do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype)
+__do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype,
+ int page_size)
{
struct memcg_batch_info *batch = NULL;
bool uncharge_memsw = true;
@@ -2053,14 +2064,14 @@ __do_uncharge(struct mem_cgroup *mem, co
if (batch->memcg != mem)
goto direct_uncharge;
/* remember freed charge and uncharge it later */
- batch->bytes += PAGE_SIZE;
+ batch->bytes += page_size;
if (uncharge_memsw)
- batch->memsw_bytes += PAGE_SIZE;
+ batch->memsw_bytes += page_size;
return;
direct_uncharge:
- res_counter_uncharge(&mem->res, PAGE_SIZE);
+ res_counter_uncharge(&mem->res, page_size);
if (uncharge_memsw)
- res_counter_uncharge(&mem->memsw, PAGE_SIZE);
+ res_counter_uncharge(&mem->memsw, page_size);
return;
}
@@ -2073,6 +2084,10 @@ __mem_cgroup_uncharge_common(struct page
struct page_cgroup *pc;
struct mem_cgroup *mem = NULL;
struct mem_cgroup_per_zone *mz;
+ int page_size = PAGE_SIZE;
+
+ if (PageTransHuge(page))
+ page_size <<= compound_order(page);
if (mem_cgroup_disabled())
return NULL;
@@ -2112,7 +2127,7 @@ __mem_cgroup_uncharge_common(struct page
}
if (!mem_cgroup_is_root(mem))
- __do_uncharge(mem, ctype);
+ __do_uncharge(mem, ctype, page_size);
if (ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
mem_cgroup_swap_statistics(mem, true);
mem_cgroup_charge_statistics(mem, pc, false);
@@ -2341,7 +2356,7 @@ int mem_cgroup_prepare_migration(struct
if (mem) {
ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false,
- page);
+ page, PAGE_SIZE);
css_put(&mem->css);
}
*ptr = mem;
@@ -2384,7 +2399,7 @@ void mem_cgroup_end_migration(struct mem
* __mem_cgroup_commit_charge() check PCG_USED bit of page_cgroup.
* So, double-counting is effectively avoided.
*/
- __mem_cgroup_commit_charge(mem, pc, ctype);
+ __mem_cgroup_commit_charge(mem, pc, ctype, PAGE_SIZE);
/*
* Both of oldpage and newpage are still under lock_page().
@@ -3864,7 +3879,7 @@ one_by_one:
cond_resched();
}
ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem,
- false, NULL);
+ false, NULL, PAGE_SIZE);
if (ret || !mem)
/* mem_cgroup_clear_mc() will do uncharge later */
return -ENOMEM;
@@ -4124,6 +4139,7 @@ static int mem_cgroup_move_charge_pte_ra
pte_t *pte;
spinlock_t *ptl;
+ VM_BUG_ON(pmd_trans_huge(*pmd));
retry:
pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
for (; addr != end; addr += PAGE_SIZE) {
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
next prev parent reply other threads:[~2010-02-21 14:18 UTC|newest]
Thread overview: 60+ messages / expand[flat|nested] mbox.gz Atom feed top
2010-02-21 14:10 [patch 00/36] Transparent Hugepage support #11 aarcange
2010-02-21 14:10 ` [patch 01/36] define MADV_HUGEPAGE aarcange
2010-02-21 14:10 ` [patch 02/36] compound_lock aarcange
2010-02-21 14:10 ` [patch 03/36] alter compound get_page/put_page aarcange
2010-02-21 14:10 ` [patch 04/36] update futex compound knowledge aarcange
2010-02-21 14:10 ` [patch 05/36] fix bad_page to show the real reason the page is bad aarcange
2010-02-21 14:10 ` [patch 06/36] clear compound mapping aarcange
2010-02-21 14:10 ` [patch 07/36] add native_set_pmd_at aarcange
2010-02-21 14:10 ` [patch 08/36] add pmd paravirt ops aarcange
2010-02-21 14:10 ` [patch 09/36] no paravirt version of pmd ops aarcange
2010-02-21 14:10 ` [patch 10/36] export maybe_mkwrite aarcange
2010-02-21 14:10 ` [patch 11/36] comment reminder in destroy_compound_page aarcange
2010-02-21 14:10 ` [patch 12/36] config_transparent_hugepage aarcange
2010-02-21 14:10 ` [patch 13/36] special pmd_trans_* functions aarcange
2010-02-21 14:10 ` [patch 14/36] add pmd mangling generic functions aarcange
2010-02-21 14:10 ` [patch 15/36] add pmd mangling functions to x86 aarcange
2010-02-21 14:10 ` [patch 16/36] bail out gup_fast on splitting pmd aarcange
2010-02-21 14:10 ` [patch 17/36] pte alloc trans splitting aarcange
2010-02-21 14:10 ` [patch 18/36] add pmd mmu_notifier helpers aarcange
2010-02-21 14:10 ` [patch 19/36] clear page compound aarcange
2010-02-21 14:10 ` [patch 20/36] add pmd_huge_pte to mm_struct aarcange
2010-02-21 14:10 ` [patch 21/36] split_huge_page_mm/vma aarcange
2010-02-21 14:10 ` [patch 22/36] split_huge_page paging aarcange
2010-02-21 14:10 ` [patch 23/36] clear_copy_huge_page aarcange
2010-02-21 14:10 ` [patch 24/36] kvm mmu transparent hugepage support aarcange
2010-02-21 14:10 ` [patch 25/36] _GFP_NO_KSWAPD aarcange
2010-02-22 17:53 ` Rik van Riel
2010-02-22 18:00 ` Andrea Arcangeli
2010-02-22 18:02 ` Avi Kivity
2010-03-01 12:14 ` Mel Gorman
2010-02-21 14:10 ` [patch 26/36] dont alloc harder for gfp nomemalloc even if nowait aarcange
2010-02-22 17:54 ` Rik van Riel
2010-02-21 14:10 ` [patch 27/36] transparent hugepage core aarcange
2010-02-21 14:10 ` [patch 28/36] adapt to mm_counter in -mm aarcange
2010-02-22 17:54 ` Rik van Riel
2010-02-21 14:10 ` [patch 29/36] page anon_vma aarcange
2010-02-22 17:55 ` Rik van Riel
2010-02-21 14:10 ` [patch 30/36] verify pmd_trans_huge isnt leaking aarcange
2010-02-22 17:56 ` Rik van Riel
2010-02-21 14:10 ` [patch 31/36] madvise(MADV_HUGEPAGE) aarcange
2010-02-21 14:10 ` [patch 32/36] pmd_trans_huge migrate bugcheck aarcange
2010-02-21 14:10 ` aarcange [this message]
2010-02-21 14:10 ` [patch 34/36] memcg huge memory aarcange
2010-02-21 14:10 ` [patch 35/36] transparent hugepage vmstat aarcange
2010-02-21 14:10 ` [patch 36/36] khugepaged aarcange
2010-02-23 7:58 ` KAMEZAWA Hiroyuki
2010-02-23 8:51 ` KAMEZAWA Hiroyuki
2010-02-23 14:26 ` Andrea Arcangeli
2010-02-23 23:57 ` KAMEZAWA Hiroyuki
2010-02-24 20:11 ` Andrew Morton
2010-02-24 20:28 ` Rik van Riel
2010-02-24 20:52 ` Andrew Morton
2010-02-24 20:57 ` Rik van Riel
2010-02-24 21:12 ` Andrew Morton
2010-02-24 21:24 ` Rik van Riel
2010-02-24 21:28 ` Andrew Morton
2010-02-24 21:58 ` Andrea Arcangeli
2010-02-24 22:52 ` Andrea Arcangeli
2010-02-24 22:56 ` Rik van Riel
2010-02-22 10:22 ` [patch 00/36] Transparent Hugepage support #11 Andrea Arcangeli
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20100221141758.070493777@redhat.com \
--to=aarcange@redhat.com \
--cc=agl@us.ibm.com \
--cc=akpm@linux-foundation.org \
--cc=arnd@arndb.de \
--cc=avi@redhat.com \
--cc=balbir@linux.vnet.ibm.com \
--cc=benh@kernel.crashing.org \
--cc=bpicco@redhat.com \
--cc=chrisw@sous-sol.org \
--cc=cl@linux-foundation.org \
--cc=dave@linux.vnet.ibm.com \
--cc=hugh.dickins@tiscali.co.uk \
--cc=ieidus@redhat.com \
--cc=kamezawa.hiroyu@jp.fujitsu.com \
--cc=kosaki.motohiro@jp.fujitsu.com \
--cc=linux-mm@kvack.org \
--cc=mel@csn.ul.ie \
--cc=mingo@elte.hu \
--cc=mtosatti@redhat.com \
--cc=npiggin@suse.de \
--cc=riel@redhat.com \
--cc=travis@sgi.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).