[PATCH][RESEND] nommu: add anonymous page memcg accounting

[PATCH][RESEND] nommu: add anonymous page memcg accounting

[Steven J. Magnani]

Add the necessary calls to track VM anonymous page usage.

Signed-off-by: Steven J. Magnani <steve@digidescorp.com>
---
diff -uprN a/mm/nommu.c b/mm/nommu.c
--- a/mm/nommu.c	2010-09-02 19:47:43.000000000 -0500
+++ b/mm/nommu.c	2010-09-02 20:07:02.000000000 -0500
@@ -524,8 +524,10 @@ static void delete_nommu_region(struct v
 /*
  * free a contiguous series of pages
  */
-static void free_page_series(unsigned long from, unsigned long to)
+static void free_page_series(unsigned long from, unsigned long to,
+			     const struct file *file)
 {
+	mem_cgroup_uncharge_start();
 	for (; from < to; from += PAGE_SIZE) {
 		struct page *page = virt_to_page(from);
 
@@ -534,8 +536,12 @@ static void free_page_series(unsigned lo
 		if (page_count(page) != 1)
 			kdebug("free page %p: refcount not one: %d",
 			       page, page_count(page));
+		if (!file)
+			mem_cgroup_uncharge_page(page);
+
 		put_page(page);
 	}
+	mem_cgroup_uncharge_end();
 }
 
 /*
@@ -563,7 +569,8 @@ static void __put_nommu_region(struct vm
 		 * from ramfs/tmpfs mustn't be released here */
 		if (region->vm_flags & VM_MAPPED_COPY) {
 			kdebug("free series");
-			free_page_series(region->vm_start, region->vm_top);
+			free_page_series(region->vm_start, region->vm_top,
+					 region->vm_file);
 		}
 		kmem_cache_free(vm_region_jar, region);
 	} else {
@@ -1117,9 +1124,26 @@ static int do_mmap_private(struct vm_are
 		set_page_refcounted(&pages[point]);
 
 	base = page_address(pages);
-	region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
+
 	region->vm_start = (unsigned long) base;
 	region->vm_end   = region->vm_start + rlen;
+
+	if (!vma->vm_file) {
+		for (point = 0; point < total; point++) {
+			int charge_failed =
+				mem_cgroup_newpage_charge(&pages[point],
+							  current->mm,
+							  GFP_KERNEL);
+			if (charge_failed) {
+				free_page_series(region->vm_start,
+						 region->vm_end, NULL);
+				region->vm_start = region->vm_end = 0;
+				goto enomem;
+			}
+		}
+	}
+
+	region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
 	region->vm_top   = region->vm_start + (total << PAGE_SHIFT);
 
 	vma->vm_start = region->vm_start;
@@ -1150,7 +1174,7 @@ static int do_mmap_private(struct vm_are
 	return 0;
 
 error_free:
-	free_page_series(region->vm_start, region->vm_end);
+	free_page_series(region->vm_start, region->vm_end, vma->vm_file);
 	region->vm_start = vma->vm_start = 0;
 	region->vm_end   = vma->vm_end = 0;
 	region->vm_top   = 0;
@@ -1555,7 +1579,7 @@ static int shrink_vma(struct mm_struct *
 	add_nommu_region(region);
 	up_write(&nommu_region_sem);
 
-	free_page_series(from, to);
+	free_page_series(from, to, vma->vm_file);
 	return 0;
 }
 

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH][RESEND] nommu: add anonymous page memcg accounting

[David Howells]

Steven J. Magnani <steve@digidescorp.com> wrote:

> Add the necessary calls to track VM anonymous page usage.

Do we really need to do memcg accounting in NOMMU mode?  Might it be better to
just apply the attached patch instead?

David
---
diff --git a/init/Kconfig b/init/Kconfig
index 2de5b1c..aecff10 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -555,7 +555,7 @@ config RESOURCE_COUNTERS
 
 config CGROUP_MEM_RES_CTLR
 	bool "Memory Resource Controller for Control Groups"
-	depends on CGROUPS && RESOURCE_COUNTERS
+	depends on CGROUPS && RESOURCE_COUNTERS && MMU
 	select MM_OWNER
 	help
 	  Provides a memory resource controller that manages both anonymous

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH][RESEND] nommu: add anonymous page memcg accounting

[Steve Magnani]

David Howells <dhowells@redhat.com> wrote:
> 
> Do we really need to do memcg accounting in NOMMU mode?  Might it be
> better to just apply the attached patch instead?
> 
> David
> ---
> diff --git a/init/Kconfig b/init/Kconfig
> index 2de5b1c..aecff10 100644
> --- a/init/Kconfig
> +++ b/init/Kconfig
> @@ -555,7 +555,7 @@ config RESOURCE_COUNTERS
>  
>  config CGROUP_MEM_RES_CTLR
>  	bool "Memory Resource Controller for Control Groups"
> -	depends on CGROUPS && RESOURCE_COUNTERS
> +	depends on CGROUPS && RESOURCE_COUNTERS && MMU
>  	select MM_OWNER
>  	help
>  	  Provides a memory resource controller that manages both anonymous

If anything I think nommu is one of the better applications of memcg. Since nommu typically == 
embedded, being able to put potential memory pigs in a sandbox so they can't destabilize the 
system is a Good Thing. That was my motivation for doing this in the first place and it works 
quite well.

If it would be better to make nommu memcg contingent on some new Kconfig option, we can do 
that. 

Steve


--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH][RESEND] nommu: add anonymous page memcg accounting

[Balbir Singh]

* Steve Magnani <steve@digidescorp.com> [2010-10-01 09:31:39]:

> David Howells <dhowells@redhat.com> wrote:
> > 
> > Do we really need to do memcg accounting in NOMMU mode?  Might it be
> > better to just apply the attached patch instead?
> > 
> > David
> > ---
> > diff --git a/init/Kconfig b/init/Kconfig
> > index 2de5b1c..aecff10 100644
> > --- a/init/Kconfig
> > +++ b/init/Kconfig
> > @@ -555,7 +555,7 @@ config RESOURCE_COUNTERS
> >  
> >  config CGROUP_MEM_RES_CTLR
> >  	bool "Memory Resource Controller for Control Groups"
> > -	depends on CGROUPS && RESOURCE_COUNTERS
> > +	depends on CGROUPS && RESOURCE_COUNTERS && MMU
> >  	select MM_OWNER
> >  	help
> >  	  Provides a memory resource controller that manages both anonymous
> 
> If anything I think nommu is one of the better applications of memcg. Since nommu typically == 
> embedded, being able to put potential memory pigs in a sandbox so they can't destabilize the 
> system is a Good Thing. That was my motivation for doing this in the first place and it works 
> quite well.

Good to know, but I want to point out that I never explictly tested it
for NOMMU when I created memcg. I thought like the rest that not
having reclaim capability would limit memcg usage in the NOMMU world.


-- 
	Three Cheers,
	Balbir

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH][RESEND] nommu: add anonymous page memcg accounting

[David Howells]

Steve Magnani <steve@digidescorp.com> wrote:

> If anything I think nommu is one of the better applications of memcg. Since
> nommu typically embedded, being able to put potential memory pigs in a
> sandbox so they can't destabilize the system is a Good Thing. That was my
> motivation for doing this in the first place and it works quite well.

I suspect it's not useful for a few reasons:

 (1) You don't normally run many applications on a NOMMU system.  Typically,
     you'll run just one, probably threaded app, I think.

 (2) In general, you won't be able to cull processes to make space.  If the OOM
     killer runs your application has a bug in it.

 (3) memcg has a huge overhead.  20 bytes per page!  On a 4K page 32-bit
     system, that's nearly 5% of your RAM, assuming I understand the
     CGROUP_MEM_RES_CTLR config help text correctly.

 (4) There's no swapping, no page faults, no migration and little shareable
     memory.  Being able to allocate large blocks of contiguous memory is much
     more important and much more of a bottleneck than this.  The 5% of RAM
     lost makes that just that little bit harder.

If it's memory sandboxing you require, ulimit might be sufficient for NOMMU
mode.

However, I suppose there's little harm in letting the patch in.  I would guess
the additions all optimise away if memcg isn't enabled.

A question for you: why does struct page_cgroup need a page pointer?  If an
array of page_cgroup structs is allocated per array of page structs, then you
should be able to use the array index to map between them.

David

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH][RESEND] nommu: add anonymous page memcg accounting

[Steven J. Magnani]

On Fri, 2010-10-01 at 16:07 +0100, David Howells wrote: 
> Steve Magnani <steve@digidescorp.com> wrote:
> 
> > If anything I think nommu is one of the better applications of memcg. Since
> > nommu typically embedded, being able to put potential memory pigs in a
> > sandbox so they can't destabilize the system is a Good Thing. That was my
> > motivation for doing this in the first place and it works quite well.
> 
> I suspect it's not useful for a few reasons:
> 
>  (1) You don't normally run many applications on a NOMMU system.  Typically,
>      you'll run just one, probably threaded app, I think.

Not always.

> 
>  (2) In general, you won't be able to cull processes to make space.  If the OOM
>      killer runs your application has a bug in it.

Not always. Every now and then applications have to deal with
user-supplied input of some sort. 

In our case it's a user-formatted disk drive that can have some
arbitrarily-sized FAT32 partition on which we are required to run
dosfsck. Now, dosfsck is the epitome of a memory pig; its memory
requirements scale with partition size, number of dentries, and any
damage encountered - none of which can be predicted. There is a set of
partitions we are able to check with no problem, but no guarantee the
user won't present us with one that would bring down the whole system,
were the OOM killer to get involved. Putting just dosfsck in its own
sandbox ensures this can't happen. See also my response to #4 below.

> 
>  (3) memcg has a huge overhead.  20 bytes per page!  On a 4K page 32-bit
>      system, that's nearly 5% of your RAM, assuming I understand the
>      CGROUP_MEM_RES_CTLR config help text correctly.

When you use 16K pages, 20 bytes/page isn't so huge :)

> 
>  (4) There's no swapping, no page faults, no migration and little shareable
>      memory.  Being able to allocate large blocks of contiguous memory is much
>      more important and much more of a bottleneck than this.  The 5% of RAM
>      lost makes that just that little bit harder.
> 
> If it's memory sandboxing you require, ulimit might be sufficient for NOMMU
> mode.

dosfsck is written to handle memory allocation failures properly
(bailing out) but I have not been able to get this code to execute when
the system runs out of memory - the OOM killer gets invoked and that's
all she wrote. Will a ulimit violation return control back to the
process, or terminate it in some graceful manner? 

> 
> However, I suppose there's little harm in letting the patch in.  I would guess
> the additions all optimise away if memcg isn't enabled.
> 
> A question for you: why does struct page_cgroup need a page pointer?  If an
> array of page_cgroup structs is allocated per array of page structs, then you
> should be able to use the array index to map between them.

Kame is probably better able to answer this.
 
Steve


--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH][RESEND] nommu: add anonymous page memcg accounting

[Balbir Singh]

* Steven J. Magnani <steve@digidescorp.com> [2010-10-01 11:41:07]:

> On Fri, 2010-10-01 at 16:07 +0100, David Howells wrote: 
> > Steve Magnani <steve@digidescorp.com> wrote:
> > 
> > > If anything I think nommu is one of the better applications of memcg. Since
> > > nommu typically embedded, being able to put potential memory pigs in a
> > > sandbox so they can't destabilize the system is a Good Thing. That was my
> > > motivation for doing this in the first place and it works quite well.
> > 
> > I suspect it's not useful for a few reasons:
> > 
> >  (1) You don't normally run many applications on a NOMMU system.  Typically,
> >      you'll run just one, probably threaded app, I think.
> 
> Not always.
> 
> > 
> >  (2) In general, you won't be able to cull processes to make space.  If the OOM
> >      killer runs your application has a bug in it.
> 
> Not always. Every now and then applications have to deal with
> user-supplied input of some sort. 
> 
> In our case it's a user-formatted disk drive that can have some
> arbitrarily-sized FAT32 partition on which we are required to run
> dosfsck. Now, dosfsck is the epitome of a memory pig; its memory
> requirements scale with partition size, number of dentries, and any
> damage encountered - none of which can be predicted. There is a set of
> partitions we are able to check with no problem, but no guarantee the
> user won't present us with one that would bring down the whole system,
> were the OOM killer to get involved. Putting just dosfsck in its own
> sandbox ensures this can't happen. See also my response to #4 below.
> 
> > 
> >  (3) memcg has a huge overhead.  20 bytes per page!  On a 4K page 32-bit
> >      system, that's nearly 5% of your RAM, assuming I understand the
> >      CGROUP_MEM_RES_CTLR config help text correctly.
> 
> When you use 16K pages, 20 bytes/page isn't so huge :)
> 
> > 
> >  (4) There's no swapping, no page faults, no migration and little shareable
> >      memory.  Being able to allocate large blocks of contiguous memory is much
> >      more important and much more of a bottleneck than this.  The 5% of RAM
> >      lost makes that just that little bit harder.
> > 
> > If it's memory sandboxing you require, ulimit might be sufficient for NOMMU
> > mode.
> 
> dosfsck is written to handle memory allocation failures properly
> (bailing out) but I have not been able to get this code to execute when
> the system runs out of memory - the OOM killer gets invoked and that's
> all she wrote. Will a ulimit violation return control back to the
> process, or terminate it in some graceful manner? 
> 
> > 
> > However, I suppose there's little harm in letting the patch in.  I would guess
> > the additions all optimise away if memcg isn't enabled.
> > 
> > A question for you: why does struct page_cgroup need a page pointer?  If an
> > array of page_cgroup structs is allocated per array of page structs, then you
> > should be able to use the array index to map between them.
> 
> Kame is probably better able to answer this.
>

To answer David's question: We have no notion of pfn in page_cgroup,
how do we do the indexing? BTW, we are moving to cgroup ids that will
take just 16 bits instead of 64 on a 64 bit system.

-- 
	Three Cheers,
	Balbir

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH][RESEND] nommu: add anonymous page memcg accounting

[KAMEZAWA Hiroyuki]

On Fri, 01 Oct 2010 11:41:07 -0500
"Steven J. Magnani" <steve@digidescorp.com> wrote:
> > However, I suppose there's little harm in letting the patch in.  I would guess
> > the additions all optimise away if memcg isn't enabled.
> > 
> > A question for you: why does struct page_cgroup need a page pointer?  If an
> > array of page_cgroup structs is allocated per array of page structs, then you
> > should be able to use the array index to map between them.
> 
No reason. It was not array in the 1st implemenation and ->page still remains. At 2nd
implementation, I didn't know embeded people has any interests on memcg. And I wasn't
sure how
	page_cgroup_to_page() : pfn_to_page(pagec_cgroup_to_pfn(pc))
will be widely used.

Now, we know page_cgroup->page is not used in very critical path _if_ node-id
and zone-id can be directly got from page_cgroup.

I'm now preparing a patch to remove struct page* pointer. I'm wondering
whether it's ok that some architecuture cannot drop struct page pointer.
If SPARSEMEM is used on 32bit arch, I'm not sure whether # of bits isn't enough.
I may have to add overhead to get nid, zid in critical path.
(for example, s390/32bit, x86-32/HIGHMEM, ARM/HIGHMEM?)

Current out priority is supporting dirty_ratio rather than memory usage diet.
Please wait. Removing page_cgroup->page patch will add something a bit complex.


Thanks,
-Kame

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH][RESEND] nommu: add anonymous page memcg accounting

[Balbir Singh]

* Steven J. Magnani <steve@digidescorp.com> [2010-10-01 05:35:15]:

> Add the necessary calls to track VM anonymous page usage.
>

Is there a motivation to have these changes for nommu as well? 
 
> Signed-off-by: Steven J. Magnani <steve@digidescorp.com>
> ---
> diff -uprN a/mm/nommu.c b/mm/nommu.c
> --- a/mm/nommu.c	2010-09-02 19:47:43.000000000 -0500
> +++ b/mm/nommu.c	2010-09-02 20:07:02.000000000 -0500
> @@ -524,8 +524,10 @@ static void delete_nommu_region(struct v
>  /*
>   * free a contiguous series of pages
>   */
> -static void free_page_series(unsigned long from, unsigned long to)
> +static void free_page_series(unsigned long from, unsigned long to,
> +			     const struct file *file)
>  {
> +	mem_cgroup_uncharge_start();
>  	for (; from < to; from += PAGE_SIZE) {
>  		struct page *page = virt_to_page(from);
> 
> @@ -534,8 +536,12 @@ static void free_page_series(unsigned lo
>  		if (page_count(page) != 1)
>  			kdebug("free page %p: refcount not one: %d",
>  			       page, page_count(page));
> +		if (!file)
> +			mem_cgroup_uncharge_page(page);
> +

I don't understand the if !file bits, do you want to selectively
enable memcg for !file for nommu?

>  		put_page(page);
>  	}
> +	mem_cgroup_uncharge_end();
>  }
> 
>  /*
> @@ -563,7 +569,8 @@ static void __put_nommu_region(struct vm
>  		 * from ramfs/tmpfs mustn't be released here */
>  		if (region->vm_flags & VM_MAPPED_COPY) {
>  			kdebug("free series");
> -			free_page_series(region->vm_start, region->vm_top);
> +			free_page_series(region->vm_start, region->vm_top,
> +					 region->vm_file);
>  		}
>  		kmem_cache_free(vm_region_jar, region);
>  	} else {
> @@ -1117,9 +1124,26 @@ static int do_mmap_private(struct vm_are
>  		set_page_refcounted(&pages[point]);
> 
>  	base = page_address(pages);
> -	region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
> +
>  	region->vm_start = (unsigned long) base;
>  	region->vm_end   = region->vm_start + rlen;
> +
> +	if (!vma->vm_file) {
> +		for (point = 0; point < total; point++) {
> +			int charge_failed =
> +				mem_cgroup_newpage_charge(&pages[point],
> +							  current->mm,
> +							  GFP_KERNEL);
> +			if (charge_failed) {
> +				free_page_series(region->vm_start,
> +						 region->vm_end, NULL);
> +				region->vm_start = region->vm_end = 0;
> +				goto enomem;
> +			}
> +		}
> +	}
> +
> +	region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY;
>  	region->vm_top   = region->vm_start + (total << PAGE_SHIFT);
> 
>  	vma->vm_start = region->vm_start;
> @@ -1150,7 +1174,7 @@ static int do_mmap_private(struct vm_are
>  	return 0;
> 
>  error_free:
> -	free_page_series(region->vm_start, region->vm_end);
> +	free_page_series(region->vm_start, region->vm_end, vma->vm_file);
>  	region->vm_start = vma->vm_start = 0;
>  	region->vm_end   = vma->vm_end = 0;
>  	region->vm_top   = 0;
> @@ -1555,7 +1579,7 @@ static int shrink_vma(struct mm_struct *
>  	add_nommu_region(region);
>  	up_write(&nommu_region_sem);
> 
> -	free_page_series(from, to);
> +	free_page_series(from, to, vma->vm_file);
>  	return 0;
>  }
> 
>

Could you please add more details on your use and update the
Documentation/cgroups/memory.txt file as well. 

-- 
	Three Cheers,
	Balbir

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>