[PATCH v2 0/2] update memcg to break its need of populate

[PATCH v2 0/2] update memcg to break its need of populate

[Glauber Costa]

Hi Tejun and Li,

I am reposting the same series I've posted recently, just rebased
(two minor conflits) ontop of your last tree.

Let me know if this is acceptable.

Glauber Costa (2):
  cgroup: pass struct mem_cgroup instead of struct cgroup to socket
    memcg
  cgroup: get rid of populate for memcg

 include/net/sock.h           |   12 ++++++------
 include/net/tcp_memcontrol.h |    4 ++--
 mm/memcontrol.c              |   32 ++++++++++----------------------
 net/core/sock.c              |   10 +++++-----
 net/ipv4/tcp_memcontrol.c    |    6 ++----
 5 files changed, 25 insertions(+), 39 deletions(-)

-- 
1.7.7.6

[PATCH v2 1/2] cgroup: pass struct mem_cgroup instead of struct cgroup to socket memcg

[Glauber Costa]

The only reason cgroup was used, was to be consistent with the populate()
interface. Now that we're getting rid of it, not only we no longer need
it, but we also *can't* call it this way.

Since we will no longer rely on populate(), this will be called from
create(). During create, the association between struct mem_cgroup
and struct cgroup does not yet exist, since cgroup internals hasn't
yet initialized its bookkeeping. This means we would not be able
to draw the memcg pointer from the cgroup pointer in these
functions, which is highly undesirable.

Signed-off-by: Glauber Costa <glommer-bzQdu9zFT3WakBO8gow8eQ@public.gmane.org>
CC: Tejun Heo <tj-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org>
CC: Li Zefan <lizefan-hv44wF8Li93QT0dZR+AlfA@public.gmane.org>
---
 include/net/sock.h           |   12 ++++++------
 include/net/tcp_memcontrol.h |    4 ++--
 mm/memcontrol.c              |   24 +++++++++---------------
 net/core/sock.c              |   10 +++++-----
 net/ipv4/tcp_memcontrol.c    |    6 ++----
 5 files changed, 24 insertions(+), 32 deletions(-)

diff --git a/include/net/sock.h b/include/net/sock.h
index a6ba1f8..b3ebe6b 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -70,16 +70,16 @@
 struct cgroup;
 struct cgroup_subsys;
 #ifdef CONFIG_NET
-int mem_cgroup_sockets_init(struct cgroup *cgrp, struct cgroup_subsys *ss);
-void mem_cgroup_sockets_destroy(struct cgroup *cgrp);
+int mem_cgroup_sockets_init(struct mem_cgroup *memcg, struct cgroup_subsys *ss);
+void mem_cgroup_sockets_destroy(struct mem_cgroup *memcg);
 #else
 static inline
-int mem_cgroup_sockets_init(struct cgroup *cgrp, struct cgroup_subsys *ss)
+int mem_cgroup_sockets_init(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
 {
 	return 0;
 }
 static inline
-void mem_cgroup_sockets_destroy(struct cgroup *cgrp)
+void mem_cgroup_sockets_destroy(struct mem_cgroup *memcg)
 {
 }
 #endif
@@ -900,9 +900,9 @@ struct proto {
 	 * This function has to setup any files the protocol want to
 	 * appear in the kmem cgroup filesystem.
 	 */
-	int			(*init_cgroup)(struct cgroup *cgrp,
+	int			(*init_cgroup)(struct mem_cgroup *memcg,
 					       struct cgroup_subsys *ss);
-	void			(*destroy_cgroup)(struct cgroup *cgrp);
+	void			(*destroy_cgroup)(struct mem_cgroup *memcg);
 	struct cg_proto		*(*proto_cgroup)(struct mem_cgroup *memcg);
 #endif
 };
diff --git a/include/net/tcp_memcontrol.h b/include/net/tcp_memcontrol.h
index 48410ff..7df18bc 100644
--- a/include/net/tcp_memcontrol.h
+++ b/include/net/tcp_memcontrol.h
@@ -12,8 +12,8 @@ struct tcp_memcontrol {
 };
 
 struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg);
-int tcp_init_cgroup(struct cgroup *cgrp, struct cgroup_subsys *ss);
-void tcp_destroy_cgroup(struct cgroup *cgrp);
+int tcp_init_cgroup(struct mem_cgroup *memcg, struct cgroup_subsys *ss);
+void tcp_destroy_cgroup(struct mem_cgroup *memcg);
 unsigned long long tcp_max_memory(const struct mem_cgroup *memcg);
 void tcp_prot_mem(struct mem_cgroup *memcg, long val, int idx);
 #endif /* _TCP_MEMCG_H */
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index bef1142..704054d 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -4640,29 +4640,22 @@ static int mem_control_numa_stat_open(struct inode *unused, struct file *file)
 #endif /* CONFIG_NUMA */
 
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_KMEM
-static int register_kmem_files(struct cgroup *cont, struct cgroup_subsys *ss)
+static int register_kmem_files(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
 {
-	/*
-	 * Part of this would be better living in a separate allocation
-	 * function, leaving us with just the cgroup tree population work.
-	 * We, however, depend on state such as network's proto_list that
-	 * is only initialized after cgroup creation. I found the less
-	 * cumbersome way to deal with it to defer it all to populate time
-	 */
-	return mem_cgroup_sockets_init(cont, ss);
+	return mem_cgroup_sockets_init(memcg, ss);
 };
 
-static void kmem_cgroup_destroy(struct cgroup *cont)
+static void kmem_cgroup_destroy(struct mem_cgroup *memcg)
 {
-	mem_cgroup_sockets_destroy(cont);
+	mem_cgroup_sockets_destroy(memcg);
 }
 #else
-static int register_kmem_files(struct cgroup *cont, struct cgroup_subsys *ss)
+static int register_kmem_files(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
 {
 	return 0;
 }
 
-static void kmem_cgroup_destroy(struct cgroup *cont)
+static void kmem_cgroup_destroy(struct mem_cgroup *memcg)
 {
 }
 #endif
@@ -5034,7 +5027,7 @@ static void mem_cgroup_destroy(struct cgroup *cont)
 {
 	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
 
-	kmem_cgroup_destroy(cont);
+	kmem_cgroup_destroy(memcg);
 
 	mem_cgroup_put(memcg);
 }
@@ -5042,7 +5035,8 @@ static void mem_cgroup_destroy(struct cgroup *cont)
 static int mem_cgroup_populate(struct cgroup_subsys *ss,
 				struct cgroup *cont)
 {
-	return register_kmem_files(cont, ss);
+	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
+	return register_kmem_files(memcg, ss);
 }
 
 #ifdef CONFIG_MMU
diff --git a/net/core/sock.c b/net/core/sock.c
index b2e14c0..878f744 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -140,7 +140,7 @@ static DEFINE_MUTEX(proto_list_mutex);
 static LIST_HEAD(proto_list);
 
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_KMEM
-int mem_cgroup_sockets_init(struct cgroup *cgrp, struct cgroup_subsys *ss)
+int mem_cgroup_sockets_init(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
 {
 	struct proto *proto;
 	int ret = 0;
@@ -148,7 +148,7 @@ int mem_cgroup_sockets_init(struct cgroup *cgrp, struct cgroup_subsys *ss)
 	mutex_lock(&proto_list_mutex);
 	list_for_each_entry(proto, &proto_list, node) {
 		if (proto->init_cgroup) {
-			ret = proto->init_cgroup(cgrp, ss);
+			ret = proto->init_cgroup(memcg, ss);
 			if (ret)
 				goto out;
 		}
@@ -159,19 +159,19 @@ int mem_cgroup_sockets_init(struct cgroup *cgrp, struct cgroup_subsys *ss)
 out:
 	list_for_each_entry_continue_reverse(proto, &proto_list, node)
 		if (proto->destroy_cgroup)
-			proto->destroy_cgroup(cgrp);
+			proto->destroy_cgroup(memcg);
 	mutex_unlock(&proto_list_mutex);
 	return ret;
 }
 
-void mem_cgroup_sockets_destroy(struct cgroup *cgrp)
+void mem_cgroup_sockets_destroy(struct mem_cgroup *memcg)
 {
 	struct proto *proto;
 
 	mutex_lock(&proto_list_mutex);
 	list_for_each_entry_reverse(proto, &proto_list, node)
 		if (proto->destroy_cgroup)
-			proto->destroy_cgroup(cgrp);
+			proto->destroy_cgroup(memcg);
 	mutex_unlock(&proto_list_mutex);
 }
 #endif
diff --git a/net/ipv4/tcp_memcontrol.c b/net/ipv4/tcp_memcontrol.c
index 8f1753d..1517037 100644
--- a/net/ipv4/tcp_memcontrol.c
+++ b/net/ipv4/tcp_memcontrol.c
@@ -18,7 +18,7 @@ static void memcg_tcp_enter_memory_pressure(struct sock *sk)
 }
 EXPORT_SYMBOL(memcg_tcp_enter_memory_pressure);
 
-int tcp_init_cgroup(struct cgroup *cgrp, struct cgroup_subsys *ss)
+int tcp_init_cgroup(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
 {
 	/*
 	 * The root cgroup does not use res_counters, but rather,
@@ -28,7 +28,6 @@ int tcp_init_cgroup(struct cgroup *cgrp, struct cgroup_subsys *ss)
 	struct res_counter *res_parent = NULL;
 	struct cg_proto *cg_proto, *parent_cg;
 	struct tcp_memcontrol *tcp;
-	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
 	struct mem_cgroup *parent = parent_mem_cgroup(memcg);
 	struct net *net = current->nsproxy->net_ns;
 
@@ -61,9 +60,8 @@ int tcp_init_cgroup(struct cgroup *cgrp, struct cgroup_subsys *ss)
 }
 EXPORT_SYMBOL(tcp_init_cgroup);
 
-void tcp_destroy_cgroup(struct cgroup *cgrp)
+void tcp_destroy_cgroup(struct mem_cgroup *memcg)
 {
-	struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
 	struct cg_proto *cg_proto;
 	struct tcp_memcontrol *tcp;
 	u64 val;
-- 
1.7.7.6

[PATCH v2 2/2] cgroup: get rid of populate for memcg

[Glauber Costa]

The last man standing justifying the need for populate() is the
sock memcg initialization functions. Now that we are able to pass
a struct mem_cgroup instead of a struct cgroup to the socket
initialization, there is nothing that stops us from initializing
everything in create().

Signed-off-by: Glauber Costa <glommer-bzQdu9zFT3WakBO8gow8eQ@public.gmane.org>
CC: Tejun Heo <tj-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org>
CC: Li Zefan <lizefan-hv44wF8Li93QT0dZR+AlfA@public.gmane.org>
---
 mm/memcontrol.c |   16 +++++-----------
 1 files changed, 5 insertions(+), 11 deletions(-)

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 704054d..64a1bcd 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -4640,7 +4640,7 @@ static int mem_control_numa_stat_open(struct inode *unused, struct file *file)
 #endif /* CONFIG_NUMA */
 
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_KMEM
-static int register_kmem_files(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
+static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
 {
 	return mem_cgroup_sockets_init(memcg, ss);
 };
@@ -4650,7 +4650,7 @@ static void kmem_cgroup_destroy(struct mem_cgroup *memcg)
 	mem_cgroup_sockets_destroy(memcg);
 }
 #else
-static int register_kmem_files(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
+static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
 {
 	return 0;
 }
@@ -5010,7 +5010,9 @@ mem_cgroup_create(struct cgroup *cont)
 	memcg->move_charge_at_immigrate = 0;
 	mutex_init(&memcg->thresholds_lock);
 	spin_lock_init(&memcg->move_lock);
-	return &memcg->css;
+
+	if (!memcg_init_kmem(memcg, &mem_cgroup_subsys))
+		return &memcg->css;
 free_out:
 	__mem_cgroup_free(memcg);
 	return ERR_PTR(error);
@@ -5032,13 +5034,6 @@ static void mem_cgroup_destroy(struct cgroup *cont)
 	mem_cgroup_put(memcg);
 }
 
-static int mem_cgroup_populate(struct cgroup_subsys *ss,
-				struct cgroup *cont)
-{
-	struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
-	return register_kmem_files(memcg, ss);
-}
-
 #ifdef CONFIG_MMU
 /* Handlers for move charge at task migration. */
 #define PRECHARGE_COUNT_AT_ONCE	256
@@ -5622,7 +5617,6 @@ struct cgroup_subsys mem_cgroup_subsys = {
 	.create = mem_cgroup_create,
 	.pre_destroy = mem_cgroup_pre_destroy,
 	.destroy = mem_cgroup_destroy,
-	.populate = mem_cgroup_populate,
 	.can_attach = mem_cgroup_can_attach,
 	.cancel_attach = mem_cgroup_cancel_attach,
 	.attach = mem_cgroup_move_task,
-- 
1.7.7.6

Re: [PATCH v2 2/2] cgroup: get rid of populate for memcg

[Tejun Heo]

(cc'ing other memcg ppl just in case)

Hello,

I don't think the error handling is correct here.

On Fri, Apr 06, 2012 at 08:04:10PM +0400, Glauber Costa wrote:
> The last man standing justifying the need for populate() is the
> sock memcg initialization functions. Now that we are able to pass
> a struct mem_cgroup instead of a struct cgroup to the socket
> initialization, there is nothing that stops us from initializing
> everything in create().
> 
> Signed-off-by: Glauber Costa <glommer-bzQdu9zFT3WakBO8gow8eQ@public.gmane.org>
> CC: Tejun Heo <tj-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org>
> CC: Li Zefan <lizefan-hv44wF8Li93QT0dZR+AlfA@public.gmane.org>
> ---
...
> @@ -5010,7 +5010,9 @@ mem_cgroup_create(struct cgroup *cont)
>  	memcg->move_charge_at_immigrate = 0;
>  	mutex_init(&memcg->thresholds_lock);
>  	spin_lock_init(&memcg->move_lock);
> -	return &memcg->css;
> +
> +	if (!memcg_init_kmem(memcg, &mem_cgroup_subsys))
> +		return &memcg->css;
>  free_out:
>  	__mem_cgroup_free(memcg);
>  	return ERR_PTR(error);

So, the control is just falling through free_out: on kmem init
failure; however, there seem to be stuff which needs to be undone -
hotcpu_notifier() registration, which BTW seems incorrect even on its
own - unmounting and mounting again would probably make the same
notifier registered multiple times corrupting notification chain, and
ref inc on the parent.

It probably would be best to reorganize the function slightly such
that, it's organized as...

	1. alloc
	2. init stuff w/o other side effects
	3. make side effects

and add kmemcg init at the end of the second step.

Also, memcg maintainers, once the patches get updated and acked, I'd
like to route them through cgroup tree so that I can kill ->populate
there.  cgroup/for-3.5 is stable branch which can be pulled into other
trees including the memcg one.  Would that be okay?

Thanks.

-- 
tejun

Re: [PATCH v2 2/2] cgroup: get rid of populate for memcg

[Glauber Costa]

On 04/09/2012 02:40 PM, Tejun Heo wrote:
> which BTW seems incorrect even on its
> own - unmounting and mounting again would probably make the same
> notifier registered multiple times corrupting notification chain, and
> ref inc on the parent.


For the maintainers: Should I fix those in a new submission, or do you 
intend to do it yourselves?

the refcnt dropping should probably be done in my patch, it is a new 
leak (sorry). The hotplug notifier, as tejun pointed, was already there.

It seems simple enough to fix, so if you guys want, I can bundle it in
a new submission.

Re: [PATCH v2 2/2] cgroup: get rid of populate for memcg

[Tejun Heo]

Hello,

On Mon, Apr 09, 2012 at 02:51:10PM -0300, Glauber Costa wrote:
> On 04/09/2012 02:40 PM, Tejun Heo wrote:
> >which BTW seems incorrect even on its
> >own - unmounting and mounting again would probably make the same
> >notifier registered multiple times corrupting notification chain, and
> >ref inc on the parent.
> 
> 
> For the maintainers: Should I fix those in a new submission, or do
> you intend to do it yourselves?
> 
> the refcnt dropping should probably be done in my patch, it is a new
> leak (sorry). The hotplug notifier, as tejun pointed, was already
> there.
> 
> It seems simple enough to fix, so if you guys want, I can bundle it in
> a new submission.

I think it would be best to create a separate patch which is routed
through the usual memcg path (I suppose memcg patches go through
-mm?).

Thanks.

-- 
tejun

Re: [PATCH v2 2/2] cgroup: get rid of populate for memcg

[Glauber Costa]

On 04/09/2012 03:13 PM, Tejun Heo wrote:
> Hello,
>
> On Mon, Apr 09, 2012 at 02:51:10PM -0300, Glauber Costa wrote:
>> On 04/09/2012 02:40 PM, Tejun Heo wrote:
>>> which BTW seems incorrect even on its
>>> own - unmounting and mounting again would probably make the same
>>> notifier registered multiple times corrupting notification chain, and
>>> ref inc on the parent.
>>
>>
>> For the maintainers: Should I fix those in a new submission, or do
>> you intend to do it yourselves?
>>
>> the refcnt dropping should probably be done in my patch, it is a new
>> leak (sorry). The hotplug notifier, as tejun pointed, was already
>> there.
>>
>> It seems simple enough to fix, so if you guys want, I can bundle it in
>> a new submission.
>
> I think it would be best to create a separate patch which is routed
> through the usual memcg path (I suppose memcg patches go through
> -mm?).
>
> Thanks.
>
Tejun,

After debugging this a bit, I found most of it not to be a problem. 
Unless I am *very* wrong (and I both read and tested stuff), mount 
operations do not recreate the root cgroup. So stuff like the hotcpu 
notifier, etc, won't be a problem.

Now, you are definitely correct in pointing out that we start leaking 
stuff now - my bad.

But I guess I can then bundle it in a new submission, after shuffling 
around stuff a bit. It is really not a bug now, so no reason to route it 
separately.

Re: [PATCH v2 2/2] cgroup: get rid of populate for memcg

[Tejun Heo]

Hello, Glauber.

On Mon, Apr 09, 2012 at 05:18:52PM -0300, Glauber Costa wrote:
> After debugging this a bit, I found most of it not to be a problem.
> Unless I am *very* wrong (and I both read and tested stuff), mount
> operations do not recreate the root cgroup. So stuff like the hotcpu
> notifier, etc, won't be a problem.

Ooh, right, root css is created during subsys init.  Please note that
the kernel will panic on subsys init failure (hmm....).

> Now, you are definitely correct in pointing out that we start
> leaking stuff now - my bad.
> 
> But I guess I can then bundle it in a new submission, after
> shuffling around stuff a bit. It is really not a bug now, so no
> reason to route it separately.

Sure, just make it a separate patch.  As long as memcg ppl are fine
with it, I can route the patches together.

Thanks.

-- 
tejun

Re: [PATCH v2 2/2] cgroup: get rid of populate for memcg

[KAMEZAWA Hiroyuki]

(2012/04/10 2:51), Glauber Costa wrote:

> On 04/09/2012 02:40 PM, Tejun Heo wrote:
>> which BTW seems incorrect even on its
>> own - unmounting and mounting again would probably make the same
>> notifier registered multiple times corrupting notification chain, and
>> ref inc on the parent.
> 
> 
> For the maintainers: Should I fix those in a new submission, or do you 
> intend to do it yourselves?
> 
> the refcnt dropping should probably be done in my patch, it is a new 
> leak (sorry). The hotplug notifier, as tejun pointed, was already there.
> 
> It seems simple enough to fix, so if you guys want, I can bundle it in
> a new submission.
> 

Please make notifier fix patch against mm tree, as an independent one.

Thanks,
-Kame

Re: [PATCH v2 2/2] cgroup: get rid of populate for memcg

[KAMEZAWA Hiroyuki]

(2012/04/10 2:40), Tejun Heo wrote:

> (cc'ing other memcg ppl just in case)
> 
> Hello,
> 
> I don't think the error handling is correct here.
> 
> On Fri, Apr 06, 2012 at 08:04:10PM +0400, Glauber Costa wrote:
>> The last man standing justifying the need for populate() is the
>> sock memcg initialization functions. Now that we are able to pass
>> a struct mem_cgroup instead of a struct cgroup to the socket
>> initialization, there is nothing that stops us from initializing
>> everything in create().
>>
>> Signed-off-by: Glauber Costa <glommer-bzQdu9zFT3WakBO8gow8eQ@public.gmane.org>
>> CC: Tejun Heo <tj-DgEjT+Ai2ygdnm+yROfE0A@public.gmane.org>
>> CC: Li Zefan <lizefan-hv44wF8Li93QT0dZR+AlfA@public.gmane.org>
>> ---
> ...
>> @@ -5010,7 +5010,9 @@ mem_cgroup_create(struct cgroup *cont)
>>  	memcg->move_charge_at_immigrate = 0;
>>  	mutex_init(&memcg->thresholds_lock);
>>  	spin_lock_init(&memcg->move_lock);
>> -	return &memcg->css;
>> +
>> +	if (!memcg_init_kmem(memcg, &mem_cgroup_subsys))
>> +		return &memcg->css;
>>  free_out:
>>  	__mem_cgroup_free(memcg);
>>  	return ERR_PTR(error);
> 
> So, the control is just falling through free_out: on kmem init
> failure; however, there seem to be stuff which needs to be undone -
> hotcpu_notifier() registration, which BTW seems incorrect even on its
> own - unmounting and mounting again would probably make the same
> notifier registered multiple times corrupting notification chain, and
> ref inc on the parent.
> 


ok, it should be fixed.

> It probably would be best to reorganize the function slightly such
> that, it's organized as...
> 
> 	1. alloc
> 	2. init stuff w/o other side effects
> 	3. make side effects
> 
> and add kmemcg init at the end of the second step.
> 
> Also, memcg maintainers, once the patches get updated and acked, I'd
> like to route them through cgroup tree so that I can kill ->populate
> there.  cgroup/for-3.5 is stable branch which can be pulled into other
> trees including the memcg one.  Would that be okay?
> 


Hm, I'm okay with that but.....Michal ? 

Thanks,
-Kame