[PATCH] [v2] nvdimm-btt: fix several memleaks

[PATCH] [v2] nvdimm-btt: fix several memleaks

[Dinghao Liu]

Resources allocated by kcalloc() in btt_freelist_init(),
btt_rtt_init(), and btt_maplocks_init() are not correctly
released in their callers when an error happens. For
example, when an error happens in btt_freelist_init(), its
caller discover_arenas() will directly free arena, which makes
arena->freelist a leaked memory. Fix these memleaks by using
devm_kcalloc() to make the memory auto-freed on driver detach.

Fixes: 5212e11fde4d ("nd_btt: atomic sector updates")
Signed-off-by: Dinghao Liu <dinghao.liu@zju.edu.cn>
---

Changelog:

v2: -Use devm_kcalloc() to fix the memleaks.
    -Fix the potential leaked memory in btt_rtt_init()
     and btt_maplocks_init().
---
 drivers/nvdimm/btt.c | 35 ++++++++++++++++-------------------
 1 file changed, 16 insertions(+), 19 deletions(-)

diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c
index d5593b0dc700..c55231f42617 100644
--- a/drivers/nvdimm/btt.c
+++ b/drivers/nvdimm/btt.c
@@ -531,13 +531,13 @@ static int arena_clear_freelist_error(struct arena_info *arena, u32 lane)
 	return ret;
 }
 
-static int btt_freelist_init(struct arena_info *arena)
+static int btt_freelist_init(struct device *dev, struct arena_info *arena)
 {
 	int new, ret;
 	struct log_entry log_new;
 	u32 i, map_entry, log_oldmap, log_newmap;
 
-	arena->freelist = kcalloc(arena->nfree, sizeof(struct free_entry),
+	arena->freelist = devm_kcalloc(dev, arena->nfree, sizeof(struct free_entry),
 					GFP_KERNEL);
 	if (!arena->freelist)
 		return -ENOMEM;
@@ -718,20 +718,20 @@ static int log_set_indices(struct arena_info *arena)
 	return 0;
 }
 
-static int btt_rtt_init(struct arena_info *arena)
+static int btt_rtt_init(struct device *dev, struct arena_info *arena)
 {
-	arena->rtt = kcalloc(arena->nfree, sizeof(u32), GFP_KERNEL);
+	arena->rtt = devm_kcalloc(dev, arena->nfree, sizeof(u32), GFP_KERNEL);
 	if (arena->rtt == NULL)
 		return -ENOMEM;
 
 	return 0;
 }
 
-static int btt_maplocks_init(struct arena_info *arena)
+static int btt_maplocks_init(struct device *dev, struct arena_info *arena)
 {
 	u32 i;
 
-	arena->map_locks = kcalloc(arena->nfree, sizeof(struct aligned_lock),
+	arena->map_locks = devm_kcalloc(dev, arena->nfree, sizeof(struct aligned_lock),
 				GFP_KERNEL);
 	if (!arena->map_locks)
 		return -ENOMEM;
@@ -805,9 +805,6 @@ static void free_arenas(struct btt *btt)
 
 	list_for_each_entry_safe(arena, next, &btt->arena_list, list) {
 		list_del(&arena->list);
-		kfree(arena->rtt);
-		kfree(arena->map_locks);
-		kfree(arena->freelist);
 		debugfs_remove_recursive(arena->debugfs_dir);
 		kfree(arena);
 	}
@@ -843,7 +840,7 @@ static void parse_arena_meta(struct arena_info *arena, struct btt_sb *super,
 	arena->flags = le32_to_cpu(super->flags);
 }
 
-static int discover_arenas(struct btt *btt)
+static int discover_arenas(struct device *dev, struct btt *btt)
 {
 	int ret = 0;
 	struct arena_info *arena;
@@ -893,15 +890,15 @@ static int discover_arenas(struct btt *btt)
 			goto out;
 		}
 
-		ret = btt_freelist_init(arena);
+		ret = btt_freelist_init(dev, arena);
 		if (ret)
 			goto out;
 
-		ret = btt_rtt_init(arena);
+		ret = btt_rtt_init(dev, arena);
 		if (ret)
 			goto out;
 
-		ret = btt_maplocks_init(arena);
+		ret = btt_maplocks_init(dev, arena);
 		if (ret)
 			goto out;
 
@@ -1022,7 +1019,7 @@ static int btt_arena_write_layout(struct arena_info *arena)
  * This function completes the initialization for the BTT namespace
  * such that it is ready to accept IOs
  */
-static int btt_meta_init(struct btt *btt)
+static int btt_meta_init(struct device *dev, struct btt *btt)
 {
 	int ret = 0;
 	struct arena_info *arena;
@@ -1033,15 +1030,15 @@ static int btt_meta_init(struct btt *btt)
 		if (ret)
 			goto unlock;
 
-		ret = btt_freelist_init(arena);
+		ret = btt_freelist_init(dev, arena);
 		if (ret)
 			goto unlock;
 
-		ret = btt_rtt_init(arena);
+		ret = btt_rtt_init(dev, arena);
 		if (ret)
 			goto unlock;
 
-		ret = btt_maplocks_init(arena);
+		ret = btt_maplocks_init(dev, arena);
 		if (ret)
 			goto unlock;
 	}
@@ -1584,7 +1581,7 @@ static struct btt *btt_init(struct nd_btt *nd_btt, unsigned long long rawsize,
 	nsio = to_nd_namespace_io(&nd_btt->ndns->dev);
 	btt->phys_bb = &nsio->bb;
 
-	ret = discover_arenas(btt);
+	ret = discover_arenas(dev, btt);
 	if (ret) {
 		dev_err(dev, "init: error in arena_discover: %d\n", ret);
 		return NULL;
@@ -1606,7 +1603,7 @@ static struct btt *btt_init(struct nd_btt *nd_btt, unsigned long long rawsize,
 			return NULL;
 		}
 
-		ret = btt_meta_init(btt);
+		ret = btt_meta_init(dev, btt);
 		if (ret) {
 			dev_err(dev, "init: error in meta_init: %d\n", ret);
 			return NULL;
-- 
2.17.1

Re: [PATCH] [v2] nvdimm-btt: fix several memleaks

[Ira Weiny]

Dinghao Liu wrote:
> Resources allocated by kcalloc() in btt_freelist_init(),
> btt_rtt_init(), and btt_maplocks_init() are not correctly
> released in their callers when an error happens. For
> example, when an error happens in btt_freelist_init(), its
> caller discover_arenas() will directly free arena, which makes
> arena->freelist a leaked memory. Fix these memleaks by using
> devm_kcalloc() to make the memory auto-freed on driver detach.

Thanks for this work!

> 
> Fixes: 5212e11fde4d ("nd_btt: atomic sector updates")
> Signed-off-by: Dinghao Liu <dinghao.liu@zju.edu.cn>
> ---
> 
> Changelog:
> 
> v2: -Use devm_kcalloc() to fix the memleaks.
>     -Fix the potential leaked memory in btt_rtt_init()
>      and btt_maplocks_init().
> ---
>  drivers/nvdimm/btt.c | 35 ++++++++++++++++-------------------
>  1 file changed, 16 insertions(+), 19 deletions(-)
> 
> diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c
> index d5593b0dc700..c55231f42617 100644
> --- a/drivers/nvdimm/btt.c
> +++ b/drivers/nvdimm/btt.c
> @@ -531,13 +531,13 @@ static int arena_clear_freelist_error(struct arena_info *arena, u32 lane)
>  	return ret;
>  }
>  
> -static int btt_freelist_init(struct arena_info *arena)
> +static int btt_freelist_init(struct device *dev, struct arena_info *arena)

Both struct arena_info and struct btt contain references to struct nd_btt
which is the device you are passing down this call stack.

Just use the device in the arena/btt rather than passing a device
structure.  That makes the code easier to read and ensures that the device
associated with this arena or btt is used.

>  {
>  	int new, ret;
>  	struct log_entry log_new;
>  	u32 i, map_entry, log_oldmap, log_newmap;
>  
> -	arena->freelist = kcalloc(arena->nfree, sizeof(struct free_entry),
> +	arena->freelist = devm_kcalloc(dev, arena->nfree, sizeof(struct free_entry),

	... devm_kcalloc(&arena->nd_btt.dev, ...)

>  					GFP_KERNEL);
>  	if (!arena->freelist)
>  		return -ENOMEM;
> @@ -718,20 +718,20 @@ static int log_set_indices(struct arena_info *arena)
>  	return 0;
>  }
>  
> -static int btt_rtt_init(struct arena_info *arena)
> +static int btt_rtt_init(struct device *dev, struct arena_info *arena)
>  {
> -	arena->rtt = kcalloc(arena->nfree, sizeof(u32), GFP_KERNEL);
> +	arena->rtt = devm_kcalloc(dev, arena->nfree, sizeof(u32), GFP_KERNEL);
>  	if (arena->rtt == NULL)
>  		return -ENOMEM;
>  
>  	return 0;
>  }
>  
> -static int btt_maplocks_init(struct arena_info *arena)
> +static int btt_maplocks_init(struct device *dev, struct arena_info *arena)
>  {
>  	u32 i;
>  
> -	arena->map_locks = kcalloc(arena->nfree, sizeof(struct aligned_lock),
> +	arena->map_locks = devm_kcalloc(dev, arena->nfree, sizeof(struct aligned_lock),
>  				GFP_KERNEL);
>  	if (!arena->map_locks)
>  		return -ENOMEM;
> @@ -805,9 +805,6 @@ static void free_arenas(struct btt *btt)
>  
>  	list_for_each_entry_safe(arena, next, &btt->arena_list, list) {
>  		list_del(&arena->list);
> -		kfree(arena->rtt);
> -		kfree(arena->map_locks);
> -		kfree(arena->freelist);

This does not quite work.

free_arenas() is used in the error paths of create_arenas() and
discover_arenas().  In those cases devm_kfree() is probably a better way
to clean up this.

However...

>  		debugfs_remove_recursive(arena->debugfs_dir);
>  		kfree(arena);

Why can't arena be allocated with devm_*?

We need to change this up a bit more to handle the error path vs regular
device shutdown free (automatic devm frees).

Ira

Re: [PATCH] [v2] nvdimm-btt: fix several memleaks

[Ira Weiny]

Ira Weiny wrote:
> Dinghao Liu wrote:

[snip]

> >  
> > -static int btt_maplocks_init(struct arena_info *arena)
> > +static int btt_maplocks_init(struct device *dev, struct arena_info *arena)
> >  {
> >  	u32 i;
> >  
> > -	arena->map_locks = kcalloc(arena->nfree, sizeof(struct aligned_lock),
> > +	arena->map_locks = devm_kcalloc(dev, arena->nfree, sizeof(struct aligned_lock),
> >  				GFP_KERNEL);
> >  	if (!arena->map_locks)
> >  		return -ENOMEM;
> > @@ -805,9 +805,6 @@ static void free_arenas(struct btt *btt)
> >  
> >  	list_for_each_entry_safe(arena, next, &btt->arena_list, list) {
> >  		list_del(&arena->list);
> > -		kfree(arena->rtt);
> > -		kfree(arena->map_locks);
> > -		kfree(arena->freelist);
> 
> This does not quite work.
> 
> free_arenas() is used in the error paths of create_arenas() and
> discover_arenas().  In those cases devm_kfree() is probably a better way
> to clean up this.
> 
> However...
> 
> >  		debugfs_remove_recursive(arena->debugfs_dir);
> >  		kfree(arena);
> 
> Why can't arena be allocated with devm_*?
> 
> We need to change this up a bit more to handle the error path vs regular
> device shutdown free (automatic devm frees).

We might want to look at using no_free_ptr() in this code.  See this
patch[1] for an example of how to inhibit the cleanup and pass the pointer
on when the function succeeds.

[1] https://lore.kernel.org/all/170261791914.1714654.6447680285357545638.stgit@dwillia2-xfh.jf.intel.com/

Ira

Re: [PATCH] [v2] nvdimm-btt: fix several memleaks

[dinghao.liu]

> Ira Weiny wrote:
> > Dinghao Liu wrote:
> 
> [snip]
> 
> -static int btt_freelist_init(struct arena_info *arena)
> +static int btt_freelist_init(struct device *dev, struct arena_info *arena)
> 
> Both struct arena_info and struct btt contain references to struct nd_btt
> which is the device you are passing down this call stack.
> 
> Just use the device in the arena/btt rather than passing a device
> structure.  That makes the code easier to read and ensures that the device
> associated with this arena or btt is used.

Thanks for this suggestion! I will fix this in the v3 patch.

> [snip]
> > >  
> > > -static int btt_maplocks_init(struct arena_info *arena)
> > > +static int btt_maplocks_init(struct device *dev, struct arena_info *arena)
> > >  {
> > >  	u32 i;
> > >  
> > > -	arena->map_locks = kcalloc(arena->nfree, sizeof(struct aligned_lock),
> > > +	arena->map_locks = devm_kcalloc(dev, arena->nfree, sizeof(struct aligned_lock),
> > >  				GFP_KERNEL);
> > >  	if (!arena->map_locks)
> > >  		return -ENOMEM;
> > > @@ -805,9 +805,6 @@ static void free_arenas(struct btt *btt)
> > >  
> > >  	list_for_each_entry_safe(arena, next, &btt->arena_list, list) {
> > >  		list_del(&arena->list);
> > > -		kfree(arena->rtt);
> > > -		kfree(arena->map_locks);
> > > -		kfree(arena->freelist);
> > 
> > This does not quite work.
> > 
> > free_arenas() is used in the error paths of create_arenas() and
> > discover_arenas().  In those cases devm_kfree() is probably a better way
> > to clean up this.

Here I'm a little confused about when devm_kfree() should be used.
Code in btt_init() implies that resources allocated by devm_* could be
auto freed in both error and success paths of probe/attach (e.g., btt 
allocated by devm_kzalloc is never freed by devm_kfree).
Using devm_kfree() in free_arenas() is ok for me, but I want to know
whether not using devm_kfree() constitutes a bug.

> > 
> > However...
> > 
> > >  		debugfs_remove_recursive(arena->debugfs_dir);
> > >  		kfree(arena);
> > 
> > Why can't arena be allocated with devm_*?
> > 
> > We need to change this up a bit more to handle the error path vs regular
> > device shutdown free (automatic devm frees).
> 

At first, I think the use of arena is correct. Therefore, allocating arena
with devm_* should be a code style optimization. However, I rechecked discover_arenas and found arena might also be leaked (e.g., if alloc_arena() fails in the second loop, the previously allocated resources in the loop is leaked). The correct code could be found in create_arenas(), where free_arenas is called on failure of alloc_arena().

To fix this issue, I think it's better to change the 'goto out_super' tag
to 'goto out'. We could also use devm_* for arena to simplify the error path
in discover_arenas(). 

> We might want to look at using no_free_ptr() in this code.  See this
> patch[1] for an example of how to inhibit the cleanup and pass the pointer
> on when the function succeeds.
> 
> [1] https://lore.kernel.org/all/170261791914.1714654.6447680285357545638.stgit@dwillia2-xfh.jf.intel.com/
> 
> Ira

Thanks for this example. But it seems that no_free_ptr() is used to handle
the scope based resource management. Changes in this patch does not introduce
this feature. Do I understand this correctly?

Regards,
Dinghao

Re: [PATCH] [v2] nvdimm-btt: fix several memleaks

[Ira Weiny]

dinghao.liu@ wrote:
> > Ira Weiny wrote:
> > > Dinghao Liu wrote:
> > 
> > [snip]
> > 
> > -static int btt_freelist_init(struct arena_info *arena)
> > +static int btt_freelist_init(struct device *dev, struct arena_info *arena)
> > 
> > Both struct arena_info and struct btt contain references to struct nd_btt
> > which is the device you are passing down this call stack.
> > 
> > Just use the device in the arena/btt rather than passing a device
> > structure.  That makes the code easier to read and ensures that the device
> > associated with this arena or btt is used.
> 
> Thanks for this suggestion! I will fix this in the v3 patch.
> 
> > [snip]
> > > >  
> > > > -static int btt_maplocks_init(struct arena_info *arena)
> > > > +static int btt_maplocks_init(struct device *dev, struct arena_info *arena)
> > > >  {
> > > >  	u32 i;
> > > >  
> > > > -	arena->map_locks = kcalloc(arena->nfree, sizeof(struct aligned_lock),
> > > > +	arena->map_locks = devm_kcalloc(dev, arena->nfree, sizeof(struct aligned_lock),
> > > >  				GFP_KERNEL);
> > > >  	if (!arena->map_locks)
> > > >  		return -ENOMEM;
> > > > @@ -805,9 +805,6 @@ static void free_arenas(struct btt *btt)
> > > >  
> > > >  	list_for_each_entry_safe(arena, next, &btt->arena_list, list) {
> > > >  		list_del(&arena->list);
> > > > -		kfree(arena->rtt);
> > > > -		kfree(arena->map_locks);
> > > > -		kfree(arena->freelist);
> > > 
> > > This does not quite work.
> > > 
> > > free_arenas() is used in the error paths of create_arenas() and
> > > discover_arenas().  In those cases devm_kfree() is probably a better way
> > > to clean up this.
> 
> Here I'm a little confused about when devm_kfree() should be used.

Over all it should be used whenever memory is allocated for the lifetime
of the device.

> Code in btt_init() implies that resources allocated by devm_* could be
> auto freed in both error and success paths of probe/attach (e.g., btt 
> allocated by devm_kzalloc is never freed by devm_kfree).
> Using devm_kfree() in free_arenas() is ok for me, but I want to know
> whether not using devm_kfree() constitutes a bug.

Unfortunately I'm not familiar enough with this code to know for sure.

After my quick checks before I thought it was.  But each time I look at it
I get confused.  This is why I was thinking maybe not using devm_*() and
using no_free_ptr() may be a better solution to make sure things don't
leak without changing the success path (which is likely working fine
because no bugs have been found.)

> 
> > > 
> > > However...
> > > 
> > > >  		debugfs_remove_recursive(arena->debugfs_dir);
> > > >  		kfree(arena);
> > > 
> > > Why can't arena be allocated with devm_*?
> > > 
> > > We need to change this up a bit more to handle the error path vs regular
> > > device shutdown free (automatic devm frees).
> > 
> 
> At first, I think the use of arena is correct. Therefore, allocating arena
> with devm_* should be a code style optimization. However, I rechecked
> discover_arenas and found arena might also be leaked (e.g., if
> alloc_arena() fails in the second loop, the previously allocated
> resources in the loop is leaked). The correct code could be found in
> create_arenas(), where free_arenas is called on failure of
> alloc_arena().
 
Yea I've not reached that level of detail in my analysis.

>
> To fix this issue, I think it's better to change the 'goto out_super'
> tag to 'goto out'. We could also use devm_* for arena to simplify the
> error path in discover_arenas(). 

I think it is your call at this point as I don't have time to dig in more
than I have.  Sorry.

> 
> > We might want to look at using no_free_ptr() in this code.  See this
> > patch[1] for an example of how to inhibit the cleanup and pass the
> > pointer on when the function succeeds.
> > 
> > [1]
> > https://lore.kernel.org/all/170261791914.1714654.6447680285357545638.stgit@dwillia2-xfh.jf.intel.com/
> > 
> > Ira
> 
> Thanks for this example. But it seems that no_free_ptr() is used to
> handle the scope based resource management. Changes in this patch does
> not introduce this feature. Do I understand this correctly?

You do understand but I was thinking that perhaps using no_free_ptr()
rather than devm_*() might be an easier way to fix this bug without trying
to decode the lifetime of everything.

Ira

> 
> Regards,
> Dinghao

Re: [PATCH] [v2] nvdimm-btt: fix several memleaks

[dinghao.liu]

> dinghao.liu@ wrote:
> > > Ira Weiny wrote:
> > > > Dinghao Liu wrote:

[snip]

> > > > 
> > > > This does not quite work.
> > > > 
> > > > free_arenas() is used in the error paths of create_arenas() and
> > > > discover_arenas().  In those cases devm_kfree() is probably a better way
> > > > to clean up this.
> > 
> > Here I'm a little confused about when devm_kfree() should be used.
> 
> Over all it should be used whenever memory is allocated for the lifetime
> of the device.
> 
> > Code in btt_init() implies that resources allocated by devm_* could be
> > auto freed in both error and success paths of probe/attach (e.g., btt 
> > allocated by devm_kzalloc is never freed by devm_kfree).
> > Using devm_kfree() in free_arenas() is ok for me, but I want to know
> > whether not using devm_kfree() constitutes a bug.
> 
> Unfortunately I'm not familiar enough with this code to know for sure.
> 
> After my quick checks before I thought it was.  But each time I look at it
> I get confused.  This is why I was thinking maybe not using devm_*() and
> using no_free_ptr() may be a better solution to make sure things don't
> leak without changing the success path (which is likely working fine
> because no bugs have been found.)

We have the same confusion here... I find a discussion about this problem,
which implies that not using devm_kfree() may delay the release, but the memory will be freed later and no memory is leaked:

https://www.mail-archive.com/linux-kernel@vger.kernel.org/msg2009561.html

> > 
> > > We might want to look at using no_free_ptr() in this code.  See this
> > > patch[1] for an example of how to inhibit the cleanup and pass the
> > > pointer on when the function succeeds.
> > > 
> > > [1]
> > > https://lore.kernel.org/all/170261791914.1714654.6447680285357545638.stgit@dwillia2-xfh.jf.intel.com/
> > > 
> > > Ira
> > 
> > Thanks for this example. But it seems that no_free_ptr() is used to
> > handle the scope based resource management. Changes in this patch does
> > not introduce this feature. Do I understand this correctly?
> 
> You do understand but I was thinking that perhaps using no_free_ptr()
> rather than devm_*() might be an easier way to fix this bug without trying
> to decode the lifetime of everything.
> 

My concern is that no_free_ptr() may not be able to completely fix all
memleaks because some of them are triggered in (part of) success paths (e.g.,
when btt_freelist_init succeeds but btt_rtt_init fails, discover_arenas still needs to clean up the memory allocated in btt_freelist_init).

I checked the design of no_free_ptr(), and it seems that it will generate
a new pointer on success and the memory still leaks in the above case. 
Therefore, I think using devm_*() is still the best solution for this bug. 

Regards,
Dinghao