[RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path

[RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path

[Mingyu Wang]

While fuzzing with Syzkaller and fault injection (failslab) enabled,
I observed a persistent resv_map memory leak in the hugetlb mmap error path.

BUG: memory leak
unreferenced object 0xffff888110b92400 (size 512):
  comm "syz.0.5386", pid 20390, jiffies 4298157188
  backtrace:
    __kmalloc_cache_noprof+0x509/0x6e0
    resv_map_alloc+0x47/0x3a0
    hugetlb_reserve_pages+0x758/0x1220
    hugetlbfs_file_mmap_prepare+0x492/0x790
    __mmap_region+0x1ae6/0x29f0

This is a regression introduced by the recent VMA iterator and mmap region
refactoring, which decoupled mmap preparation from VMA completion.

In `__mmap_region()`, `call_mmap_prepare()` triggers `hugetlbfs_file_mmap_prepare()`,
which successfully allocates the `resv_map` and registers a `success_hook`
in `desc->action`.

If `__mmap_new_vma()` subsequently fails (e.g., `vma_iter_prealloc()`
returns -ENOMEM due to failslab), the code jumps to `abort_munmap`.
However, the `desc` structure is completely discarded without invoking
any cleanup. The newly allocated empty VMA is freed, but since
`set_vma_user_defined_fields()` was never reached, `vm_area_free()`
doesn't call `hugetlb_vm_close()`. Thus, the `resv_map` is permanently leaked.

This RFC proposes adding an `abort_hook` to `struct mmap_action`
so that subsystems can properly clean up resources allocated during the
`mmap_prepare` phase if VMA creation fails.

Any feedback on whether this architectural approach is correct, or how to 
properly implement the hugetlb unreserve rollback, would be highly appreciated.

Signed-off-by: Mingyu Wang <25181214217@stu.xidian.edu.cn>
---
 fs/hugetlbfs/inode.c     | 9 +++++++++
 include/linux/mm_types.h | 2 ++
 mm/vma.c                 | 4 ++++
 3 files changed, 15 insertions(+)

diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
index 8b05bec08e04..002bb6d9ca23 100644
--- a/fs/hugetlbfs/inode.c
+++ b/fs/hugetlbfs/inode.c
@@ -102,6 +102,14 @@ static int hugetlb_file_mmap_prepare_success(const struct vm_area_struct *vma)
 	return hugetlb_vma_lock_alloc((struct vm_area_struct *)vma);
 }
 
+static void hugetlb_file_mmap_prepare_abort(struct vm_area_desc *desc)
+{
+	/*
+	 * TODO: Implement the proper rollback for hugetlb_reserve_pages()
+	 * and drop the resv_map reference held in the desc here.
+	 */
+}
+
 static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
 {
 	struct file *file = desc->file;
@@ -172,6 +180,7 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
 	if (!ret) {
 		/* Allocate the VMA lock after we set it up. */
 		desc->action.success_hook = hugetlb_file_mmap_prepare_success;
+		desc->action.abort_hook = hugetlb_file_mmap_prepare_abort;
 		/*
 		 * We cannot permit the rmap finding this VMA in the time
 		 * between the VMA being inserted into the VMA tree and the
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index a308e2c23b82..9320f6699fa9 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -861,6 +861,8 @@ struct mmap_action {
 	 * it is not valid to clear the error here.
 	 */
 	int (*error_hook)(int err);
+	
+	void (*abort_hook)(struct vm_area_desc *desc);
 
 	/*
 	 * This should be set in rare instances where the operation required
diff --git a/mm/vma.c b/mm/vma.c
index 377321b48734..d64cea5b4335 100644
--- a/mm/vma.c
+++ b/mm/vma.c
@@ -2799,6 +2799,10 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr,
 	 */
 	if (map.file_doesnt_need_get)
 		fput(map.file);
+	
+	if (have_mmap_prepare && desc.action.abort_hook)
+		desc.action.abort_hook(&desc);
+	
 	vms_abort_munmap_vmas(&map.vms, &map.mas_detach);
 	return error;
 }
-- 
2.34.1

Re: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path

[Muchun Song]

> On Apr 25, 2026, at 15:07, Mingyu Wang <25181214217@stu.xidian.edu.cn> wrote:
> 
> While fuzzing with Syzkaller and fault injection (failslab) enabled,
> I observed a persistent resv_map memory leak in the hugetlb mmap error path.
> 
> BUG: memory leak
> unreferenced object 0xffff888110b92400 (size 512):
>  comm "syz.0.5386", pid 20390, jiffies 4298157188
>  backtrace:
>    __kmalloc_cache_noprof+0x509/0x6e0
>    resv_map_alloc+0x47/0x3a0
>    hugetlb_reserve_pages+0x758/0x1220
>    hugetlbfs_file_mmap_prepare+0x492/0x790
>    __mmap_region+0x1ae6/0x29f0
> 
> This is a regression introduced by the recent VMA iterator and mmap region
> refactoring, which decoupled mmap preparation from VMA completion.
> 
> In `__mmap_region()`, `call_mmap_prepare()` triggers `hugetlbfs_file_mmap_prepare()`,
> which successfully allocates the `resv_map` and registers a `success_hook`
> in `desc->action`.
> 
> If `__mmap_new_vma()` subsequently fails (e.g., `vma_iter_prealloc()`
> returns -ENOMEM due to failslab), the code jumps to `abort_munmap`.
> However, the `desc` structure is completely discarded without invoking
> any cleanup. The newly allocated empty VMA is freed, but since
> `set_vma_user_defined_fields()` was never reached, `vm_area_free()`
> doesn't call `hugetlb_vm_close()`. Thus, the `resv_map` is permanently leaked.
> 
> This RFC proposes adding an `abort_hook` to `struct mmap_action`
> so that subsystems can properly clean up resources allocated during the
> `mmap_prepare` phase if VMA creation fails.
> 
> Any feedback on whether this architectural approach is correct, or how to 
> properly implement the hugetlb unreserve rollback, would be highly appreciated.

Please use ./scripts/get_maintainer.pl to get full mail list for Cc/To since
it is not only related to HugeTLB subsystem. It will also consider the author
of commit introducing the problem.

> 
> Signed-off-by: Mingyu Wang <25181214217@stu.xidian.edu.cn>
> ---
> fs/hugetlbfs/inode.c     | 9 +++++++++
> include/linux/mm_types.h | 2 ++
> mm/vma.c                 | 4 ++++
> 3 files changed, 15 insertions(+)
> 
> diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
> index 8b05bec08e04..002bb6d9ca23 100644
> --- a/fs/hugetlbfs/inode.c
> +++ b/fs/hugetlbfs/inode.c
> @@ -102,6 +102,14 @@ static int hugetlb_file_mmap_prepare_success(const struct vm_area_struct *vma)
> return hugetlb_vma_lock_alloc((struct vm_area_struct *)vma);
> }
> 
> +static void hugetlb_file_mmap_prepare_abort(struct vm_area_desc *desc)
> +{
> + 	/*
> +	 * TODO: Implement the proper rollback for hugetlb_reserve_pages()
> +	 * and drop the resv_map reference held in the desc here.
> +	 */
> +}
> +
> static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
> {
> struct file *file = desc->file;
> @@ -172,6 +180,7 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
> 	if (!ret) {
> 	/* Allocate the VMA lock after we set it up. */
> 	desc->action.success_hook = hugetlb_file_mmap_prepare_success;
> + 	desc->action.abort_hook = hugetlb_file_mmap_prepare_abort;
> 	/*
> 	 * We cannot permit the rmap finding this VMA in the time
> 	 * between the VMA being inserted into the VMA tree and the
> diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
> index a308e2c23b82..9320f6699fa9 100644
> --- a/include/linux/mm_types.h
> +++ b/include/linux/mm_types.h
> @@ -861,6 +861,8 @@ struct mmap_action {
> * it is not valid to clear the error here.
> */
> 	int (*error_hook)(int err);
> + 
> + 	void (*abort_hook)(struct vm_area_desc *desc);

At least for me, it is not good name to distinguish it from error_hook.
abort_mmap_prepare? I am not sure if it is a good solution, Cc other
MM maintainers as well.

Muchun,
Thanks.

> 
> /*
> * This should be set in rare instances where the operation required
> diff --git a/mm/vma.c b/mm/vma.c
> index 377321b48734..d64cea5b4335 100644
> --- a/mm/vma.c
> +++ b/mm/vma.c
> @@ -2799,6 +2799,10 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr,
> */
> 	if (map.file_doesnt_need_get)
> 		fput(map.file);
> + 
> + 	if (have_mmap_prepare && desc.action.abort_hook)
> + 		desc.action.abort_hook(&desc);
> + 
> 	vms_abort_munmap_vmas(&map.vms, &map.mas_detach);
> 	return error;
> }
> -- 
> 2.34.1
> 

Re: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path

[David Hildenbrand (Arm)]

>> 	/*
>> 	 * We cannot permit the rmap finding this VMA in the time
>> 	 * between the VMA being inserted into the VMA tree and the
>> diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
>> index a308e2c23b82..9320f6699fa9 100644
>> --- a/include/linux/mm_types.h
>> +++ b/include/linux/mm_types.h
>> @@ -861,6 +861,8 @@ struct mmap_action {
>> * it is not valid to clear the error here.
>> */
>> 	int (*error_hook)(int err);
>> + 
>> + 	void (*abort_hook)(struct vm_area_desc *desc);
> 
> At least for me, it is not good name to distinguish it from error_hook.
> abort_mmap_prepare? I am not sure if it is a good solution, Cc other
> MM maintainers as well.
>

I'm sure Lorenzo will provide feedback here :)

-- 
Cheers,

David

Re: Re: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path

[王明煜]

Hi Muchun and David,

Thanks a lot for taking the time to review!

> At least for me, it is not good name to distinguish it from error_hook.
> abort_mmap_prepare? I am not sure if it is a good solution, Cc other
> MM maintainers as well.

I completely agree. `abort_mmap_prepare` is indeed much clearer and more specific to its purpose. I will update the naming in the v2 patch.

> Please use ./scripts/get_maintainer.pl to get full mail list for Cc/To since
> it is not only related to HugeTLB subsystem.

I apologize for the missing CCs. I did generate the full list using `get_maintainer.pl`, but the SMTP server of Xidian University drops emails with too many foreign recipients due to strict anti-spam policies. I had to heavily trim the CC list just to get the RFC patch delivered. I will try to use an alternative email or chunk the CCs to ensure all relevant authors are included in the next version.

I will hold off on sending the v2 patch until Lorenzo shares his thoughts on whether this architectural approach is the right way to go.

Best regards,
Mingyu


> -----原始邮件-----
> 发件人: "David Hildenbrand (Arm)" <david@kernel.org>
> 发送时间:2026-04-27 16:17:23 (星期一)
> 收件人: "Muchun Song" <muchun.song@linux.dev>, "Mingyu Wang" <25181214217@stu.xidian.edu.cn>
> 抄送: Liam.Howlett@oracle.com, akpm@linux-foundation.org, linux-mm@kvack.org, linux-kernel@vger.kernel.org, ljs@kernel.org, vbabka@kernel.org, jannh@google.com, pfalcato@suse.de, osalvador@suse.de
> 主题: Re: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path
> 
> 
> >> 	/*
> >> 	 * We cannot permit the rmap finding this VMA in the time
> >> 	 * between the VMA being inserted into the VMA tree and the
> >> diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
> >> index a308e2c23b82..9320f6699fa9 100644
> >> --- a/include/linux/mm_types.h
> >> +++ b/include/linux/mm_types.h
> >> @@ -861,6 +861,8 @@ struct mmap_action {
> >> * it is not valid to clear the error here.
> >> */
> >> 	int (*error_hook)(int err);
> >> + 
> >> + 	void (*abort_hook)(struct vm_area_desc *desc);
> > 
> > At least for me, it is not good name to distinguish it from error_hook.
> > abort_mmap_prepare? I am not sure if it is a good solution, Cc other
> > MM maintainers as well.
> >
> 
> I'm sure Lorenzo will provide feedback here :)
> 
> -- 
> Cheers,
> 
> David

Re: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path

[Lorenzo Stoakes]

On Mon, Apr 27, 2026 at 03:55:00PM +0800, Muchun Song wrote:
>
> > On Apr 25, 2026, at 15:07, Mingyu Wang <25181214217@stu.xidian.edu.cn> wrote:
> >
> > While fuzzing with Syzkaller and fault injection (failslab) enabled,
> > I observed a persistent resv_map memory leak in the hugetlb mmap error path.
> >
> > BUG: memory leak
> > unreferenced object 0xffff888110b92400 (size 512):
> >  comm "syz.0.5386", pid 20390, jiffies 4298157188
> >  backtrace:
> >    __kmalloc_cache_noprof+0x509/0x6e0
> >    resv_map_alloc+0x47/0x3a0
> >    hugetlb_reserve_pages+0x758/0x1220
> >    hugetlbfs_file_mmap_prepare+0x492/0x790
> >    __mmap_region+0x1ae6/0x29f0
> >
> > This is a regression introduced by the recent VMA iterator and mmap region
> > refactoring, which decoupled mmap preparation from VMA completion.
> >
> > In `__mmap_region()`, `call_mmap_prepare()` triggers `hugetlbfs_file_mmap_prepare()`,
> > which successfully allocates the `resv_map` and registers a `success_hook`
> > in `desc->action`.
> >
> > If `__mmap_new_vma()` subsequently fails (e.g., `vma_iter_prealloc()`
> > returns -ENOMEM due to failslab), the code jumps to `abort_munmap`.
> > However, the `desc` structure is completely discarded without invoking
> > any cleanup. The newly allocated empty VMA is freed, but since
> > `set_vma_user_defined_fields()` was never reached, `vm_area_free()`
> > doesn't call `hugetlb_vm_close()`. Thus, the `resv_map` is permanently leaked.
> >
> > This RFC proposes adding an `abort_hook` to `struct mmap_action`
> > so that subsystems can properly clean up resources allocated during the
> > `mmap_prepare` phase if VMA creation fails.

Yeah sorry, this is a general problem that I addressed already with the
vm_ops->mapped callback.

I'll come up with a patch to fix this up for hugetlb, thanks for highlighting
this.

I plan to get rid of the success hook in any case, it's only because hugetlb is
doing something really... not great with the 'VMA lock' (really hugetlb VMA lock
I suppose) that we need a VMA pointer at the point.

> >
> > Any feedback on whether this architectural approach is correct, or how to
> > properly implement the hugetlb unreserve rollback, would be highly appreciated.
>
> Please use ./scripts/get_maintainer.pl to get full mail list for Cc/To since
> it is not only related to HugeTLB subsystem. It will also consider the author
> of commit introducing the problem.

Please do do that, I wrote this whole thing, so y'know :)

Especially at the moment I am very very busy with LSF coming up so you need to
make sure you include me in the mail.

>
> >
> > Signed-off-by: Mingyu Wang <25181214217@stu.xidian.edu.cn>
> > ---
> > fs/hugetlbfs/inode.c     | 9 +++++++++
> > include/linux/mm_types.h | 2 ++
> > mm/vma.c                 | 4 ++++
> > 3 files changed, 15 insertions(+)
> >
> > diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
> > index 8b05bec08e04..002bb6d9ca23 100644
> > --- a/fs/hugetlbfs/inode.c
> > +++ b/fs/hugetlbfs/inode.c
> > @@ -102,6 +102,14 @@ static int hugetlb_file_mmap_prepare_success(const struct vm_area_struct *vma)
> > return hugetlb_vma_lock_alloc((struct vm_area_struct *)vma);
> > }
> >
> > +static void hugetlb_file_mmap_prepare_abort(struct vm_area_desc *desc)
> > +{
> > + 	/*
> > +	 * TODO: Implement the proper rollback for hugetlb_reserve_pages()
> > +	 * and drop the resv_map reference held in the desc here.
> > +	 */
> > +}
> > +
> > static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
> > {
> > struct file *file = desc->file;
> > @@ -172,6 +180,7 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
> > 	if (!ret) {
> > 	/* Allocate the VMA lock after we set it up. */
> > 	desc->action.success_hook = hugetlb_file_mmap_prepare_success;
> > + 	desc->action.abort_hook = hugetlb_file_mmap_prepare_abort;
> > 	/*
> > 	 * We cannot permit the rmap finding this VMA in the time
> > 	 * between the VMA being inserted into the VMA tree and the
> > diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
> > index a308e2c23b82..9320f6699fa9 100644
> > --- a/include/linux/mm_types.h
> > +++ b/include/linux/mm_types.h
> > @@ -861,6 +861,8 @@ struct mmap_action {
> > * it is not valid to clear the error here.
> > */
> > 	int (*error_hook)(int err);
> > +
> > + 	void (*abort_hook)(struct vm_area_desc *desc);
>
> At least for me, it is not good name to distinguish it from error_hook.
> abort_mmap_prepare? I am not sure if it is a good solution, Cc other
> MM maintainers as well.

Yeah no we definitely don't want this, I plan to eliminate these hooks
eventually.

Really intend on adding no others, these were to work around effectively legacy
issues in mmap callbacks.

>
> Muchun,
> Thanks.
>
> >
> > /*
> > * This should be set in rare instances where the operation required
> > diff --git a/mm/vma.c b/mm/vma.c
> > index 377321b48734..d64cea5b4335 100644
> > --- a/mm/vma.c
> > +++ b/mm/vma.c
> > @@ -2799,6 +2799,10 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr,
> > */
> > 	if (map.file_doesnt_need_get)
> > 		fput(map.file);
> > +
> > + 	if (have_mmap_prepare && desc.action.abort_hook)
> > + 		desc.action.abort_hook(&desc);
> > +
> > 	vms_abort_munmap_vmas(&map.vms, &map.mas_detach);
> > 	return error;
> > }
> > --
> > 2.34.1
> >
>

Thanks, Lorenzo

Re: Re: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path

[王明煜]

Hi Lorenzo,

Thanks for the prompt response and for taking over the fix! 

I completely understand the architectural direction to eliminate the hooks, which makes perfect sense. I'll watch out for your patch.

Also, my sincere apologies for omitting you from the CC list originally. My university's SMTP server blocked the email due to the long recipient list from `get_maintainer.pl`, forcing me to trim it heavily just to get the report out. I'll make sure to explicitly include the original authors in the future.

Good luck with the upcoming LSF/MM!

Best regards,
Mingyu


> -----原始邮件-----
> 发件人: "Lorenzo Stoakes" <ljs@kernel.org>
> 发送时间:2026-04-27 22:18:34 (星期一)
> 收件人: "Muchun Song" <muchun.song@linux.dev>
> 抄送: "Mingyu Wang" <25181214217@stu.xidian.edu.cn>, Liam.Howlett@oracle.com, akpm@linux-foundation.org, linux-mm@kvack.org, linux-kernel@vger.kernel.org, vbabka@kernel.org, jannh@google.com, pfalcato@suse.de, osalvador@suse.de, david@kernel.org
> 主题: Re: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path
> 
> 
> On Mon, Apr 27, 2026 at 03:55:00PM +0800, Muchun Song wrote:
> >
> > > On Apr 25, 2026, at 15:07, Mingyu Wang <25181214217@stu.xidian.edu.cn> wrote:
> > >
> > > While fuzzing with Syzkaller and fault injection (failslab) enabled,
> > > I observed a persistent resv_map memory leak in the hugetlb mmap error path.
> > >
> > > BUG: memory leak
> > > unreferenced object 0xffff888110b92400 (size 512):
> > >  comm "syz.0.5386", pid 20390, jiffies 4298157188
> > >  backtrace:
> > >    __kmalloc_cache_noprof+0x509/0x6e0
> > >    resv_map_alloc+0x47/0x3a0
> > >    hugetlb_reserve_pages+0x758/0x1220
> > >    hugetlbfs_file_mmap_prepare+0x492/0x790
> > >    __mmap_region+0x1ae6/0x29f0
> > >
> > > This is a regression introduced by the recent VMA iterator and mmap region
> > > refactoring, which decoupled mmap preparation from VMA completion.
> > >
> > > In `__mmap_region()`, `call_mmap_prepare()` triggers `hugetlbfs_file_mmap_prepare()`,
> > > which successfully allocates the `resv_map` and registers a `success_hook`
> > > in `desc->action`.
> > >
> > > If `__mmap_new_vma()` subsequently fails (e.g., `vma_iter_prealloc()`
> > > returns -ENOMEM due to failslab), the code jumps to `abort_munmap`.
> > > However, the `desc` structure is completely discarded without invoking
> > > any cleanup. The newly allocated empty VMA is freed, but since
> > > `set_vma_user_defined_fields()` was never reached, `vm_area_free()`
> > > doesn't call `hugetlb_vm_close()`. Thus, the `resv_map` is permanently leaked.
> > >
> > > This RFC proposes adding an `abort_hook` to `struct mmap_action`
> > > so that subsystems can properly clean up resources allocated during the
> > > `mmap_prepare` phase if VMA creation fails.
> 
> Yeah sorry, this is a general problem that I addressed already with the
> vm_ops->mapped callback.
> 
> I'll come up with a patch to fix this up for hugetlb, thanks for highlighting
> this.
> 
> I plan to get rid of the success hook in any case, it's only because hugetlb is
> doing something really... not great with the 'VMA lock' (really hugetlb VMA lock
> I suppose) that we need a VMA pointer at the point.
> 
> > >
> > > Any feedback on whether this architectural approach is correct, or how to
> > > properly implement the hugetlb unreserve rollback, would be highly appreciated.
> >
> > Please use ./scripts/get_maintainer.pl to get full mail list for Cc/To since
> > it is not only related to HugeTLB subsystem. It will also consider the author
> > of commit introducing the problem.
> 
> Please do do that, I wrote this whole thing, so y'know :)
> 
> Especially at the moment I am very very busy with LSF coming up so you need to
> make sure you include me in the mail.
> 
> >
> > >
> > > Signed-off-by: Mingyu Wang <25181214217@stu.xidian.edu.cn>
> > > ---
> > > fs/hugetlbfs/inode.c     | 9 +++++++++
> > > include/linux/mm_types.h | 2 ++
> > > mm/vma.c                 | 4 ++++
> > > 3 files changed, 15 insertions(+)
> > >
> > > diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
> > > index 8b05bec08e04..002bb6d9ca23 100644
> > > --- a/fs/hugetlbfs/inode.c
> > > +++ b/fs/hugetlbfs/inode.c
> > > @@ -102,6 +102,14 @@ static int hugetlb_file_mmap_prepare_success(const struct vm_area_struct *vma)
> > > return hugetlb_vma_lock_alloc((struct vm_area_struct *)vma);
> > > }
> > >
> > > +static void hugetlb_file_mmap_prepare_abort(struct vm_area_desc *desc)
> > > +{
> > > + 	/*
> > > +	 * TODO: Implement the proper rollback for hugetlb_reserve_pages()
> > > +	 * and drop the resv_map reference held in the desc here.
> > > +	 */
> > > +}
> > > +
> > > static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
> > > {
> > > struct file *file = desc->file;
> > > @@ -172,6 +180,7 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
> > > 	if (!ret) {
> > > 	/* Allocate the VMA lock after we set it up. */
> > > 	desc->action.success_hook = hugetlb_file_mmap_prepare_success;
> > > + 	desc->action.abort_hook = hugetlb_file_mmap_prepare_abort;
> > > 	/*
> > > 	 * We cannot permit the rmap finding this VMA in the time
> > > 	 * between the VMA being inserted into the VMA tree and the
> > > diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
> > > index a308e2c23b82..9320f6699fa9 100644
> > > --- a/include/linux/mm_types.h
> > > +++ b/include/linux/mm_types.h
> > > @@ -861,6 +861,8 @@ struct mmap_action {
> > > * it is not valid to clear the error here.
> > > */
> > > 	int (*error_hook)(int err);
> > > +
> > > + 	void (*abort_hook)(struct vm_area_desc *desc);
> >
> > At least for me, it is not good name to distinguish it from error_hook.
> > abort_mmap_prepare? I am not sure if it is a good solution, Cc other
> > MM maintainers as well.
> 
> Yeah no we definitely don't want this, I plan to eliminate these hooks
> eventually.
> 
> Really intend on adding no others, these were to work around effectively legacy
> issues in mmap callbacks.
> 
> >
> > Muchun,
> > Thanks.
> >
> > >
> > > /*
> > > * This should be set in rare instances where the operation required
> > > diff --git a/mm/vma.c b/mm/vma.c
> > > index 377321b48734..d64cea5b4335 100644
> > > --- a/mm/vma.c
> > > +++ b/mm/vma.c
> > > @@ -2799,6 +2799,10 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr,
> > > */
> > > 	if (map.file_doesnt_need_get)
> > > 		fput(map.file);
> > > +
> > > + 	if (have_mmap_prepare && desc.action.abort_hook)
> > > + 		desc.action.abort_hook(&desc);
> > > +
> > > 	vms_abort_munmap_vmas(&map.vms, &map.mas_detach);
> > > 	return error;
> > > }
> > > --
> > > 2.34.1
> > >
> >
> 
> Thanks, Lorenzo

Re: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path

[Lorenzo Stoakes]

On Mon, Apr 27, 2026 at 10:39:37PM +0800, 王明煜 wrote:
> Hi Lorenzo,
>
> Thanks for the prompt response and for taking over the fix!
>
> I completely understand the architectural direction to eliminate the hooks, which makes perfect sense. I'll watch out for your patch.
>
> Also, my sincere apologies for omitting you from the CC list originally. My university's SMTP server blocked the email due to the long recipient list from `get_maintainer.pl`, forcing me to trim it heavily just to get the report out. I'll make sure to explicitly include the original authors in the future.

Ugh sorry to hear!

>
> Good luck with the upcoming LSF/MM!

Thanks!

I will come back to this, it's not going to be straightforward, hugetlbfs is
a... pain.

But thanks again for raising this, there is very much an issue here that I need
to address.

vm_ops->mapped won't do here, as hugetlbfs (of course) requires data that the
interface (intentionally) doesn't provide.

I may have to revert this (ugh), because mmap_prepare should be idempotent and
this is just an error on my part converting things a little too one-to-one.

Anyway, shall return to this! :)

>
> Best regards,
> Mingyu

Thanks, Lorenzo

>
>
> > -----原始邮件-----
> > 发件人: "Lorenzo Stoakes" <ljs@kernel.org>
> > 发送时间:2026-04-27 22:18:34 (星期一)
> > 收件人: "Muchun Song" <muchun.song@linux.dev>
> > 抄送: "Mingyu Wang" <25181214217@stu.xidian.edu.cn>, Liam.Howlett@oracle.com, akpm@linux-foundation.org, linux-mm@kvack.org, linux-kernel@vger.kernel.org, vbabka@kernel.org, jannh@google.com, pfalcato@suse.de, osalvador@suse.de, david@kernel.org
> > 主题: Re: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path
> >
> >
> > On Mon, Apr 27, 2026 at 03:55:00PM +0800, Muchun Song wrote:
> > >
> > > > On Apr 25, 2026, at 15:07, Mingyu Wang <25181214217@stu.xidian.edu.cn> wrote:
> > > >
> > > > While fuzzing with Syzkaller and fault injection (failslab) enabled,
> > > > I observed a persistent resv_map memory leak in the hugetlb mmap error path.
> > > >
> > > > BUG: memory leak
> > > > unreferenced object 0xffff888110b92400 (size 512):
> > > >  comm "syz.0.5386", pid 20390, jiffies 4298157188
> > > >  backtrace:
> > > >    __kmalloc_cache_noprof+0x509/0x6e0
> > > >    resv_map_alloc+0x47/0x3a0
> > > >    hugetlb_reserve_pages+0x758/0x1220
> > > >    hugetlbfs_file_mmap_prepare+0x492/0x790
> > > >    __mmap_region+0x1ae6/0x29f0
> > > >
> > > > This is a regression introduced by the recent VMA iterator and mmap region
> > > > refactoring, which decoupled mmap preparation from VMA completion.
> > > >
> > > > In `__mmap_region()`, `call_mmap_prepare()` triggers `hugetlbfs_file_mmap_prepare()`,
> > > > which successfully allocates the `resv_map` and registers a `success_hook`
> > > > in `desc->action`.
> > > >
> > > > If `__mmap_new_vma()` subsequently fails (e.g., `vma_iter_prealloc()`
> > > > returns -ENOMEM due to failslab), the code jumps to `abort_munmap`.
> > > > However, the `desc` structure is completely discarded without invoking
> > > > any cleanup. The newly allocated empty VMA is freed, but since
> > > > `set_vma_user_defined_fields()` was never reached, `vm_area_free()`
> > > > doesn't call `hugetlb_vm_close()`. Thus, the `resv_map` is permanently leaked.
> > > >
> > > > This RFC proposes adding an `abort_hook` to `struct mmap_action`
> > > > so that subsystems can properly clean up resources allocated during the
> > > > `mmap_prepare` phase if VMA creation fails.
> >
> > Yeah sorry, this is a general problem that I addressed already with the
> > vm_ops->mapped callback.
> >
> > I'll come up with a patch to fix this up for hugetlb, thanks for highlighting
> > this.
> >
> > I plan to get rid of the success hook in any case, it's only because hugetlb is
> > doing something really... not great with the 'VMA lock' (really hugetlb VMA lock
> > I suppose) that we need a VMA pointer at the point.
> >
> > > >
> > > > Any feedback on whether this architectural approach is correct, or how to
> > > > properly implement the hugetlb unreserve rollback, would be highly appreciated.
> > >
> > > Please use ./scripts/get_maintainer.pl to get full mail list for Cc/To since
> > > it is not only related to HugeTLB subsystem. It will also consider the author
> > > of commit introducing the problem.
> >
> > Please do do that, I wrote this whole thing, so y'know :)
> >
> > Especially at the moment I am very very busy with LSF coming up so you need to
> > make sure you include me in the mail.
> >
> > >
> > > >
> > > > Signed-off-by: Mingyu Wang <25181214217@stu.xidian.edu.cn>
> > > > ---
> > > > fs/hugetlbfs/inode.c     | 9 +++++++++
> > > > include/linux/mm_types.h | 2 ++
> > > > mm/vma.c                 | 4 ++++
> > > > 3 files changed, 15 insertions(+)
> > > >
> > > > diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
> > > > index 8b05bec08e04..002bb6d9ca23 100644
> > > > --- a/fs/hugetlbfs/inode.c
> > > > +++ b/fs/hugetlbfs/inode.c
> > > > @@ -102,6 +102,14 @@ static int hugetlb_file_mmap_prepare_success(const struct vm_area_struct *vma)
> > > > return hugetlb_vma_lock_alloc((struct vm_area_struct *)vma);
> > > > }
> > > >
> > > > +static void hugetlb_file_mmap_prepare_abort(struct vm_area_desc *desc)
> > > > +{
> > > > + 	/*
> > > > +	 * TODO: Implement the proper rollback for hugetlb_reserve_pages()
> > > > +	 * and drop the resv_map reference held in the desc here.
> > > > +	 */
> > > > +}
> > > > +
> > > > static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
> > > > {
> > > > struct file *file = desc->file;
> > > > @@ -172,6 +180,7 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
> > > > 	if (!ret) {
> > > > 	/* Allocate the VMA lock after we set it up. */
> > > > 	desc->action.success_hook = hugetlb_file_mmap_prepare_success;
> > > > + 	desc->action.abort_hook = hugetlb_file_mmap_prepare_abort;
> > > > 	/*
> > > > 	 * We cannot permit the rmap finding this VMA in the time
> > > > 	 * between the VMA being inserted into the VMA tree and the
> > > > diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
> > > > index a308e2c23b82..9320f6699fa9 100644
> > > > --- a/include/linux/mm_types.h
> > > > +++ b/include/linux/mm_types.h
> > > > @@ -861,6 +861,8 @@ struct mmap_action {
> > > > * it is not valid to clear the error here.
> > > > */
> > > > 	int (*error_hook)(int err);
> > > > +
> > > > + 	void (*abort_hook)(struct vm_area_desc *desc);
> > >
> > > At least for me, it is not good name to distinguish it from error_hook.
> > > abort_mmap_prepare? I am not sure if it is a good solution, Cc other
> > > MM maintainers as well.
> >
> > Yeah no we definitely don't want this, I plan to eliminate these hooks
> > eventually.
> >
> > Really intend on adding no others, these were to work around effectively legacy
> > issues in mmap callbacks.
> >
> > >
> > > Muchun,
> > > Thanks.
> > >
> > > >
> > > > /*
> > > > * This should be set in rare instances where the operation required
> > > > diff --git a/mm/vma.c b/mm/vma.c
> > > > index 377321b48734..d64cea5b4335 100644
> > > > --- a/mm/vma.c
> > > > +++ b/mm/vma.c
> > > > @@ -2799,6 +2799,10 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr,
> > > > */
> > > > 	if (map.file_doesnt_need_get)
> > > > 		fput(map.file);
> > > > +
> > > > + 	if (have_mmap_prepare && desc.action.abort_hook)
> > > > + 		desc.action.abort_hook(&desc);
> > > > +
> > > > 	vms_abort_munmap_vmas(&map.vms, &map.mas_detach);
> > > > 	return error;
> > > > }
> > > > --
> > > > 2.34.1
> > > >
> > >
> >
> > Thanks, Lorenzo

Re: Re: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path

[王明煜]

Hi Lorenzo,

No worries at all! Enjoy the LSF/MM summit.

I completely understand the pain with hugetlbfs. Since this specific memory leak is deeply tied to the error path and requires fault injection (failslab) under heavy fuzzing to hit, it has been notoriously difficult to extract a clean C reproducer. 

Because of this, I'd be more than happy to help test any fix or revert you eventually come up with. Whenever you have a prototype patch ready (even if it's just a rough draft), feel free to ping me or send it my way. I can easily plug it into my Syzkaller environment and verify if the resv_map leak is fully resolved under pressure.

Good luck with the conference, and looking forward to your updates when you're back!

Best regards,
Mingyu


> -----原始邮件-----
> 发件人: "Lorenzo Stoakes" <ljs@kernel.org>
> 发送时间:2026-04-27 23:20:13 (星期一)
> 收件人: 王明煜 <25181214217@stu.xidian.edu.cn>
> 抄送: "Muchun Song" <muchun.song@linux.dev>, Liam.Howlett@oracle.com, akpm@linux-foundation.org, linux-mm@kvack.org, linux-kernel@vger.kernel.org, vbabka@kernel.org, jannh@google.com, pfalcato@suse.de, osalvador@suse.de, david@kernel.org
> 主题: Re: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path
> 
> On Mon, Apr 27, 2026 at 10:39:37PM +0800, 王明煜 wrote:
> > Hi Lorenzo,
> >
> > Thanks for the prompt response and for taking over the fix!
> >
> > I completely understand the architectural direction to eliminate the hooks, which makes perfect sense. I'll watch out for your patch.
> >
> > Also, my sincere apologies for omitting you from the CC list originally. My university's SMTP server blocked the email due to the long recipient list from `get_maintainer.pl`, forcing me to trim it heavily just to get the report out. I'll make sure to explicitly include the original authors in the future.
> 
> Ugh sorry to hear!
> 
> >
> > Good luck with the upcoming LSF/MM!
> 
> Thanks!
> 
> I will come back to this, it's not going to be straightforward, hugetlbfs is
> a... pain.
> 
> But thanks again for raising this, there is very much an issue here that I need
> to address.
> 
> vm_ops->mapped won't do here, as hugetlbfs (of course) requires data that the
> interface (intentionally) doesn't provide.
> 
> I may have to revert this (ugh), because mmap_prepare should be idempotent and
> this is just an error on my part converting things a little too one-to-one.
> 
> Anyway, shall return to this! :)
> 
> >
> > Best regards,
> > Mingyu
> 
> Thanks, Lorenzo
> 
> >
> >
> > > -----原始邮件-----
> > > 发件人: "Lorenzo Stoakes" <ljs@kernel.org>
> > > 发送时间:2026-04-27 22:18:34 (星期一)
> > > 收件人: "Muchun Song" <muchun.song@linux.dev>
> > > 抄送: "Mingyu Wang" <25181214217@stu.xidian.edu.cn>, Liam.Howlett@oracle.com, akpm@linux-foundation.org, linux-mm@kvack.org, linux-kernel@vger.kernel.org, vbabka@kernel.org, jannh@google.com, pfalcato@suse.de, osalvador@suse.de, david@kernel.org
> > > 主题: Re: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path
> > >
> > >
> > > On Mon, Apr 27, 2026 at 03:55:00PM +0800, Muchun Song wrote:
> > > >
> > > > > On Apr 25, 2026, at 15:07, Mingyu Wang <25181214217@stu.xidian.edu.cn> wrote:
> > > > >
> > > > > While fuzzing with Syzkaller and fault injection (failslab) enabled,
> > > > > I observed a persistent resv_map memory leak in the hugetlb mmap error path.
> > > > >
> > > > > BUG: memory leak
> > > > > unreferenced object 0xffff888110b92400 (size 512):
> > > > >  comm "syz.0.5386", pid 20390, jiffies 4298157188
> > > > >  backtrace:
> > > > >    __kmalloc_cache_noprof+0x509/0x6e0
> > > > >    resv_map_alloc+0x47/0x3a0
> > > > >    hugetlb_reserve_pages+0x758/0x1220
> > > > >    hugetlbfs_file_mmap_prepare+0x492/0x790
> > > > >    __mmap_region+0x1ae6/0x29f0
> > > > >
> > > > > This is a regression introduced by the recent VMA iterator and mmap region
> > > > > refactoring, which decoupled mmap preparation from VMA completion.
> > > > >
> > > > > In `__mmap_region()`, `call_mmap_prepare()` triggers `hugetlbfs_file_mmap_prepare()`,
> > > > > which successfully allocates the `resv_map` and registers a `success_hook`
> > > > > in `desc->action`.
> > > > >
> > > > > If `__mmap_new_vma()` subsequently fails (e.g., `vma_iter_prealloc()`
> > > > > returns -ENOMEM due to failslab), the code jumps to `abort_munmap`.
> > > > > However, the `desc` structure is completely discarded without invoking
> > > > > any cleanup. The newly allocated empty VMA is freed, but since
> > > > > `set_vma_user_defined_fields()` was never reached, `vm_area_free()`
> > > > > doesn't call `hugetlb_vm_close()`. Thus, the `resv_map` is permanently leaked.
> > > > >
> > > > > This RFC proposes adding an `abort_hook` to `struct mmap_action`
> > > > > so that subsystems can properly clean up resources allocated during the
> > > > > `mmap_prepare` phase if VMA creation fails.
> > >
> > > Yeah sorry, this is a general problem that I addressed already with the
> > > vm_ops->mapped callback.
> > >
> > > I'll come up with a patch to fix this up for hugetlb, thanks for highlighting
> > > this.
> > >
> > > I plan to get rid of the success hook in any case, it's only because hugetlb is
> > > doing something really... not great with the 'VMA lock' (really hugetlb VMA lock
> > > I suppose) that we need a VMA pointer at the point.
> > >
> > > > >
> > > > > Any feedback on whether this architectural approach is correct, or how to
> > > > > properly implement the hugetlb unreserve rollback, would be highly appreciated.
> > > >
> > > > Please use ./scripts/get_maintainer.pl to get full mail list for Cc/To since
> > > > it is not only related to HugeTLB subsystem. It will also consider the author
> > > > of commit introducing the problem.
> > >
> > > Please do do that, I wrote this whole thing, so y'know :)
> > >
> > > Especially at the moment I am very very busy with LSF coming up so you need to
> > > make sure you include me in the mail.
> > >
> > > >
> > > > >
> > > > > Signed-off-by: Mingyu Wang <25181214217@stu.xidian.edu.cn>
> > > > > ---
> > > > > fs/hugetlbfs/inode.c     | 9 +++++++++
> > > > > include/linux/mm_types.h | 2 ++
> > > > > mm/vma.c                 | 4 ++++
> > > > > 3 files changed, 15 insertions(+)
> > > > >
> > > > > diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
> > > > > index 8b05bec08e04..002bb6d9ca23 100644
> > > > > --- a/fs/hugetlbfs/inode.c
> > > > > +++ b/fs/hugetlbfs/inode.c
> > > > > @@ -102,6 +102,14 @@ static int hugetlb_file_mmap_prepare_success(const struct vm_area_struct *vma)
> > > > > return hugetlb_vma_lock_alloc((struct vm_area_struct *)vma);
> > > > > }
> > > > >
> > > > > +static void hugetlb_file_mmap_prepare_abort(struct vm_area_desc *desc)
> > > > > +{
> > > > > + 	/*
> > > > > +	 * TODO: Implement the proper rollback for hugetlb_reserve_pages()
> > > > > +	 * and drop the resv_map reference held in the desc here.
> > > > > +	 */
> > > > > +}
> > > > > +
> > > > > static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
> > > > > {
> > > > > struct file *file = desc->file;
> > > > > @@ -172,6 +180,7 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
> > > > > 	if (!ret) {
> > > > > 	/* Allocate the VMA lock after we set it up. */
> > > > > 	desc->action.success_hook = hugetlb_file_mmap_prepare_success;
> > > > > + 	desc->action.abort_hook = hugetlb_file_mmap_prepare_abort;
> > > > > 	/*
> > > > > 	 * We cannot permit the rmap finding this VMA in the time
> > > > > 	 * between the VMA being inserted into the VMA tree and the
> > > > > diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
> > > > > index a308e2c23b82..9320f6699fa9 100644
> > > > > --- a/include/linux/mm_types.h
> > > > > +++ b/include/linux/mm_types.h
> > > > > @@ -861,6 +861,8 @@ struct mmap_action {
> > > > > * it is not valid to clear the error here.
> > > > > */
> > > > > 	int (*error_hook)(int err);
> > > > > +
> > > > > + 	void (*abort_hook)(struct vm_area_desc *desc);
> > > >
> > > > At least for me, it is not good name to distinguish it from error_hook.
> > > > abort_mmap_prepare? I am not sure if it is a good solution, Cc other
> > > > MM maintainers as well.
> > >
> > > Yeah no we definitely don't want this, I plan to eliminate these hooks
> > > eventually.
> > >
> > > Really intend on adding no others, these were to work around effectively legacy
> > > issues in mmap callbacks.
> > >
> > > >
> > > > Muchun,
> > > > Thanks.
> > > >
> > > > >
> > > > > /*
> > > > > * This should be set in rare instances where the operation required
> > > > > diff --git a/mm/vma.c b/mm/vma.c
> > > > > index 377321b48734..d64cea5b4335 100644
> > > > > --- a/mm/vma.c
> > > > > +++ b/mm/vma.c
> > > > > @@ -2799,6 +2799,10 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr,
> > > > > */
> > > > > 	if (map.file_doesnt_need_get)
> > > > > 		fput(map.file);
> > > > > +
> > > > > + 	if (have_mmap_prepare && desc.action.abort_hook)
> > > > > + 		desc.action.abort_hook(&desc);
> > > > > +
> > > > > 	vms_abort_munmap_vmas(&map.vms, &map.mas_detach);
> > > > > 	return error;
> > > > > }
> > > > > --
> > > > > 2.34.1
> > > > >
> > > >
> > >
> > > Thanks, Lorenzo

Re: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path

[kernel test robot]

Hi Mingyu,

[This is a private test report for your RFC patch.]
kernel test robot noticed the following build warnings:

[auto build test WARNING on akpm-mm/mm-everything]

url:    https://github.com/intel-lab-lkp/linux/commits/Mingyu-Wang/mm-hugetlb-fix-resv_map-memory-leak-in-__mmap_region-error-path/20260425-203812
base:   https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-everything
patch link:    https://lore.kernel.org/r/20260425070700.562229-1-25181214217%40stu.xidian.edu.cn
patch subject: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path
config: arm-allnoconfig (https://download.01.org/0day-ci/archive/20260502/202605020650.VNBzVdYb-lkp@intel.com/config)
compiler: clang version 23.0.0git (https://github.com/llvm/llvm-project 5bac06718f502014fade905512f1d26d578a18f3)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260502/202605020650.VNBzVdYb-lkp@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202605020650.VNBzVdYb-lkp@intel.com/

All warnings (new ones prefixed by >>):

   In file included from arch/arm/kernel/asm-offsets.c:14:
   In file included from include/linux/mm.h:8:
   In file included from include/linux/gfp.h:7:
   In file included from include/linux/mmzone.h:22:
>> include/linux/mm_types.h:865:28: warning: declaration of 'struct vm_area_desc' will not be visible outside of this function [-Wvisibility]
     865 |         void (*abort_hook)(struct vm_area_desc *desc);
         |                                   ^
   1 warning generated.
--
   In file included from kernel/fork.c:16:
   In file included from include/linux/slab.h:17:
   In file included from include/linux/gfp.h:7:
   In file included from include/linux/mmzone.h:22:
>> include/linux/mm_types.h:865:28: warning: declaration of 'struct vm_area_desc' will not be visible outside of this function [-Wvisibility]
     865 |         void (*abort_hook)(struct vm_area_desc *desc);
         |                                   ^
   In file included from kernel/fork.c:34:
   In file included from include/linux/mempolicy.h:16:
   include/linux/pagemap.h:521:45: error: use of undeclared identifier 'PMD_SHIFT'
     521 |         return mapping_max_folio_order(mapping) >= PMD_ORDER;
         |                                                    ^~~~~~~~~
   include/linux/pgtable.h:8:20: note: expanded from macro 'PMD_ORDER'
       8 | #define PMD_ORDER       (PMD_SHIFT - PAGE_SHIFT)
         |                          ^~~~~~~~~
   In file included from kernel/fork.c:42:
   include/linux/mman.h:157:9: warning: division by zero is undefined [-Wdivision-by-zero]
     157 |                _calc_vm_trans(flags, MAP_SYNC,       VM_SYNC      ) |
         |                ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
   include/linux/mman.h:135:21: note: expanded from macro '_calc_vm_trans'
     135 |    : ((x) & (bit1)) / ((bit1) / (bit2))))
         |                     ^ ~~~~~~~~~~~~~~~~~
   2 warnings and 1 error generated.
--
   In file included from kernel/cpu.c:5:
   In file included from include/linux/sched/mm.h:8:
>> include/linux/mm_types.h:865:28: warning: declaration of 'struct vm_area_desc' will not be visible outside of this function [-Wvisibility]
     865 |         void (*abort_hook)(struct vm_area_desc *desc);
         |                                   ^
   In file included from kernel/cpu.c:26:
   In file included from include/linux/suspend.h:5:
   In file included from include/linux/swap.h:13:
   include/linux/pagemap.h:521:45: error: use of undeclared identifier 'PMD_SHIFT'
     521 |         return mapping_max_folio_order(mapping) >= PMD_ORDER;
         |                                                    ^~~~~~~~~
   include/linux/pgtable.h:8:20: note: expanded from macro 'PMD_ORDER'
       8 | #define PMD_ORDER       (PMD_SHIFT - PAGE_SHIFT)
         |                          ^~~~~~~~~
   1 warning and 1 error generated.
--
   In file included from kernel/sched/core.c:12:
   In file included from include/linux/highmem.h:5:
   In file included from include/linux/fs.h:5:
   In file included from include/linux/fs/super.h:5:
   In file included from include/linux/fs/super_types.h:7:
   In file included from include/linux/list_lru.h:14:
   In file included from include/linux/xarray.h:16:
   In file included from include/linux/gfp.h:7:
   In file included from include/linux/mmzone.h:22:
>> include/linux/mm_types.h:865:28: warning: declaration of 'struct vm_area_desc' will not be visible outside of this function [-Wvisibility]
     865 |         void (*abort_hook)(struct vm_area_desc *desc);
         |                                   ^
   In file included from kernel/sched/core.c:83:
   In file included from arch/arm/include/asm/tlb.h:21:
   include/linux/pagemap.h:521:45: error: use of undeclared identifier 'PMD_SHIFT'
     521 |         return mapping_max_folio_order(mapping) >= PMD_ORDER;
         |                                                    ^~~~~~~~~
   include/linux/pgtable.h:8:20: note: expanded from macro 'PMD_ORDER'
       8 | #define PMD_ORDER       (PMD_SHIFT - PAGE_SHIFT)
         |                          ^~~~~~~~~
   kernel/sched/core.c:8053:12: warning: array index -1 is before the beginning of the array [-Warray-bounds]
    8053 |                                        preempt_modes[preempt_dynamic_mode] : "undef",
         |                                        ^             ~~~~~~~~~~~~~~~~~~~~
   kernel/sched/core.c:8028:1: note: array 'preempt_modes' declared here
    8028 | const char *preempt_modes[] = {
         | ^
   2 warnings and 1 error generated.
--
   In file included from kernel/time/timekeeping.c:6:
   In file included from include/linux/audit.h:13:
   In file included from include/linux/ptrace.h:7:
   In file included from include/linux/sched/signal.h:9:
   In file included from include/linux/sched/task.h:13:
   In file included from include/linux/uaccess.h:13:
   In file included from arch/arm/include/asm/uaccess.h:17:
   In file included from arch/arm/include/asm/pgtable.h:16:
   In file included from arch/arm/include/asm/pgtable-nommu.h:13:
   In file included from include/linux/slab.h:17:
   In file included from include/linux/gfp.h:7:
   In file included from include/linux/mmzone.h:22:
>> include/linux/mm_types.h:865:28: warning: declaration of 'struct vm_area_desc' will not be visible outside of this function [-Wvisibility]
     865 |         void (*abort_hook)(struct vm_area_desc *desc);
         |                                   ^
   kernel/time/timekeeping.c:1948:13: warning: variable 'suspend_timing_needed' set but not used [-Wunused-but-set-global]
    1948 | static bool suspend_timing_needed;
         |             ^
   2 warnings generated.
--
   In file included from kernel/time/alarmtimer.c:18:
   In file included from include/linux/rtc.h:17:
   In file included from include/linux/interrupt.h:11:
   In file included from include/linux/hardirq.h:11:
   In file included from arch/arm/include/asm/hardirq.h:10:
   In file included from include/asm-generic/hardirq.h:17:
   In file included from include/linux/irq.h:19:
   In file included from include/linux/topology.h:34:
   In file included from include/linux/mmzone.h:22:
>> include/linux/mm_types.h:865:28: warning: declaration of 'struct vm_area_desc' will not be visible outside of this function [-Wvisibility]
     865 |         void (*abort_hook)(struct vm_area_desc *desc);
         |                                   ^
   kernel/time/alarmtimer.c:54:29: warning: variable 'freezer_alarmtype' set but not used [-Wunused-but-set-global]
      54 | static enum alarmtimer_type freezer_alarmtype;
         |                             ^
   kernel/time/alarmtimer.c:55:16: warning: variable 'freezer_expires' set but not used [-Wunused-but-set-global]
      55 | static ktime_t freezer_expires;
         |                ^
   3 warnings generated.
--
   In file included from arch/arm/kernel/asm-offsets.c:14:
   In file included from include/linux/mm.h:8:
   In file included from include/linux/gfp.h:7:
   In file included from include/linux/mmzone.h:22:
>> include/linux/mm_types.h:865:28: warning: declaration of 'struct vm_area_desc' will not be visible outside of this function [-Wvisibility]
     865 |         void (*abort_hook)(struct vm_area_desc *desc);
         |                                   ^
   1 warning generated.
   In file included from kernel/sched/rq-offsets.c:5:
   In file included from kernel/sched/sched.h:15:
   In file included from include/linux/sched/mm.h:8:
>> include/linux/mm_types.h:865:28: warning: declaration of 'struct vm_area_desc' will not be visible outside of this function [-Wvisibility]
     865 |         void (*abort_hook)(struct vm_area_desc *desc);
         |                                   ^
   1 warning generated.


vim +865 include/linux/mm_types.h

   820	
   821	/*
   822	 * Describes an action an mmap_prepare hook can instruct to be taken to complete
   823	 * the mapping of a VMA. Specified in vm_area_desc.
   824	 */
   825	struct mmap_action {
   826		union {
   827			struct {
   828				unsigned long start;
   829				unsigned long start_pfn;
   830				unsigned long size;
   831				pgprot_t pgprot;
   832			} remap;
   833			struct {
   834				phys_addr_t start_phys_addr;
   835				unsigned long size;
   836			} simple_ioremap;
   837			struct {
   838				unsigned long start;
   839				struct page **pages;
   840				unsigned long nr_pages;
   841				pgoff_t pgoff;
   842			} map_kernel;
   843		};
   844		enum mmap_action_type type;
   845	
   846		/*
   847		 * If specified, this hook is invoked after the selected action has been
   848		 * successfully completed. Note that the VMA write lock still held.
   849		 *
   850		 * The absolute minimum ought to be done here.
   851		 *
   852		 * Returns 0 on success, or an error code.
   853		 */
   854		int (*success_hook)(const struct vm_area_struct *vma);
   855	
   856		/*
   857		 * If specified, this hook is invoked when an error occurred when
   858		 * attempting the selected action.
   859		 *
   860		 * The hook can return an error code in order to filter the error, but
   861		 * it is not valid to clear the error here.
   862		 */
   863		int (*error_hook)(int err);
   864		
 > 865		void (*abort_hook)(struct vm_area_desc *desc);
   866	
   867		/*
   868		 * This should be set in rare instances where the operation required
   869		 * that the rmap should not be able to access the VMA until
   870		 * completely set up.
   871		 */
   872		bool hide_from_rmap_until_complete :1;
   873	};
   874	

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki

Re: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path

[kernel test robot]

Hi Mingyu,

[This is a private test report for your RFC patch.]
kernel test robot noticed the following build errors:

[auto build test ERROR on akpm-mm/mm-everything]

url:    https://github.com/intel-lab-lkp/linux/commits/Mingyu-Wang/mm-hugetlb-fix-resv_map-memory-leak-in-__mmap_region-error-path/20260425-203812
base:   https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-everything
patch link:    https://lore.kernel.org/r/20260425070700.562229-1-25181214217%40stu.xidian.edu.cn
patch subject: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path
config: alpha-allnoconfig (https://download.01.org/0day-ci/archive/20260502/202605020656.9mdM8VGh-lkp@intel.com/config)
compiler: alpha-linux-gcc (GCC) 15.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260502/202605020656.9mdM8VGh-lkp@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202605020656.9mdM8VGh-lkp@intel.com/

All error/warnings (new ones prefixed by >>):

   In file included from include/linux/sched/signal.h:14,
                    from include/linux/ptrace.h:7,
                    from arch/alpha/kernel/asm-offsets.c:12:
>> include/linux/mm_types.h:865:35: warning: 'struct vm_area_desc' declared inside parameter list will not be visible outside of this definition or declaration
     865 |         void (*abort_hook)(struct vm_area_desc *desc);
         |                                   ^~~~~~~~~~~~
--
   In file included from include/linux/mmzone.h:22,
                    from include/linux/gfp.h:7,
                    from include/linux/xarray.h:16,
                    from include/linux/list_lru.h:14,
                    from include/linux/fs/super_types.h:7,
                    from include/linux/fs/super.h:5,
                    from include/linux/fs.h:5,
                    from include/linux/backing-dev.h:13,
                    from mm/vma_internal.h:12,
                    from mm/vma.c:7:
>> include/linux/mm_types.h:865:35: warning: 'struct vm_area_desc' declared inside parameter list will not be visible outside of this definition or declaration
     865 |         void (*abort_hook)(struct vm_area_desc *desc);
         |                                   ^~~~~~~~~~~~
   mm/vma.c: In function '__mmap_region':
>> mm/vma.c:2805:40: error: passing argument 1 of 'desc.action.abort_hook' from incompatible pointer type [-Wincompatible-pointer-types]
    2805 |                 desc.action.abort_hook(&desc);
         |                                        ^~~~~
         |                                        |
         |                                        struct vm_area_desc *
   mm/vma.c:2805:40: note: expected 'struct vm_area_desc *' but argument is of type 'struct vm_area_desc *'
--
   In file included from include/linux/sched/signal.h:14,
                    from include/linux/ptrace.h:7,
                    from arch/alpha/kernel/asm-offsets.c:12:
>> include/linux/mm_types.h:865:35: warning: 'struct vm_area_desc' declared inside parameter list will not be visible outside of this definition or declaration
     865 |         void (*abort_hook)(struct vm_area_desc *desc);
         |                                   ^~~~~~~~~~~~
   In file included from include/linux/sched/mm.h:8,
                    from kernel/sched/sched.h:15,
                    from kernel/sched/rq-offsets.c:5:
>> include/linux/mm_types.h:865:35: warning: 'struct vm_area_desc' declared inside parameter list will not be visible outside of this definition or declaration
     865 |         void (*abort_hook)(struct vm_area_desc *desc);
         |                                   ^~~~~~~~~~~~


vim +2805 mm/vma.c

  2731	
  2732	static unsigned long __mmap_region(struct file *file, unsigned long addr,
  2733			unsigned long len, vma_flags_t vma_flags,
  2734			unsigned long pgoff, struct list_head *uf)
  2735	{
  2736		struct mm_struct *mm = current->mm;
  2737		struct vm_area_struct *vma = NULL;
  2738		bool have_mmap_prepare = file && file->f_op->mmap_prepare;
  2739		VMA_ITERATOR(vmi, mm, addr);
  2740		MMAP_STATE(map, mm, &vmi, addr, len, pgoff, vma_flags, file);
  2741		struct vm_area_desc desc = {
  2742			.mm = mm,
  2743			.file = file,
  2744			.action = {
  2745				.type = MMAP_NOTHING, /* Default to no further action. */
  2746			},
  2747		};
  2748		bool allocated_new = false;
  2749		int error;
  2750	
  2751		map.check_ksm_early = can_set_ksm_flags_early(&map);
  2752	
  2753		error = __mmap_setup(&map, &desc, uf);
  2754		if (!error && have_mmap_prepare)
  2755			error = call_mmap_prepare(&map, &desc);
  2756		if (error)
  2757			goto abort_munmap;
  2758	
  2759		if (map.check_ksm_early)
  2760			update_ksm_flags(&map);
  2761	
  2762		/* Attempt to merge with adjacent VMAs... */
  2763		if (map.prev || map.next) {
  2764			VMG_MMAP_STATE(vmg, &map, /* vma = */ NULL);
  2765	
  2766			vma = vma_merge_new_range(&vmg);
  2767		}
  2768	
  2769		/* ...but if we can't, allocate a new VMA. */
  2770		if (!vma) {
  2771			error = __mmap_new_vma(&map, &vma, &desc.action);
  2772			if (error)
  2773				goto unacct_error;
  2774			allocated_new = true;
  2775		}
  2776	
  2777		if (have_mmap_prepare)
  2778			set_vma_user_defined_fields(vma, &map);
  2779	
  2780		__mmap_complete(&map, vma);
  2781	
  2782		if (have_mmap_prepare && allocated_new) {
  2783			error = mmap_action_complete(vma, &desc.action,
  2784						     /*is_compat=*/false);
  2785			if (error)
  2786				return error;
  2787		}
  2788	
  2789		return addr;
  2790	
  2791		/* Accounting was done by __mmap_setup(). */
  2792	unacct_error:
  2793		if (map.charged)
  2794			vm_unacct_memory(map.charged);
  2795	abort_munmap:
  2796		/*
  2797		 * This indicates that .mmap_prepare has set a new file, differing from
  2798		 * desc->vm_file. But since we're aborting the operation, only the
  2799		 * original file will be cleaned up. Ensure we clean up both.
  2800		 */
  2801		if (map.file_doesnt_need_get)
  2802			fput(map.file);
  2803		
  2804		if (have_mmap_prepare && desc.action.abort_hook)
> 2805			desc.action.abort_hook(&desc);
  2806		
  2807		vms_abort_munmap_vmas(&map.vms, &map.mas_detach);
  2808		return error;
  2809	}
  2810	

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki

Re: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path

[kernel test robot]

Hi Mingyu,

[This is a private test report for your RFC patch.]
kernel test robot noticed the following build errors:

[auto build test ERROR on akpm-mm/mm-everything]

url:    https://github.com/intel-lab-lkp/linux/commits/Mingyu-Wang/mm-hugetlb-fix-resv_map-memory-leak-in-__mmap_region-error-path/20260425-203812
base:   https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-everything
patch link:    https://lore.kernel.org/r/20260425070700.562229-1-25181214217%40stu.xidian.edu.cn
patch subject: [RFC PATCH] mm/hugetlb: fix resv_map memory leak in __mmap_region error path
config: hexagon-allnoconfig (https://download.01.org/0day-ci/archive/20260502/202605020645.8318Xwwx-lkp@intel.com/config)
compiler: clang version 23.0.0git (https://github.com/llvm/llvm-project 5bac06718f502014fade905512f1d26d578a18f3)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260502/202605020645.8318Xwwx-lkp@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202605020645.8318Xwwx-lkp@intel.com/

All errors (new ones prefixed by >>):

   In file included from mm/vma.c:7:
   In file included from mm/vma_internal.h:12:
   In file included from include/linux/backing-dev.h:13:
   In file included from include/linux/fs.h:5:
   In file included from include/linux/fs/super.h:5:
   In file included from include/linux/fs/super_types.h:7:
   In file included from include/linux/list_lru.h:14:
   In file included from include/linux/xarray.h:16:
   In file included from include/linux/gfp.h:7:
   In file included from include/linux/mmzone.h:22:
   include/linux/mm_types.h:865:28: warning: declaration of 'struct vm_area_desc' will not be visible outside of this function [-Wvisibility]
     865 |         void (*abort_hook)(struct vm_area_desc *desc);
         |                                   ^
>> mm/vma.c:2805:26: error: incompatible pointer types passing 'struct vm_area_desc *' to parameter of type 'struct vm_area_desc *' [-Wincompatible-pointer-types]
    2805 |                 desc.action.abort_hook(&desc);
         |                                        ^~~~~
   1 warning and 1 error generated.


vim +2805 mm/vma.c

  2731	
  2732	static unsigned long __mmap_region(struct file *file, unsigned long addr,
  2733			unsigned long len, vma_flags_t vma_flags,
  2734			unsigned long pgoff, struct list_head *uf)
  2735	{
  2736		struct mm_struct *mm = current->mm;
  2737		struct vm_area_struct *vma = NULL;
  2738		bool have_mmap_prepare = file && file->f_op->mmap_prepare;
  2739		VMA_ITERATOR(vmi, mm, addr);
  2740		MMAP_STATE(map, mm, &vmi, addr, len, pgoff, vma_flags, file);
  2741		struct vm_area_desc desc = {
  2742			.mm = mm,
  2743			.file = file,
  2744			.action = {
  2745				.type = MMAP_NOTHING, /* Default to no further action. */
  2746			},
  2747		};
  2748		bool allocated_new = false;
  2749		int error;
  2750	
  2751		map.check_ksm_early = can_set_ksm_flags_early(&map);
  2752	
  2753		error = __mmap_setup(&map, &desc, uf);
  2754		if (!error && have_mmap_prepare)
  2755			error = call_mmap_prepare(&map, &desc);
  2756		if (error)
  2757			goto abort_munmap;
  2758	
  2759		if (map.check_ksm_early)
  2760			update_ksm_flags(&map);
  2761	
  2762		/* Attempt to merge with adjacent VMAs... */
  2763		if (map.prev || map.next) {
  2764			VMG_MMAP_STATE(vmg, &map, /* vma = */ NULL);
  2765	
  2766			vma = vma_merge_new_range(&vmg);
  2767		}
  2768	
  2769		/* ...but if we can't, allocate a new VMA. */
  2770		if (!vma) {
  2771			error = __mmap_new_vma(&map, &vma, &desc.action);
  2772			if (error)
  2773				goto unacct_error;
  2774			allocated_new = true;
  2775		}
  2776	
  2777		if (have_mmap_prepare)
  2778			set_vma_user_defined_fields(vma, &map);
  2779	
  2780		__mmap_complete(&map, vma);
  2781	
  2782		if (have_mmap_prepare && allocated_new) {
  2783			error = mmap_action_complete(vma, &desc.action,
  2784						     /*is_compat=*/false);
  2785			if (error)
  2786				return error;
  2787		}
  2788	
  2789		return addr;
  2790	
  2791		/* Accounting was done by __mmap_setup(). */
  2792	unacct_error:
  2793		if (map.charged)
  2794			vm_unacct_memory(map.charged);
  2795	abort_munmap:
  2796		/*
  2797		 * This indicates that .mmap_prepare has set a new file, differing from
  2798		 * desc->vm_file. But since we're aborting the operation, only the
  2799		 * original file will be cleaned up. Ensure we clean up both.
  2800		 */
  2801		if (map.file_doesnt_need_get)
  2802			fput(map.file);
  2803		
  2804		if (have_mmap_prepare && desc.action.abort_hook)
> 2805			desc.action.abort_hook(&desc);
  2806		
  2807		vms_abort_munmap_vmas(&map.vms, &map.mas_detach);
  2808		return error;
  2809	}
  2810	

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki