[PATCH 0/6] mm/mremap: report a failed split before moving anything

[PATCH 0/6] mm/mremap: report a failed split before moving anything

[Josef Bacik]

Hello,

I went through the callers of __split_vma() to see who can't cope with
it failing, and mremap() is the bad one.  If you move part of a VMA we
don't split it until the very end, when we unmap the old range.  By then
the page tables have moved and there's nothing to go back to, so we fix
up the accounting and return success with the old range still mapped.

Lorenzo recently added a way to split ahead of the move for the mlock()
+ MREMAP_DONTUNMAP case.  This series uses that for every move that
unmaps its source, so a split that fails is reported before we've done
anything.

Doing that turned up a real bug in hugetlb.  We take the reservation map
away from the whole old VMA before the part that's moving is split off,
so the parts that stay never give their reservations back.  That fix is
the first patch so it can go to stable by itself, and the second patch
is a selftest for it.

The next two fix the accounting for when the unmap at the end does fail,
we either charged memory that was never accounted or lost track of the
charge for the whole VMA.  That gets a lot harder to hit with the last
patch but it's still possible.  There's also a test for an old size of
0, which nothing covered.

I tested this with the mm selftests and with a loop that fails each
allocation inside mremap() in turn and checks that we either moved or
didn't.  The numbers for moving part of a mapping don't change.  Thanks,

Josef

---
Josef Bacik (6):
      mm/mremap: fix hugetlb reservation leak on partial move
      selftests/mm: test a partial mremap() of hugetlb memory
      mm/mremap: reaccount on unmap failure only if accounted
      mm/mremap: restore VM_ACCOUNT if the unmap fails
      selftests/mm: test mremap() with an old size of 0
      mm/mremap: split first so a failed split is reported

 mm/mremap.c                                 | 55 +++++++++++++++++++++++------
 tools/testing/selftests/mm/hugetlb-mremap.c | 52 +++++++++++++++++++++++++--
 tools/testing/selftests/mm/mremap_test.c    | 52 ++++++++++++++++++++++++++-
 3 files changed, 145 insertions(+), 14 deletions(-)
---
base-commit: cd1d0b147afa81f80927f9711e53401703964217
change-id: 20261030-b4-mremap-pre-split-13a3d54d6dbd

[PATCH 1/6] mm/mremap: fix hugetlb reservation leak on partial move

[Josef Bacik]

Moving a part of a private hugetlb mapping with mremap() leaks the huge
page reservations of the parts that stay. They are not returned when the
mapping is unmapped, when the process exits or when the pool is resized.

fixup_hugetlb_reservations() takes the reservation map away from the old
VMA while that VMA still spans its whole range, because the moved part is
only split off by the unmap at the end of the move. hugetlb_vm_op_close()
then finds no map in the parts that remain and returns nothing for them:

    mmap() of 4 pages, MAP_PRIVATE | MAP_HUGETLB    HugePages_Rsvd: 4
    mremap() of the second page to a new address    HugePages_Rsvd: 4
    munmap() of all of it                           HugePages_Rsvd: 3

The source is unmapped in this case, so the number of VMAs ends up where it
did before and the check against sysctl_max_map_count stays as it is.

Fix by splitting a hugetlb VMA in prep_move_vma() before a part of it is
moved, as commit 597e4b9ab48a ("mm/mremap: fix locked_vm leak by splitting
VMA for MREMAP_DONTUNMAP") does for an mlock()'d VMA. A backport needs that
commit, which is tagged for stable itself.

Fixes: 550a7d60bd5e ("mm, hugepages: add mremap() support for hugepage backed vma")
Cc: <stable@vger.kernel.org>
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
---
 mm/mremap.c | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/mm/mremap.c b/mm/mremap.c
index 0d06d7f37f19..cf746bafdee4 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -1038,7 +1038,7 @@ static void vrm_stat_account(struct vma_remap_struct *vrm,
 }
 
 static bool __check_map_count_against_split(struct mm_struct *mm,
-					    bool pre_split,
+					    bool source_stays,
 					    bool before_unmaps)
 {
 	const int sys_map_count = get_sysctl_max_map_count();
@@ -1092,14 +1092,15 @@ static bool __check_map_count_against_split(struct mm_struct *mm,
 	 */
 	map_count += 2;
 
-	/* If pre-split, the -1 observed above doesn't apply. */
-	if (pre_split)
+	/* If the split off source stays, the -1 observed above doesn't apply. */
+	if (source_stays)
 		map_count++;
 
 	return map_count <= sys_map_count;
 }
 
-static bool needs_pre_split(struct vma_remap_struct *vrm)
+/* Is the source range split off its VMA, to then stay mapped? */
+static bool split_source_stays(struct vma_remap_struct *vrm)
 {
 	/*
 	 * An MREMAP_DONTUNMAP of a mlock()'d VMA needs to unlock the
@@ -1109,11 +1110,23 @@ static bool needs_pre_split(struct vma_remap_struct *vrm)
 		vma_test(vrm->vma, VMA_LOCKED_BIT);
 }
 
+static bool needs_pre_split(struct vma_remap_struct *vrm)
+{
+	/*
+	 * fixup_hugetlb_reservations() takes the reservations away from the
+	 * source VMA.  The parts of it that do not move have to keep theirs.
+	 */
+	if (vma_is_hugetlb(vrm->vma))
+		return true;
+
+	return split_source_stays(vrm);
+}
+
 /* Do we violate the map count limit if we split VMAs when moving the VMA? */
 static bool check_map_count_against_split(struct vma_remap_struct *vrm)
 {
 	return __check_map_count_against_split(current->mm,
-		needs_pre_split(vrm), /*before_unmaps=*/false);
+		split_source_stays(vrm), /*before_unmaps=*/false);
 }
 
 /* Do we violate the map count limit if we split VMAs prior to early unmaps? */
@@ -1162,10 +1175,7 @@ static unsigned long prep_move_vma(struct vma_remap_struct *vrm)
 	if (err)
 		return err;
 
-	/*
-	 * To account mlock()'d pages correctly in the MREMAP_DONTUNMAP
-	 * case perform any split ahead of time for an mlock()'d VMA.
-	 */
+	/* Perform any split ahead of time if the source VMA has to be distinct. */
 	if (needs_pre_split(vrm)) {
 		VMA_ITERATOR(vmi, vma->vm_mm, old_addr);
 

-- 
2.55.0

Re: [PATCH 0/6] mm/mremap: report a failed split before moving anything

[Lorenzo Stoakes (ARM)]

On Fri, Oct 30, 2026 at 02:42:45PM +0000, Josef Bacik wrote:
> Hello,
>
> I went through the callers of __split_vma() to see who can't cope with
> it failing, and mremap() is the bad one.  If you move part of a VMA we

Yeah, what's frustrating there is that, AFAICT, the only really relevant
case is DAX.

But maybe hugetlb somehow raises its ugly head as you allude to.

> don't split it until the very end, when we unmap the old range.  By then
> the page tables have moved and there's nothing to go back to, so we fix
> up the accounting and return success with the old range still mapped.
>
> Lorenzo recently added a way to split ahead of the move for the mlock()
> + MREMAP_DONTUNMAP case.  This series uses that for every move that

MREMAP_DONTUNMAP is another real horror show for bugs. Candidate for
'should have never been merged'...

> unmaps its source, so a split that fails is reported before we've done
> anything.
>
> Doing that turned up a real bug in hugetlb.  We take the reservation map
> away from the whole old VMA before the part that's moving is split off,
> so the parts that stay never give their reservations back.  That fix is
> the first patch so it can go to stable by itself, and the second patch
> is a selftest for it.
>
> The next two fix the accounting for when the unmap at the end does fail,
> we either charged memory that was never accounted or lost track of the
> charge for the whole VMA.  That gets a lot harder to hit with the last
> patch but it's still possible.  There's also a test for an old size of
> 0, which nothing covered.

I feel like the charging logic needs to be reworked anyway. This whole
'drop VMA_ACCOUNT_BIT' hack is extremely stupid.

>
> I tested this with the mm selftests and with a loop that fails each
> allocation inside mremap() in turn and checks that we either moved or
> didn't.  The numbers for moving part of a mapping don't change.  Thanks,
>
> Josef

Anyway, thanks for these, will go through.

>
> ---
> Josef Bacik (6):
>       mm/mremap: fix hugetlb reservation leak on partial move
>       selftests/mm: test a partial mremap() of hugetlb memory
>       mm/mremap: reaccount on unmap failure only if accounted
>       mm/mremap: restore VM_ACCOUNT if the unmap fails
>       selftests/mm: test mremap() with an old size of 0
>       mm/mremap: split first so a failed split is reported
>
>  mm/mremap.c                                 | 55 +++++++++++++++++++++++------
>  tools/testing/selftests/mm/hugetlb-mremap.c | 52 +++++++++++++++++++++++++--
>  tools/testing/selftests/mm/mremap_test.c    | 52 ++++++++++++++++++++++++++-
>  3 files changed, 145 insertions(+), 14 deletions(-)
> ---
> base-commit: cd1d0b147afa81f80927f9711e53401703964217
> change-id: 20261030-b4-mremap-pre-split-13a3d54d6dbd
>

--
Cheers, Lorenzo

Re: [PATCH 1/6] mm/mremap: fix hugetlb reservation leak on partial move

[Lorenzo Stoakes (ARM)]

On Fri, Oct 30, 2026 at 02:42:46PM +0000, Josef Bacik wrote:
> Moving a part of a private hugetlb mapping with mremap() leaks the huge
> page reservations of the parts that stay. They are not returned when the
> mapping is unmapped, when the process exits or when the pool is resized.
>
> fixup_hugetlb_reservations() takes the reservation map away from the old
> VMA while that VMA still spans its whole range, because the moved part is
> only split off by the unmap at the end of the move. hugetlb_vm_op_close()
> then finds no map in the parts that remain and returns nothing for them:
>
>     mmap() of 4 pages, MAP_PRIVATE | MAP_HUGETLB    HugePages_Rsvd: 4
>     mremap() of the second page to a new address    HugePages_Rsvd: 4
>     munmap() of all of it                           HugePages_Rsvd: 3
>
> The source is unmapped in this case, so the number of VMAs ends up where it
> did before and the check against sysctl_max_map_count stays as it is.
>
> Fix by splitting a hugetlb VMA in prep_move_vma() before a part of it is
> moved, as commit 597e4b9ab48a ("mm/mremap: fix locked_vm leak by splitting
> VMA for MREMAP_DONTUNMAP") does for an mlock()'d VMA. A backport needs that
> commit, which is tagged for stable itself.

Ack, that one goes back to 5.7 in theory, this one 5.16 so all good.

>
> Fixes: 550a7d60bd5e ("mm, hugepages: add mremap() support for hugepage backed vma")
> Cc: <stable@vger.kernel.org>
> Signed-off-by: Josef Bacik <josef@toxicpanda.com>

Some nits below but LGTM, so:

Reviewed-by: Lorenzo Stoakes (ARM) <ljs@kernel.org>

> ---
>  mm/mremap.c | 28 +++++++++++++++++++---------
>  1 file changed, 19 insertions(+), 9 deletions(-)
>
> diff --git a/mm/mremap.c b/mm/mremap.c
> index 0d06d7f37f19..cf746bafdee4 100644
> --- a/mm/mremap.c
> +++ b/mm/mremap.c
> @@ -1038,7 +1038,7 @@ static void vrm_stat_account(struct vma_remap_struct *vrm,
>  }
>
>  static bool __check_map_count_against_split(struct mm_struct *mm,
> -					    bool pre_split,
> +					    bool source_stays,
>  					    bool before_unmaps)
>  {
>  	const int sys_map_count = get_sysctl_max_map_count();
> @@ -1092,14 +1092,15 @@ static bool __check_map_count_against_split(struct mm_struct *mm,
>  	 */
>  	map_count += 2;
>
> -	/* If pre-split, the -1 observed above doesn't apply. */
> -	if (pre_split)
> +	/* If the split off source stays, the -1 observed above doesn't apply. */
> +	if (source_stays)
>  		map_count++;
>
>  	return map_count <= sys_map_count;
>  }
>
> -static bool needs_pre_split(struct vma_remap_struct *vrm)
> +/* Is the source range split off its VMA, to then stay mapped? */

NIT, but slightly awkward wording. Honestly I think we can just do without a
comment here, the name self-documents, and the hugetlb 'stuff' is documented in
the comment below.

> +static bool split_source_stays(struct vma_remap_struct *vrm)
>  {
>  	/*
>  	 * An MREMAP_DONTUNMAP of a mlock()'d VMA needs to unlock the
> @@ -1109,11 +1110,23 @@ static bool needs_pre_split(struct vma_remap_struct *vrm)
>  		vma_test(vrm->vma, VMA_LOCKED_BIT);
>  }
>
> +static bool needs_pre_split(struct vma_remap_struct *vrm)
> +{
> +	/*
> +	 * fixup_hugetlb_reservations() takes the reservations away from the
> +	 * source VMA.  The parts of it that do not move have to keep theirs.
> +	 */

If only we could swear in comments... :)

> +	if (vma_is_hugetlb(vrm->vma))
> +		return true;
> +
> +	return split_source_stays(vrm);
> +}
> +
>  /* Do we violate the map count limit if we split VMAs when moving the VMA? */
>  static bool check_map_count_against_split(struct vma_remap_struct *vrm)
>  {
>  	return __check_map_count_against_split(current->mm,
> -		needs_pre_split(vrm), /*before_unmaps=*/false);
> +		split_source_stays(vrm), /*before_unmaps=*/false);
>  }
>
>  /* Do we violate the map count limit if we split VMAs prior to early unmaps? */
> @@ -1162,10 +1175,7 @@ static unsigned long prep_move_vma(struct vma_remap_struct *vrm)
>  	if (err)
>  		return err;
>
> -	/*
> -	 * To account mlock()'d pages correctly in the MREMAP_DONTUNMAP
> -	 * case perform any split ahead of time for an mlock()'d VMA.
> -	 */
> +	/* Perform any split ahead of time if the source VMA has to be distinct. */

Nice reduction in comment size too :)

>  	if (needs_pre_split(vrm)) {
>  		VMA_ITERATOR(vmi, vma->vm_mm, old_addr);
>
>
> --
> 2.55.0
>

--
Cheers, Lorenzo