* [PATCH] selftests/mm: Fix incorrect mmap() error handling with NULL instead of MAP_FAILED
@ 2026-05-13 2:52 Hongfu Li
2026-05-13 4:27 ` Dev Jain
2026-05-13 9:25 ` Lorenzo Stoakes
0 siblings, 2 replies; 4+ messages in thread
From: Hongfu Li @ 2026-05-13 2:52 UTC (permalink / raw)
To: akpm, david, ljs, liam, vbabka, rppt, surenb, mhocko, shuah
Cc: linux-mm, linux-kselftest, linux-kernel, Hongfu Li
mmap() returns MAP_FAILED, which is defined as (void *)-1, on error,
not NULL. Several selftests incorrectly check the return value of
mmap() using !ptr or ptr == NULL, which would erroneously treat
MAP_FAILED as a valid pointer since MAP_FAILED is non-zero and
non-NULL. This can lead to segfaults when mmap() actually fails
under memory pressure.
Signed-off-by: Hongfu Li <lihongfu@kylinos.cn>
---
tools/testing/selftests/mm/ksm_tests.c | 2 +-
tools/testing/selftests/mm/madv_populate.c | 2 +-
tools/testing/selftests/mm/soft-dirty.c | 4 ++--
tools/testing/selftests/mm/vm_util.c | 2 +-
4 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c
index a0b48b839d54..ed481f817282 100644
--- a/tools/testing/selftests/mm/ksm_tests.c
+++ b/tools/testing/selftests/mm/ksm_tests.c
@@ -174,7 +174,7 @@ static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_
{
void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0);
- if (!map_ptr) {
+ if (map_ptr == MAP_FAILED) {
perror("mmap");
return NULL;
}
diff --git a/tools/testing/selftests/mm/madv_populate.c b/tools/testing/selftests/mm/madv_populate.c
index 88050e0f829a..7fce5d0b622b 100644
--- a/tools/testing/selftests/mm/madv_populate.c
+++ b/tools/testing/selftests/mm/madv_populate.c
@@ -34,7 +34,7 @@ static void sense_support(void)
addr = mmap(0, pagesize, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, 0, 0);
- if (!addr)
+ if (addr == MAP_FAILED)
ksft_exit_fail_msg("mmap failed\n");
ret = madvise(addr, pagesize, MADV_POPULATE_READ);
diff --git a/tools/testing/selftests/mm/soft-dirty.c b/tools/testing/selftests/mm/soft-dirty.c
index bcfcac99b436..67c26c265880 100644
--- a/tools/testing/selftests/mm/soft-dirty.c
+++ b/tools/testing/selftests/mm/soft-dirty.c
@@ -143,7 +143,7 @@ static void test_mprotect(int pagemap_fd, int pagesize, bool anon)
if (anon) {
map = mmap(NULL, pagesize, PROT_READ|PROT_WRITE,
MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
- if (!map)
+ if (map == MAP_FAILED)
ksft_exit_fail_msg("anon mmap failed\n");
} else {
test_fd = open(fname, O_RDWR | O_CREAT, 0664);
@@ -155,7 +155,7 @@ static void test_mprotect(int pagemap_fd, int pagesize, bool anon)
ftruncate(test_fd, pagesize);
map = mmap(NULL, pagesize, PROT_READ|PROT_WRITE,
MAP_SHARED, test_fd, 0);
- if (!map)
+ if (map == MAP_FAILED)
ksft_exit_fail_msg("file mmap failed\n");
}
diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c
index db94564f4431..63aaa2d9ec0b 100644
--- a/tools/testing/selftests/mm/vm_util.c
+++ b/tools/testing/selftests/mm/vm_util.c
@@ -463,7 +463,7 @@ bool softdirty_supported(void)
/* New mappings are expected to be marked with VM_SOFTDIRTY (sd). */
addr = mmap(0, pagesize, PROT_READ | PROT_WRITE,
MAP_ANONYMOUS | MAP_PRIVATE, 0, 0);
- if (!addr)
+ if (addr == MAP_FAILED)
ksft_exit_fail_msg("mmap failed\n");
supported = check_vmflag(addr, "sd");
--
2.25.1
^ permalink raw reply related [flat|nested] 4+ messages in thread
* Re: [PATCH] selftests/mm: Fix incorrect mmap() error handling with NULL instead of MAP_FAILED
2026-05-13 2:52 [PATCH] selftests/mm: Fix incorrect mmap() error handling with NULL instead of MAP_FAILED Hongfu Li
@ 2026-05-13 4:27 ` Dev Jain
2026-05-13 9:09 ` Hongfu Li
2026-05-13 9:25 ` Lorenzo Stoakes
1 sibling, 1 reply; 4+ messages in thread
From: Dev Jain @ 2026-05-13 4:27 UTC (permalink / raw)
To: Hongfu Li, akpm, david, ljs, liam, vbabka, rppt, surenb, mhocko,
shuah
Cc: linux-mm, linux-kselftest, linux-kernel
On 13/05/26 8:22 am, Hongfu Li wrote:
> mmap() returns MAP_FAILED, which is defined as (void *)-1, on error,
> not NULL. Several selftests incorrectly check the return value of
> mmap() using !ptr or ptr == NULL, which would erroneously treat
> MAP_FAILED as a valid pointer since MAP_FAILED is non-zero and
> non-NULL. This can lead to segfaults when mmap() actually fails
> under memory pressure.
>
> Signed-off-by: Hongfu Li <lihongfu@kylinos.cn>
Hopefully you have fixed all instances in selftests/mm :)
Reviewed-by: Dev Jain <dev.jain@arm.com>
> ---
> tools/testing/selftests/mm/ksm_tests.c | 2 +-
> tools/testing/selftests/mm/madv_populate.c | 2 +-
> tools/testing/selftests/mm/soft-dirty.c | 4 ++--
> tools/testing/selftests/mm/vm_util.c | 2 +-
> 4 files changed, 5 insertions(+), 5 deletions(-)
>
> diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c
> index a0b48b839d54..ed481f817282 100644
> --- a/tools/testing/selftests/mm/ksm_tests.c
> +++ b/tools/testing/selftests/mm/ksm_tests.c
> @@ -174,7 +174,7 @@ static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_
> {
> void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0);
>
> - if (!map_ptr) {
> + if (map_ptr == MAP_FAILED) {
> perror("mmap");
> return NULL;
> }
> diff --git a/tools/testing/selftests/mm/madv_populate.c b/tools/testing/selftests/mm/madv_populate.c
> index 88050e0f829a..7fce5d0b622b 100644
> --- a/tools/testing/selftests/mm/madv_populate.c
> +++ b/tools/testing/selftests/mm/madv_populate.c
> @@ -34,7 +34,7 @@ static void sense_support(void)
>
> addr = mmap(0, pagesize, PROT_READ | PROT_WRITE,
> MAP_ANONYMOUS | MAP_PRIVATE, 0, 0);
> - if (!addr)
> + if (addr == MAP_FAILED)
> ksft_exit_fail_msg("mmap failed\n");
>
> ret = madvise(addr, pagesize, MADV_POPULATE_READ);
> diff --git a/tools/testing/selftests/mm/soft-dirty.c b/tools/testing/selftests/mm/soft-dirty.c
> index bcfcac99b436..67c26c265880 100644
> --- a/tools/testing/selftests/mm/soft-dirty.c
> +++ b/tools/testing/selftests/mm/soft-dirty.c
> @@ -143,7 +143,7 @@ static void test_mprotect(int pagemap_fd, int pagesize, bool anon)
> if (anon) {
> map = mmap(NULL, pagesize, PROT_READ|PROT_WRITE,
> MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
> - if (!map)
> + if (map == MAP_FAILED)
> ksft_exit_fail_msg("anon mmap failed\n");
> } else {
> test_fd = open(fname, O_RDWR | O_CREAT, 0664);
> @@ -155,7 +155,7 @@ static void test_mprotect(int pagemap_fd, int pagesize, bool anon)
> ftruncate(test_fd, pagesize);
> map = mmap(NULL, pagesize, PROT_READ|PROT_WRITE,
> MAP_SHARED, test_fd, 0);
> - if (!map)
> + if (map == MAP_FAILED)
> ksft_exit_fail_msg("file mmap failed\n");
> }
>
> diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c
> index db94564f4431..63aaa2d9ec0b 100644
> --- a/tools/testing/selftests/mm/vm_util.c
> +++ b/tools/testing/selftests/mm/vm_util.c
> @@ -463,7 +463,7 @@ bool softdirty_supported(void)
> /* New mappings are expected to be marked with VM_SOFTDIRTY (sd). */
> addr = mmap(0, pagesize, PROT_READ | PROT_WRITE,
> MAP_ANONYMOUS | MAP_PRIVATE, 0, 0);
> - if (!addr)
> + if (addr == MAP_FAILED)
> ksft_exit_fail_msg("mmap failed\n");
>
> supported = check_vmflag(addr, "sd");
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [PATCH] selftests/mm: Fix incorrect mmap() error handling with NULL instead of MAP_FAILED
2026-05-13 4:27 ` Dev Jain
@ 2026-05-13 9:09 ` Hongfu Li
0 siblings, 0 replies; 4+ messages in thread
From: Hongfu Li @ 2026-05-13 9:09 UTC (permalink / raw)
To: dev.jain
Cc: akpm, david, liam, lihongfu, linux-kernel, linux-kselftest,
linux-mm, ljs, mhocko, rppt, shuah, surenb, vbabka
> > mmap() returns MAP_FAILED, which is defined as (void *)-1, on error,
> > not NULL. Several selftests incorrectly check the return value of
> > mmap() using !ptr or ptr == NULL, which would erroneously treat
> > MAP_FAILED as a valid pointer since MAP_FAILED is non-zero and
> > non-NULL. This can lead to segfaults when mmap() actually fails
> > under memory pressure.
> >
> > Signed-off-by: Hongfu Li <lihongfu@kylinos.cn>
>
> Hopefully you have fixed all instances in selftests/mm :)
>
> Reviewed-by: Dev Jain <dev.jain@arm.com>
Thanks a lot for the reminder.
I missed some cases in selftests/mm, will fix all of them and post
v2 shortly.
Thanks again for your review.
Best regards,
Hongfu
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [PATCH] selftests/mm: Fix incorrect mmap() error handling with NULL instead of MAP_FAILED
2026-05-13 2:52 [PATCH] selftests/mm: Fix incorrect mmap() error handling with NULL instead of MAP_FAILED Hongfu Li
2026-05-13 4:27 ` Dev Jain
@ 2026-05-13 9:25 ` Lorenzo Stoakes
1 sibling, 0 replies; 4+ messages in thread
From: Lorenzo Stoakes @ 2026-05-13 9:25 UTC (permalink / raw)
To: Hongfu Li
Cc: akpm, david, liam, vbabka, rppt, surenb, mhocko, shuah, linux-mm,
linux-kselftest, linux-kernel
On Wed, May 13, 2026 at 10:52:23AM +0800, Hongfu Li wrote:
> mmap() returns MAP_FAILED, which is defined as (void *)-1, on error,
> not NULL. Several selftests incorrectly check the return value of
> mmap() using !ptr or ptr == NULL, which would erroneously treat
> MAP_FAILED as a valid pointer since MAP_FAILED is non-zero and
> non-NULL. This can lead to segfaults when mmap() actually fails
> under memory pressure.
>
> Signed-off-by: Hongfu Li <lihongfu@kylinos.cn>
Ugh how did we end up with all of these :) thanks for the fix, assuming you get
all of the other cases on respin, feel free to attach my tag to that:
Reviewed-by: Lorenzo Stoakes <ljs@kernel.org>
Cheers, Lorenzo
> ---
> tools/testing/selftests/mm/ksm_tests.c | 2 +-
> tools/testing/selftests/mm/madv_populate.c | 2 +-
> tools/testing/selftests/mm/soft-dirty.c | 4 ++--
> tools/testing/selftests/mm/vm_util.c | 2 +-
> 4 files changed, 5 insertions(+), 5 deletions(-)
>
> diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c
> index a0b48b839d54..ed481f817282 100644
> --- a/tools/testing/selftests/mm/ksm_tests.c
> +++ b/tools/testing/selftests/mm/ksm_tests.c
> @@ -174,7 +174,7 @@ static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_
> {
> void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0);
>
> - if (!map_ptr) {
> + if (map_ptr == MAP_FAILED) {
> perror("mmap");
> return NULL;
> }
> diff --git a/tools/testing/selftests/mm/madv_populate.c b/tools/testing/selftests/mm/madv_populate.c
> index 88050e0f829a..7fce5d0b622b 100644
> --- a/tools/testing/selftests/mm/madv_populate.c
> +++ b/tools/testing/selftests/mm/madv_populate.c
> @@ -34,7 +34,7 @@ static void sense_support(void)
>
> addr = mmap(0, pagesize, PROT_READ | PROT_WRITE,
> MAP_ANONYMOUS | MAP_PRIVATE, 0, 0);
> - if (!addr)
> + if (addr == MAP_FAILED)
> ksft_exit_fail_msg("mmap failed\n");
>
> ret = madvise(addr, pagesize, MADV_POPULATE_READ);
> diff --git a/tools/testing/selftests/mm/soft-dirty.c b/tools/testing/selftests/mm/soft-dirty.c
> index bcfcac99b436..67c26c265880 100644
> --- a/tools/testing/selftests/mm/soft-dirty.c
> +++ b/tools/testing/selftests/mm/soft-dirty.c
> @@ -143,7 +143,7 @@ static void test_mprotect(int pagemap_fd, int pagesize, bool anon)
> if (anon) {
> map = mmap(NULL, pagesize, PROT_READ|PROT_WRITE,
> MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
> - if (!map)
> + if (map == MAP_FAILED)
> ksft_exit_fail_msg("anon mmap failed\n");
> } else {
> test_fd = open(fname, O_RDWR | O_CREAT, 0664);
> @@ -155,7 +155,7 @@ static void test_mprotect(int pagemap_fd, int pagesize, bool anon)
> ftruncate(test_fd, pagesize);
> map = mmap(NULL, pagesize, PROT_READ|PROT_WRITE,
> MAP_SHARED, test_fd, 0);
> - if (!map)
> + if (map == MAP_FAILED)
> ksft_exit_fail_msg("file mmap failed\n");
> }
>
> diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c
> index db94564f4431..63aaa2d9ec0b 100644
> --- a/tools/testing/selftests/mm/vm_util.c
> +++ b/tools/testing/selftests/mm/vm_util.c
> @@ -463,7 +463,7 @@ bool softdirty_supported(void)
> /* New mappings are expected to be marked with VM_SOFTDIRTY (sd). */
> addr = mmap(0, pagesize, PROT_READ | PROT_WRITE,
> MAP_ANONYMOUS | MAP_PRIVATE, 0, 0);
> - if (!addr)
> + if (addr == MAP_FAILED)
> ksft_exit_fail_msg("mmap failed\n");
>
> supported = check_vmflag(addr, "sd");
> --
> 2.25.1
>
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2026-05-13 9:25 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-05-13 2:52 [PATCH] selftests/mm: Fix incorrect mmap() error handling with NULL instead of MAP_FAILED Hongfu Li
2026-05-13 4:27 ` Dev Jain
2026-05-13 9:09 ` Hongfu Li
2026-05-13 9:25 ` Lorenzo Stoakes
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox