[LTP] [PATCH] hugemmap37: migrated task-size-overrun.c from libhugetlbfs v4

[LTP] [PATCH] hugemmap37: migrated task-size-overrun.c from libhugetlbfs v4

[Pavithra]

This test verifies that the kernel correctly prevents mmap operations
from creating mappings that straddle the TASK_SIZE boundary when using
hugepages. Such mappings could lead to undefined behavior or security
issues if allowed.

Signed-off-by: Pavithra <pavrampu@linux.ibm.com>
---
 runtest/hugetlb                               |   1 +
 testcases/kernel/mem/.gitignore               |   1 +
 .../kernel/mem/hugetlb/hugemmap/hugemmap37.c  | 164 ++++++++++++++++++
 3 files changed, 166 insertions(+)
 create mode 100644 testcases/kernel/mem/hugetlb/hugemmap/hugemmap37.c

diff --git a/runtest/hugetlb b/runtest/hugetlb
index 0896d3c94..8aaafeee3 100644
--- a/runtest/hugetlb
+++ b/runtest/hugetlb
@@ -36,6 +36,7 @@ hugemmap30 hugemmap30
 hugemmap31 hugemmap31
 hugemmap32 hugemmap32
 hugemmap34 hugemmap34
+hugemmap37 hugemmap37
 hugemmap05_1 hugemmap05 -m
 hugemmap05_2 hugemmap05 -s
 hugemmap05_3 hugemmap05 -s -m
diff --git a/testcases/kernel/mem/.gitignore b/testcases/kernel/mem/.gitignore
index b4455de51..38d428fe8 100644
--- a/testcases/kernel/mem/.gitignore
+++ b/testcases/kernel/mem/.gitignore
@@ -36,6 +36,7 @@
 /hugetlb/hugemmap/hugemmap31
 /hugetlb/hugemmap/hugemmap32
 /hugetlb/hugemmap/hugemmap34
+/hugetlb/hugemmap/hugemmap37
 /hugetlb/hugeshmat/hugeshmat01
 /hugetlb/hugeshmat/hugeshmat02
 /hugetlb/hugeshmat/hugeshmat03
diff --git a/testcases/kernel/mem/hugetlb/hugemmap/hugemmap37.c b/testcases/kernel/mem/hugetlb/hugemmap/hugemmap37.c
new file mode 100644
index 000000000..9a9d45de5
--- /dev/null
+++ b/testcases/kernel/mem/hugetlb/hugemmap/hugemmap37.c
@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: LGPL-2.1-or-later
+/*
+ * Copyright (C) 2005-2006 David Gibson & Adam Litke, IBM Corporation.
+ * Copyright (c) 2026 Pavithra <pavrampu@linux.ibm.com>
+ */
+
+/*
+ * Origin: https://github.com/libhugetlbfs/libhugetlbfs/blob/master/tests/task-size-overrun.c
+ *
+ * This test verifies the behavior of mmap across the TASK_SIZE boundary.
+ * It checks whether mmap with and without MAP_FIXED correctly handles
+ * mappings that straddle the TASK_SIZE boundary.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <errno.h>
+
+#include "hugetlb.h"
+#include "tst_test.h"
+#include "tst_safe_stdio.h"
+#include "tst_safe_macros.h"
+
+#define MAPS_BUF_SZ 4096
+#define MNTPOINT "hugetlbfs/"
+
+static long hpage_size;
+static int fd = -1;
+
+static unsigned long find_last_mapped(void)
+{
+	char line[MAPS_BUF_SZ];
+	unsigned long start, end, off, ino;
+	FILE *f;
+	int found = 0;
+
+	f = SAFE_FOPEN("/proc/self/maps", "r");
+
+	/* Read all lines and keep the last non-special mapping */
+	while (fgets(line, MAPS_BUF_SZ, f)) {
+		/* Skip special mappings like [vsyscall], [vdso], [vvar] */
+		if (strstr(line, "[vsyscall]") || strstr(line, "[vdso]") ||
+		    strstr(line, "[vvar]"))
+			continue;
+
+		found = 1;
+	}
+
+	SAFE_FCLOSE(f);
+
+	if (!found)
+		tst_brk(TBROK, "Could not find any valid mapping in /proc/self/maps");
+
+	tst_res(TINFO, "Last map: %s", line);
+	if (sscanf(line, "%lx-%lx %*s %lx %*s %ld", &start, &end, &off, &ino) != 4)
+		tst_brk(TBROK, "Failed to parse /proc/self/maps line");
+
+	tst_res(TINFO, "Last map: at 0x%lx-0x%lx", start, end);
+	return end;
+}
+
+static unsigned long find_task_size(void)
+{
+	unsigned long low, high;
+	void *p;
+	int page_size = getpagesize();
+
+	low = find_last_mapped();
+	if (!low || ((low % page_size) != 0))
+		tst_brk(TBROK, "Bogus stack end address, 0x%lx!?", low);
+
+	/* Convert to page frame number */
+	low = low / page_size;
+
+	/*
+	 * Set high to maximum possible address space
+	 * For 64-bit: (2^64 - 1) / page_size
+	 * We use -1UL which gives us the maximum unsigned long value
+	 */
+	high = (-1UL) / page_size;
+
+	tst_res(TINFO, "Binary searching for task size PFNs 0x%lx..0x%lx", low, high);
+
+	while (high > low + 1) {
+		unsigned long pfn = (low + high) / 2;
+		unsigned long addr = pfn * page_size;
+
+		p = mmap((void *)addr, page_size, PROT_READ,
+			   MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0);
+		if (p == MAP_FAILED) {
+			tst_res(TINFO | TERRNO, "Map failed at 0x%lx", addr);
+			high = pfn;
+		} else {
+			tst_res(TINFO, "Map succeeded at 0x%lx", addr);
+			SAFE_MUNMAP(p, page_size);
+			low = pfn;
+		}
+	}
+
+	return low * page_size;
+}
+
+static void run_test(void)
+{
+	void *p;
+	unsigned long task_size;
+	unsigned long straddle_addr;
+
+	task_size = find_task_size();
+	tst_res(TINFO, "TASK_SIZE = 0x%lx", task_size);
+
+	straddle_addr = task_size - hpage_size;
+	straddle_addr = LTP_ALIGN(straddle_addr, hpage_size);
+
+	tst_res(TINFO, "Mapping without MAP_FIXED at %lx...", straddle_addr);
+	errno = 0;
+	p = mmap((void *)straddle_addr, 2*hpage_size, PROT_READ|PROT_WRITE,
+		 MAP_SHARED, fd, 0);
+	if (p == (void *)straddle_addr) {
+		tst_res(TFAIL, "Apparently succeeded in mapping across TASK_SIZE boundary");
+		SAFE_MUNMAP(p, 2*hpage_size);
+	} else if (p != MAP_FAILED) {
+		tst_res(TPASS, "mmap without MAP_FIXED correctly avoided TASK_SIZE boundary");
+		SAFE_MUNMAP(p, 2*hpage_size);
+	} else {
+		tst_res(TPASS, "mmap without MAP_FIXED correctly failed");
+	}
+
+	tst_res(TINFO, "Mapping with MAP_FIXED at %lx...", straddle_addr);
+	errno = 0;
+	p = mmap((void *)straddle_addr, 2*hpage_size, PROT_READ|PROT_WRITE,
+		 MAP_SHARED|MAP_FIXED, fd, 0);
+	if (p != MAP_FAILED) {
+		tst_res(TFAIL, "Apparently succeeded in mapping across TASK_SIZE boundary");
+		SAFE_MUNMAP(p, 2*hpage_size);
+	} else {
+		tst_res(TPASS, "mmap with MAP_FIXED correctly failed");
+	}
+}
+
+static void setup(void)
+{
+	hpage_size = tst_get_hugepage_size();
+	fd = tst_creat_unlinked(MNTPOINT, 0, 0600);
+}
+
+static void cleanup(void)
+{
+	if (fd != -1)
+		SAFE_CLOSE(fd);
+}
+
+static struct tst_test test = {
+	.needs_root = 1,
+	.mntpoint = MNTPOINT,
+	.needs_hugetlbfs = 1,
+	.setup = setup,
+	.cleanup = cleanup,
+	.test_all = run_test,
+	.hugepages = {3, TST_NEEDS},
+};
-- 
2.53.0


-- 
Mailing list info: https://lists.linux.it/listinfo/ltp

Re: [LTP] hugemmap37: migrated task-size-overrun.c from libhugetlbfs v4

[linuxtestproject.agent]

Hi Pavithra,

On Sat Apr 4 2026, Pavithra wrote:
> hugemmap37: migrated task-size-overrun.c from libhugetlbfs v4

> +static unsigned long find_last_mapped(void)
> +{
> +	char line[MAPS_BUF_SZ];
> +	...
> +	/* Read all lines and keep the last non-special mapping */
> +	while (fgets(line, MAPS_BUF_SZ, f)) {
> +		/* Skip special mappings like [vsyscall], [vdso], [vvar] */
> +		if (strstr(line, "[vsyscall]") || strstr(line, "[vdso]") ||
> +		    strstr(line, "[vvar]"))
> +			continue;
> +
> +		found = 1;
> +	}
> +	...
> +	tst_res(TINFO, "Last map: %s", line);
> +	if (sscanf(line, "%lx-%lx %*s %lx %*s %ld", &start, &end, &off, &ino) != 4)

The comment says "keep the last non-special mapping" but the code does
not do that.  fgets() overwrites `line` on every iteration, including
iterations that immediately hit the `continue`.  After the loop, `line`
holds the last line read overall, not the last non-special line.

On x86_64 with vsyscall support, [vsyscall] appears as the final entry
in /proc/self/maps at address 0xffffffffff600000.  In that case sscanf
parses that address and find_last_mapped() returns 0xffffffffff601000.
find_task_size() then starts its binary search with a low-PFN value
already above TASK_SIZE, all probe mmaps fail, and an incorrect
TASK_SIZE is returned -- making straddle_addr wrong and the test
results unreliable.

The fix is to copy `line` into a separate buffer when a non-special
line is encountered, and use that saved buffer after the loop.

> +// SPDX-License-Identifier: LGPL-2.1-or-later

Other tests in this directory that were migrated from the same
libhugetlbfs upstream (e.g. hugemmap34.c, which credits the same
original authors) carry GPL-2.0-or-later.  LTP relicenses migrated
code under GPL-2.0-or-later for consistency.

> +/*
> + * Origin: https://github.com/libhugetlbfs/...
> + *
> + * This test verifies the behavior of mmap across the TASK_SIZE boundary.

The high-level description block must open with /*\ (backslash after
the slash-star) so that the LTP documentation build recognises it as
an RST-formatted block and includes it in the test catalog.  A plain
/* comment is silently ignored by the doc toolchain.

Verdict - Needs revision

---
Note:

The agent can sometimes produce false positives although often its
findings are genuine. If you find issues with the review, please
comment this email or ignore the suggestions.

Regards,
LTP AI Reviewer

-- 
Mailing list info: https://lists.linux.it/listinfo/ltp