[PATCH v2 0/4] tools/nolibc: pass mode to open syscall if O_TMPFILE is set

[PATCH v2 0/4] tools/nolibc: pass mode to open syscall if O_TMPFILE is set

[Thomas Weißschuh]

When O_TMPFILE is set, the open mode needs to be passed to the kernel as
per the documentation.

---
Changes in v2:
- Always pass the mode to the kernel
- Rebase on latest nolibc/for-next
- Link to v1: https://patch.msgid.link/20260429-nolibc-open-tmpfile-v1-0-63d19c95c0a5@weissschuh.net

To: Willy Tarreau <w@1wt.eu>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Thomas Weißschuh <linux@weissschuh.net>

---
Thomas Weißschuh (4):
      tools/nolibc: split implicit open flags into a macro
      tools/nolibc: split open mode handling into a macro
      tools/nolibc: always pass mode to open syscall
      selftests/nolibc: test open mode handling

 tools/include/nolibc/fcntl.h                 | 43 +++++++++++-----------------
 tools/testing/selftests/nolibc/nolibc-test.c | 23 +++++++++++++++
 2 files changed, 40 insertions(+), 26 deletions(-)
---
base-commit: bb2d82d41894cb30d836e9796ff67d2f9a71eccf
change-id: 20260429-nolibc-open-tmpfile-19e677e33c8f

Best regards,
--  
Thomas Weißschuh <linux@weissschuh.net>

[PATCH v2 1/4] tools/nolibc: split implicit open flags into a macro

[Thomas Weißschuh]

This logic is duplicated and its current form will be in the way of some
upcoming simplificiations.

Move it into a macro to avoid the duplication and enable some cleanups.

Signed-off-by: Thomas Weißschuh <linux@weissschuh.net>
Acked-by: Willy Tarreau <w@1wt.eu>
---
 tools/include/nolibc/fcntl.h | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tools/include/nolibc/fcntl.h b/tools/include/nolibc/fcntl.h
index 014910a8e928..46f591cf82fd 100644
--- a/tools/include/nolibc/fcntl.h
+++ b/tools/include/nolibc/fcntl.h
@@ -14,6 +14,8 @@
 #include "types.h"
 #include "sys.h"
 
+#define __nolibc_open_flags(_flags) ((_flags) | O_LARGEFILE)
+
 /*
  * int openat(int dirfd, const char *path, int flags[, mode_t mode]);
  */
@@ -29,8 +31,6 @@ int openat(int dirfd, const char *path, int flags, ...)
 {
 	mode_t mode = 0;
 
-	flags |= O_LARGEFILE;
-
 	if (flags & O_CREAT) {
 		va_list args;
 
@@ -39,7 +39,7 @@ int openat(int dirfd, const char *path, int flags, ...)
 		va_end(args);
 	}
 
-	return __sysret(_sys_openat(dirfd, path, flags, mode));
+	return __sysret(_sys_openat(dirfd, path, __nolibc_open_flags(flags), mode));
 }
 
 /*
@@ -57,8 +57,6 @@ int open(const char *path, int flags, ...)
 {
 	mode_t mode = 0;
 
-	flags |= O_LARGEFILE;
-
 	if (flags & O_CREAT) {
 		va_list args;
 
@@ -67,7 +65,7 @@ int open(const char *path, int flags, ...)
 		va_end(args);
 	}
 
-	return __sysret(_sys_open(path, flags, mode));
+	return __sysret(_sys_open(path, __nolibc_open_flags(flags), mode));
 }
 
 /*

-- 
2.54.0

[PATCH v2 2/4] tools/nolibc: split open mode handling into a macro

[Thomas Weißschuh]

This logic is duplicated and some upcoming extensions would require even
more duplicated logic.

Move it into a macro to avoid the duplication and allow cleaner changes.

Signed-off-by: Thomas Weißschuh <linux@weissschuh.net>
Acked-by: Willy Tarreau <w@1wt.eu>
---
 tools/include/nolibc/fcntl.h | 40 ++++++++++++++++++----------------------
 1 file changed, 18 insertions(+), 22 deletions(-)

diff --git a/tools/include/nolibc/fcntl.h b/tools/include/nolibc/fcntl.h
index 46f591cf82fd..d7ea02e1332d 100644
--- a/tools/include/nolibc/fcntl.h
+++ b/tools/include/nolibc/fcntl.h
@@ -16,6 +16,21 @@
 
 #define __nolibc_open_flags(_flags) ((_flags) | O_LARGEFILE)
 
+#define __nolibc_open_mode(_flags)							\
+({											\
+	mode_t _mode = 0;								\
+											\
+	if ((_flags) & O_CREAT) {							\
+		va_list args;								\
+											\
+		va_start(args, (_flags));						\
+		_mode = va_arg(args, mode_t);						\
+		va_end(args);								\
+	}										\
+											\
+	_mode;										\
+})
+
 /*
  * int openat(int dirfd, const char *path, int flags[, mode_t mode]);
  */
@@ -29,17 +44,8 @@ int _sys_openat(int dirfd, const char *path, int flags, mode_t mode)
 static __attribute__((unused))
 int openat(int dirfd, const char *path, int flags, ...)
 {
-	mode_t mode = 0;
-
-	if (flags & O_CREAT) {
-		va_list args;
-
-		va_start(args, flags);
-		mode = va_arg(args, mode_t);
-		va_end(args);
-	}
-
-	return __sysret(_sys_openat(dirfd, path, __nolibc_open_flags(flags), mode));
+	return __sysret(_sys_openat(dirfd, path, __nolibc_open_flags(flags),
+						 __nolibc_open_mode(flags)));
 }
 
 /*
@@ -55,17 +61,7 @@ int _sys_open(const char *path, int flags, mode_t mode)
 static __attribute__((unused))
 int open(const char *path, int flags, ...)
 {
-	mode_t mode = 0;
-
-	if (flags & O_CREAT) {
-		va_list args;
-
-		va_start(args, flags);
-		mode = va_arg(args, mode_t);
-		va_end(args);
-	}
-
-	return __sysret(_sys_open(path, __nolibc_open_flags(flags), mode));
+	return __sysret(_sys_open(path, __nolibc_open_flags(flags), __nolibc_open_mode(flags)));
 }
 
 /*

-- 
2.54.0

[PATCH v2 3/4] tools/nolibc: always pass mode to open syscall

[Thomas Weißschuh]

When O_TMPFILE is set, the open mode needs to be passed to the kernel as
per the documentation. Currently this is not done.
Instead of checking for O_TMPFILE explicitly and making the conditionals
more complex, just always pass the mode to the kernel. If no value was
passed the mode will be garbage, but the kernel will ignore it anyways.

Fixes: a7604ba149e7 ("tools/nolibc/sys: make open() take a vararg on the 3rd argument")
Suggested-by: Willy Tarreau <w@1wt.eu>
Link: https://lore.kernel.org/lkml/afRfjdovT6pNtwtP@1wt.eu/
Signed-off-by: Thomas Weißschuh <linux@weissschuh.net>
---
 tools/include/nolibc/fcntl.h | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/tools/include/nolibc/fcntl.h b/tools/include/nolibc/fcntl.h
index d7ea02e1332d..d4b6af60d4cc 100644
--- a/tools/include/nolibc/fcntl.h
+++ b/tools/include/nolibc/fcntl.h
@@ -18,15 +18,12 @@
 
 #define __nolibc_open_mode(_flags)							\
 ({											\
-	mode_t _mode = 0;								\
+	mode_t _mode;									\
+	va_list args;									\
 											\
-	if ((_flags) & O_CREAT) {							\
-		va_list args;								\
-											\
-		va_start(args, (_flags));						\
-		_mode = va_arg(args, mode_t);						\
-		va_end(args);								\
-	}										\
+	va_start(args, (_flags));							\
+	_mode = va_arg(args, mode_t);							\
+	va_end(args);									\
 											\
 	_mode;										\
 })

-- 
2.54.0

[PATCH v2 4/4] selftests/nolibc: test open mode handling

[Thomas Weißschuh]

Add a selftest for the new O_TMPFILE open mode handling.
While O_CREAT or openat() are not tested, the code is the same,
so assume these also work.

Signed-off-by: Thomas Weißschuh <linux@weissschuh.net>
Acked-by: Willy Tarreau <w@1wt.eu>
---
 tools/testing/selftests/nolibc/nolibc-test.c | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/tools/testing/selftests/nolibc/nolibc-test.c b/tools/testing/selftests/nolibc/nolibc-test.c
index 1db6e8d55c16..c3867cc570c6 100644
--- a/tools/testing/selftests/nolibc/nolibc-test.c
+++ b/tools/testing/selftests/nolibc/nolibc-test.c
@@ -1305,6 +1305,28 @@ int test_openat(void)
 	return 0;
 }
 
+int test_open_mode(void)
+{
+	const mode_t mode = 0444;
+	struct stat stat_buf;
+	int fd, ret;
+
+	fd = open("/tmp", O_TMPFILE | O_RDWR, mode);
+	if (fd == -1)
+		return -1;
+
+	ret = fstat(fd, &stat_buf);
+	close(fd);
+
+	if (ret == -1)
+		return -1;
+
+	if ((stat_buf.st_mode & 0777) != mode)
+		return -1;
+
+	return 0;
+}
+
 int test_nolibc_enosys(void)
 {
 	if (true)
@@ -1540,6 +1562,7 @@ int run_syscall(int min, int max)
 		CASE_TEST(open_tty);          EXPECT_SYSNE(1, tmp = open("/dev/null", O_RDONLY), -1); if (tmp != -1) close(tmp); break;
 		CASE_TEST(open_blah);         EXPECT_SYSER(1, tmp = open("/proc/self/blah", O_RDONLY), -1, ENOENT); if (tmp != -1) close(tmp); break;
 		CASE_TEST(openat_dir);        EXPECT_SYSZR(1, test_openat()); break;
+		CASE_TEST(open_mode);         EXPECT_SYSZR(1, test_open_mode()); break;
 		CASE_TEST(pipe);              EXPECT_SYSZR(1, test_pipe()); break;
 		CASE_TEST(poll_null);         EXPECT_SYSZR(1, poll(NULL, 0, 0)); break;
 		CASE_TEST(poll_stdout);       EXPECT_SYSNE(1, ({ struct pollfd fds = { 1, POLLOUT, 0}; poll(&fds, 1, 0); }), -1); break;

-- 
2.54.0

Re: [PATCH v2 3/4] tools/nolibc: always pass mode to open syscall

[Willy Tarreau]

Hi Thomas,

On Thu, May 14, 2026 at 02:05:13PM +0200, Thomas Weißschuh wrote:
> When O_TMPFILE is set, the open mode needs to be passed to the kernel as
> per the documentation. Currently this is not done.
> Instead of checking for O_TMPFILE explicitly and making the conditionals
> more complex, just always pass the mode to the kernel. If no value was
> passed the mode will be garbage, but the kernel will ignore it anyways.
> 
> Fixes: a7604ba149e7 ("tools/nolibc/sys: make open() take a vararg on the 3rd argument")
> Suggested-by: Willy Tarreau <w@1wt.eu>
> Link: https://lore.kernel.org/lkml/afRfjdovT6pNtwtP@1wt.eu/
> Signed-off-by: Thomas Weißschuh <linux@weissschuh.net>

Looks good, thank you for making this one move forward!

Acked-by: Willy Tarreau <w@1wt.eu>
Willy