[PATCH] selftests: core: Fix unshare_test on systems with max nr_open

["Jin Li" <jinli@lanxincomputing.com>]

When nr_open is already set to INT_MAX, nr_open + 1024 overflows
and causes setrlimit to fail. This happens on systems where
fs.nr_open has been configured to its maximum value.

Add a check to detect this condition. If nr_open is near INT_MAX,
skip increasing fs.nr_open and use a fallback value of 1048576
for RLIMIT_NOFILE instead, which is large enough for the test.

This fixes the test failure on systems with maxed-out nr_open.

Signed-off-by: Jin Li <jinli@lanxincomputing.com>
---
 tools/testing/selftests/core/unshare_test.c | 38 +++++++++++++++------
 1 file changed, 27 insertions(+), 11 deletions(-)

diff --git a/tools/testing/selftests/core/unshare_test.c b/tools/testing/selftests/core/unshare_test.c
index ffce75a6c228..dcc61488e4dd 100644
--- a/tools/testing/selftests/core/unshare_test.c
+++ b/tools/testing/selftests/core/unshare_test.c
@@ -42,18 +42,34 @@ TEST(unshare_EMFILE)
 
 	ASSERT_EQ(0, getrlimit(RLIMIT_NOFILE, &rlimit));
 
-	/* bump fs.nr_open */
-	n2 = sprintf(buf2, "%d\n", nr_open + 1024);
-	lseek(fd, 0, SEEK_SET);
-	write(fd, buf2, n2);
-
-	/* bump ulimit -n */
-	rlimit.rlim_cur = nr_open + 1024;
-	rlimit.rlim_max = nr_open + 1024;
-	EXPECT_EQ(0, setrlimit(RLIMIT_NOFILE, &rlimit)) {
+	/*
+	 * Only bump fs.nr_open and RLIMIT_NOFILE if nr_open is not already
+	 * at or near INT_MAX. Adding 1024 to INT_MAX would overflow.
+	 */
+	if (nr_open < INT_MAX - 1024) {
+		n2 = sprintf(buf2, "%d\n", nr_open + 1024);
 		lseek(fd, 0, SEEK_SET);
-		write(fd, buf, n);
-		exit(EXIT_FAILURE);
+		write(fd, buf2, n2);
+
+		rlimit.rlim_cur = nr_open + 1024;
+		rlimit.rlim_max = nr_open + 1024;
+		EXPECT_EQ(0, setrlimit(RLIMIT_NOFILE, &rlimit)) {
+			lseek(fd, 0, SEEK_SET);
+			write(fd, buf, n);
+			exit(EXIT_FAILURE);
+		}
+	} else {
+		/*
+		 * If nr_open is already at maximum, use a fallback value
+		 * for RLIMIT_NOFILE that is large enough for the test.
+		 */
+		rlimit.rlim_cur = 1048576;
+		rlimit.rlim_max = 1048576;
+		EXPECT_EQ(0, setrlimit(RLIMIT_NOFILE, &rlimit)) {
+			lseek(fd, 0, SEEK_SET);
+			write(fd, buf, n);
+			exit(EXIT_FAILURE);
+		}
 	}
 
 	/* get a descriptor past the old fs.nr_open */
-- 
2.53.0