[PATCH liburing 0/2] liburing IORING_SETUP_NO_SQARRAY support

[PATCH liburing 0/2] liburing IORING_SETUP_NO_SQARRAY support

[Pavel Begunkov]

Patch 1 adds support for IORING_SETUP_NO_SQARRAY, i.e. not using and
mmaping the first SQ indirection level sq_array.

Patch 2 defaults liburing to using IORING_SETUP_NO_SQARRAY. If it's
not supported by the kernel we'll fallback to a setup without the
flag. If the user specifically asks for IORING_SETUP_NO_SQARRAY,
it'll also fail if the feature is unsupported.

Note: two tests need sqarray, and so there is a new helper
__io_uring_queue_init_params(), which is not static but not
exported by the library. Further, we don't declare it in
liburing.h but only under tests to prevent misuse.

Pavel Begunkov (2):
  setup: add IORING_SETUP_NO_SQARRAY support
  setup: default to IORING_SETUP_NO_SQARRAY

 src/include/liburing/io_uring.h |  5 ++++
 src/setup.c                     | 42 +++++++++++++++++++++++++--------
 test/accept-reuse.c             |  2 +-
 test/helpers.h                  | 13 ++++++++++
 test/io_uring_enter.c           |  7 ++++--
 5 files changed, 56 insertions(+), 13 deletions(-)

-- 
2.41.0

[PATCH liburing 1/2] setup: add IORING_SETUP_NO_SQARRAY support

[Pavel Begunkov]

IORING_SETUP_NO_SQARRAY removes sq_array from the kernel, and hence when
set liburing should not try to mmap and initialise it.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 src/include/liburing/io_uring.h |  5 +++++
 src/setup.c                     | 14 +++++++++-----
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/include/liburing/io_uring.h b/src/include/liburing/io_uring.h
index 10a6ef0..793d64f 100644
--- a/src/include/liburing/io_uring.h
+++ b/src/include/liburing/io_uring.h
@@ -185,6 +185,11 @@ enum {
  */
 #define IORING_SETUP_REGISTERED_FD_ONLY	(1U << 15)
 
+/*
+ * Removes indirection through the SQ index array.
+ */
+#define IORING_SETUP_NO_SQARRAY		(1U << 16)
+
 enum io_uring_op {
 	IORING_OP_NOP,
 	IORING_OP_READV,
diff --git a/src/setup.c b/src/setup.c
index 2dcb5aa..a0e8296 100644
--- a/src/setup.c
+++ b/src/setup.c
@@ -76,7 +76,8 @@ static void io_uring_setup_ring_pointers(struct io_uring_params *p,
 	sq->kring_entries = sq->ring_ptr + p->sq_off.ring_entries;
 	sq->kflags = sq->ring_ptr + p->sq_off.flags;
 	sq->kdropped = sq->ring_ptr + p->sq_off.dropped;
-	sq->array = sq->ring_ptr + p->sq_off.array;
+	if (!(p->flags & IORING_SETUP_NO_SQARRAY))
+		sq->array = sq->ring_ptr + p->sq_off.array;
 
 	cq->khead = cq->ring_ptr + p->cq_off.head;
 	cq->ktail = cq->ring_ptr + p->cq_off.tail;
@@ -220,7 +221,8 @@ static int io_uring_alloc_huge(unsigned entries, struct io_uring_params *p,
 	ring_mem = cq_entries * sizeof(struct io_uring_cqe);
 	if (p->flags & IORING_SETUP_CQE32)
 		ring_mem *= 2;
-	ring_mem += sq_entries * sizeof(unsigned);
+	if (!(p->flags & IORING_SETUP_NO_SQARRAY))
+		ring_mem += sq_entries * sizeof(unsigned);
 	mem_used = sqes_mem + ring_mem;
 	mem_used = (mem_used + page_size - 1) & ~(page_size - 1);
 
@@ -335,11 +337,13 @@ static int __io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
 	/*
 	 * Directly map SQ slots to SQEs
 	 */
-	sq_array = ring->sq.array;
 	sq_entries = ring->sq.ring_entries;
-	for (index = 0; index < sq_entries; index++)
-		sq_array[index] = index;
 
+	if (!(p->flags & IORING_SETUP_NO_SQARRAY)) {
+		sq_array = ring->sq.array;
+		for (index = 0; index < sq_entries; index++)
+			sq_array[index] = index;
+	}
 	ring->features = p->features;
 	ring->flags = p->flags;
 	ring->enter_ring_fd = fd;
-- 
2.41.0

[PATCH liburing 2/2] setup: default to IORING_SETUP_NO_SQARRAY

[Pavel Begunkov]

We have never exposed sq_array to users, so we can safely add
IORING_SETUP_NO_SQARRAY to all rings. If the kernel is too old and it
fails, we'll retry creating a ring without it.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 src/setup.c           | 28 +++++++++++++++++++++++-----
 test/accept-reuse.c   |  2 +-
 test/helpers.h        | 13 +++++++++++++
 test/io_uring_enter.c |  7 +++++--
 4 files changed, 42 insertions(+), 8 deletions(-)

diff --git a/src/setup.c b/src/setup.c
index a0e8296..6b3f538 100644
--- a/src/setup.c
+++ b/src/setup.c
@@ -287,9 +287,9 @@ static int io_uring_alloc_huge(unsigned entries, struct io_uring_params *p,
 	return (int) mem_used;
 }
 
-static int __io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
-					struct io_uring_params *p, void *buf,
-					size_t buf_size)
+int __io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
+				 struct io_uring_params *p, void *buf,
+				 size_t buf_size)
 {
 	int fd, ret = 0;
 	unsigned *sq_array;
@@ -357,6 +357,24 @@ static int __io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
 	return ret;
 }
 
+static int io_uring_queue_init_try_nosqarr(unsigned entries, struct io_uring *ring,
+					   struct io_uring_params *p, void *buf,
+					   size_t buf_size)
+{
+	unsigned flags = p->flags;
+	int ret;
+
+	p->flags |= IORING_SETUP_NO_SQARRAY;
+	ret = __io_uring_queue_init_params(entries, ring, p, buf, buf_size);
+
+	/* don't fallback if explicitly asked for NOSQARRAY */
+	if (ret != -EINVAL || (flags & IORING_SETUP_NO_SQARRAY))
+		return ret;
+
+	p->flags = flags;
+	return __io_uring_queue_init_params(entries, ring, p, buf, buf_size);
+}
+
 /*
  * Like io_uring_queue_init_params(), except it allows the application to pass
  * in a pre-allocated memory range that is used for the shared data between
@@ -375,7 +393,7 @@ int io_uring_queue_init_mem(unsigned entries, struct io_uring *ring,
 {
 	/* should already be set... */
 	p->flags |= IORING_SETUP_NO_MMAP;
-	return __io_uring_queue_init_params(entries, ring, p, buf, buf_size);
+	return io_uring_queue_init_try_nosqarr(entries, ring, p, buf, buf_size);
 }
 
 int io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
@@ -383,7 +401,7 @@ int io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
 {
 	int ret;
 
-	ret = __io_uring_queue_init_params(entries, ring, p, NULL, 0);
+	ret = io_uring_queue_init_try_nosqarr(entries, ring, p, NULL, 0);
 	return ret >= 0 ? 0 : ret;
 }
 
diff --git a/test/accept-reuse.c b/test/accept-reuse.c
index 1f10b45..716f201 100644
--- a/test/accept-reuse.c
+++ b/test/accept-reuse.c
@@ -45,7 +45,7 @@ int main(int argc, char **argv)
 		return T_EXIT_SKIP;
 
 	memset(&params, 0, sizeof(params));
-	ret = io_uring_queue_init_params(4, &io_uring, &params);
+	ret = t_io_uring_init_sqarray(4, &io_uring, &params);
 	if (ret) {
 		fprintf(stderr, "io_uring_init_failed: %d\n", ret);
 		return T_EXIT_FAIL;
diff --git a/test/helpers.h b/test/helpers.h
index 5307324..d91c1dc 100644
--- a/test/helpers.h
+++ b/test/helpers.h
@@ -87,6 +87,19 @@ bool t_probe_defer_taskrun(void);
 
 unsigned __io_uring_flush_sq(struct io_uring *ring);
 
+int __io_uring_queue_init_params(unsigned entries, struct io_uring *ring,
+				 struct io_uring_params *p, void *buf,
+				 size_t buf_size);
+
+static inline int t_io_uring_init_sqarray(unsigned entries, struct io_uring *ring,
+					struct io_uring_params *p)
+{
+	int ret;
+
+	ret = __io_uring_queue_init_params(entries, ring, p, NULL, 0);
+	return ret >= 0 ? 0 : ret;
+}
+
 #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
 
 void t_error(int status, int errnum, const char *format, ...);
diff --git a/test/io_uring_enter.c b/test/io_uring_enter.c
index ecd54ff..2a6b6be 100644
--- a/test/io_uring_enter.c
+++ b/test/io_uring_enter.c
@@ -183,13 +183,16 @@ int main(int argc, char **argv)
 	unsigned ktail, mask, index;
 	unsigned sq_entries;
 	unsigned completed, dropped;
+	struct io_uring_params p;
 
 	if (argc > 1)
 		return T_EXIT_SKIP;
 
-	ret = io_uring_queue_init(IORING_MAX_ENTRIES, &ring, 0);
+	memset(&p, 0, sizeof(p));
+	ret = t_io_uring_init_sqarray(IORING_MAX_ENTRIES, &ring, &p);
 	if (ret == -ENOMEM)
-		ret = io_uring_queue_init(IORING_MAX_ENTRIES_FALLBACK, &ring, 0);
+		ret = t_io_uring_init_sqarray(IORING_MAX_ENTRIES_FALLBACK,
+					      &ring, &p);
 	if (ret < 0) {
 		perror("io_uring_queue_init");
 		exit(T_EXIT_FAIL);
-- 
2.41.0

Re: [PATCH liburing 0/2] liburing IORING_SETUP_NO_SQARRAY support

[Jens Axboe]

On Fri, 29 Sep 2023 13:09:39 +0100, Pavel Begunkov wrote:
> Patch 1 adds support for IORING_SETUP_NO_SQARRAY, i.e. not using and
> mmaping the first SQ indirection level sq_array.
> 
> Patch 2 defaults liburing to using IORING_SETUP_NO_SQARRAY. If it's
> not supported by the kernel we'll fallback to a setup without the
> flag. If the user specifically asks for IORING_SETUP_NO_SQARRAY,
> it'll also fail if the feature is unsupported.
> 
> [...]

Applied, thanks!

[1/2] setup: add IORING_SETUP_NO_SQARRAY support
      commit: 74c1191cbfa2b552b3ceaa63386d871c2d5d2136
[2/2] setup: default to IORING_SETUP_NO_SQARRAY
      commit: 3401b06c5e8291a2ad946bb181ab347f18a4c8c3

Best regards,
-- 
Jens Axboe