[PATCH liburing 0/4] vectored registered buffer support and tests

[PATCH liburing 0/4] vectored registered buffer support and tests

[Pavel Begunkov]

It should give good coverage, I plan to extend read-write.c to
use it as well.

Pavel Begunkov (4):
  Add vectored registered buffer req init helpers
  test/sendzc: test registered vectored buffers
  tests/helpers: add t_create_socketpair_ip
  tests: targeted registered vector tests

 src/include/liburing.h |  31 +++
 test/Makefile          |   1 +
 test/helpers.c         | 111 ++++++++
 test/helpers.h         |   5 +
 test/regvec.c          | 604 +++++++++++++++++++++++++++++++++++++++++
 test/send-zerocopy.c   | 138 ++--------
 6 files changed, 776 insertions(+), 114 deletions(-)
 create mode 100644 test/regvec.c

-- 
2.48.1

[PATCH liburing 1/4] Add vectored registered buffer req init helpers

[Pavel Begunkov]

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 src/include/liburing.h | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/src/include/liburing.h b/src/include/liburing.h
index d162d0e6..e71551ed 100644
--- a/src/include/liburing.h
+++ b/src/include/liburing.h
@@ -556,6 +556,16 @@ IOURINGINLINE void io_uring_prep_read_fixed(struct io_uring_sqe *sqe, int fd,
 	sqe->buf_index = (__u16) buf_index;
 }
 
+IOURINGINLINE void io_uring_prep_readv_fixed(struct io_uring_sqe *sqe, int fd,
+					     const struct iovec *iovecs,
+					     unsigned nr_vecs, __u64 offset,
+					     int flags, int buf_index)
+{
+	io_uring_prep_readv2(sqe, fd, iovecs, nr_vecs, offset, flags);
+	sqe->opcode = IORING_OP_WRITE_FIXED;
+	sqe->buf_index = (__u16)buf_index;
+}
+
 IOURINGINLINE void io_uring_prep_writev(struct io_uring_sqe *sqe, int fd,
 					const struct iovec *iovecs,
 					unsigned nr_vecs, __u64 offset)
@@ -580,6 +590,16 @@ IOURINGINLINE void io_uring_prep_write_fixed(struct io_uring_sqe *sqe, int fd,
 	sqe->buf_index = (__u16) buf_index;
 }
 
+IOURINGINLINE void io_uring_prep_writev2_fixed(struct io_uring_sqe *sqe, int fd,
+				       const struct iovec *iovecs,
+				       unsigned nr_vecs, __u64 offset,
+				       int flags, int buf_index)
+{
+	io_uring_prep_writev2(sqe, fd, iovecs, nr_vecs, offset, flags);
+	sqe->opcode = IORING_OP_WRITE_FIXED;
+	sqe->buf_index = (__u16)buf_index;
+}
+
 IOURINGINLINE void io_uring_prep_recvmsg(struct io_uring_sqe *sqe, int fd,
 					 struct msghdr *msg, unsigned flags)
 {
@@ -964,6 +984,17 @@ IOURINGINLINE void io_uring_prep_sendmsg_zc(struct io_uring_sqe *sqe, int fd,
 	sqe->opcode = IORING_OP_SENDMSG_ZC;
 }
 
+IOURINGINLINE void io_uring_prep_sendmsg_zc_fixed(struct io_uring_sqe *sqe,
+						int fd,
+						const struct msghdr *msg,
+						unsigned flags,
+						unsigned buf_index)
+{
+	io_uring_prep_sendmsg_zc(sqe, fd, msg, flags);
+	sqe->ioprio |= IORING_RECVSEND_FIXED_BUF;
+	sqe->buf_index = buf_index;
+}
+
 IOURINGINLINE void io_uring_prep_recv(struct io_uring_sqe *sqe, int sockfd,
 				      void *buf, size_t len, int flags)
 {
-- 
2.48.1

[PATCH liburing 2/4] test/sendzc: test registered vectored buffers

[Pavel Begunkov]

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 test/send-zerocopy.c | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/test/send-zerocopy.c b/test/send-zerocopy.c
index c8eafe28..680481a0 100644
--- a/test/send-zerocopy.c
+++ b/test/send-zerocopy.c
@@ -69,6 +69,7 @@ static size_t page_sz;
 static char *tx_buffer, *rx_buffer;
 static struct iovec buffers_iov[__BUF_NR];
 
+static bool has_regvec;
 static bool has_sendzc;
 static bool has_sendmsg;
 static bool hit_enomem;
@@ -96,6 +97,7 @@ static int probe_zc_support(void)
 
 	has_sendzc = p->ops_len > IORING_OP_SEND_ZC;
 	has_sendmsg = p->ops_len > IORING_OP_SENDMSG_ZC;
+	has_regvec = p->ops_len > IORING_OP_READV_FIXED;
 	io_uring_queue_exit(&ring);
 	free(p);
 	return 0;
@@ -448,6 +450,11 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se
 			else
 				io_uring_prep_sendmsg(sqe, sock_client, &msghdr[i], msg_flags);
 
+			if (real_fixed_buf) {
+				sqe->ioprio |= IORING_RECVSEND_FIXED_BUF;
+				sqe->buf_index = conf->buf_index;
+			}
+
 			if (!conf->iovec) {
 				io = &iov[i];
 				iov_len = 1;
@@ -619,7 +626,11 @@ static int test_inet_send(struct io_uring *ring)
 			conf.tcp = tcp;
 			regbuf = conf.mix_register || conf.fixed_buf;
 
-			if (conf.iovec && (!conf.use_sendmsg || regbuf || conf.cork))
+			if (!tcp && conf.long_iovec)
+				continue;
+			if (conf.use_sendmsg && regbuf && !has_regvec)
+				continue;
+			if (conf.iovec && (!conf.use_sendmsg || conf.cork))
 				continue;
 			if (!conf.zc) {
 				if (regbuf)
@@ -637,7 +648,7 @@ static int test_inet_send(struct io_uring *ring)
 				continue;
 			if (!client_connect && conf.addr == NULL)
 				continue;
-			if (conf.use_sendmsg && (regbuf || !has_sendmsg))
+			if (conf.use_sendmsg && !has_sendmsg)
 				continue;
 			if (msg_zc_set && !conf.zc)
 				continue;
-- 
2.48.1

[PATCH liburing 3/4] tests/helpers: add t_create_socketpair_ip

[Pavel Begunkov]

create_socketpair_ip() is useful for zerocopy tx testing as it needs TCP
sockets and not just AF_UNIX. There will be tx zc tests in more files,
so move the function to helpers.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 test/helpers.c       | 111 ++++++++++++++++++++++++++++++++++++++
 test/helpers.h       |   5 ++
 test/send-zerocopy.c | 123 ++++---------------------------------------
 3 files changed, 127 insertions(+), 112 deletions(-)

diff --git a/test/helpers.c b/test/helpers.c
index 07186911..b2d386f4 100644
--- a/test/helpers.c
+++ b/test/helpers.c
@@ -373,3 +373,114 @@ void *aligned_alloc(size_t alignment, size_t size)
 
 	return ret;
 }
+
+int t_create_socketpair_ip(struct sockaddr_storage *addr,
+				int *sock_client, int *sock_server,
+				bool ipv6, bool client_connect,
+				bool msg_zc, bool tcp, const char *name)
+{
+	socklen_t addr_size;
+	int family, sock, listen_sock = -1;
+	int ret;
+
+	memset(addr, 0, sizeof(*addr));
+	if (ipv6) {
+		struct sockaddr_in6 *saddr = (struct sockaddr_in6 *)addr;
+
+		family = AF_INET6;
+		saddr->sin6_family = family;
+		saddr->sin6_port = htons(0);
+		addr_size = sizeof(*saddr);
+	} else {
+		struct sockaddr_in *saddr = (struct sockaddr_in *)addr;
+
+		family = AF_INET;
+		saddr->sin_family = family;
+		saddr->sin_port = htons(0);
+		saddr->sin_addr.s_addr = htonl(INADDR_ANY);
+		addr_size = sizeof(*saddr);
+	}
+
+	/* server sock setup */
+	if (tcp) {
+		sock = listen_sock = socket(family, SOCK_STREAM, IPPROTO_TCP);
+	} else {
+		sock = *sock_server = socket(family, SOCK_DGRAM, 0);
+	}
+	if (sock < 0) {
+		perror("socket");
+		return 1;
+	}
+
+	ret = bind(sock, (struct sockaddr *)addr, addr_size);
+	if (ret < 0) {
+		perror("bind");
+		return 1;
+	}
+
+	ret = getsockname(sock, (struct sockaddr *)addr, &addr_size);
+	if (ret < 0) {
+		fprintf(stderr, "getsockname failed %i\n", errno);
+		return 1;
+	}
+
+	if (tcp) {
+		ret = listen(sock, 128);
+		assert(ret != -1);
+	}
+
+	if (ipv6) {
+		struct sockaddr_in6 *saddr = (struct sockaddr_in6 *)addr;
+
+		inet_pton(AF_INET6, name, &(saddr->sin6_addr));
+	} else {
+		struct sockaddr_in *saddr = (struct sockaddr_in *)addr;
+
+		inet_pton(AF_INET, name, &saddr->sin_addr);
+	}
+
+	/* client sock setup */
+	if (tcp) {
+		*sock_client = socket(family, SOCK_STREAM, IPPROTO_TCP);
+		assert(client_connect);
+	} else {
+		*sock_client = socket(family, SOCK_DGRAM, 0);
+	}
+	if (*sock_client < 0) {
+		perror("socket");
+		return 1;
+	}
+	if (client_connect) {
+		ret = connect(*sock_client, (struct sockaddr *)addr, addr_size);
+		if (ret < 0) {
+			perror("connect");
+			return 1;
+		}
+	}
+	if (msg_zc) {
+#ifdef SO_ZEROCOPY
+		int val = 1;
+
+		/*
+		 * NOTE: apps must not set SO_ZEROCOPY when using io_uring zc.
+		 * It's only here to test interactions with MSG_ZEROCOPY.
+		 */
+		if (setsockopt(*sock_client, SOL_SOCKET, SO_ZEROCOPY, &val, sizeof(val))) {
+			perror("setsockopt zc");
+			return 1;
+		}
+#else
+		fprintf(stderr, "no SO_ZEROCOPY\n");
+		return 1;
+#endif
+	}
+	if (tcp) {
+		*sock_server = accept(listen_sock, NULL, NULL);
+		if (!*sock_server) {
+			fprintf(stderr, "can't accept\n");
+			return 1;
+		}
+		close(listen_sock);
+	}
+	return 0;
+}
diff --git a/test/helpers.h b/test/helpers.h
index d0294eba..f8a5c7f2 100644
--- a/test/helpers.h
+++ b/test/helpers.h
@@ -81,6 +81,11 @@ struct iovec *t_create_buffers(size_t buf_num, size_t buf_size);
  */
 int t_create_socket_pair(int fd[2], bool stream);
 
+int t_create_socketpair_ip(struct sockaddr_storage *addr,
+				int *sock_client, int *sock_server,
+				bool ipv6, bool client_connect,
+				bool msg_zc, bool tcp, const char *name);
+
 /*
  * Helper for setting up a ring and checking for user privs
  */
diff --git a/test/send-zerocopy.c b/test/send-zerocopy.c
index 680481a0..b505e4d0 100644
--- a/test/send-zerocopy.c
+++ b/test/send-zerocopy.c
@@ -258,117 +258,6 @@ static int test_send_faults(int sock_tx, int sock_rx)
 	return T_EXIT_PASS;
 }
 
-static int create_socketpair_ip(struct sockaddr_storage *addr,
-				int *sock_client, int *sock_server,
-				bool ipv6, bool client_connect,
-				bool msg_zc, bool tcp)
-{
-	socklen_t addr_size;
-	int family, sock, listen_sock = -1;
-	int ret;
-
-	memset(addr, 0, sizeof(*addr));
-	if (ipv6) {
-		struct sockaddr_in6 *saddr = (struct sockaddr_in6 *)addr;
-
-		family = AF_INET6;
-		saddr->sin6_family = family;
-		saddr->sin6_port = htons(0);
-		addr_size = sizeof(*saddr);
-	} else {
-		struct sockaddr_in *saddr = (struct sockaddr_in *)addr;
-
-		family = AF_INET;
-		saddr->sin_family = family;
-		saddr->sin_port = htons(0);
-		saddr->sin_addr.s_addr = htonl(INADDR_ANY);
-		addr_size = sizeof(*saddr);
-	}
-
-	/* server sock setup */
-	if (tcp) {
-		sock = listen_sock = socket(family, SOCK_STREAM, IPPROTO_TCP);
-	} else {
-		sock = *sock_server = socket(family, SOCK_DGRAM, 0);
-	}
-	if (sock < 0) {
-		perror("socket");
-		return 1;
-	}
-
-	ret = bind(sock, (struct sockaddr *)addr, addr_size);
-	if (ret < 0) {
-		perror("bind");
-		return 1;
-	}
-
-	ret = getsockname(sock, (struct sockaddr *)addr, &addr_size);
-	if (ret < 0) {
-		fprintf(stderr, "getsockname failed %i\n", errno);
-		return 1;
-	}
-
-	if (tcp) {
-		ret = listen(sock, 128);
-		assert(ret != -1);
-	}
-
-	if (ipv6) {
-		struct sockaddr_in6 *saddr = (struct sockaddr_in6 *)addr;
-
-		inet_pton(AF_INET6, HOSTV6, &(saddr->sin6_addr));
-	} else {
-		struct sockaddr_in *saddr = (struct sockaddr_in *)addr;
-
-		inet_pton(AF_INET, HOST, &saddr->sin_addr);
-	}
-
-	/* client sock setup */
-	if (tcp) {
-		*sock_client = socket(family, SOCK_STREAM, IPPROTO_TCP);
-		assert(client_connect);
-	} else {
-		*sock_client = socket(family, SOCK_DGRAM, 0);
-	}
-	if (*sock_client < 0) {
-		perror("socket");
-		return 1;
-	}
-	if (client_connect) {
-		ret = connect(*sock_client, (struct sockaddr *)addr, addr_size);
-		if (ret < 0) {
-			perror("connect");
-			return 1;
-		}
-	}
-	if (msg_zc) {
-#ifdef SO_ZEROCOPY
-		int val = 1;
-
-		/*
-		 * NOTE: apps must not set SO_ZEROCOPY when using io_uring zc.
-		 * It's only here to test interactions with MSG_ZEROCOPY.
-		 */
-		if (setsockopt(*sock_client, SOL_SOCKET, SO_ZEROCOPY, &val, sizeof(val))) {
-			perror("setsockopt zc");
-			return 1;
-		}
-#else
-		fprintf(stderr, "no SO_ZEROCOPY\n");
-		return 1;
-#endif
-	}
-	if (tcp) {
-		*sock_server = accept(listen_sock, NULL, NULL);
-		if (!*sock_server) {
-			fprintf(stderr, "can't accept\n");
-			return 1;
-		}
-		close(listen_sock);
-	}
-	return 0;
-}
-
 struct send_conf {
 	bool fixed_buf;
 	bool mix_register;
@@ -574,6 +463,16 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se
 	return 0;
 }
 
+static int create_socketpair_ip(struct sockaddr_storage *addr,
+				int *sock_client, int *sock_server,
+				bool ipv6, bool client_connect,
+				bool msg_zc, bool tcp)
+{
+	return t_create_socketpair_ip(addr, sock_client, sock_server, ipv6,
+					client_connect, msg_zc, tcp,
+					ipv6 ? HOSTV6 : HOST);
+}
+
 static int test_inet_send(struct io_uring *ring)
 {
 	struct send_conf conf;
@@ -598,7 +497,7 @@ static int test_inet_send(struct io_uring *ring)
 			continue;
 #endif
 		ret = create_socketpair_ip(&addr, &sock_client, &sock_server, ipv6,
-				 client_connect, msg_zc_set, tcp);
+					client_connect, msg_zc_set, tcp);
 		if (ret) {
 			fprintf(stderr, "sock prep failed %d\n", ret);
 			return 1;
-- 
2.48.1

[PATCH liburing 4/4] tests: targeted registered vector tests

[Pavel Begunkov]

The main part here is to exercise various iovec->bvec reallocation
scenarios, but it also checks edge cases and validates final data.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 test/Makefile |   1 +
 test/regvec.c | 604 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 605 insertions(+)
 create mode 100644 test/regvec.c

diff --git a/test/Makefile b/test/Makefile
index 0367ef72..e937c852 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -241,6 +241,7 @@ test_srcs := \
 	wq-aff.c \
 	xattr.c \
 	zcrx.c \
+	regvec.c \
 	# EOL
 
 # Please keep this list sorted alphabetically.
diff --git a/test/regvec.c b/test/regvec.c
new file mode 100644
index 00000000..3afa82b3
--- /dev/null
+++ b/test/regvec.c
@@ -0,0 +1,604 @@
+/* SPDX-License-Identifier: MIT */
+#include <sys/mman.h>
+#include <linux/mman.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <poll.h>
+#include <pthread.h>
+#include <errno.h>
+
+#include "helpers.h"
+#include "liburing.h"
+
+static bool has_regvec;
+
+struct buf_desc {
+	char			*buf_wr;
+	char			*buf_rd;
+	size_t			size;
+
+	struct io_uring 	ring;
+	bool			ring_init;
+	bool			fixed;
+	int			buf_idx;
+	bool			rw;
+};
+
+#define BUF_BASE_IDX	1
+static int page_sz;
+
+static void probe_support(void)
+{
+	struct io_uring_probe *p;
+	struct io_uring ring;
+	int ret = 0;
+
+	ret = io_uring_queue_init(1, &ring, 0);
+	if (ret) {
+		fprintf(stderr, "queue init failed: %d\n", ret);
+		exit(ret);
+	}
+
+	p = t_calloc(1, sizeof(*p) + 256 * sizeof(struct io_uring_probe_op));
+	ret = io_uring_register_probe(&ring, p, 256);
+
+	/* if we don't have PROBE_REGISTER, we don't have OP_READ/WRITE */
+	if (ret == -EINVAL)
+		goto out;
+	if (ret) {
+		fprintf(stderr, "register_probe: %d\n", ret);
+		goto out;
+	}
+
+	has_regvec = p->ops_len > IORING_OP_READV_FIXED &&
+		     (p->ops[IORING_OP_READV_FIXED].flags & IO_URING_OP_SUPPORTED);
+out:
+	io_uring_queue_exit(&ring);
+	if (p)
+		free(p);
+}
+
+static void bind_ring(struct buf_desc *bd, struct io_uring *ring, unsigned buf_idx)
+{
+	size_t size = bd->size;
+	struct iovec iov;
+	int ret;
+
+	iov.iov_len = size;
+	iov.iov_base = bd->buf_wr;
+
+	ret = io_uring_register_buffers_update_tag(ring, buf_idx, &iov, NULL, 1);
+	if (ret != 1) {
+		if (geteuid()) {
+			fprintf(stderr, "Not root, skipping\n");
+			exit(0);
+		}
+		fprintf(stderr, "buf reg failed %i\n", ret);
+		exit(1);
+	}
+	bd->buf_idx = buf_idx;
+}
+
+static void reinit_ring(struct buf_desc *bd)
+{
+	struct io_uring *ring = &bd->ring;
+	int ret;
+
+	if (bd->ring_init) {
+		io_uring_queue_exit(ring);
+		bd->ring_init = false;
+	}
+
+	ret = io_uring_queue_init(32, ring, 0);
+	if (ret) {
+		fprintf(stderr, "ring init error %i\n", ret);
+		exit(1);
+	}
+
+	ret = io_uring_register_buffers_sparse(ring, 128);
+	if (ret) {
+		fprintf(stderr, "table reg error %i\n", ret);
+		exit(1);
+	}
+
+	bind_ring(bd, &bd->ring, BUF_BASE_IDX);
+	bd->ring_init = true;
+}
+
+static void init_buffers(struct buf_desc *bd, size_t size)
+{
+	void *start;
+	void *mem;
+
+	start = mmap(NULL, size + page_sz * 2, PROT_NONE,
+			MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
+	if (start == MAP_FAILED) {
+		fprintf(stderr, "Unable to preserve the page mixture memory.\n");
+		exit(1);
+	}
+
+	mem = mmap(start + page_sz, size, PROT_READ | PROT_WRITE,
+		   MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
+	if (mem == MAP_FAILED) {
+		fprintf(stderr, "mmap fail\n");
+		exit(1);
+	}
+
+	memset(bd, 0, sizeof(*bd));
+	bd->size = size;
+	bd->buf_wr = mem;
+	bd->buf_rd = malloc(size);
+	if (!bd->buf_rd) {
+		fprintf(stderr, "malloc fail\n");
+		exit(1);
+	}
+}
+
+static int verify_data(struct buf_desc *bd, struct iovec *wr_vecs, int nr_iovec,
+			int fd)
+{
+	int iov_idx, ret;
+
+	for (iov_idx = 0; iov_idx < nr_iovec; iov_idx++) {
+		struct iovec *vec = &wr_vecs[iov_idx];
+		size_t seg_size = vec->iov_len;
+		size_t read_bytes = 0;
+
+		while (1) {
+			ret = read(fd, bd->buf_rd + read_bytes, seg_size - read_bytes);
+			if (ret < 0) {
+				fprintf(stderr, "read error %i", ret);
+				return 1;
+			}
+			read_bytes += ret;
+			if (read_bytes == seg_size)
+				break;
+			if (ret == 0) {
+				fprintf(stderr, "can't read %i", ret);
+				return 2;
+			}
+		}
+
+		ret = memcmp(bd->buf_rd, vec->iov_base, seg_size);
+		if (ret != 0) {
+			fprintf(stderr, "data mismatch %i\n", ret);
+			return 3;
+		}
+	}
+	return 0;
+}
+
+struct verify_data {
+	struct buf_desc *bd;
+	struct iovec *vecs;
+	int nr_vec;
+	int fd;
+};
+
+static void *verify_thread_cb(void *data)
+{
+	struct verify_data *vd = data;
+	int ret;
+
+	ret = verify_data(vd->bd, vd->vecs, vd->nr_vec, vd->fd);
+	return (void *)(unsigned long)ret;
+}
+
+static int test_rw(struct buf_desc *bd, struct iovec *vecs, int nr_vec, int fd_wr)
+{
+	unsigned buf_idx = bd->buf_idx;
+	struct io_uring *ring = &bd->ring;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int ret;
+
+	sqe = io_uring_get_sqe(ring);
+	io_uring_prep_writev(sqe, fd_wr, vecs, nr_vec, 0);
+	if (bd->fixed)
+		sqe->buf_index = buf_idx;
+
+	ret = io_uring_submit(ring);
+	if (ret != 1) {
+		fprintf(stderr, "submit failed %i\n", ret);
+		exit(1);
+	}
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait_cqe=%d\n", ret);
+		exit(1);
+	}
+
+	ret = cqe->res;
+	io_uring_cqe_seen(ring, cqe);
+	return ret;
+}
+
+static int test_sendzc(struct buf_desc *bd, struct iovec *vecs, int nr_vec, int fd_wr)
+{
+	unsigned buf_idx = bd->buf_idx;
+	struct io_uring *ring = &bd->ring;
+	struct io_uring_sqe *sqe;
+	struct io_uring_cqe *cqe;
+	int ret, cqe_ret, more;
+	struct msghdr msghdr;
+
+	memset(&msghdr, 0, sizeof(msghdr));
+	msghdr.msg_iov = vecs;
+	msghdr.msg_iovlen = nr_vec;
+
+	sqe = io_uring_get_sqe(ring);
+	if (bd->fixed)
+		io_uring_prep_sendmsg_zc_fixed(sqe, fd_wr, &msghdr, 0, buf_idx);
+	else
+		io_uring_prep_sendmsg_zc(sqe, fd_wr, &msghdr, 0);
+
+	ret = io_uring_submit(ring);
+	if (ret != 1) {
+		fprintf(stderr, "submit failed %i\n", ret);
+		exit(1);
+	}
+	ret = io_uring_wait_cqe(ring, &cqe);
+	if (ret) {
+		fprintf(stderr, "wait_cqe=%d\n", ret);
+		exit(1);
+	}
+
+	cqe_ret = cqe->res;
+	more = cqe->flags & IORING_CQE_F_MORE;
+	io_uring_cqe_seen(ring, cqe);
+
+	if (more) {
+		ret = io_uring_wait_cqe(ring, &cqe);
+		if (ret) {
+			fprintf(stderr, "wait_cqe=%d\n", ret);
+			exit(1);
+		}
+		io_uring_cqe_seen(ring, cqe);
+	}
+	return cqe_ret;
+}
+
+static int test_vec(struct buf_desc *bd, struct iovec *vecs, int nr_vec,
+		    bool expect_fail, int *cqe_ret)
+{
+	struct sockaddr_storage addr;
+	int sock_server, sock_client;
+	struct verify_data vd;
+	size_t total_len = 0;
+	int i, ret;
+	void *verify_res;
+	pthread_t th;
+
+	ret = t_create_socketpair_ip(&addr, &sock_client, &sock_server,
+					true, true, false, true, "::1");
+	if (ret) {
+		fprintf(stderr, "sock prep failed %d\n", ret);
+		return 1;
+	}
+
+	for (i = 0; i < bd->size; i++)
+		bd->buf_wr[i] = i;
+	memset(bd->buf_rd, 0, bd->size);
+
+	for (i = 0; i < nr_vec; i++)
+		total_len += vecs[i].iov_len;
+
+	vd.bd = bd;
+	vd.vecs = vecs;
+	vd.nr_vec = nr_vec;
+	vd.fd = sock_server;
+
+	if (!expect_fail) {
+		ret = pthread_create(&th, NULL, verify_thread_cb, &vd);
+		if (ret) {
+			fprintf(stderr, "pthread_create failed %i\n", ret);
+			return ret;
+		}
+	}
+
+	if (bd->rw)
+		ret = test_rw(bd, vecs, nr_vec, sock_client);
+	else
+		ret = test_sendzc(bd, vecs, nr_vec, sock_client);
+
+	*cqe_ret = ret;
+
+	if (!expect_fail && ret != total_len) {
+		fprintf(stderr, "invalid cqe %i, expected %lu\n",
+				 ret, (unsigned long)total_len);
+		return 1;
+	}
+
+	if (!expect_fail) {
+		pthread_join(th, &verify_res);
+		ret = (int)(unsigned long)verify_res;
+		if (ret) {
+			fprintf(stderr, "verify failed  %i\n", ret);
+			return 1;
+		}
+	}
+	close(sock_client);
+	close(sock_server);
+	return 0;
+}
+
+struct work {
+	struct iovec	*vecs;
+	unsigned	nr_vecs;
+};
+
+static int test_sequence(struct buf_desc *bd, unsigned nr, struct work *ws)
+{
+	int i, ret;
+	int cqe_ret;
+
+	reinit_ring(bd);
+
+	for (i = 0; i < nr; i++) {
+		ret = test_vec(bd, ws[i].vecs, ws[i].nr_vecs, false, &cqe_ret);
+		if (ret) {
+			fprintf(stderr, "sequence failed, idx %i/%i\n", i, nr);
+			return ret;
+		}
+	}
+	return 0;
+}
+
+static void test_basic(struct buf_desc *bd)
+{
+	void *p = bd->buf_wr;
+	int ret;
+	struct iovec iov_page =		{ .iov_base = p,
+					  .iov_len = page_sz, };
+	struct iovec iov_inner =	{ .iov_base = p + 1,
+					  .iov_len = 3, };
+	struct iovec iov_maxbvec =	{ .iov_base = p + page_sz - 1,
+					  .iov_len = page_sz + 2, };
+	struct iovec iov_big =		{ .iov_base = p,
+					  .iov_len = page_sz * 12 + 33, };
+	struct iovec iov_big_unalign =	{ .iov_base = p + 10,
+					  .iov_len = page_sz * 7 + 41, };
+	struct iovec iov_full =		{ .iov_base = p,
+					  .iov_len = bd->size, };
+	struct iovec iov_right1 =	{ .iov_base = p + bd->size - page_sz + 5,
+					  .iov_len = page_sz - 5 };
+	struct iovec iov_right2 =	{ .iov_base = p + bd->size - page_sz - 5,
+					  .iov_len = page_sz + 5 };
+	struct iovec iov_full_unalign = { .iov_base = p + 1,
+					  .iov_len = bd->size - 1, };
+	struct iovec vecs[] = {
+		iov_page,
+		iov_big,
+		iov_inner,
+		iov_big_unalign,
+		iov_big_unalign,
+	};
+	struct iovec vecs_basic[] = { iov_page, iov_page, iov_page };
+	struct iovec vecs_full[] = { iov_full, iov_full, iov_full };
+	struct iovec vecs_full_unalign[] = { iov_full_unalign, iov_full_unalign,
+					     iov_full_unalign };
+	struct iovec vecs_maxsegs[] = { iov_maxbvec, iov_maxbvec, iov_maxbvec,
+				      iov_maxbvec, iov_maxbvec, iov_maxbvec};
+
+	ret = test_sequence(bd, 1, (struct work[]) {
+			{ &iov_page, 1 },
+			{ vecs, 1 }});
+	if (ret) {
+		fprintf(stderr, "seq failure: basic aligned, %i\n", ret);
+		exit(1);
+	}
+
+	ret = test_sequence(bd, 2, (struct work[]) {
+			{ vecs, 1 },
+			{ vecs, 1 }});
+	if (ret) {
+		fprintf(stderr, "seq failure: basic aligned, %i\n", ret);
+		exit(1);
+	}
+
+	ret = test_sequence(bd, 2, (struct work[]) {
+			{ vecs + 1, 1 },
+			{ vecs + 1, 1 }});
+	if (ret) {
+		fprintf(stderr, "seq failure: multi page buffer, %i\n", ret);
+		exit(1);
+	}
+
+	ret = test_sequence(bd, 2, (struct work[]) {
+			{ vecs + 2, 1 },
+			{ vecs + 2, 1 }});
+	if (ret) {
+		fprintf(stderr, "seq failure: misaligned buffer, %i\n", ret);
+		exit(1);
+	}
+
+	ret = test_sequence(bd, 2, (struct work[]) {
+			{ vecs + 3, 1 },
+			{ vecs + 3, 1 }});
+	if (ret) {
+		fprintf(stderr, "seq failure: misaligned multipage buffer, %i\n", ret);
+		exit(1);
+	}
+
+	ret = test_sequence(bd, 2, (struct work[]) {
+			{ vecs, 1 },
+			{ vecs + 3, 1 }});
+	if (ret) {
+		fprintf(stderr, "seq failure: realloc + increase bvec, %i\n", ret);
+		exit(1);
+	}
+
+	ret = test_sequence(bd, 2, (struct work[]) {
+			{ vecs + 3, 1 },
+			{ vecs + 0, 1 }});
+	if (ret) {
+		fprintf(stderr, "seq failure: realloc + decrease bvec, %i\n", ret);
+		exit(1);
+	}
+
+	ret = test_sequence(bd, 2, (struct work[]) {
+			{ vecs, 4 },
+			{ vecs, 4 }});
+	if (ret) {
+		fprintf(stderr, "seq failure: multisegment, %i\n", ret);
+		exit(1);
+	}
+
+	ret = test_sequence(bd, 3, (struct work[]) {
+			{ vecs, 2 },
+			{ vecs, 3 },
+			{ vecs, 4 }});
+	if (ret) {
+		fprintf(stderr, "seq failure: multisegment 2, %i\n", ret);
+		exit(1);
+	}
+
+	ret = test_sequence(bd, 3, (struct work[]) {
+			{ vecs_basic, 1 },
+			{ vecs_basic, 2 },
+			{ vecs_basic, 3 }});
+	if (ret) {
+		fprintf(stderr, "seq failure: increase iovec, %i\n", ret);
+		exit(1);
+	}
+
+	ret = test_sequence(bd, 3, (struct work[]) {
+			{ vecs_basic, 3 },
+			{ vecs_basic, 2 },
+			{ vecs_basic, 1 }});
+	if (ret) {
+		fprintf(stderr, "seq failure: decrease iovec, %i\n", ret);
+		exit(1);
+	}
+
+	ret = test_sequence(bd, 3, (struct work[]) {
+			{ &iov_right1, 1 },
+			{ &iov_right2, 1 },
+			{ &iov_right1, 1 }});
+	if (ret) {
+		fprintf(stderr, "seq failure: right aligned, %i\n", ret);
+		exit(1);
+	}
+
+	ret = test_sequence(bd, 3, (struct work[]) {
+			{ vecs_full, 1 },
+			{ vecs_full, 1 },
+			{ vecs_full, 3 }});
+	if (ret) {
+		fprintf(stderr, "seq failure: full size, %i\n", ret);
+		exit(1);
+	}
+
+	ret = test_sequence(bd, 3, (struct work[]) {
+			{ vecs_full_unalign, 1 },
+			{ vecs_full_unalign, 1 },
+			{ vecs_full_unalign, 3 }});
+	if (ret) {
+		fprintf(stderr, "seq failure: full size unsigned, %i\n", ret);
+		exit(1);
+	}
+
+	ret = test_sequence(bd, 3, (struct work[]) {
+			{ vecs_maxsegs, 1 },
+			{ vecs_maxsegs, 2 },
+			{ vecs_maxsegs, 3 }});
+	if (ret) {
+		fprintf(stderr, "seq failure: overestimated segments, %i\n", ret);
+		exit(1);
+	}
+
+	ret = test_sequence(bd, 3, (struct work[]) {
+			{ vecs_maxsegs, 6 },
+			{ vecs_maxsegs, 6 },
+			{ vecs_maxsegs, 6 }});
+	if (ret) {
+		fprintf(stderr, "seq failure: overestimated segments 2, %i\n", ret);
+		exit(1);
+	}
+}
+
+static void test_fail(struct buf_desc *bd)
+{
+	int ret, cqe_ret;
+	void *p = bd->buf_wr;
+	struct iovec iov_0len = { .iov_base = p, .iov_len = 0 };
+	struct iovec iov_0buf = { .iov_base = 0, .iov_len = 1 };
+	struct iovec iov_inv = { .iov_base = (void *)-1U, .iov_len = 1 };
+	struct iovec iov_under = { .iov_base = p - 1, .iov_len = 1 };
+	struct iovec iov_over = { .iov_base = p + bd->size, .iov_len = 1 };
+	struct iovec vecs_0[] = { iov_0len, iov_0len, iov_0len, iov_0len,
+				   iov_0len, iov_0len, iov_0len, iov_0len };
+
+	reinit_ring(bd);
+	ret = test_vec(bd, vecs_0, 8, true, &cqe_ret);
+	if (ret || cqe_ret > 0) {
+		fprintf(stderr, "0 length test failed %i, cqe %i\n",
+				ret, cqe_ret);
+		exit(1);
+	}
+
+	reinit_ring(bd);
+	ret = test_vec(bd, &iov_0buf, 1, true, &cqe_ret);
+	if (ret || cqe_ret >= 0) {
+		fprintf(stderr, "0 buf test failed %i, cqe %i\n",
+				ret, cqe_ret);
+		exit(1);
+	}
+
+	reinit_ring(bd);
+	ret = test_vec(bd, &iov_inv, 1, true, &cqe_ret);
+	if (ret || cqe_ret >= 0) {
+		fprintf(stderr, "inv buf test failed %i, cqe %i\n",
+				ret, cqe_ret);
+		exit(1);
+	}
+
+	reinit_ring(bd);
+	ret = test_vec(bd, &iov_under, 1, true, &cqe_ret);
+	if (ret || cqe_ret >= 0) {
+		fprintf(stderr, "inv buf underflow failed %i, cqe %i\n",
+				ret, cqe_ret);
+		exit(1);
+	}
+
+	reinit_ring(bd);
+	ret = test_vec(bd, &iov_over, 1, true, &cqe_ret);
+	if (ret || cqe_ret >= 0) {
+		fprintf(stderr, "inv buf overflow failed %i, cqe %i\n",
+				ret, cqe_ret);
+		exit(1);
+	}
+}
+
+int main(void)
+{
+	struct buf_desc bd = {};
+	int i = 0;
+
+	page_sz = sysconf(_SC_PAGESIZE);
+
+	probe_support();
+	if (!has_regvec) {
+		printf("doesn't support registered vector ops, skip\n");
+		return 0;
+	}
+
+	init_buffers(&bd, page_sz * 32);
+	bd.fixed = true;
+
+	for (i = 0; i < 2; i++) {
+		bool rw = i & 1;
+
+		bd.rw = rw;
+
+		test_basic(&bd);
+		test_fail(&bd);
+	}
+
+	io_uring_queue_exit(&bd.ring);
+	return 0;
+}
-- 
2.48.1

Re: [PATCH liburing 1/4] Add vectored registered buffer req init helpers

[Caleb Sander Mateos]

On Fri, Mar 7, 2025 at 8:22 AM Pavel Begunkov <asml.silence@gmail.com> wrote:
>
> Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
> ---
>  src/include/liburing.h | 31 +++++++++++++++++++++++++++++++
>  1 file changed, 31 insertions(+)
>
> diff --git a/src/include/liburing.h b/src/include/liburing.h
> index d162d0e6..e71551ed 100644
> --- a/src/include/liburing.h
> +++ b/src/include/liburing.h
> @@ -556,6 +556,16 @@ IOURINGINLINE void io_uring_prep_read_fixed(struct io_uring_sqe *sqe, int fd,
>         sqe->buf_index = (__u16) buf_index;
>  }
>
> +IOURINGINLINE void io_uring_prep_readv_fixed(struct io_uring_sqe *sqe, int fd,
> +                                            const struct iovec *iovecs,
> +                                            unsigned nr_vecs, __u64 offset,
> +                                            int flags, int buf_index)
> +{
> +       io_uring_prep_readv2(sqe, fd, iovecs, nr_vecs, offset, flags);
> +       sqe->opcode = IORING_OP_WRITE_FIXED;

Presumably should be IORING_OP_READV_FIXED? You'll probably need to
copy the UAPI header changes to liburing.

> +       sqe->buf_index = (__u16)buf_index;
> +}
> +
>  IOURINGINLINE void io_uring_prep_writev(struct io_uring_sqe *sqe, int fd,
>                                         const struct iovec *iovecs,
>                                         unsigned nr_vecs, __u64 offset)
> @@ -580,6 +590,16 @@ IOURINGINLINE void io_uring_prep_write_fixed(struct io_uring_sqe *sqe, int fd,
>         sqe->buf_index = (__u16) buf_index;
>  }
>
> +IOURINGINLINE void io_uring_prep_writev2_fixed(struct io_uring_sqe *sqe, int fd,
> +                                      const struct iovec *iovecs,
> +                                      unsigned nr_vecs, __u64 offset,
> +                                      int flags, int buf_index)
> +{
> +       io_uring_prep_writev2(sqe, fd, iovecs, nr_vecs, offset, flags);
> +       sqe->opcode = IORING_OP_WRITE_FIXED;

IORING_OP_WRITEV_FIXED?

Best,
Caleb

> +       sqe->buf_index = (__u16)buf_index;
> +}
> +
>  IOURINGINLINE void io_uring_prep_recvmsg(struct io_uring_sqe *sqe, int fd,
>                                          struct msghdr *msg, unsigned flags)
>  {
> @@ -964,6 +984,17 @@ IOURINGINLINE void io_uring_prep_sendmsg_zc(struct io_uring_sqe *sqe, int fd,
>         sqe->opcode = IORING_OP_SENDMSG_ZC;
>  }
>
> +IOURINGINLINE void io_uring_prep_sendmsg_zc_fixed(struct io_uring_sqe *sqe,
> +                                               int fd,
> +                                               const struct msghdr *msg,
> +                                               unsigned flags,
> +                                               unsigned buf_index)
> +{
> +       io_uring_prep_sendmsg_zc(sqe, fd, msg, flags);
> +       sqe->ioprio |= IORING_RECVSEND_FIXED_BUF;
> +       sqe->buf_index = buf_index;
> +}
> +
>  IOURINGINLINE void io_uring_prep_recv(struct io_uring_sqe *sqe, int sockfd,
>                                       void *buf, size_t len, int flags)
>  {
> --
> 2.48.1
>
>

Re: [PATCH liburing 1/4] Add vectored registered buffer req init helpers

[Pavel Begunkov]

On 3/7/25 17:10, Caleb Sander Mateos wrote:
> On Fri, Mar 7, 2025 at 8:22 AM Pavel Begunkov <asml.silence@gmail.com> wrote:
...
>> +IOURINGINLINE void io_uring_prep_writev2_fixed(struct io_uring_sqe *sqe, int fd,
>> +                                      const struct iovec *iovecs,
>> +                                      unsigned nr_vecs, __u64 offset,
>> +                                      int flags, int buf_index)
>> +{
>> +       io_uring_prep_writev2(sqe, fd, iovecs, nr_vecs, offset, flags);
>> +       sqe->opcode = IORING_OP_WRITE_FIXED;
> 
> IORING_OP_WRITEV_FIXED?

Good catch, that came from a prototype where it was based on
that opcode, I should just use the helper in the test.

-- 
Pavel Begunkov

Re: [PATCH liburing 1/4] Add vectored registered buffer req init helpers

[Jens Axboe]

On 3/7/25 10:26 AM, Pavel Begunkov wrote:
> On 3/7/25 17:10, Caleb Sander Mateos wrote:
>> On Fri, Mar 7, 2025 at 8:22 AM Pavel Begunkov <asml.silence@gmail.com> wrote:
> ...
>>> +IOURINGINLINE void io_uring_prep_writev2_fixed(struct io_uring_sqe *sqe, int fd,
>>> +                                      const struct iovec *iovecs,
>>> +                                      unsigned nr_vecs, __u64 offset,
>>> +                                      int flags, int buf_index)
>>> +{
>>> +       io_uring_prep_writev2(sqe, fd, iovecs, nr_vecs, offset, flags);
>>> +       sqe->opcode = IORING_OP_WRITE_FIXED;
>>
>> IORING_OP_WRITEV_FIXED?
> 
> Good catch, that came from a prototype where it was based on
> that opcode, I should just use the helper in the test.

Both also need an ffi addition in the map.

-- 
Jens Axboe