Recent changes (master)

[Jens Axboe <axboe@kernel.dk>]

The following changes since commit a7abc9fb769596d3bbf6d779e99d1cb8c1fcd49b:

  t/io_uring: add support for registered files (2019-01-10 22:29:27 -0700)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to e0abe38815e1c8cf7a319c6fbd0b1d60691db3d5:

  t/io_uring: only set IORING_ENTER_GETEVENTS when actively reaping (2019-01-11 14:40:16 -0700)

----------------------------------------------------------------
Jens Axboe (4):
      io_uring: update to newer API
      t/io_uring: remember to set p->sq_thread_cpu
      engines/io_uring: remove unused ld->io_us array
      t/io_uring: only set IORING_ENTER_GETEVENTS when actively reaping

 engines/io_uring.c | 27 +++++++++++++++------------
 os/io_uring.h      | 31 ++++++++++++++++++++-----------
 t/io_uring.c       | 47 ++++++++++++++++++++++++++++-------------------
 3 files changed, 63 insertions(+), 42 deletions(-)

---

Diff of recent changes:

diff --git a/engines/io_uring.c b/engines/io_uring.c
index 7591190a..ca3e157f 100644
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -48,7 +48,6 @@ struct ioring_mmap {
 struct ioring_data {
 	int ring_fd;
 
-	struct io_u **io_us;
 	struct io_u **io_u_index;
 
 	struct io_sq_ring sq_ring;
@@ -150,25 +149,31 @@ static int fio_ioring_prep(struct thread_data *td, struct io_u *io_u)
 	sqe->buf_index = 0;
 
 	if (io_u->ddir == DDIR_READ || io_u->ddir == DDIR_WRITE) {
-		if (io_u->ddir == DDIR_READ)
-			sqe->opcode = IORING_OP_READV;
-		else
-			sqe->opcode = IORING_OP_WRITEV;
-
 		if (o->fixedbufs) {
-			sqe->flags |= IOSQE_FIXED_BUFFER;
+			if (io_u->ddir == DDIR_READ)
+				sqe->opcode = IORING_OP_READ_FIXED;
+			else
+				sqe->opcode = IORING_OP_WRITE_FIXED;
 			sqe->addr = io_u->xfer_buf;
 			sqe->len = io_u->xfer_buflen;
 			sqe->buf_index = io_u->index;
 		} else {
+			if (io_u->ddir == DDIR_READ)
+				sqe->opcode = IORING_OP_READV;
+			else
+				sqe->opcode = IORING_OP_WRITEV;
 			sqe->addr = &ld->iovecs[io_u->index];
 			sqe->len = 1;
 		}
 		sqe->off = io_u->offset;
-	} else if (ddir_sync(io_u->ddir))
+	} else if (ddir_sync(io_u->ddir)) {
+		sqe->fsync_flags = 0;
+		if (io_u->ddir == DDIR_DATASYNC)
+			sqe->fsync_flags |= IORING_FSYNC_DATASYNC;
 		sqe->opcode = IORING_OP_FSYNC;
+	}
 
-	sqe->data = (unsigned long) io_u;
+	sqe->user_data = (unsigned long) io_u;
 	return 0;
 }
 
@@ -182,7 +187,7 @@ static struct io_u *fio_ioring_event(struct thread_data *td, int event)
 	index = (event + ld->cq_ring_off) & ld->cq_ring_mask;
 
 	cqe = &ld->cq_ring.cqes[index];
-	io_u = (struct io_u *) cqe->data;
+	io_u = (struct io_u *) cqe->user_data;
 
 	if (cqe->res != io_u->xfer_buflen) {
 		if (cqe->res > io_u->xfer_buflen)
@@ -390,7 +395,6 @@ static void fio_ioring_cleanup(struct thread_data *td)
 			fio_ioring_unmap(ld);
 
 		free(ld->io_u_index);
-		free(ld->io_us);
 		free(ld->iovecs);
 		free(ld);
 	}
@@ -526,7 +530,6 @@ static int fio_ioring_init(struct thread_data *td)
 
 	/* io_u index */
 	ld->io_u_index = calloc(td->o.iodepth, sizeof(struct io_u *));
-	ld->io_us = calloc(td->o.iodepth, sizeof(struct io_u *));
 	ld->iovecs = calloc(td->o.iodepth, sizeof(struct iovec));
 
 	td->io_ops_data = ld;
diff --git a/os/io_uring.h b/os/io_uring.h
index e1d3df0b..74370aed 100644
--- a/os/io_uring.h
+++ b/os/io_uring.h
@@ -16,7 +16,7 @@
  */
 struct io_uring_sqe {
 	__u8	opcode;		/* type of operation for this sqe */
-	__u8	flags;		/* IOSQE_ flags below */
+	__u8	flags;		/* IOSQE_ flags */
 	__u16	ioprio;		/* ioprio for the request */
 	__s32	fd;		/* file descriptor to do IO on */
 	__u64	off;		/* offset into file */
@@ -27,18 +27,18 @@ struct io_uring_sqe {
 	__u32	len;		/* buffer size or number of iovecs */
 	union {
 		__kernel_rwf_t	rw_flags;
-		__u32		__resv;
+		__u32		fsync_flags;
 	};
 	__u16	buf_index;	/* index into fixed buffers, if used */
-	__u16	__pad2[3];
-	__u64	data;		/* data to be passed back at completion time */
+	__u16	__pad2;
+	__u32	__pad3;
+	__u64	user_data;	/* data to be passed back at completion time */
 };
 
 /*
  * sqe->flags
  */
-#define IOSQE_FIXED_BUFFER	(1 << 0)	/* use fixed buffer */
-#define IOSQE_FIXED_FILE	(1 << 1)	/* use fixed fileset */
+#define IOSQE_FIXED_FILE	(1 << 0)	/* use fixed fileset */
 
 /*
  * io_uring_setup() flags
@@ -50,13 +50,19 @@ struct io_uring_sqe {
 #define IORING_OP_READV		1
 #define IORING_OP_WRITEV	2
 #define IORING_OP_FSYNC		3
-#define IORING_OP_FDSYNC	4
+#define IORING_OP_READ_FIXED	4
+#define IORING_OP_WRITE_FIXED	5
+
+/*
+ * sqe->fsync_flags
+ */
+#define IORING_FSYNC_DATASYNC	(1 << 0)
 
 /*
  * IO completion data structure (Completion Queue Entry)
  */
 struct io_uring_cqe {
-	__u64	data;		/* sqe->data submission passed back */
+	__u64	user_data;	/* sqe->data submission passed back */
 	__s32	res;		/* result code for this event */
 	__u32	flags;
 };
@@ -87,6 +93,9 @@ struct io_sqring_offsets {
 	__u32 resv[3];
 };
 
+/*
+ * sq_ring->flags
+ */
 #define IORING_SQ_NEED_WAKEUP	(1 << 0) /* needs io_uring_enter wakeup */
 
 struct io_cqring_offsets {
@@ -127,12 +136,12 @@ struct io_uring_params {
 
 struct io_uring_register_buffers {
 	struct iovec *iovecs;
-	unsigned nr_iovecs;
+	__u32 nr_iovecs;
 };
 
 struct io_uring_register_files {
-	int *fds;
-	unsigned nr_fds;
+	__s32 *fds;
+	__u32 nr_fds;
 };
 
 #endif
diff --git a/t/io_uring.c b/t/io_uring.c
index 0461329b..d4160c3d 100644
--- a/t/io_uring.c
+++ b/t/io_uring.c
@@ -59,7 +59,8 @@ static unsigned sq_ring_mask, cq_ring_mask;
 struct file {
 	unsigned long max_blocks;
 	unsigned pending_ios;
-	int fd;
+	int real_fd;
+	int fixed_fd;
 };
 
 struct submitter {
@@ -77,6 +78,8 @@ struct submitter {
 	unsigned long cachehit, cachemiss;
 	volatile int finish;
 
+	__s32 *fds;
+
 	struct file files[MAX_FDS];
 	unsigned nr_files;
 	unsigned cur_file;
@@ -105,17 +108,18 @@ static int io_uring_register_buffers(struct submitter *s)
 static int io_uring_register_files(struct submitter *s)
 {
 	struct io_uring_register_files reg;
-	int i, ret;
+	int i;
 
-	reg.fds = calloc(s->nr_files, sizeof(int));
-	for (i = 0; i < s->nr_files; i++)
-		reg.fds[i] = s->files[i].fd;
+	s->fds = calloc(s->nr_files, sizeof(__s32));
+	for (i = 0; i < s->nr_files; i++) {
+		s->fds[i] = s->files[i].real_fd;
+		s->files[i].fixed_fd = i;
+	}
+	reg.fds = s->fds;
 	reg.nr_fds = s->nr_files;
 
-	ret = syscall(__NR_sys_io_uring_register, s->ring_fd,
+	return syscall(__NR_sys_io_uring_register, s->ring_fd,
 			IORING_REGISTER_FILES, &reg);
-	free(reg.fds);
-	return ret;
 }
 
 static int io_uring_setup(unsigned entries, struct io_uring_params *p)
@@ -163,21 +167,21 @@ static void init_io(struct submitter *s, unsigned index)
 	offset = (r % (f->max_blocks - 1)) * BS;
 
 	sqe->flags = IOSQE_FIXED_FILE;
-	sqe->opcode = IORING_OP_READV;
 	if (fixedbufs) {
+		sqe->opcode = IORING_OP_READ_FIXED;
 		sqe->addr = s->iovecs[index].iov_base;
 		sqe->len = BS;
 		sqe->buf_index = index;
-		sqe->flags |= IOSQE_FIXED_BUFFER;
 	} else {
+		sqe->opcode = IORING_OP_READV;
 		sqe->addr = &s->iovecs[index];
 		sqe->len = 1;
 		sqe->buf_index = 0;
 	}
 	sqe->ioprio = 0;
-	sqe->fd = f->fd;
+	sqe->fd = f->fixed_fd;
 	sqe->off = offset;
-	sqe->data = (unsigned long) f;
+	sqe->user_data = (unsigned long) f;
 }
 
 static int prep_more_ios(struct submitter *s, int max_ios)
@@ -212,12 +216,12 @@ static int get_file_size(struct file *f)
 {
 	struct stat st;
 
-	if (fstat(f->fd, &st) < 0)
+	if (fstat(f->real_fd, &st) < 0)
 		return -1;
 	if (S_ISBLK(st.st_mode)) {
 		unsigned long long bytes;
 
-		if (ioctl(f->fd, BLKGETSIZE64, &bytes) != 0)
+		if (ioctl(f->real_fd, BLKGETSIZE64, &bytes) != 0)
 			return -1;
 
 		f->max_blocks = bytes / BS;
@@ -244,7 +248,7 @@ static int reap_events(struct submitter *s)
 		if (head == *ring->tail)
 			break;
 		cqe = &ring->cqes[head & cq_ring_mask];
-		f = (struct file *) cqe->data;
+		f = (struct file *) cqe->user_data;
 		f->pending_ios--;
 		if (cqe->res != BS) {
 			printf("io: unexpected ret=%d\n", cqe->res);
@@ -296,8 +300,11 @@ submit:
 		 * poll, or if IORING_SQ_NEED_WAKEUP is set.
 		 */
 		if (!sq_thread_poll || (*ring->flags & IORING_SQ_NEED_WAKEUP)) {
-			ret = io_uring_enter(s, to_submit, to_wait,
-						IORING_ENTER_GETEVENTS);
+			unsigned flags = 0;
+
+			if (to_wait)
+				flags = IORING_ENTER_GETEVENTS;
+			ret = io_uring_enter(s, to_submit, to_wait, flags);
 			s->calls++;
 		}
 
@@ -382,8 +389,10 @@ static int setup_ring(struct submitter *s)
 		p.flags |= IORING_SETUP_IOPOLL;
 	if (sq_thread_poll) {
 		p.flags |= IORING_SETUP_SQPOLL;
-		if (sq_thread_cpu != -1)
+		if (sq_thread_cpu != -1) {
 			p.flags |= IORING_SETUP_SQ_AFF;
+			p.sq_thread_cpu = sq_thread_cpu;
+		}
 	}
 
 	fd = io_uring_setup(DEPTH, &p);
@@ -463,7 +472,7 @@ int main(int argc, char *argv[])
 			perror("open");
 			return 1;
 		}
-		f->fd = fd;
+		f->real_fd = fd;
 		if (get_file_size(f)) {
 			printf("failed getting size of device/file\n");
 			return 1;