Recent changes (master)

[Jens Axboe <axboe@kernel.dk>]

The following changes since commit 93d1811ce6a510d36d61381840283d1ae9933b37:

  t/io_uring: pick next file if we're over the limti (2019-01-15 05:57:54 -0700)

are available in the Git repository at:

  git://git.kernel.dk/fio.git master

for you to fetch changes up to 2b9415ddc260726c3ea9ae3436826f9181811143:

  engines/io_uring: ensure sqe stores are ordered SQ ring tail update (2019-01-15 22:06:05 -0700)

----------------------------------------------------------------
Jens Axboe (7):
      t/io_uring: print file depths
      t/io_uring: wait if we're at queue limit
      t/io_uring: terminate buf[] file depth string
      t/io_uring: fixes
      x86-64: correct read/write barriers
      t/io_uring: use fio provided memory barriers
      engines/io_uring: ensure sqe stores are ordered SQ ring tail update

 arch/arch-x86_64.h |  4 ++--
 engines/io_uring.c |  2 ++
 t/io_uring.c       | 51 ++++++++++++++++++++++++++++++++++++---------------
 3 files changed, 40 insertions(+), 17 deletions(-)

---

Diff of recent changes:

diff --git a/arch/arch-x86_64.h b/arch/arch-x86_64.h
index 665c6b04..0cd21b8f 100644
--- a/arch/arch-x86_64.h
+++ b/arch/arch-x86_64.h
@@ -27,8 +27,8 @@ static inline void do_cpuid(unsigned int *eax, unsigned int *ebx,
 #define	FIO_HUGE_PAGE		2097152
 
 #define nop		__asm__ __volatile__("rep;nop": : :"memory")
-#define read_barrier()	__asm__ __volatile__("lfence":::"memory")
-#define write_barrier()	__asm__ __volatile__("sfence":::"memory")
+#define read_barrier()	__asm__ __volatile__("":::"memory")
+#define write_barrier()	__asm__ __volatile__("":::"memory")
 
 static inline unsigned long arch_ffz(unsigned long bitmask)
 {
diff --git a/engines/io_uring.c b/engines/io_uring.c
index 56af8d71..8c5d9deb 100644
--- a/engines/io_uring.c
+++ b/engines/io_uring.c
@@ -289,6 +289,8 @@ static enum fio_q_status fio_ioring_queue(struct thread_data *td,
 	if (next_tail == *ring->head)
 		return FIO_Q_BUSY;
 
+	/* ensure sqe stores are ordered with tail update */
+	write_barrier();
 	ring->array[tail & ld->sq_ring_mask] = io_u->index;
 	*ring->tail = next_tail;
 	write_barrier();
diff --git a/t/io_uring.c b/t/io_uring.c
index b5f1e094..ef5d52d1 100644
--- a/t/io_uring.c
+++ b/t/io_uring.c
@@ -24,8 +24,6 @@
 #include "../lib/types.h"
 #include "../os/linux/io_uring.h"
 
-#define barrier()	__asm__ __volatile__("": : :"memory")
-
 #define min(a, b)		((a < b) ? (a) : (b))
 
 struct io_sq_ring {
@@ -47,8 +45,8 @@ struct io_cq_ring {
 
 #define DEPTH			128
 
-#define BATCH_SUBMIT		64
-#define BATCH_COMPLETE		64
+#define BATCH_SUBMIT		32
+#define BATCH_COMPLETE		32
 
 #define BS			4096
 
@@ -211,7 +209,7 @@ static int prep_more_ios(struct submitter *s, int max_ios)
 	next_tail = tail = *ring->tail;
 	do {
 		next_tail++;
-		barrier();
+		read_barrier();
 		if (next_tail == *ring->head)
 			break;
 
@@ -224,9 +222,9 @@ static int prep_more_ios(struct submitter *s, int max_ios)
 
 	if (*ring->tail != tail) {
 		/* order tail store with writes to sqes above */
-		barrier();
+		write_barrier();
 		*ring->tail = tail;
-		barrier();
+		write_barrier();
 	}
 	return prepped;
 }
@@ -263,7 +261,7 @@ static int reap_events(struct submitter *s)
 	do {
 		struct file *f;
 
-		barrier();
+		read_barrier();
 		if (head == *ring->tail)
 			break;
 		cqe = &ring->cqes[head & cq_ring_mask];
@@ -285,7 +283,7 @@ static int reap_events(struct submitter *s)
 
 	s->inflight -= reaped;
 	*ring->head = head;
-	barrier();
+	write_barrier();
 	return reaped;
 }
 
@@ -311,7 +309,7 @@ static void *submitter_fn(void *data)
 submit_more:
 		to_submit = prepped;
 submit:
-		if (to_submit && (s->inflight + to_submit < DEPTH))
+		if (to_submit && (s->inflight + to_submit <= DEPTH))
 			to_wait = 0;
 		else
 			to_wait = min(s->inflight + to_submit, BATCH_COMPLETE);
@@ -338,9 +336,10 @@ submit:
 		do {
 			int r;
 			r = reap_events(s);
-			if (r == -1)
+			if (r == -1) {
+				s->finish = 1;
 				break;
-			else if (r > 0)
+			} else if (r > 0)
 				this_reap += r;
 		} while (sq_thread_poll && this_reap < to_wait);
 		s->reaps += this_reap;
@@ -406,7 +405,7 @@ static int setup_ring(struct submitter *s)
 
 	memset(&p, 0, sizeof(p));
 
-	if (polled)
+	if (polled && !do_nop)
 		p.flags |= IORING_SETUP_IOPOLL;
 	if (sq_thread_poll) {
 		p.flags |= IORING_SETUP_SQPOLL;
@@ -469,11 +468,30 @@ static int setup_ring(struct submitter *s)
 	return 0;
 }
 
+static void file_depths(char *buf)
+{
+	struct submitter *s = &submitters[0];
+	char *p;
+	int i;
+
+	buf[0] = '\0';
+	p = buf;
+	for (i = 0; i < s->nr_files; i++) {
+		struct file *f = &s->files[i];
+
+		if (i + 1 == s->nr_files)
+			p += sprintf(p, "%d", f->pending_ios);
+		else
+			p += sprintf(p, "%d, ", f->pending_ios);
+	}
+}
+
 int main(int argc, char *argv[])
 {
 	struct submitter *s = &submitters[0];
 	unsigned long done, calls, reap, cache_hit, cache_miss;
 	int err, i, flags, fd;
+	char *fdepths;
 	void *ret;
 
 	if (!do_nop && argc < 2) {
@@ -544,6 +562,7 @@ int main(int argc, char *argv[])
 
 	pthread_create(&s->thread, NULL, submitter_fn, s);
 
+	fdepths = malloc(8 * s->nr_files);
 	cache_hit = cache_miss = reap = calls = done = 0;
 	do {
 		unsigned long this_done = 0;
@@ -573,9 +592,10 @@ int main(int argc, char *argv[])
 			ipc = (this_reap - reap) / (this_call - calls);
 		} else
 			rpc = ipc = -1;
-		printf("IOPS=%lu, IOS/call=%ld/%ld, inflight=%u (head=%u tail=%u), Cachehit=%0.2f%%\n",
+		file_depths(fdepths);
+		printf("IOPS=%lu, IOS/call=%ld/%ld, inflight=%u (%s), Cachehit=%0.2f%%\n",
 				this_done - done, rpc, ipc, s->inflight,
-				*s->cq_ring.head, *s->cq_ring.tail, hit);
+				fdepths, hit);
 		done = this_done;
 		calls = this_call;
 		reap = this_reap;
@@ -585,5 +605,6 @@ int main(int argc, char *argv[])
 
 	pthread_join(s->thread, &ret);
 	close(s->ring_fd);
+	free(fdepths);
 	return 0;
 }