[PATCH v2 12/14] fuse: add pinned payload buffers capability for io-uring buffer rings

[Joanne Koong <joannelkoong@gmail.com>]

Extend the buffer ring pinning capability to payload buffers via the
FUSE_URING_PINNED_BUFFERS flag. When set alongside FUSE_URING_BUFRING,
the kernel pins and vmaps the payload buffer region during queue setup.

With pinned payloads, the kernel uses direct memcpy for all payload
buffer copies, avoiding the per-request overhead of pinning/unpinning
user pages and translating virtual addresses. This is particularly
beneficial for large payload copies.

As with pinned headers, buffers must be page-aligned. Pinned pages are
accounted against RLIMIT_MEMLOCK (bypassed with CAP_IPC_LOCK) and
unpinned in process context during connection abort.

In benchmarks using passthrough_hp on a high-performance NVMe-backed
system, pinned headers and pinned payload buffers showed around a 10%
throughput improvement for direct randreads (~2150 MiB/s to ~2400
MiB/s), a 4% improvement for direct sequential reads (~2510 MiB/s to
~2620 MiB/s), a 8% improvement for buffered randreads (~2100 MiB/s to
~2280 MiB/s), and a 6% improvement for buffered sequential reads (~2500
MiB/s to ~2670 MiB/s).

Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
---
 fs/fuse/dev_uring.c       | 54 +++++++++++++++++++++++++++++++++------
 fs/fuse/dev_uring_i.h     |  4 +++
 include/uapi/linux/fuse.h |  2 ++
 3 files changed, 52 insertions(+), 8 deletions(-)

diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c
index 79736b02cf9f..06d3d8dc1c82 100644
--- a/fs/fuse/dev_uring.c
+++ b/fs/fuse/dev_uring.c
@@ -52,6 +52,11 @@ static inline bool bufring_pinned_headers(struct fuse_ring_queue *queue)
 	return queue->bufring->use_pinned_headers;
 }
 
+static inline bool bufring_pinned_buffers(struct fuse_ring_queue *queue)
+{
+	return queue->bufring->use_pinned_buffers;
+}
+
 static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd,
 				   struct fuse_ring_ent *ring_ent)
 {
@@ -235,6 +240,11 @@ static void fuse_uring_bufring_unpin(struct fuse_ring_queue *queue)
 		fuse_bufring_unpin_mem(&br->pinned_headers);
 		br->use_pinned_headers = false;
 	}
+
+	if (bufring_pinned_buffers(queue)) {
+		fuse_bufring_unpin_mem(&br->pinned_bufs);
+		br->use_pinned_buffers = false;
+	}
 }
 
 void fuse_uring_destruct(struct fuse_conn *fc)
@@ -474,6 +484,7 @@ static int fuse_uring_bufring_setup(struct io_uring_cmd *cmd,
 	unsigned int buf_size = READ_ONCE(cmd_req->init.buf_size);
 	struct iovec iov[FUSE_URING_IOV_SEGS];
 	bool pinned_headers = init_flags & FUSE_URING_PINNED_HEADERS;
+	bool pinned_bufs = init_flags & FUSE_URING_PINNED_BUFFERS;
 	void __user *payload, *headers;
 	size_t headers_size, payload_size, ring_size;
 	struct fuse_bufring *br;
@@ -523,7 +534,22 @@ static int fuse_uring_bufring_setup(struct io_uring_cmd *cmd,
 		br->headers = headers;
 	}
 
-	payload_addr = (uintptr_t)payload;
+	if (pinned_bufs) {
+		err = fuse_bufring_pin_mem(&br->pinned_bufs, payload,
+					   payload_size);
+		if (err) {
+			if (pinned_headers)
+				fuse_bufring_unpin_mem(&br->pinned_headers);
+			kfree(br);
+			return err;
+		}
+		br->use_pinned_buffers = true;
+	}
+
+	if (pinned_bufs)
+		payload_addr = (uintptr_t)br->pinned_bufs.addr;
+	else
+		payload_addr = (uintptr_t)payload;
 
 	/* populate the ring buffer */
 	for (i = 0; i < nr_bufs; i++, payload_addr += buf_size) {
@@ -553,6 +579,7 @@ static bool queue_init_flags_consistent(struct fuse_ring_queue *queue,
 {
 	bool bufring = init_flags & FUSE_URING_BUFRING;
 	bool pinned_headers = init_flags & FUSE_URING_PINNED_HEADERS;
+	bool pinned_bufs = init_flags & FUSE_URING_PINNED_BUFFERS;
 
 	if (bufring_enabled(queue) != bufring)
 		return false;
@@ -560,7 +587,8 @@ static bool queue_init_flags_consistent(struct fuse_ring_queue *queue,
 	if (!bufring)
 		return true;
 
-	return bufring_pinned_headers(queue) == pinned_headers;
+	return bufring_pinned_headers(queue) == pinned_headers &&
+		bufring_pinned_buffers(queue) == pinned_bufs;
 }
 
 static struct fuse_ring_queue *
@@ -1011,13 +1039,15 @@ static int setup_fuse_copy_state(struct fuse_copy_state *cs,
 				 struct fuse_ring_ent *ent, int dir,
 				 struct iov_iter *iter)
 {
-	void __user *payload;
+	void __user *payload = NULL;
+	bool use_bufring = bufring_enabled(ent->queue);
+	bool pinned_buffers = use_bufring && bufring_pinned_buffers(ent->queue);
 	int err;
 
-	if (bufring_enabled(ent->queue))
-		payload = (void __user *)ent->payload_buf.addr;
-	else
+	if (!use_bufring)
 		payload = ent->payload;
+	else if (!pinned_buffers)
+		payload = (void __user *)ent->payload_buf.addr;
 
 	if (payload) {
 		err = import_ubuf(dir, payload, ring->max_payload_sz, iter);
@@ -1029,6 +1059,12 @@ static int setup_fuse_copy_state(struct fuse_copy_state *cs,
 
 	fuse_copy_init(cs, dir == ITER_DEST, iter);
 
+	if (pinned_buffers) {
+		cs->is_kaddr = true;
+		cs->kaddr = (void *)ent->payload_buf.addr;
+		cs->len = ent->payload_buf.len;
+	}
+
 	cs->is_uring = true;
 	cs->req = req;
 
@@ -1608,11 +1644,13 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd,
 static bool init_flags_valid(u64 init_flags)
 {
 	u64 valid_flags =
-		FUSE_URING_BUFRING | FUSE_URING_PINNED_HEADERS;
+		FUSE_URING_BUFRING | FUSE_URING_PINNED_HEADERS |
+		FUSE_URING_PINNED_BUFFERS;
 	bool bufring = init_flags & FUSE_URING_BUFRING;
 	bool pinned_headers = init_flags & FUSE_URING_PINNED_HEADERS;
+	bool pinned_buffers = init_flags & FUSE_URING_PINNED_BUFFERS;
 
-	if (pinned_headers && !bufring)
+	if (!bufring && (pinned_headers || pinned_buffers))
 		return false;
 
 	return !(init_flags & ~valid_flags);
diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h
index 05c0f061a882..859ee4e6ba03 100644
--- a/fs/fuse/dev_uring_i.h
+++ b/fs/fuse/dev_uring_i.h
@@ -57,6 +57,7 @@ struct fuse_bufring_pinned {
 
 struct fuse_bufring {
 	bool use_pinned_headers: 1;
+	bool use_pinned_buffers: 1;
 	unsigned int queue_depth;
 
 	union {
@@ -65,6 +66,9 @@ struct fuse_bufring {
 		struct fuse_bufring_pinned pinned_headers;
 	};
 
+	/* only used if the buffers are pinned */
+	struct fuse_bufring_pinned pinned_bufs;
+
 	/* metadata tracking state of the bufring */
 	unsigned int nbufs;
 	unsigned int head;
diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h
index e57244c03d42..51ecb66dd6eb 100644
--- a/include/uapi/linux/fuse.h
+++ b/include/uapi/linux/fuse.h
@@ -245,6 +245,7 @@
  *  - add FUSE_URING_BUFRING flag
  *  - add fuse_uring_cmd_req init struct
  *  - add FUSE_URING_PINNED_HEADERS flag
+ *  - add FUSE_URING_PINNED_BUFFERS flag
  */
 
 #ifndef _LINUX_FUSE_H
@@ -1308,6 +1309,7 @@ enum fuse_uring_cmd {
 /* fuse_uring_cmd_req flags */
 #define FUSE_URING_BUFRING		(1 << 0)
 #define FUSE_URING_PINNED_HEADERS	(1 << 1)
+#define FUSE_URING_PINNED_BUFFERS	(1 << 2)
 
 /**
  * In the 80B command area of the SQE.
-- 
2.52.0