Re: [PATCH v2 12/14] fuse: add pinned payload buffers capability for io-uring buffer rings

[Jeff Layton <jlayton@kernel.org>]

On Thu, 2026-04-02 at 09:28 -0700, Joanne Koong wrote:
> Extend the buffer ring pinning capability to payload buffers via the
> FUSE_URING_PINNED_BUFFERS flag. When set alongside FUSE_URING_BUFRING,
> the kernel pins and vmaps the payload buffer region during queue setup.
> 
> With pinned payloads, the kernel uses direct memcpy for all payload
> buffer copies, avoiding the per-request overhead of pinning/unpinning
> user pages and translating virtual addresses. This is particularly
> beneficial for large payload copies.
> 
> As with pinned headers, buffers must be page-aligned. Pinned pages are
> accounted against RLIMIT_MEMLOCK (bypassed with CAP_IPC_LOCK) and
> unpinned in process context during connection abort.
> 
> In benchmarks using passthrough_hp on a high-performance NVMe-backed
> system, pinned headers and pinned payload buffers showed around a 10%
> throughput improvement for direct randreads (~2150 MiB/s to ~2400
> MiB/s), a 4% improvement for direct sequential reads (~2510 MiB/s to
> ~2620 MiB/s), a 8% improvement for buffered randreads (~2100 MiB/s to
> ~2280 MiB/s), and a 6% improvement for buffered sequential reads (~2500
> MiB/s to ~2670 MiB/s).
> 
> Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
> ---
>  fs/fuse/dev_uring.c       | 54 +++++++++++++++++++++++++++++++++------
>  fs/fuse/dev_uring_i.h     |  4 +++
>  include/uapi/linux/fuse.h |  2 ++
>  3 files changed, 52 insertions(+), 8 deletions(-)
> 
> diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c
> index 79736b02cf9f..06d3d8dc1c82 100644
> --- a/fs/fuse/dev_uring.c
> +++ b/fs/fuse/dev_uring.c
> @@ -52,6 +52,11 @@ static inline bool bufring_pinned_headers(struct fuse_ring_queue *queue)
>  	return queue->bufring->use_pinned_headers;
>  }
>  
> +static inline bool bufring_pinned_buffers(struct fuse_ring_queue *queue)
> +{
> +	return queue->bufring->use_pinned_buffers;
> +}
> +
>  static void uring_cmd_set_ring_ent(struct io_uring_cmd *cmd,
>  				   struct fuse_ring_ent *ring_ent)
>  {
> @@ -235,6 +240,11 @@ static void fuse_uring_bufring_unpin(struct fuse_ring_queue *queue)
>  		fuse_bufring_unpin_mem(&br->pinned_headers);
>  		br->use_pinned_headers = false;
>  	}
> +
> +	if (bufring_pinned_buffers(queue)) {
> +		fuse_bufring_unpin_mem(&br->pinned_bufs);
> +		br->use_pinned_buffers = false;
> +	}
>  }
>  
>  void fuse_uring_destruct(struct fuse_conn *fc)
> @@ -474,6 +484,7 @@ static int fuse_uring_bufring_setup(struct io_uring_cmd *cmd,
>  	unsigned int buf_size = READ_ONCE(cmd_req->init.buf_size);
>  	struct iovec iov[FUSE_URING_IOV_SEGS];
>  	bool pinned_headers = init_flags & FUSE_URING_PINNED_HEADERS;
> +	bool pinned_bufs = init_flags & FUSE_URING_PINNED_BUFFERS;
>  	void __user *payload, *headers;
>  	size_t headers_size, payload_size, ring_size;
>  	struct fuse_bufring *br;
> @@ -523,7 +534,22 @@ static int fuse_uring_bufring_setup(struct io_uring_cmd *cmd,
>  		br->headers = headers;
>  	}
>  
> -	payload_addr = (uintptr_t)payload;
> +	if (pinned_bufs) {
> +		err = fuse_bufring_pin_mem(&br->pinned_bufs, payload,
> +					   payload_size);
> +		if (err) {
> +			if (pinned_headers)
> +				fuse_bufring_unpin_mem(&br->pinned_headers);
> +			kfree(br);
> +			return err;
> +		}
> +		br->use_pinned_buffers = true;
> +	}
> +
> +	if (pinned_bufs)
> +		payload_addr = (uintptr_t)br->pinned_bufs.addr;
> +	else
> +		payload_addr = (uintptr_t)payload;
>  
>  	/* populate the ring buffer */
>  	for (i = 0; i < nr_bufs; i++, payload_addr += buf_size) {
> @@ -553,6 +579,7 @@ static bool queue_init_flags_consistent(struct fuse_ring_queue *queue,
>  {
>  	bool bufring = init_flags & FUSE_URING_BUFRING;
>  	bool pinned_headers = init_flags & FUSE_URING_PINNED_HEADERS;
> +	bool pinned_bufs = init_flags & FUSE_URING_PINNED_BUFFERS;
>  
>  	if (bufring_enabled(queue) != bufring)
>  		return false;
> @@ -560,7 +587,8 @@ static bool queue_init_flags_consistent(struct fuse_ring_queue *queue,
>  	if (!bufring)
>  		return true;
>  
> -	return bufring_pinned_headers(queue) == pinned_headers;
> +	return bufring_pinned_headers(queue) == pinned_headers &&
> +		bufring_pinned_buffers(queue) == pinned_bufs;
>  }
>  
>  static struct fuse_ring_queue *
> @@ -1011,13 +1039,15 @@ static int setup_fuse_copy_state(struct fuse_copy_state *cs,
>  				 struct fuse_ring_ent *ent, int dir,
>  				 struct iov_iter *iter)
>  {
> -	void __user *payload;
> +	void __user *payload = NULL;
> +	bool use_bufring = bufring_enabled(ent->queue);
> +	bool pinned_buffers = use_bufring && bufring_pinned_buffers(ent->queue);
>  	int err;
>  
> -	if (bufring_enabled(ent->queue))
> -		payload = (void __user *)ent->payload_buf.addr;
> -	else
> +	if (!use_bufring)
>  		payload = ent->payload;
> +	else if (!pinned_buffers)
> +		payload = (void __user *)ent->payload_buf.addr;
>  
>  	if (payload) {
>  		err = import_ubuf(dir, payload, ring->max_payload_sz, iter);
> @@ -1029,6 +1059,12 @@ static int setup_fuse_copy_state(struct fuse_copy_state *cs,
>  
>  	fuse_copy_init(cs, dir == ITER_DEST, iter);
>  
> +	if (pinned_buffers) {
> +		cs->is_kaddr = true;
> +		cs->kaddr = (void *)ent->payload_buf.addr;
> +		cs->len = ent->payload_buf.len;
> +	}
> +
>  	cs->is_uring = true;
>  	cs->req = req;
>  
> @@ -1608,11 +1644,13 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd,
>  static bool init_flags_valid(u64 init_flags)
>  {
>  	u64 valid_flags =
> -		FUSE_URING_BUFRING | FUSE_URING_PINNED_HEADERS;
> +		FUSE_URING_BUFRING | FUSE_URING_PINNED_HEADERS |
> +		FUSE_URING_PINNED_BUFFERS;
>  	bool bufring = init_flags & FUSE_URING_BUFRING;
>  	bool pinned_headers = init_flags & FUSE_URING_PINNED_HEADERS;
> +	bool pinned_buffers = init_flags & FUSE_URING_PINNED_BUFFERS;
>  
> -	if (pinned_headers && !bufring)
> +	if (!bufring && (pinned_headers || pinned_buffers))
>  		return false;
>  
>  	return !(init_flags & ~valid_flags);
> diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h
> index 05c0f061a882..859ee4e6ba03 100644
> --- a/fs/fuse/dev_uring_i.h
> +++ b/fs/fuse/dev_uring_i.h
> @@ -57,6 +57,7 @@ struct fuse_bufring_pinned {
>  
>  struct fuse_bufring {
>  	bool use_pinned_headers: 1;
> +	bool use_pinned_buffers: 1;
>  	unsigned int queue_depth;
>  
>  	union {
> @@ -65,6 +66,9 @@ struct fuse_bufring {
>  		struct fuse_bufring_pinned pinned_headers;
>  	};
>  
> +	/* only used if the buffers are pinned */
> +	struct fuse_bufring_pinned pinned_bufs;
> +
>  	/* metadata tracking state of the bufring */
>  	unsigned int nbufs;
>  	unsigned int head;
> diff --git a/include/uapi/linux/fuse.h b/include/uapi/linux/fuse.h
> index e57244c03d42..51ecb66dd6eb 100644
> --- a/include/uapi/linux/fuse.h
> +++ b/include/uapi/linux/fuse.h
> @@ -245,6 +245,7 @@
>   *  - add FUSE_URING_BUFRING flag
>   *  - add fuse_uring_cmd_req init struct
>   *  - add FUSE_URING_PINNED_HEADERS flag
> + *  - add FUSE_URING_PINNED_BUFFERS flag
>   */
>  
>  #ifndef _LINUX_FUSE_H
> @@ -1308,6 +1309,7 @@ enum fuse_uring_cmd {
>  /* fuse_uring_cmd_req flags */
>  #define FUSE_URING_BUFRING		(1 << 0)
>  #define FUSE_URING_PINNED_HEADERS	(1 << 1)
> +#define FUSE_URING_PINNED_BUFFERS	(1 << 2)
>  
>  /**
>   * In the 80B command area of the SQE.

Reviewed-by: Jeff Layton <jlayton@kernel.org>