[PATCH v1 07/30] io_uring/rsrc: add fixed buffer table pinning/unpinning

[Joanne Koong <joannelkoong@gmail.com>]

Add kernel APIs to pin and unpin the buffer table for fixed buffers,
preventing userspace from unregistering or updating the fixed buffers
table while it is pinned by the kernel.

This has two advantages:
a) Eliminating the overhead of having to fetch and construct an iter for
a fixed buffer per every cmd. Instead, the caller can pin the buffer
table, fetch/construct the iter once, and use that across cmds for
however long it needs to until it is ready to unpin the buffer table.

b) Allowing a fixed buffer lookup at any index. The buffer table must be
pinned in order to allow this, otherwise we would have to keep track of
all the nodes that have been looked up by the io_kiocb so that we can
properly adjust the refcounts for those nodes. Ensuring that the buffer
table must first be pinned before being able to fetch a buffer at any
index makes things logistically a lot neater.

This is a preparatory patch for fuse io-uring's usage of fixed buffers.

Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
---
 include/linux/io_uring/buf.h   | 13 +++++++++++
 include/linux/io_uring_types.h |  9 ++++++++
 io_uring/rsrc.c                | 42 ++++++++++++++++++++++++++++++++++
 3 files changed, 64 insertions(+)

diff --git a/include/linux/io_uring/buf.h b/include/linux/io_uring/buf.h
index 7a1cf197434d..c997c01c24c4 100644
--- a/include/linux/io_uring/buf.h
+++ b/include/linux/io_uring/buf.h
@@ -9,6 +9,9 @@ int io_uring_buf_ring_pin(struct io_ring_ctx *ctx, unsigned buf_group,
 			  unsigned issue_flags, struct io_buffer_list **bl);
 int io_uring_buf_ring_unpin(struct io_ring_ctx *ctx, unsigned buf_group,
 			    unsigned issue_flags);
+
+int io_uring_buf_table_pin(struct io_ring_ctx *ctx, unsigned issue_flags);
+int io_uring_buf_table_unpin(struct io_ring_ctx *ctx, unsigned issue_flags);
 #else
 static inline int io_uring_buf_ring_pin(struct io_ring_ctx *ctx,
 					unsigned buf_group,
@@ -23,6 +26,16 @@ static inline int io_uring_buf_ring_unpin(struct io_ring_ctx *ctx,
 {
 	return -EOPNOTSUPP;
 }
+static inline int io_uring_buf_table_pin(struct io_ring_ctx *ctx,
+					 unsigned issue_flags)
+{
+	return -EOPNOTSUPP;
+}
+static inline int io_uring_buf_table_unpin(struct io_ring_ctx *ctx,
+					   unsigned issue_flags)
+{
+	return -EOPNOTSUPP;
+}
 #endif /* CONFIG_IO_URING */
 
 #endif /* _LINUX_IO_URING_BUF_H */
diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 36fac08db636..e1a75cfe57d9 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -57,8 +57,17 @@ struct io_wq_work {
 	int cancel_seq;
 };
 
+/*
+ * struct io_rsrc_data flag values:
+ *
+ * IO_RSRC_DATA_PINNED: data is pinned and cannot be unregistered by userspace
+ * until it has been unpinned. Currently this is only possible on buffer tables.
+ */
+#define IO_RSRC_DATA_PINNED		BIT(0)
+
 struct io_rsrc_data {
 	unsigned int			nr;
+	u8				flags;
 	struct io_rsrc_node		**nodes;
 };
 
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 3765a50329a8..67331cae0a5a 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -9,6 +9,7 @@
 #include <linux/hugetlb.h>
 #include <linux/compat.h>
 #include <linux/io_uring.h>
+#include <linux/io_uring/buf.h>
 #include <linux/io_uring/cmd.h>
 
 #include <uapi/linux/io_uring.h>
@@ -304,6 +305,8 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
 		return -ENXIO;
 	if (up->offset + nr_args > ctx->buf_table.nr)
 		return -EINVAL;
+	if (ctx->buf_table.flags & IO_RSRC_DATA_PINNED)
+		return -EBUSY;
 
 	for (done = 0; done < nr_args; done++) {
 		struct io_rsrc_node *node;
@@ -615,6 +618,8 @@ int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
 {
 	if (!ctx->buf_table.nr)
 		return -ENXIO;
+	if (ctx->buf_table.flags & IO_RSRC_DATA_PINNED)
+		return -EBUSY;
 	io_rsrc_data_free(ctx, &ctx->buf_table);
 	return 0;
 }
@@ -1580,3 +1585,40 @@ int io_prep_reg_iovec(struct io_kiocb *req, struct iou_vec *iv,
 	req->flags |= REQ_F_IMPORT_BUFFER;
 	return 0;
 }
+
+int io_uring_buf_table_pin(struct io_ring_ctx *ctx, unsigned issue_flags)
+{
+	struct io_rsrc_data *data;
+	int err = 0;
+
+	io_ring_submit_lock(ctx, issue_flags);
+
+	data = &ctx->buf_table;
+	/* There was nothing registered. There is nothing to pin */
+	if (!data->nr)
+		err = -ENXIO;
+	else
+		data->flags |= IO_RSRC_DATA_PINNED;
+
+	io_ring_submit_unlock(ctx, issue_flags);
+	return err;
+}
+EXPORT_SYMBOL_GPL(io_uring_buf_table_pin);
+
+int io_uring_buf_table_unpin(struct io_ring_ctx *ctx, unsigned issue_flags)
+{
+	struct io_rsrc_data *data;
+	int err = 0;
+
+	io_ring_submit_lock(ctx, issue_flags);
+
+	data = &ctx->buf_table;
+	if (WARN_ON_ONCE(!(data->flags & IO_RSRC_DATA_PINNED)))
+		err = -EINVAL;
+	else
+		data->flags &= ~IO_RSRC_DATA_PINNED;
+
+	io_ring_submit_unlock(ctx, issue_flags);
+	return err;
+}
+EXPORT_SYMBOL_GPL(io_uring_buf_table_unpin);
-- 
2.47.3