[RFC PATCH 0/5] ceph: convert to netfs_direct_read_iter

[RFC PATCH 0/5] ceph: convert to netfs_direct_read_iter

[Jeff Layton]

This patch is based on top of David Howells' netfs-lib branch [1]. That
tree adds new O_DIRECT read helpers to netfs.ko. This patchset converts
ceph to use it. With this, all of the usual xfstests pass for me on
ceph.

We should be able to rip out a large part of ceph_direct_read_write with
this set. I haven't done that here, and will probably wait until we have
converted ceph to use netfs DIO write helpers (which don't exist yet).
Once that's in place, we can just remove that function and related
infrastructure wholesale.

I'd like to get this into our testing branch for an eventual merge into
v5.19. We need it in our testing branch for a bit though.

David, in the past, I think we just based our master branch on top of
whatever branch you were feeding to -next. Around -rc3, could you
rebase netfs-lib on top of that and use that as a base for what you're
feeding into -next? Then we can just base our -next feeder branch onto
yours.

[1]: https://git.kernel.org/pub/scm/linux/kernel/git/dhowells/linux-fs.git/log/?h=netfs-lib

David Howells (1):
  ceph: Use the provided iterator in ceph_netfs_issue_op()

Jeff Layton (4):
  netfs: don't error out on short DIO reads
  ceph: set rsize in netfs_i_context from mount options
  ceph: enhance dout messages in issue_read codepaths
  ceph: switch to netfs_direct_read_iter

 fs/ceph/addr.c  | 55 ++++++++++++++++++++++++++++++++-----------------
 fs/ceph/file.c  |  3 +--
 fs/ceph/inode.c |  3 ++-
 fs/netfs/io.c   |  5 -----
 4 files changed, 39 insertions(+), 27 deletions(-)
-- 
2.35.1

[RFC PATCH 1/5] netfs: don't error out on short DIO reads

[Jeff Layton]

There's no reason that userland can't request to read beyond the EOF. A
short read is expected in that situation.

Signed-off-by: Jeff Layton <jlayton@kernel.org>
---
 fs/netfs/io.c | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/fs/netfs/io.c b/fs/netfs/io.c
index aaaafc3e1601..b94f2d27127e 100644
--- a/fs/netfs/io.c
+++ b/fs/netfs/io.c
@@ -728,11 +728,6 @@ ssize_t netfs_begin_read(struct netfs_io_request *rreq, bool sync)
 		}
 
 		ret = rreq->error;
-		if (ret == 0 && rreq->submitted < rreq->len &&
-		    rreq->origin == NETFS_DIO_READ) {
-			trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_read);
-			ret = -EIO;
-		}
 		if (ret == 0)
 			ret = netfs_dio_copy_to_dest(rreq);
 		if (ret == 0) {
-- 
2.35.1

[RFC PATCH 2/5] ceph: set rsize in netfs_i_context from mount options

[Jeff Layton]

Signed-off-by: Jeff Layton <jlayton@kernel.org>
---
 fs/ceph/inode.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c
index 1dad69a0ab70..8ea1b53b6ce9 100644
--- a/fs/ceph/inode.c
+++ b/fs/ceph/inode.c
@@ -450,6 +450,7 @@ static int ceph_fill_fragtree(struct inode *inode,
  */
 struct inode *ceph_alloc_inode(struct super_block *sb)
 {
+	struct ceph_mount_options *fsopt = ceph_sb_to_client(sb)->mount_options;
 	struct ceph_inode_info *ci;
 	struct netfs_i_context *ctx;
 	int i;
@@ -463,7 +464,7 @@ struct inode *ceph_alloc_inode(struct super_block *sb)
 	/* Set parameters for the netfs library */
 	ctx = netfs_i_context(&ci->vfs_inode);
 	netfs_i_context_init(&ci->vfs_inode, &ceph_netfs_ops);
-	ctx->rsize = 1024 * 1024;
+	ctx->rsize = fsopt->rsize;
 
 	spin_lock_init(&ci->i_ceph_lock);
 
-- 
2.35.1

[RFC PATCH 3/5] ceph: Use the provided iterator in ceph_netfs_issue_op()

[Jeff Layton]

From: David Howells <dhowells@redhat.com>

The netfs_read_subrequest struct now contains a persistent iterator
representing the destination buffer for a read that the network filesystem
should use.  Make ceph use this.

Signed-off-by: David Howells <dhowells@redhat.com>
Signed-off-by: Jeff Layton <jlayton@kernel.org>
---
 fs/ceph/addr.c | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c
index f0af362f8756..e7a7b5d29c7d 100644
--- a/fs/ceph/addr.c
+++ b/fs/ceph/addr.c
@@ -233,7 +233,6 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq)
 	struct ceph_mds_request *req;
 	struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
 	struct ceph_inode_info *ci = ceph_inode(inode);
-	struct iov_iter iter;
 	ssize_t err = 0;
 	size_t len;
 
@@ -266,8 +265,7 @@ static bool ceph_netfs_issue_op_inline(struct netfs_io_subrequest *subreq)
 	}
 
 	len = min_t(size_t, iinfo->inline_len - subreq->start, subreq->len);
-	iov_iter_xarray(&iter, READ, &rreq->mapping->i_pages, subreq->start, len);
-	err = copy_to_iter(iinfo->inline_data + subreq->start, len, &iter);
+	err = copy_to_iter(iinfo->inline_data + subreq->start, len, &subreq->iter);
 	if (err == 0)
 		err = -EFAULT;
 
@@ -285,7 +283,6 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq)
 	struct ceph_fs_client *fsc = ceph_inode_to_client(inode);
 	struct ceph_osd_request *req;
 	struct ceph_vino vino = ceph_vino(inode);
-	struct iov_iter iter;
 	struct page **pages;
 	size_t page_off;
 	int err = 0;
@@ -306,15 +303,14 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq)
 	}
 
 	dout("%s: pos=%llu orig_len=%zu len=%llu\n", __func__, subreq->start, subreq->len, len);
-	iov_iter_xarray(&iter, READ, &rreq->mapping->i_pages, subreq->start, len);
-	err = iov_iter_get_pages_alloc(&iter, &pages, len, &page_off);
+
+	err = iov_iter_get_pages_alloc(&subreq->iter, &pages, len, &page_off);
 	if (err < 0) {
 		dout("%s: iov_ter_get_pages_alloc returned %d\n", __func__, err);
 		goto out;
 	}
 
-	/* should always give us a page-aligned read */
-	WARN_ON_ONCE(page_off);
+	/* FIXME: adjust the len in req downward if necessary */
 	len = err;
 
 	osd_req_op_extent_osd_data_pages(req, 0, pages, len, 0, false, false);
-- 
2.35.1

[RFC PATCH 4/5] ceph: enhance dout messages in issue_read codepaths

[Jeff Layton]

Signed-off-by: Jeff Layton <jlayton@kernel.org>
---
 fs/ceph/addr.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c
index e7a7b5d29c7d..0726494a0981 100644
--- a/fs/ceph/addr.c
+++ b/fs/ceph/addr.c
@@ -190,6 +190,8 @@ static bool ceph_netfs_clamp_length(struct netfs_io_subrequest *subreq)
 	/* Truncate the extent at the end of the current block */
 	ceph_calc_file_object_mapping(&ci->i_layout, subreq->start, subreq->len,
 				      &objno, &objoff, &xlen);
+	dout("%s: subreq->len=0x%zx xlen=0x%x rsize=0x%x",
+		__func__, subreq->len, xlen, fsc->mount_options->rsize);
 	subreq->len = min(xlen, fsc->mount_options->rsize);
 	return true;
 }
@@ -302,7 +304,9 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq)
 		goto out;
 	}
 
-	dout("%s: pos=%llu orig_len=%zu len=%llu\n", __func__, subreq->start, subreq->len, len);
+	dout("%s: pos=%llu orig_len=%zu len=%llu debug_id=%x debug_idx=%hx iter->count=%zx\n",
+		__func__, subreq->start, subreq->len, len, rreq->debug_id,
+		subreq->debug_index, iov_iter_count(&subreq->iter));
 
 	err = iov_iter_get_pages_alloc(&subreq->iter, &pages, len, &page_off);
 	if (err < 0) {
-- 
2.35.1

[RFC PATCH 5/5] ceph: switch to netfs_direct_read_iter

[Jeff Layton]

Signed-off-by: Jeff Layton <jlayton@kernel.org>
---
 fs/ceph/addr.c | 41 +++++++++++++++++++++++++++++------------
 fs/ceph/file.c |  3 +--
 2 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c
index 0726494a0981..bc575bbbf8b7 100644
--- a/fs/ceph/addr.c
+++ b/fs/ceph/addr.c
@@ -201,7 +201,6 @@ static void finish_netfs_read(struct ceph_osd_request *req)
 	struct ceph_fs_client *fsc = ceph_inode_to_client(req->r_inode);
 	struct ceph_osd_data *osd_data = osd_req_op_extent_osd_data(req, 0);
 	struct netfs_io_subrequest *subreq = req->r_priv;
-	int num_pages;
 	int err = req->r_result;
 
 	ceph_update_read_metrics(&fsc->mdsc->metric, req->r_start_latency,
@@ -216,13 +215,18 @@ static void finish_netfs_read(struct ceph_osd_request *req)
 	else if (err == -EBLOCKLISTED)
 		fsc->blocklisted = true;
 
-	if (err >= 0 && err < subreq->len)
-		__set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
+	if (err >= 0) {
+		if (err < subreq->len)
+			__set_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags);
+		iov_iter_advance(&subreq->iter, err);
+	}
+	if (!iov_iter_is_bvec(&subreq->iter))
+		ceph_put_page_vector(osd_data->pages,
+				     calc_pages_for(osd_data->alignment,
+				     osd_data->length),
+				     false);
 
 	netfs_subreq_terminated(subreq, err, true);
-
-	num_pages = calc_pages_for(osd_data->alignment, osd_data->length);
-	ceph_put_page_vector(osd_data->pages, num_pages, false);
 	iput(req->r_inode);
 }
 
@@ -285,6 +289,7 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq)
 	struct ceph_fs_client *fsc = ceph_inode_to_client(inode);
 	struct ceph_osd_request *req;
 	struct ceph_vino vino = ceph_vino(inode);
+	struct iov_iter *iter = &subreq->iter;
 	struct page **pages;
 	size_t page_off;
 	int err = 0;
@@ -308,16 +313,28 @@ static void ceph_netfs_issue_read(struct netfs_io_subrequest *subreq)
 		__func__, subreq->start, subreq->len, len, rreq->debug_id,
 		subreq->debug_index, iov_iter_count(&subreq->iter));
 
-	err = iov_iter_get_pages_alloc(&subreq->iter, &pages, len, &page_off);
-	if (err < 0) {
-		dout("%s: iov_ter_get_pages_alloc returned %d\n", __func__, err);
-		goto out;
+	if (iov_iter_is_bvec(iter)) {
+		/*
+		 * FIXME: remove force cast, ideally by plumbing an IOV_ITER osd_data
+		 * 	  variant.
+		 */
+		osd_req_op_extent_osd_data_bvecs(req, 0, (__force struct bio_vec *)iter->bvec,
+				iter->nr_segs, len);
+		goto submit;
 	}
 
-	/* FIXME: adjust the len in req downward if necessary */
-	len = err;
+	err = iov_iter_get_pages_alloc(&subreq->iter, &pages, len, &page_off);
+	if (err < len) {
+		if (err < 0) {
+			dout("%s: iov_ter_get_pages_alloc returned %d\n", __func__, err);
+			goto out;
+		}
+		len = err;
+		req->r_ops[0].extent.length = err;
+	}
 
 	osd_req_op_extent_osd_data_pages(req, 0, pages, len, 0, false, false);
+submit:
 	req->r_callback = finish_netfs_read;
 	req->r_priv = subreq;
 	req->r_inode = inode;
diff --git a/fs/ceph/file.c b/fs/ceph/file.c
index 6c9e837aa1d3..8271459b36dc 100644
--- a/fs/ceph/file.c
+++ b/fs/ceph/file.c
@@ -1624,8 +1624,7 @@ static ssize_t ceph_read_iter(struct kiocb *iocb, struct iov_iter *to)
 
 		if (ci->i_inline_version == CEPH_INLINE_NONE) {
 			if (!retry_op && (iocb->ki_flags & IOCB_DIRECT)) {
-				ret = ceph_direct_read_write(iocb, to,
-							     NULL, NULL);
+				ret = netfs_direct_read_iter(iocb, to);
 				if (ret >= 0 && ret < len)
 					retry_op = CHECK_EOF;
 			} else {
-- 
2.35.1