[PATCH-RFC 0/2] pNFS: recoalesce pages when ld read/write fails

[PATCH-RFC 0/2] pNFS: recoalesce pages when ld read/write fails

[Peng Tao]

Hi,

I start to look at the error handling for ld_read/write_done.
The two patches try to address the error handling for read/write pagelist
failures, via nfs readpage/writepage interface. Please see if this method
is proper. Thanks!

Cheers,
Tao

[PATCH 1/2] pNFS: deal with ld write failure

[Peng Tao]

For pnfs pagelist write failure, we need to pg_recoalesce and resend
IO to mds.

Signed-off-by: Peng Tao <peng_tao@emc.com>
---
 fs/nfs/internal.h |    4 ++++
 fs/nfs/pnfs.c     |   35 ++++++++++++++++++++++++++++++++---
 fs/nfs/write.c    |   21 ++++++++++++++-------
 3 files changed, 50 insertions(+), 10 deletions(-)

diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index ab12913..62f183d 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -305,6 +305,10 @@ extern void nfs_readdata_release(struct nfs_read_data *rdata);
 /* write.c */
 extern int nfs_generic_flush(struct nfs_pageio_descriptor *desc,
 		struct list_head *head);
+extern int do_nfs_writepage(struct page *page, struct writeback_control *wbc,
+		struct nfs_pageio_descriptor *pgio);
+extern void nfs_pageio_init_write_mds(struct nfs_pageio_descriptor *pgio,
+		struct inode *inode, int ioflags);
 extern void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio);
 extern void nfs_writedata_release(struct nfs_write_data *wdata);
 extern void nfs_commit_free(struct nfs_write_data *p);
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index e550e88..08aba45 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -1172,6 +1172,13 @@ int
 pnfs_ld_write_done(struct nfs_write_data *data)
 {
 	int status;
+	struct nfs_pageio_descriptor pgio;
+	struct writeback_control wbc = {
+		.sync_mode = WB_SYNC_ALL,
+		.range_start = data->mds_offset,
+		.nr_to_write = data->npages,
+		.range_end = LLONG_MAX,
+	};
 
 	if (!data->pnfs_error) {
 		pnfs_set_layoutcommit(data);
@@ -1180,11 +1187,33 @@ pnfs_ld_write_done(struct nfs_write_data *data)
 		return 0;
 	}
 
+	put_lseg(data->lseg);
+	data->lseg = NULL;
 	dprintk("%s: pnfs_error=%d, retry via MDS\n", __func__,
 		data->pnfs_error);
-	status = nfs_initiate_write(data, NFS_CLIENT(data->inode),
-				    data->mds_ops, NFS_FILE_SYNC);
-	return status ? : -EAGAIN;
+	nfs_pageio_init_write_mds(&pgio, data->inode, FLUSH_STABLE);
+	pgio.pg_recoalesce = 1;
+	while (!list_empty(&data->pages)) {
+		struct nfs_page *req = nfs_list_entry(data->pages.next);
+		struct page *page = req->wb_page;
+
+		nfs_list_remove_request(req);
+		nfs_clear_page_tag_locked(req);
+
+		end_page_writeback(page);
+
+		lock_page(page);
+		status = do_nfs_writepage(page, &wbc, &pgio);
+		if (status) {
+			/* FIXME: is this enough?? */
+			set_page_dirty(page);
+		}
+		unlock_page(page);
+	}
+	nfs_pageio_complete(&pgio);
+	nfs_writedata_release(data);
+
+	return 0;
 }
 EXPORT_SYMBOL_GPL(pnfs_ld_write_done);
 
diff --git a/fs/nfs/write.c b/fs/nfs/write.c
index b39b37f..0ccdf98 100644
--- a/fs/nfs/write.c
+++ b/fs/nfs/write.c
@@ -285,14 +285,9 @@ out:
 	return ret;
 }
 
-static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, struct nfs_pageio_descriptor *pgio)
+int do_nfs_writepage(struct page *page, struct writeback_control *wbc, struct nfs_pageio_descriptor *pgio)
 {
-	struct inode *inode = page->mapping->host;
 	int ret;
-
-	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
-	nfs_add_stats(inode, NFSIOS_WRITEPAGES, 1);
-
 	nfs_pageio_cond_complete(pgio, page->index);
 	ret = nfs_page_async_flush(pgio, page, wbc->sync_mode == WB_SYNC_NONE);
 	if (ret == -EAGAIN) {
@@ -301,6 +296,17 @@ static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, st
 	}
 	return ret;
 }
+EXPORT_SYMBOL_GPL(do_nfs_writepage);
+
+static int nfs_do_writepage(struct page *page, struct writeback_control *wbc, struct nfs_pageio_descriptor *pgio)
+{
+	struct inode *inode = page->mapping->host;
+
+	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
+	nfs_add_stats(inode, NFSIOS_WRITEPAGES, 1);
+
+	return do_nfs_writepage(page, wbc, pgio);
+}
 
 /*
  * Write an mmapped page to the server.
@@ -1051,12 +1057,13 @@ static const struct nfs_pageio_ops nfs_pageio_write_ops = {
 	.pg_doio = nfs_generic_pg_writepages,
 };
 
-static void nfs_pageio_init_write_mds(struct nfs_pageio_descriptor *pgio,
+void nfs_pageio_init_write_mds(struct nfs_pageio_descriptor *pgio,
 				  struct inode *inode, int ioflags)
 {
 	nfs_pageio_init(pgio, inode, &nfs_pageio_write_ops,
 				NFS_SERVER(inode)->wsize, ioflags);
 }
+EXPORT_SYMBOL_GPL(nfs_pageio_init_write_mds);
 
 void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio)
 {
-- 
1.7.1.262.g5ef3d

[PATCH 2/2] pNFS: deal with ld read failure

[Peng Tao]

For pnfs pagelist read failure, we need to pg_recoalesce and resend
IO to mds.

Signed-off-by: Peng Tao <peng_tao@emc.com>
---
 fs/nfs/internal.h |    2 ++
 fs/nfs/pnfs.c     |   18 ++++++++++++++----
 fs/nfs/read.c     |    3 ++-
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index 62f183d..78b662e 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -307,6 +307,8 @@ extern int nfs_generic_flush(struct nfs_pageio_descriptor *desc,
 		struct list_head *head);
 extern int do_nfs_writepage(struct page *page, struct writeback_control *wbc,
 		struct nfs_pageio_descriptor *pgio);
+extern void nfs_pageio_init_read_mds(struct nfs_pageio_descriptor *pgio,
+		struct inode *inode);
 extern void nfs_pageio_init_write_mds(struct nfs_pageio_descriptor *pgio,
 		struct inode *inode, int ioflags);
 extern void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio);
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 08aba45..66fc854 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -1300,7 +1300,7 @@ EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);
 int
 pnfs_ld_read_done(struct nfs_read_data *data)
 {
-	int status;
+	struct nfs_pageio_descriptor pgio;
 
 	if (!data->pnfs_error) {
 		__nfs4_read_done_cb(data);
@@ -1309,11 +1309,21 @@ pnfs_ld_read_done(struct nfs_read_data *data)
 		return 0;
 	}
 
+	put_lseg(data->lseg);
+	data->lseg = NULL;
 	dprintk("%s: pnfs_error=%d, retry via MDS\n", __func__,
 		data->pnfs_error);
-	status = nfs_initiate_read(data, NFS_CLIENT(data->inode),
-				   data->mds_ops);
-	return status ? : -EAGAIN;
+	nfs_pageio_init_read_mds(&pgio, data->inode);
+	pgio.pg_recoalesce = 1;
+	while (!list_empty(&data->pages)) {
+		struct nfs_page *req = nfs_list_entry(data->pages.next);
+
+		nfs_pageio_add_request(&pgio, req);
+	}
+	nfs_pageio_complete(&pgio);
+	nfs_readdata_release(data);
+
+	return 0;
 }
 EXPORT_SYMBOL_GPL(pnfs_ld_read_done);
 
diff --git a/fs/nfs/read.c b/fs/nfs/read.c
index 2171c04..2484131 100644
--- a/fs/nfs/read.c
+++ b/fs/nfs/read.c
@@ -112,12 +112,13 @@ static void nfs_readpage_truncate_uninitialised_page(struct nfs_read_data *data)
 	}
 }
 
-static void nfs_pageio_init_read_mds(struct nfs_pageio_descriptor *pgio,
+void nfs_pageio_init_read_mds(struct nfs_pageio_descriptor *pgio,
 		struct inode *inode)
 {
 	nfs_pageio_init(pgio, inode, &nfs_pageio_read_ops,
 			NFS_SERVER(inode)->rsize, 0);
 }
+EXPORT_SYMBOL_GPL(nfs_pageio_init_read_mds);
 
 void nfs_pageio_reset_read_mds(struct nfs_pageio_descriptor *pgio)
 {
-- 
1.7.1.262.g5ef3d

Re: [PATCH-RFC 0/2] pNFS: recoalesce pages when ld read/write fails

[Peng Tao]

On Sat, Aug 6, 2011 at 11:52 PM, Peng Tao <bergwolf@gmail.com> wrote:
> Hi,
>
> I start to look at the error handling for ld_read/write_done.
> The two patches try to address the error handling for read/write pagelist
> failures, via nfs readpage/writepage interface. Please see if this method
> is proper. Thanks!
FWIW, this is just what we need for recoalescing pages when IO fails.
Currently ld_read/write_done is called in default workqueue, which is
not really a good place for IO path code. Trond mentioned that nfsiod
is not an ideal place for it either. So do we need to create a pnfs
private workqueue for this? Any other suggestions?

-- 
Thanks,
-Bergwolf