[PATCH 02/22] Revert "pnfs: Enable O_DIRECT write path."

[PATCH 02/22] Revert "pnfs: Enable O_DIRECT write path."

[Fred Isaman]

This reverts commit 2faf680af973895bdfe19f2254b59dc1a153dd82.
---
 fs/nfs/direct.c |   41 +----------------------------------------
 1 files changed, 1 insertions(+), 40 deletions(-)

diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c
index 02e5918..1148214 100644
--- a/fs/nfs/direct.c
+++ b/fs/nfs/direct.c
@@ -505,7 +505,6 @@ static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
 		.workqueue = nfsiod_workqueue,
 		.flags = RPC_TASK_ASYNC,
 	};
-	enum pnfs_try_status trypnfs;
 
 	dreq->count = 0;
 	get_dreq(dreq);
@@ -529,11 +528,6 @@ static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
 		 * Reuse data->task; data->args should not have changed
 		 * since the original request was sent.
 		 */
-		trypnfs = pnfs_try_to_write_data(data, &nfs_write_direct_ops,
-						 NFS_FILE_SYNC);
-		if (trypnfs == PNFS_ATTEMPTED)
-			continue;
-
 		nfs_direct_write_execute(data, &task_setup_data, &msg);
 	}
 
@@ -616,7 +610,6 @@ static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
 		.workqueue = nfsiod_workqueue,
 		.flags = RPC_TASK_ASYNC,
 	};
-	enum pnfs_try_status trypnfs;
 
 	data->inode = dreq->inode;
 	data->cred = msg.rpc_cred;
@@ -630,11 +623,6 @@ static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
 	data->res.verf = &data->verf;
 	nfs_fattr_init(&data->fattr);
 
-	trypnfs = pnfs_try_to_commit(data, &nfs_commit_direct_ops,
-				     RPC_TASK_ASYNC);
-	if (trypnfs == PNFS_ATTEMPTED)
-		return;
-
 	nfs_direct_commit_execute(dreq, data, &task_setup_data, &msg);
 }
 
@@ -683,9 +671,6 @@ static void nfs_direct_write_result(struct rpc_task *task, void *calldata)
 {
 	struct nfs_write_data *data = calldata;
 
-	dprintk("%s: verf: %d stable %d\n", __func__,
-		data->res.verf->committed, data->args.stable);
-
 	if (nfs_writeback_done(task, data) != 0)
 		return;
 }
@@ -799,17 +784,6 @@ static ssize_t nfs_direct_write_schedule_segment(struct nfs_direct_req *dreq,
 	unsigned int pgbase;
 	int result;
 	ssize_t started = 0;
-	size_t pnfs_stripe_rem = count;
-	enum pnfs_try_status trypnfs;
-
-	/* pnfs_stripe_rem will be set to the remaining bytes in
-	 * the first stripe_unit (which for standard nfs is count)
-	 */
-	pnfs_direct_init_io(inode, ctx, count, pos, 1,
-			    &wsize, &pnfs_stripe_rem);
-
-	dprintk("%s: pos %llu count %Zu wsize %Zu\n",
-		__func__, pos, count, wsize);
 
 	do {
 		struct nfs_write_data *data;
@@ -818,12 +792,6 @@ static ssize_t nfs_direct_write_schedule_segment(struct nfs_direct_req *dreq,
 		pgbase = user_addr & ~PAGE_MASK;
 		bytes = min(wsize,count);
 
-#if defined(CONFIG_NFS_V4_1)
-		if (pnfs_enabled_sb(NFS_SERVER(inode))) {
-			bytes = min(bytes, pnfs_stripe_rem);
-			pnfs_stripe_rem = wsize;
-		}
-#endif /* CONFIG_NFS_V4_1 */
 		result = -ENOMEM;
 		data = nfs_writedata_alloc(nfs_page_array_len(pgbase, bytes));
 		if (unlikely(!data))
@@ -867,15 +835,8 @@ static ssize_t nfs_direct_write_schedule_segment(struct nfs_direct_req *dreq,
 		data->res.verf = &data->verf;
 		nfs_fattr_init(&data->fattr);
 
-		trypnfs = pnfs_try_to_write_data(data, &nfs_write_direct_ops,
-						 sync);
-		if (trypnfs == PNFS_ATTEMPTED) {
-			result = pnfs_get_write_status(data);
-			if (result)
-				break;
-		} else if (nfs_direct_write_execute(data, &task_setup_data, &msg)) {
+		if (nfs_direct_write_execute(data, &task_setup_data, &msg))
 			break;
-		}
 
 		started += bytes;
 		user_addr += bytes;
-- 
1.6.6.1

[PATCH 03/22] Revert "pnfs: Enable O_DIRECT read path."

[Fred Isaman]

This reverts commit fe1dbd120b6a94bbacec205d0a4ae40d36e314b5.
---
 fs/nfs/direct.c |   26 +-------------------------
 1 files changed, 1 insertions(+), 25 deletions(-)

diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c
index 1148214..3ef9b0c 100644
--- a/fs/nfs/direct.c
+++ b/fs/nfs/direct.c
@@ -56,7 +56,6 @@
 
 #include "internal.h"
 #include "iostat.h"
-#include "pnfs.h"
 
 #define NFSDBG_FACILITY		NFSDBG_VFS
 
@@ -329,17 +328,6 @@ static ssize_t nfs_direct_read_schedule_segment(struct nfs_direct_req *dreq,
 	unsigned int pgbase;
 	int result;
 	ssize_t started = 0;
-	size_t pnfs_stripe_rem = count;
-	enum pnfs_try_status trypnfs;
-
-	/* pnfs_stripe_rem will be set to the remaining bytes in
-	 * the first stripe_unit (which for standard nfs is count)
-	 */
-	pnfs_direct_init_io(inode, ctx, count, pos, 0, &rsize,
-			    &pnfs_stripe_rem);
-
-	dprintk("%s: pos %llu count %Zu wsize %Zu\n",
-		__func__, pos, count, rsize);
 
 	do {
 		struct nfs_read_data *data;
@@ -347,12 +335,6 @@ static ssize_t nfs_direct_read_schedule_segment(struct nfs_direct_req *dreq,
 
 		pgbase = user_addr & ~PAGE_MASK;
 		bytes = min(rsize,count);
-#if defined(CONFIG_NFS_V4_1)
-		if (pnfs_enabled_sb(NFS_SERVER(inode))) {
-			bytes = min(bytes, pnfs_stripe_rem);
-			pnfs_stripe_rem = rsize;
-		}
-#endif /* CONFIG_NFS_V4_1 */
 
 		result = -ENOMEM;
 		data = nfs_readdata_alloc(nfs_page_array_len(pgbase, bytes));
@@ -393,14 +375,8 @@ static ssize_t nfs_direct_read_schedule_segment(struct nfs_direct_req *dreq,
 		data->res.eof = 0;
 		data->res.count = bytes;
 
-		trypnfs = pnfs_try_to_read_data(data, &nfs_read_direct_ops);
-		if (trypnfs == PNFS_ATTEMPTED) {
-			result = pnfs_get_read_status(data);
-			if (result)
-				break;
-		} else if (nfs_direct_read_execute(data, &task_setup_data, &msg)) {
+		if (nfs_direct_read_execute(data, &task_setup_data, &msg))
 			break;
-		}
 
 		started += bytes;
 		user_addr += bytes;
-- 
1.6.6.1

[PATCH 04/22] Revert "pnfs: Add function to set up O_DIRECT I/O"

[Fred Isaman]

This reverts commit 4bc73cd4118b5d5b710c28c83a750bf4e02e8269.

Conflicts:

	fs/nfs/pnfs.c
	fs/nfs/pnfs.h

Signed-off-by: Fred Isaman <iisaman@netapp.com>
---
 fs/nfs/pnfs.c |   31 -------------------------------
 fs/nfs/pnfs.h |   25 -------------------------
 2 files changed, 0 insertions(+), 56 deletions(-)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 20285bc..8dbf740 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -1399,37 +1399,6 @@ pnfs_pageio_init_write(struct nfs_pageio_descriptor *pgio, struct inode *inode)
 	pnfs_set_pg_test(inode, pgio);
 }
 
-/* Retrieve I/O parameters for O_DIRECT.
- * Out Args:
- * iosize    - min of boundary and (rsize or wsize)
- * remaining - # bytes remaining in the current stripe unit
- */
-void
-_pnfs_direct_init_io(struct inode *inode, struct nfs_open_context *ctx,
-		     size_t count, loff_t loff, int iswrite, size_t *iosize,
-		     size_t *remaining)
-{
-	struct nfs_server *nfss = NFS_SERVER(inode);
-	u32 boundary;
-	unsigned int rwsize;
-
-	if (count <= 0 ||
-	    pnfs_update_layout(inode, ctx, count, loff, IOMODE_READ, NULL))
-		return;
-
-	if (iswrite)
-		rwsize = nfss->wsize;
-	else
-		rwsize = nfss->rsize;
-
-	boundary = pnfs_getboundary(inode);
-
-	*iosize = min(rwsize, boundary);
-	*remaining = boundary - (do_div(loff, boundary));
-
-	dprintk("%s Rem %Zu iosize %Zu\n", __func__, *remaining, *iosize);
-}
-
 /*
  * Get a layoutout for COMMIT
  */
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 8edca30..5e9b06b 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -66,9 +66,6 @@ void pnfs_layout_release(struct pnfs_layout_type *, atomic_t *,
 void pnfs_set_layout_stateid(struct pnfs_layout_type *lo,
 			     const nfs4_stateid *stateid);
 void pnfs_destroy_layout(struct nfs_inode *);
-void _pnfs_direct_init_io(struct inode *inode, struct nfs_open_context *ctx,
-			  size_t count, loff_t loff, int iswrite,
-			  size_t *rwsize, size_t *remaining);
 
 #define PNFS_EXISTS_LDIO_OP(srv, opname) ((srv)->pnfs_curr_ld &&	\
 				     (srv)->pnfs_curr_ld->ld_io_ops &&	\
@@ -182,20 +179,6 @@ static inline int pnfs_get_read_status(struct nfs_read_data *data)
 	return data->pdata.pnfs_error;
 }
 
-static inline void pnfs_direct_init_io(struct inode *inode,
-				       struct nfs_open_context *ctx,
-				       size_t count, loff_t loff, int iswrite,
-				       size_t *iosize, size_t *remaining)
-{
-	struct nfs_server *nfss = NFS_SERVER(inode);
-
-	if (pnfs_enabled_sb(nfss))
-		return _pnfs_direct_init_io(inode, ctx, count, loff, iswrite,
-					    iosize, remaining);
-
-	return;
-}
-
 static inline int pnfs_use_rpc(struct nfs_server *nfss)
 {
 	if (pnfs_enabled_sb(nfss))
@@ -241,14 +224,6 @@ static inline int pnfs_get_read_status(struct nfs_read_data *data)
 	return 0;
 }
 
-/* Set num of remaining bytes, which is everything */
-static inline void pnfs_direct_init_io(struct inode *inode,
-				       struct nfs_open_context *ctx,
-				       size_t count, loff_t loff, int iswrite,
-				       size_t *iosize, size_t *remaining)
-{
-}
-
 static inline int pnfs_use_rpc(struct nfs_server *nfss)
 {
 	return 1;
-- 
1.6.6.1

[PATCH 05/22] pnfs: filelayout: clean and breakup nfs4_pnfs_dserver_get

[Fred Isaman]

Rewrite nfs4_pnfs_dserver_get as two functions, nfs4_fl_calc_ds_index() and
nfs4_fl_prepare_ds().  This cleans up the code a bit and prepares for more
extensive rewrite of filelayout_commit().
---
 fs/nfs/nfs4filelayout.c    |   74 ++++++++++++-----------------------
 fs/nfs/nfs4filelayout.h    |   33 ++++++++--------
 fs/nfs/nfs4filelayoutdev.c |   93 +++++++++++++++-----------------------------
 3 files changed, 73 insertions(+), 127 deletions(-)

diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
index 200bbc2..1e99df0 100644
--- a/fs/nfs/nfs4filelayout.c
+++ b/fs/nfs/nfs4filelayout.c
@@ -196,8 +196,8 @@ filelayout_read_pagelist(struct pnfs_layout_type *layoutid,
 {
 	struct inode *inode = PNFS_INODE(layoutid);
 	struct nfs4_filelayout_segment *flseg;
-	struct nfs4_pnfs_dserver dserver;
-	int status;
+	struct nfs4_pnfs_ds *ds;
+	u32 idx;
 
 	dprintk("--> %s ino %lu nr_pages %d pgbase %u req %Zu@%llu\n",
 		__func__, inode->i_ino, nr_pages, pgbase, count, offset);
@@ -205,23 +205,19 @@ filelayout_read_pagelist(struct pnfs_layout_type *layoutid,
 	flseg = LSEG_LD_DATA(data->pdata.lseg);
 
 	/* Retrieve the correct rpc_client for the byte range */
-	status = nfs4_pnfs_dserver_get(data->pdata.lseg,
-				       offset,
-				       count,
-				       &dserver);
-	if (status) {
-		printk(KERN_ERR "%s: dserver get failed status %d use MDS\n",
-		       __func__, status);
+	idx = nfs4_fl_calc_ds_index(data->pdata.lseg, offset);
+	ds = nfs4_fl_prepare_ds(data->pdata.lseg, idx);
+	if (!ds) {
+		printk(KERN_ERR "%s: prepare_ds failed, use MDS\n", __func__);
 		return PNFS_NOT_ATTEMPTED;
 	}
-
 	dprintk("%s USE DS:ip %x %s\n", __func__,
-		htonl(dserver.ds->ds_ip_addr), dserver.ds->r_addr);
+		htonl(ds->ds_ip_addr), ds->r_addr);
 
 	/* just try the first data server for the index..*/
-	data->fldata.pnfs_client = dserver.ds->ds_clp->cl_rpcclient;
-	data->fldata.ds_nfs_client = dserver.ds->ds_clp;
-	data->args.fh = dserver.fh;
+	data->fldata.ds_nfs_client = ds->ds_clp;
+	data->fldata.pnfs_client = ds->ds_clp->cl_rpcclient;
+	data->args.fh = nfs4_fl_select_ds_fh(flseg, idx);
 
 	/* Now get the file offset on the dserver
 	 * Set the read offset to this offset, and
@@ -255,32 +251,26 @@ filelayout_write_pagelist(struct pnfs_layout_type *layoutid,
 {
 	struct inode *inode = PNFS_INODE(layoutid);
 	struct nfs4_filelayout_segment *flseg = LSEG_LD_DATA(data->pdata.lseg);
-	struct nfs4_pnfs_dserver dserver;
-	int status;
+	struct nfs4_pnfs_ds *ds;
+	u32 idx;
 
 	dprintk("--> %s ino %lu nr_pages %d pgbase %u req %Zu@%llu sync %d\n",
 		__func__, inode->i_ino, nr_pages, pgbase, count, offset, sync);
 
 	/* Retrieve the correct rpc_client for the byte range */
-	status = nfs4_pnfs_dserver_get(data->pdata.lseg,
-				       offset,
-				       count,
-				       &dserver);
-
-	if (status) {
-		printk(KERN_ERR "%s: dserver get failed status %d use MDS\n",
-		       __func__, status);
+	idx = nfs4_fl_calc_ds_index(data->pdata.lseg, offset);
+	ds = nfs4_fl_prepare_ds(data->pdata.lseg, idx);
+	if (!ds) {
+		printk(KERN_ERR "%s: prepare_ds failed, use MDS\n", __func__);
 		return PNFS_NOT_ATTEMPTED;
 	}
-
 	dprintk("%s ino %lu %Zu@%llu DS:%x:%hu %s\n",
 		__func__, inode->i_ino, count, offset,
-		htonl(dserver.ds->ds_ip_addr), ntohs(dserver.ds->ds_port),
-		dserver.ds->r_addr);
+		htonl(ds->ds_ip_addr), ntohs(ds->ds_port), ds->r_addr);
 
-	data->fldata.pnfs_client = dserver.ds->ds_clp->cl_rpcclient;
-	data->fldata.ds_nfs_client = dserver.ds->ds_clp;
-	data->args.fh = dserver.fh;
+	data->fldata.ds_nfs_client = ds->ds_clp;
+	data->fldata.pnfs_client = ds->ds_clp->cl_rpcclient;
+	data->args.fh = nfs4_fl_select_ds_fh(flseg, idx);
 
 	/* Get the file offset on the dserver. Set the write offset to
 	 * this offset and save the original offset.
@@ -568,15 +558,12 @@ filelayout_commit(struct pnfs_layout_type *layoutid, int sync,
 {
 	struct nfs4_filelayout_segment *nfslay;
 	struct nfs_write_data   *dsdata = NULL;
-	struct nfs4_pnfs_dserver dserver;
 	struct nfs4_pnfs_ds *ds;
 	struct nfs_page *req, *reqt;
 	struct list_head *pos, *tmp, head, head2;
 	loff_t file_offset, comp_offset;
 	size_t stripesz, cbytes;
-	int status;
 	enum pnfs_try_status trypnfs = PNFS_ATTEMPTED;
-	struct nfs4_file_layout_dsaddr *dsaddr;
 	u32 idx1, idx2;
 
 	nfslay = LSEG_LD_DATA(data->pdata.lseg);
@@ -593,9 +580,6 @@ filelayout_commit(struct pnfs_layout_type *layoutid, int sync,
 	stripesz = filelayout_get_stripesize(layoutid);
 	dprintk("%s stripesize %Zd\n", __func__, stripesz);
 
-	dsaddr = container_of(data->pdata.lseg->deviceid,
-			      struct nfs4_file_layout_dsaddr, deviceid);
-
 	INIT_LIST_HEAD(&head);
 	INIT_LIST_HEAD(&head2);
 	list_add(&head, &data->pages);
@@ -609,19 +593,13 @@ filelayout_commit(struct pnfs_layout_type *layoutid, int sync,
 		file_offset = (loff_t)req->wb_index << PAGE_CACHE_SHIFT;
 
 		/* Get dserver for the current page */
-		status = nfs4_pnfs_dserver_get(data->pdata.lseg,
-					       file_offset,
-					       req->wb_bytes,
-					       &dserver);
-		if (status) {
+		idx1 = nfs4_fl_calc_ds_index(data->pdata.lseg, file_offset);
+		ds = nfs4_fl_prepare_ds(data->pdata.lseg, idx1);
+		if (!ds) {
 			data->pdata.pnfs_error = -EIO;
 			goto err_rewind;
 		}
 
-		/* Get its index */
-		idx1 = filelayout_dserver_get_index(file_offset, dsaddr,
-						    nfslay);
-
 		/* Gather all pages going to the current data server by
 		 * comparing their indices.
 		 * XXX: This recalculates the indices unecessarily.
@@ -630,8 +608,7 @@ filelayout_commit(struct pnfs_layout_type *layoutid, int sync,
 		list_for_each_safe(pos, tmp, &head) {
 			reqt = nfs_list_entry(pos);
 			comp_offset = (loff_t)reqt->wb_index << PAGE_CACHE_SHIFT;
-			idx2 = filelayout_dserver_get_index(comp_offset,
-							    dsaddr, nfslay);
+			idx2 = nfs4_fl_calc_ds_index(data->pdata.lseg, comp_offset);
 			if (idx1 == idx2) {
 				nfs_list_remove_request(reqt);
 				nfs_list_add_request(reqt, &head2);
@@ -655,10 +632,9 @@ filelayout_commit(struct pnfs_layout_type *layoutid, int sync,
 		list_add(&dsdata->pages, &head2);
 		list_del_init(&head2);
 
-		ds = dserver.ds;
 		dsdata->fldata.pnfs_client = ds->ds_clp->cl_rpcclient;
 		dsdata->fldata.ds_nfs_client = ds->ds_clp;
-		dsdata->args.fh = dserver.fh;
+		dsdata->args.fh = nfs4_fl_select_ds_fh(nfslay, idx1);
 
 		dprintk("%s: Initiating commit: %Zu@%llu USE DS:\n",
 			__func__, cbytes, file_offset);
diff --git a/fs/nfs/nfs4filelayout.h b/fs/nfs/nfs4filelayout.h
index fbf307c..3697926 100644
--- a/fs/nfs/nfs4filelayout.h
+++ b/fs/nfs/nfs4filelayout.h
@@ -26,6 +26,9 @@
 
 #define FILE_MT(inode) ((struct filelayout_mount_type *) \
 			(NFS_SERVER(inode)->pnfs_mountid->mountid))
+#define FILE_DSADDR(lseg) (container_of(lseg->deviceid, \
+					struct nfs4_file_layout_dsaddr, \
+					deviceid))
 
 enum stripetype4 {
 	STRIPE_SPARSE = 1,
@@ -55,16 +58,6 @@ struct nfs4_pnfs_dev_hlist {
 	struct hlist_head	dev_list[NFS4_PNFS_DEV_HASH_SIZE];
 };
 
-/*
- * Used for I/O, Maps a stripe index to a layout file handle and a
- * multipath data server.
- */
-
-struct nfs4_pnfs_dserver {
-	struct nfs_fh        *fh;
-	struct nfs4_pnfs_ds	*ds;
-};
-
 struct nfs4_filelayout_segment {
 	u32 stripe_type;
 	u32 commit_through_mds;
@@ -87,18 +80,24 @@ struct filelayout_mount_type {
 	struct super_block *fl_sb;
 };
 
+static inline struct nfs_fh *
+nfs4_fl_select_ds_fh(struct nfs4_filelayout_segment *flseg, u32 idx)
+{
+	/* FRED - what about case == 0??? */
+	if (flseg->num_fh == 1)
+		return &flseg->fh_array[0];
+	else
+		return &flseg->fh_array[idx];
+}
+
 extern struct pnfs_client_operations *pnfs_callback_ops;
 
 extern void nfs4_fl_free_deviceid_callback(struct kref *);
 extern void print_ds(struct nfs4_pnfs_ds *ds);
 char *deviceid_fmt(const struct pnfs_deviceid *dev_id);
-int nfs4_pnfs_dserver_get(struct pnfs_layout_segment *lseg,
-			  loff_t offset,
-			  size_t count,
-			  struct nfs4_pnfs_dserver *dserver);
-u32 filelayout_dserver_get_index(loff_t offset,
-				 struct nfs4_file_layout_dsaddr *di,
-				 struct nfs4_filelayout_segment *layout);
+u32 nfs4_fl_calc_ds_index(struct pnfs_layout_segment *lseg, loff_t offset);
+struct nfs4_pnfs_ds *nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg,
+					u32 ds_idx);
 extern struct nfs4_file_layout_dsaddr *
 nfs4_pnfs_device_item_find(struct nfs_client *, struct pnfs_deviceid *dev_id);
 struct nfs4_file_layout_dsaddr *
diff --git a/fs/nfs/nfs4filelayoutdev.c b/fs/nfs/nfs4filelayoutdev.c
index b04c9d9..6721aca 100644
--- a/fs/nfs/nfs4filelayoutdev.c
+++ b/fs/nfs/nfs4filelayoutdev.c
@@ -554,90 +554,61 @@ nfs4_pnfs_device_item_find(struct nfs_client *clp, struct pnfs_deviceid *id)
 		container_of(d, struct nfs4_file_layout_dsaddr, deviceid);
 }
 
-/* Want res = ((offset / layout->stripe_unit) % dsaddr->stripe_count)
+/* Want res = (offset - layout->pattern_offset)/ layout->stripe_unit
  * Then: ((res + fsi) % dsaddr->stripe_count)
  */
-u32
-filelayout_dserver_get_index(loff_t offset,
-			     struct nfs4_file_layout_dsaddr *dsaddr,
-			     struct nfs4_filelayout_segment *layout)
+static inline u32
+_nfs4_fl_calc_j_index(loff_t offset,
+		      struct nfs4_file_layout_dsaddr *dsaddr,
+		      struct nfs4_filelayout_segment *layout)
 {
-	u64 tmp, tmp2;
+	u64 tmp;
 
-	tmp = offset;
+	tmp = offset - layout->pattern_offset;
 	do_div(tmp, layout->stripe_unit);
-	tmp2 = do_div(tmp, dsaddr->stripe_count) + layout->first_stripe_index;
-	return do_div(tmp2, dsaddr->stripe_count);
+	tmp += layout->first_stripe_index;
+	return do_div(tmp, dsaddr->stripe_count);
 }
 
-/* Retrieve the rpc client for a specified byte range
- * in 'inode' by filling in the contents of 'dserver'.
- */
-int
-nfs4_pnfs_dserver_get(struct pnfs_layout_segment *lseg,
-		      loff_t offset,
-		      size_t count,
-		      struct nfs4_pnfs_dserver *dserver)
+u32
+nfs4_fl_calc_ds_index(struct pnfs_layout_segment *lseg, loff_t offset)
 {
-	struct nfs4_filelayout_segment *layout = LSEG_LD_DATA(lseg);
-	struct inode *inode = PNFS_INODE(lseg->layout);
-	struct nfs_server *mds_srv = NFS_SERVER(inode);
+	struct nfs4_filelayout_segment *flseg = LSEG_LD_DATA(lseg);
 	struct nfs4_file_layout_dsaddr *dsaddr;
-	u64 tmp, tmp2;
-	u32 stripe_idx, end_idx, ds_idx;
-
-	if (!layout)
-		return 1;
-
-	dsaddr = container_of(lseg->deviceid, struct nfs4_file_layout_dsaddr,
-			      deviceid);
-
-	stripe_idx = filelayout_dserver_get_index(offset, dsaddr, layout);
-
-	/* For debugging, ensure entire requested range is in this dserver */
-	tmp = offset + count - 1;
-	do_div(tmp, layout->stripe_unit);
-	tmp2 = do_div(tmp, dsaddr->stripe_count) + layout->first_stripe_index;
-	end_idx = do_div(tmp2, dsaddr->stripe_count);
+	u32 j;
 
-	dprintk("%s: offset=%Lu, count=%Zu, si=%u, dsi=%u, "
-		"stripe_count=%u, stripe_unit=%u first_stripe_index %u\n",
-		__func__,
-		offset, count, stripe_idx, end_idx, dsaddr->stripe_count,
-		layout->stripe_unit, layout->first_stripe_index);
+	dsaddr = FILE_DSADDR(lseg);
+	j = _nfs4_fl_calc_j_index(offset, dsaddr, flseg);
+	return dsaddr->stripe_indices[j];
+}
 
-	BUG_ON(end_idx != stripe_idx);
-	BUG_ON(stripe_idx >= dsaddr->stripe_count);
+struct nfs4_pnfs_ds *
+nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx)
+{
+	struct nfs4_filelayout_segment *flseg = LSEG_LD_DATA(lseg);
+	struct nfs4_file_layout_dsaddr *dsaddr;
 
-	ds_idx = dsaddr->stripe_indices[stripe_idx];
+	dsaddr = FILE_DSADDR(lseg);
 	if (dsaddr->ds_list[ds_idx] == NULL) {
 		printk(KERN_ERR "%s: No data server for device id (%s)!! \n",
-			__func__, deviceid_fmt(&layout->dev_id));
-		return 1;
+			__func__, deviceid_fmt(&flseg->dev_id));
+		return NULL;
 	}
 
 	if (!dsaddr->ds_list[ds_idx]->ds_clp) {
 		int err;
 
-		err = nfs4_pnfs_ds_create(mds_srv, dsaddr->ds_list[ds_idx]);
+		err = nfs4_pnfs_ds_create(PNFS_NFS_SERVER(lseg->layout),
+					  dsaddr->ds_list[ds_idx]);
 		if (err) {
 			printk(KERN_ERR "%s nfs4_pnfs_ds_create error %d\n",
 			       __func__, err);
-			return 1;
+			return NULL;
 		}
 	}
-	dserver->ds = dsaddr->ds_list[ds_idx];
+	dprintk("%s: dev_id=%s, ds_idx=%u\n",
+		__func__, deviceid_fmt(&flseg->dev_id), ds_idx);
 
-	if (layout->num_fh == 1)
-		dserver->fh = &layout->fh_array[0];
-	else
-		dserver->fh = &layout->fh_array[ds_idx];
-
-	dprintk("%s: dev_id=%s, ip:port=%s, ds_idx=%u stripe_idx=%u, "
-		"offset=%llu, count=%Zu\n",
-		__func__, deviceid_fmt(&layout->dev_id),
-		dserver->ds->r_addr,
-		ds_idx, stripe_idx, offset, count);
-
-	return 0;
+	return dsaddr->ds_list[ds_idx];
 }
+
-- 
1.6.6.1

[PATCH 05/22] pnfs: filelayout: clean and breakup nfs4_pnfs_dserver_get

[Fred Isaman]

Rewrite nfs4_pnfs_dserver_get as two functions, nfs4_fl_calc_ds_index() and
nfs4_fl_prepare_ds().  This cleans up the code a bit and prepares for more
extensive rewrite of filelayout_commit().

Signed-off-by: Fred Isaman <iisaman@netapp.com>
---
 fs/nfs/nfs4filelayout.c    |   75 ++++++++++++----------------------
 fs/nfs/nfs4filelayout.h    |   33 +++++++--------
 fs/nfs/nfs4filelayoutdev.c |   95 +++++++++++++++----------------------------
 3 files changed, 75 insertions(+), 128 deletions(-)

diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c
index 200bbc2..2d7f634 100644
--- a/fs/nfs/nfs4filelayout.c
+++ b/fs/nfs/nfs4filelayout.c
@@ -196,8 +196,8 @@ filelayout_read_pagelist(struct pnfs_layout_type *layoutid,
 {
 	struct inode *inode = PNFS_INODE(layoutid);
 	struct nfs4_filelayout_segment *flseg;
-	struct nfs4_pnfs_dserver dserver;
-	int status;
+	struct nfs4_pnfs_ds *ds;
+	u32 idx;
 
 	dprintk("--> %s ino %lu nr_pages %d pgbase %u req %Zu@%llu\n",
 		__func__, inode->i_ino, nr_pages, pgbase, count, offset);
@@ -205,23 +205,19 @@ filelayout_read_pagelist(struct pnfs_layout_type *layoutid,
 	flseg = LSEG_LD_DATA(data->pdata.lseg);
 
 	/* Retrieve the correct rpc_client for the byte range */
-	status = nfs4_pnfs_dserver_get(data->pdata.lseg,
-				       offset,
-				       count,
-				       &dserver);
-	if (status) {
-		printk(KERN_ERR "%s: dserver get failed status %d use MDS\n",
-		       __func__, status);
+	idx = nfs4_fl_calc_ds_index(data->pdata.lseg, offset);
+	ds = nfs4_fl_prepare_ds(data->pdata.lseg, idx);
+	if (!ds) {
+		printk(KERN_ERR "%s: prepare_ds failed, use MDS\n", __func__);
 		return PNFS_NOT_ATTEMPTED;
 	}
-
 	dprintk("%s USE DS:ip %x %s\n", __func__,
-		htonl(dserver.ds->ds_ip_addr), dserver.ds->r_addr);
+		htonl(ds->ds_ip_addr), ds->r_addr);
 
 	/* just try the first data server for the index..*/
-	data->fldata.pnfs_client = dserver.ds->ds_clp->cl_rpcclient;
-	data->fldata.ds_nfs_client = dserver.ds->ds_clp;
-	data->args.fh = dserver.fh;
+	data->fldata.ds_nfs_client = ds->ds_clp;
+	data->fldata.pnfs_client = ds->ds_clp->cl_rpcclient;
+	data->args.fh = nfs4_fl_select_ds_fh(flseg, idx);
 
 	/* Now get the file offset on the dserver
 	 * Set the read offset to this offset, and
@@ -255,32 +251,26 @@ filelayout_write_pagelist(struct pnfs_layout_type *layoutid,
 {
 	struct inode *inode = PNFS_INODE(layoutid);
 	struct nfs4_filelayout_segment *flseg = LSEG_LD_DATA(data->pdata.lseg);
-	struct nfs4_pnfs_dserver dserver;
-	int status;
+	struct nfs4_pnfs_ds *ds;
+	u32 idx;
 
 	dprintk("--> %s ino %lu nr_pages %d pgbase %u req %Zu@%llu sync %d\n",
 		__func__, inode->i_ino, nr_pages, pgbase, count, offset, sync);
 
 	/* Retrieve the correct rpc_client for the byte range */
-	status = nfs4_pnfs_dserver_get(data->pdata.lseg,
-				       offset,
-				       count,
-				       &dserver);
-
-	if (status) {
-		printk(KERN_ERR "%s: dserver get failed status %d use MDS\n",
-		       __func__, status);
+	idx = nfs4_fl_calc_ds_index(data->pdata.lseg, offset);
+	ds = nfs4_fl_prepare_ds(data->pdata.lseg, idx);
+	if (!ds) {
+		printk(KERN_ERR "%s: prepare_ds failed, use MDS\n", __func__);
 		return PNFS_NOT_ATTEMPTED;
 	}
-
 	dprintk("%s ino %lu %Zu@%llu DS:%x:%hu %s\n",
 		__func__, inode->i_ino, count, offset,
-		htonl(dserver.ds->ds_ip_addr), ntohs(dserver.ds->ds_port),
-		dserver.ds->r_addr);
+		htonl(ds->ds_ip_addr), ntohs(ds->ds_port), ds->r_addr);
 
-	data->fldata.pnfs_client = dserver.ds->ds_clp->cl_rpcclient;
-	data->fldata.ds_nfs_client = dserver.ds->ds_clp;
-	data->args.fh = dserver.fh;
+	data->fldata.ds_nfs_client = ds->ds_clp;
+	data->fldata.pnfs_client = ds->ds_clp->cl_rpcclient;
+	data->args.fh = nfs4_fl_select_ds_fh(flseg, idx);
 
 	/* Get the file offset on the dserver. Set the write offset to
 	 * this offset and save the original offset.
@@ -568,15 +558,12 @@ filelayout_commit(struct pnfs_layout_type *layoutid, int sync,
 {
 	struct nfs4_filelayout_segment *nfslay;
 	struct nfs_write_data   *dsdata = NULL;
-	struct nfs4_pnfs_dserver dserver;
 	struct nfs4_pnfs_ds *ds;
 	struct nfs_page *req, *reqt;
 	struct list_head *pos, *tmp, head, head2;
 	loff_t file_offset, comp_offset;
 	size_t stripesz, cbytes;
-	int status;
 	enum pnfs_try_status trypnfs = PNFS_ATTEMPTED;
-	struct nfs4_file_layout_dsaddr *dsaddr;
 	u32 idx1, idx2;
 
 	nfslay = LSEG_LD_DATA(data->pdata.lseg);
@@ -593,9 +580,6 @@ filelayout_commit(struct pnfs_layout_type *layoutid, int sync,
 	stripesz = filelayout_get_stripesize(layoutid);
 	dprintk("%s stripesize %Zd\n", __func__, stripesz);
 
-	dsaddr = container_of(data->pdata.lseg->deviceid,
-			      struct nfs4_file_layout_dsaddr, deviceid);
-
 	INIT_LIST_HEAD(&head);
 	INIT_LIST_HEAD(&head2);
 	list_add(&head, &data->pages);
@@ -609,19 +593,13 @@ filelayout_commit(struct pnfs_layout_type *layoutid, int sync,
 		file_offset = (loff_t)req->wb_index << PAGE_CACHE_SHIFT;
 
 		/* Get dserver for the current page */
-		status = nfs4_pnfs_dserver_get(data->pdata.lseg,
-					       file_offset,
-					       req->wb_bytes,
-					       &dserver);
-		if (status) {
+		idx1 = nfs4_fl_calc_ds_index(data->pdata.lseg, file_offset);
+		ds = nfs4_fl_prepare_ds(data->pdata.lseg, idx1);
+		if (!ds) {
 			data->pdata.pnfs_error = -EIO;
 			goto err_rewind;
 		}
 
-		/* Get its index */
-		idx1 = filelayout_dserver_get_index(file_offset, dsaddr,
-						    nfslay);
-
 		/* Gather all pages going to the current data server by
 		 * comparing their indices.
 		 * XXX: This recalculates the indices unecessarily.
@@ -630,8 +608,8 @@ filelayout_commit(struct pnfs_layout_type *layoutid, int sync,
 		list_for_each_safe(pos, tmp, &head) {
 			reqt = nfs_list_entry(pos);
 			comp_offset = (loff_t)reqt->wb_index << PAGE_CACHE_SHIFT;
-			idx2 = filelayout_dserver_get_index(comp_offset,
-							    dsaddr, nfslay);
+			idx2 = nfs4_fl_calc_ds_index(data->pdata.lseg,
+						     comp_offset);
 			if (idx1 == idx2) {
 				nfs_list_remove_request(reqt);
 				nfs_list_add_request(reqt, &head2);
@@ -655,10 +633,9 @@ filelayout_commit(struct pnfs_layout_type *layoutid, int sync,
 		list_add(&dsdata->pages, &head2);
 		list_del_init(&head2);
 
-		ds = dserver.ds;
 		dsdata->fldata.pnfs_client = ds->ds_clp->cl_rpcclient;
 		dsdata->fldata.ds_nfs_client = ds->ds_clp;
-		dsdata->args.fh = dserver.fh;
+		dsdata->args.fh = nfs4_fl_select_ds_fh(nfslay, idx1);
 
 		dprintk("%s: Initiating commit: %Zu@%llu USE DS:\n",
 			__func__, cbytes, file_offset);
diff --git a/fs/nfs/nfs4filelayout.h b/fs/nfs/nfs4filelayout.h
index fbf307c..3697926 100644
--- a/fs/nfs/nfs4filelayout.h
+++ b/fs/nfs/nfs4filelayout.h
@@ -26,6 +26,9 @@
 
 #define FILE_MT(inode) ((struct filelayout_mount_type *) \
 			(NFS_SERVER(inode)->pnfs_mountid->mountid))
+#define FILE_DSADDR(lseg) (container_of(lseg->deviceid, \
+					struct nfs4_file_layout_dsaddr, \
+					deviceid))
 
 enum stripetype4 {
 	STRIPE_SPARSE = 1,
@@ -55,16 +58,6 @@ struct nfs4_pnfs_dev_hlist {
 	struct hlist_head	dev_list[NFS4_PNFS_DEV_HASH_SIZE];
 };
 
-/*
- * Used for I/O, Maps a stripe index to a layout file handle and a
- * multipath data server.
- */
-
-struct nfs4_pnfs_dserver {
-	struct nfs_fh        *fh;
-	struct nfs4_pnfs_ds	*ds;
-};
-
 struct nfs4_filelayout_segment {
 	u32 stripe_type;
 	u32 commit_through_mds;
@@ -87,18 +80,24 @@ struct filelayout_mount_type {
 	struct super_block *fl_sb;
 };
 
+static inline struct nfs_fh *
+nfs4_fl_select_ds_fh(struct nfs4_filelayout_segment *flseg, u32 idx)
+{
+	/* FRED - what about case == 0??? */
+	if (flseg->num_fh == 1)
+		return &flseg->fh_array[0];
+	else
+		return &flseg->fh_array[idx];
+}
+
 extern struct pnfs_client_operations *pnfs_callback_ops;
 
 extern void nfs4_fl_free_deviceid_callback(struct kref *);
 extern void print_ds(struct nfs4_pnfs_ds *ds);
 char *deviceid_fmt(const struct pnfs_deviceid *dev_id);
-int nfs4_pnfs_dserver_get(struct pnfs_layout_segment *lseg,
-			  loff_t offset,
-			  size_t count,
-			  struct nfs4_pnfs_dserver *dserver);
-u32 filelayout_dserver_get_index(loff_t offset,
-				 struct nfs4_file_layout_dsaddr *di,
-				 struct nfs4_filelayout_segment *layout);
+u32 nfs4_fl_calc_ds_index(struct pnfs_layout_segment *lseg, loff_t offset);
+struct nfs4_pnfs_ds *nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg,
+					u32 ds_idx);
 extern struct nfs4_file_layout_dsaddr *
 nfs4_pnfs_device_item_find(struct nfs_client *, struct pnfs_deviceid *dev_id);
 struct nfs4_file_layout_dsaddr *
diff --git a/fs/nfs/nfs4filelayoutdev.c b/fs/nfs/nfs4filelayoutdev.c
index b04c9d9..cd39a6a 100644
--- a/fs/nfs/nfs4filelayoutdev.c
+++ b/fs/nfs/nfs4filelayoutdev.c
@@ -554,90 +554,61 @@ nfs4_pnfs_device_item_find(struct nfs_client *clp, struct pnfs_deviceid *id)
 		container_of(d, struct nfs4_file_layout_dsaddr, deviceid);
 }
 
-/* Want res = ((offset / layout->stripe_unit) % dsaddr->stripe_count)
+/* Want res = (offset - layout->pattern_offset)/ layout->stripe_unit
  * Then: ((res + fsi) % dsaddr->stripe_count)
  */
-u32
-filelayout_dserver_get_index(loff_t offset,
-			     struct nfs4_file_layout_dsaddr *dsaddr,
-			     struct nfs4_filelayout_segment *layout)
+static inline u32
+_nfs4_fl_calc_j_index(loff_t offset,
+		      struct nfs4_file_layout_dsaddr *dsaddr,
+		      struct nfs4_filelayout_segment *layout)
 {
-	u64 tmp, tmp2;
+	u64 tmp;
 
-	tmp = offset;
+	tmp = offset - layout->pattern_offset;
 	do_div(tmp, layout->stripe_unit);
-	tmp2 = do_div(tmp, dsaddr->stripe_count) + layout->first_stripe_index;
-	return do_div(tmp2, dsaddr->stripe_count);
+	tmp += layout->first_stripe_index;
+	return do_div(tmp, dsaddr->stripe_count);
 }
 
-/* Retrieve the rpc client for a specified byte range
- * in 'inode' by filling in the contents of 'dserver'.
- */
-int
-nfs4_pnfs_dserver_get(struct pnfs_layout_segment *lseg,
-		      loff_t offset,
-		      size_t count,
-		      struct nfs4_pnfs_dserver *dserver)
+u32
+nfs4_fl_calc_ds_index(struct pnfs_layout_segment *lseg, loff_t offset)
 {
-	struct nfs4_filelayout_segment *layout = LSEG_LD_DATA(lseg);
-	struct inode *inode = PNFS_INODE(lseg->layout);
-	struct nfs_server *mds_srv = NFS_SERVER(inode);
+	struct nfs4_filelayout_segment *flseg = LSEG_LD_DATA(lseg);
 	struct nfs4_file_layout_dsaddr *dsaddr;
-	u64 tmp, tmp2;
-	u32 stripe_idx, end_idx, ds_idx;
-
-	if (!layout)
-		return 1;
-
-	dsaddr = container_of(lseg->deviceid, struct nfs4_file_layout_dsaddr,
-			      deviceid);
-
-	stripe_idx = filelayout_dserver_get_index(offset, dsaddr, layout);
-
-	/* For debugging, ensure entire requested range is in this dserver */
-	tmp = offset + count - 1;
-	do_div(tmp, layout->stripe_unit);
-	tmp2 = do_div(tmp, dsaddr->stripe_count) + layout->first_stripe_index;
-	end_idx = do_div(tmp2, dsaddr->stripe_count);
+	u32 j;
 
-	dprintk("%s: offset=%Lu, count=%Zu, si=%u, dsi=%u, "
-		"stripe_count=%u, stripe_unit=%u first_stripe_index %u\n",
-		__func__,
-		offset, count, stripe_idx, end_idx, dsaddr->stripe_count,
-		layout->stripe_unit, layout->first_stripe_index);
+	dsaddr = FILE_DSADDR(lseg);
+	j = _nfs4_fl_calc_j_index(offset, dsaddr, flseg);
+	return dsaddr->stripe_indices[j];
+}
 
-	BUG_ON(end_idx != stripe_idx);
-	BUG_ON(stripe_idx >= dsaddr->stripe_count);
+struct nfs4_pnfs_ds *
+nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg, u32 ds_idx)
+{
+	struct nfs4_filelayout_segment *flseg = LSEG_LD_DATA(lseg);
+	struct nfs4_file_layout_dsaddr *dsaddr;
 
-	ds_idx = dsaddr->stripe_indices[stripe_idx];
+	dsaddr = FILE_DSADDR(lseg);
 	if (dsaddr->ds_list[ds_idx] == NULL) {
-		printk(KERN_ERR "%s: No data server for device id (%s)!! \n",
-			__func__, deviceid_fmt(&layout->dev_id));
-		return 1;
+		printk(KERN_ERR "%s: No data server for device id (%s)!!\n",
+			__func__, deviceid_fmt(&flseg->dev_id));
+		return NULL;
 	}
 
 	if (!dsaddr->ds_list[ds_idx]->ds_clp) {
 		int err;
 
-		err = nfs4_pnfs_ds_create(mds_srv, dsaddr->ds_list[ds_idx]);
+		err = nfs4_pnfs_ds_create(PNFS_NFS_SERVER(lseg->layout),
+					  dsaddr->ds_list[ds_idx]);
 		if (err) {
 			printk(KERN_ERR "%s nfs4_pnfs_ds_create error %d\n",
 			       __func__, err);
-			return 1;
+			return NULL;
 		}
 	}
-	dserver->ds = dsaddr->ds_list[ds_idx];
+	dprintk("%s: dev_id=%s, ds_idx=%u\n",
+		__func__, deviceid_fmt(&flseg->dev_id), ds_idx);
 
-	if (layout->num_fh == 1)
-		dserver->fh = &layout->fh_array[0];
-	else
-		dserver->fh = &layout->fh_array[ds_idx];
-
-	dprintk("%s: dev_id=%s, ip:port=%s, ds_idx=%u stripe_idx=%u, "
-		"offset=%llu, count=%Zu\n",
-		__func__, deviceid_fmt(&layout->dev_id),
-		dserver->ds->r_addr,
-		ds_idx, stripe_idx, offset, count);
-
-	return 0;
+	return dsaddr->ds_list[ds_idx];
 }
+
-- 
1.6.6.1