[PATCH 0/9] fixes for layoutrecall callback pnfs-all-2.6.37 and -latest

[PATCH 0/9] fixes for layoutrecall callback pnfs-all-2.6.37 and -latest

[Benny Halevy]

The following patches fix bugs on the layout recall callback path both at
the server an client

[PATCH 1/9] pnfsd: Initialize cb_layout clr_recall.cb_work
[PATCH 2/9] pnfsd: add default error case in nfsd4_cb_layout_done

These two are for pnfs-all-2.6.37 only,
The rest are for both 37 and 38 (pnfs-all-latest)

[PATCH 3/9] pnfsd: do not clear rpc_resp in nfsd4_cb_done_sequence
[PATCH 4/9] pnfs: fix missing unlock in nfs_client_return_layouts
[PATCH 5/9] pnfs: return bool status from nfs4_asynch_forget_layouts
[PATCH 6/9] pnfs: return NFS4ERR_DELAY from layoutrecall while waiting on lsegs
[PATCH 7/9] pnfs: fix cl_drain_notification indexing
[PATCH 8/9] pnfs: fix nfs4_asynch_forget_layouts accounting
[PATCH 9/9] pnfs: improve nfs4_layoutget_done debug printouts

I've pushed them to a git tree I set up for the Connectathon:

git://linux-nfs.org/~bhalevy/linux-pnfs-cthon.git
or
ssh://linux-nfs.org/~bhalevy/exports/linux-pnfs-cthon.git
if you don't have git access.

Benny

[PATCH 1/9] pnfsd: Initialize cb_layout clr_recall.cb_work

[Benny Halevy]

Fixes a 2,6,37-only bug

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfsd/nfs4pnfsd.c |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diff --git a/fs/nfsd/nfs4pnfsd.c b/fs/nfsd/nfs4pnfsd.c
index 4aa7999..507e3e2 100644
--- a/fs/nfsd/nfs4pnfsd.c
+++ b/fs/nfsd/nfs4pnfsd.c
@@ -579,6 +579,7 @@ alloc_init_layoutrecall(struct nfsd4_pnfs_cb_layout *cbl,
 
 	kref_init(&clr->clr_ref);
 	INIT_LIST_HEAD(&clr->clr_perclnt);
+	INIT_WORK(&clr->clr_recall.cb_work, nfsd4_do_callback_rpc);
 
 	dprintk("NFSD %s return %p\n", __func__, clr);
 	return clr;
-- 
1.7.3.4

[PATCH 2/9] pnfsd: add default error case in nfsd4_cb_layout_done

[Benny Halevy]

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfsd/nfs4callback.c |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c
index 55f27d1..8351f67 100644
--- a/fs/nfsd/nfs4callback.c
+++ b/fs/nfsd/nfs4callback.c
@@ -1072,6 +1072,7 @@ static void nfsd4_cb_layout_done(struct rpc_task *task, void *calldata)
 
 	switch (task->tk_status) {
 	case -EIO:
+	default:
 		/* Network partition? */
 		atomic_set(&clp->cl_cb_set, 0);
 		warn_no_callback_path(clp, task->tk_status);
-- 
1.7.3.4

[PATCH 3/9] pnfsd: do not clear rpc_resp in nfsd4_cb_done_sequence

[Benny Halevy]

This is not needed anymore and is causes bug on the retry path
after getting NFS4ERR_DELAY when nfs4_xdr_dec_cb_layout sees cb == NULL

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfsd/nfs4callback.c |    3 ---
 1 files changed, 0 insertions(+), 3 deletions(-)

diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c
index 8351f67..3fb989a 100644
--- a/fs/nfsd/nfs4callback.c
+++ b/fs/nfsd/nfs4callback.c
@@ -876,9 +876,6 @@ static void nfsd4_cb_done_sequence(struct rpc_task *task,
 		rpc_wake_up_next(&clp->cl_cb_waitq);
 		dprintk("%s: freed slot, new seqid=%d\n", __func__,
 			clp->cl_cb_session->se_cb_seq_nr);
-
-		/* We're done looking into the sequence information */
-		task->tk_msg.rpc_resp = NULL;
 	}
 }
 
-- 
1.7.3.4

[PATCH 4/9] pnfs: fix missing unlock in nfs_client_return_layouts

[Benny Halevy]

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback_proc.c |    1 +
 1 files changed, 1 insertions(+), 0 deletions(-)

diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
index 48c61c2..9547c34 100644
--- a/fs/nfs/callback_proc.c
+++ b/fs/nfs/callback_proc.c
@@ -209,6 +209,7 @@ void nfs_client_return_layouts(struct nfs_client *clp)
 		list_del(&cb_info->pcl_list);
 		clp->cl_cb_lrecall_count--;
 		clp->cl_drain_notification[1 << cb_info->pcl_notify_bit] = NULL;
+		spin_unlock(&clp->cl_lock);
 		rpc_wake_up(&clp->cl_rpcwaitq_recall);
 		kfree(cb_info);
 	}
-- 
1.7.3.4

[PATCH 5/9] pnfs: return bool status from nfs4_asynch_forget_layouts

[Benny Halevy]

return true if segments were marked invalid

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/pnfs.c |   10 ++++++++--
 fs/nfs/pnfs.h |    2 +-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index beebc85..7507054 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -475,7 +475,7 @@ pnfs_clear_lseg_list(struct pnfs_layout_hdr *lo, struct list_head *tmp_list,
 			mark_lseg_invalid(lseg, tmp_list);
 			rv = true;
 		}
-	dprintk("%s:Return\n", __func__);
+	dprintk("%s:Return %d\n", __func__, rv);
 	return rv;
 }
 
@@ -640,20 +640,26 @@ send_layoutget(struct pnfs_layout_hdr *lo,
 	return lseg;
 }
 
-void nfs4_asynch_forget_layouts(struct pnfs_layout_hdr *lo,
+bool nfs4_asynch_forget_layouts(struct pnfs_layout_hdr *lo,
 				struct pnfs_layout_range *range,
 				int notify_bit, atomic_t *notify_count,
 				struct list_head *tmp_list)
 {
+	bool rv = false;
 	struct pnfs_layout_segment *lseg, *tmp;
 
 	assert_spin_locked(&lo->inode->i_lock);
+	dprintk("%s\n", __func__);
 	list_for_each_entry_safe(lseg, tmp, &lo->segs, fi_list)
 		if (should_free_lseg(&lseg->range, range)) {
 			lseg->pls_notify_mask |= (1 << notify_bit);
 			atomic_inc(notify_count);
 			mark_lseg_invalid(lseg, tmp_list);
+			rv = true;
 		}
+
+	dprintk("%s:Return %d\n", __func__, rv);
+	return rv;
 }
 
 /* Return true if there is layout based io in progress in the given range.
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 1ee59c2..11c6ee4 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -298,7 +298,7 @@ void pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo,
 int pnfs_choose_layoutget_stateid(nfs4_stateid *dst,
 				  struct pnfs_layout_hdr *lo,
 				  struct nfs4_state *open_state);
-void nfs4_asynch_forget_layouts(struct pnfs_layout_hdr *lo,
+bool nfs4_asynch_forget_layouts(struct pnfs_layout_hdr *lo,
 				struct pnfs_layout_range *range,
 				int notify_bit, atomic_t *notify_count,
 				struct list_head *tmp_list);
-- 
1.7.3.4

[PATCH 6/9] pnfs: return NFS4ERR_DELAY from layoutrecall while waiting on lsegs

[Benny Halevy]

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback_proc.c |   23 +++++++++++++++--------
 1 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
index 9547c34..5e4b5d1 100644
--- a/fs/nfs/callback_proc.c
+++ b/fs/nfs/callback_proc.c
@@ -186,6 +186,7 @@ void nfs_client_return_layouts(struct nfs_client *clp)
 {
 	struct pnfs_cb_lrecall_info *cb_info;
 
+	dprintk("%s\n", __func__);
 	spin_lock(&clp->cl_lock);
 	while (true) {
 		if (list_empty(&clp->cl_layoutrecalls)) {
@@ -270,10 +271,13 @@ static int initiate_layout_draining(struct pnfs_cb_lrecall_info *cb_info)
 		spin_lock(&lo->inode->i_lock);
 		if (rv == NFS4_OK) {
 			lo->plh_block_lgets++;
-			nfs4_asynch_forget_layouts(lo, &args->cbl_range,
-						   cb_info->pcl_notify_bit,
-						   &cb_info->pcl_count,
-						   &free_me_list);
+			if (nfs4_asynch_forget_layouts(lo, &args->cbl_range,
+						       cb_info->pcl_notify_bit,
+						       &cb_info->pcl_count,
+						       &free_me_list))
+				rv = NFS4ERR_DELAY;
+			else
+				rv = NFS4ERR_NOMATCHING_LAYOUT;
 		}
 		pnfs_set_layout_stateid(lo, &args->cbl_stateid, true);
 		spin_unlock(&lo->inode->i_lock);
@@ -309,10 +313,13 @@ static int initiate_layout_draining(struct pnfs_cb_lrecall_info *cb_info)
 					 &recall_list, plh_bulk_recall) {
 			spin_lock(&lo->inode->i_lock);
 			set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
-			nfs4_asynch_forget_layouts(lo, &range,
-						   cb_info->pcl_notify_bit,
-						   &cb_info->pcl_count,
-						   &free_me_list);
+			if (nfs4_asynch_forget_layouts(lo, &range,
+						       cb_info->pcl_notify_bit,
+						       &cb_info->pcl_count,
+						       &free_me_list))
+				rv = NFS4ERR_DELAY;
+			else
+				rv = NFS4ERR_NOMATCHING_LAYOUT;
 			list_del_init(&lo->plh_bulk_recall);
 			spin_unlock(&lo->inode->i_lock);
 			put_layout_hdr(lo);
-- 
1.7.3.4

[PATCH 7/9] pnfs: fix cl_drain_notification indexing

[Benny Halevy]

cl_drain_notification is an actual index into cl_drain_notification
and a bit index into pls_notify_mask.

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/callback_proc.c |   27 ++++++++++++++++-----------
 fs/nfs/pnfs.c          |    4 ++--
 fs/nfs/pnfs.h          |    2 +-
 3 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/fs/nfs/callback_proc.c b/fs/nfs/callback_proc.c
index 5e4b5d1..dd295e2 100644
--- a/fs/nfs/callback_proc.c
+++ b/fs/nfs/callback_proc.c
@@ -197,8 +197,10 @@ void nfs_client_return_layouts(struct nfs_client *clp)
 					   struct pnfs_cb_lrecall_info,
 					   pcl_list);
 		spin_unlock(&clp->cl_lock);
+		/* Were all recalled lsegs already forgotten */
 		if (atomic_read(&cb_info->pcl_count) != 0)
 			break;
+
 		/* What do on error return?  These layoutreturns are
 		 * required by the protocol.  So if do not get
 		 * successful reply, probably have to do something
@@ -209,7 +211,7 @@ void nfs_client_return_layouts(struct nfs_client *clp)
 		/* Removing from the list unblocks LAYOUTGETs */
 		list_del(&cb_info->pcl_list);
 		clp->cl_cb_lrecall_count--;
-		clp->cl_drain_notification[1 << cb_info->pcl_notify_bit] = NULL;
+		clp->cl_drain_notification[cb_info->pcl_notify_idx] = NULL;
 		spin_unlock(&clp->cl_lock);
 		rpc_wake_up(&clp->cl_rpcwaitq_recall);
 		kfree(cb_info);
@@ -224,7 +226,7 @@ void notify_drained(struct nfs_client *clp, u64 mask)
 	/* clp lock not needed except to remove used up entries */
 	/* Should probably use functions defined in bitmap.h */
 	while (mask) {
-		if ((mask & 1) && (atomic_dec_and_test(*ptr)))
+		if ((mask & 1) && atomic_dec_and_test(*ptr))
 			done = true;
 		mask >>= 1;
 		ptr++;
@@ -272,7 +274,7 @@ static int initiate_layout_draining(struct pnfs_cb_lrecall_info *cb_info)
 		if (rv == NFS4_OK) {
 			lo->plh_block_lgets++;
 			if (nfs4_asynch_forget_layouts(lo, &args->cbl_range,
-						       cb_info->pcl_notify_bit,
+						       cb_info->pcl_notify_idx,
 						       &cb_info->pcl_count,
 						       &free_me_list))
 				rv = NFS4ERR_DELAY;
@@ -314,7 +316,7 @@ static int initiate_layout_draining(struct pnfs_cb_lrecall_info *cb_info)
 			spin_lock(&lo->inode->i_lock);
 			set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
 			if (nfs4_asynch_forget_layouts(lo, &range,
-						       cb_info->pcl_notify_bit,
+						       cb_info->pcl_notify_idx,
 						       &cb_info->pcl_count,
 						       &free_me_list))
 				rv = NFS4ERR_DELAY;
@@ -334,8 +336,7 @@ static u32 do_callback_layoutrecall(struct nfs_client *clp,
 				    struct cb_layoutrecallargs *args)
 {
 	struct pnfs_cb_lrecall_info *new;
-	atomic_t **ptr;
-	int bit_num;
+	int i;
 	u32 res;
 
 	dprintk("%s enter, type=%i\n", __func__, args->cbl_recall_type);
@@ -353,22 +354,26 @@ static u32 do_callback_layoutrecall(struct nfs_client *clp,
 		kfree(new);
 		res = NFS4ERR_DELAY;
 		spin_unlock(&clp->cl_lock);
+		dprintk("%s: too many layout recalls\n", __func__);
 		goto out;
 	}
 	clp->cl_cb_lrecall_count++;
 	/* Adding to the list will block conflicting LGET activity */
 	list_add_tail(&new->pcl_list, &clp->cl_layoutrecalls);
-	for (bit_num = 0, ptr = clp->cl_drain_notification; *ptr; ptr++)
-		bit_num++;
-	*ptr = &new->pcl_count;
-	new->pcl_notify_bit = bit_num;
+	for (i = 0; i < PNFS_MAX_CB_LRECALLS; i++)
+		if (!clp->cl_drain_notification[i]) {
+			clp->cl_drain_notification[i] = &new->pcl_count;
+			break;
+		}
+	BUG_ON(i >= PNFS_MAX_CB_LRECALLS);
+	new->pcl_notify_idx = i;
 	spin_unlock(&clp->cl_lock);
 	res = initiate_layout_draining(new);
 	if (res || atomic_dec_and_test(&new->pcl_count)) {
 		spin_lock(&clp->cl_lock);
 		list_del(&new->pcl_list);
 		clp->cl_cb_lrecall_count--;
-		clp->cl_drain_notification[1 << bit_num] = NULL;
+		clp->cl_drain_notification[new->pcl_notify_idx] = NULL;
 		rpc_wake_up(&clp->cl_rpcwaitq_recall);
 		spin_unlock(&clp->cl_lock);
 		if (res == NFS4_OK) {
diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 7507054..8c5f4b4 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -642,7 +642,7 @@ send_layoutget(struct pnfs_layout_hdr *lo,
 
 bool nfs4_asynch_forget_layouts(struct pnfs_layout_hdr *lo,
 				struct pnfs_layout_range *range,
-				int notify_bit, atomic_t *notify_count,
+				int notify_idx, atomic_t *notify_count,
 				struct list_head *tmp_list)
 {
 	bool rv = false;
@@ -652,7 +652,7 @@ bool nfs4_asynch_forget_layouts(struct pnfs_layout_hdr *lo,
 	dprintk("%s\n", __func__);
 	list_for_each_entry_safe(lseg, tmp, &lo->segs, fi_list)
 		if (should_free_lseg(&lseg->range, range)) {
-			lseg->pls_notify_mask |= (1 << notify_bit);
+			lseg->pls_notify_mask |= (1 << notify_idx);
 			atomic_inc(notify_count);
 			mark_lseg_invalid(lseg, tmp_list);
 			rv = true;
diff --git a/fs/nfs/pnfs.h b/fs/nfs/pnfs.h
index 11c6ee4..092559b 100644
--- a/fs/nfs/pnfs.h
+++ b/fs/nfs/pnfs.h
@@ -179,7 +179,7 @@ struct pnfs_device {
 struct pnfs_cb_lrecall_info {
 	struct list_head	pcl_list; /* hook into cl_layoutrecalls list */
 	atomic_t		pcl_count;
-	int			pcl_notify_bit;
+	int			pcl_notify_idx;
 	struct nfs_client	*pcl_clp;
 	struct inode		*pcl_ino;
 	struct cb_layoutrecallargs pcl_args;
-- 
1.7.3.4

[PATCH 8/9] pnfs: fix nfs4_asynch_forget_layouts accounting

[Benny Halevy]

increment notify_count only the first time is it associated with a callback.

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/pnfs.c |   15 +++++++++++----
 1 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
index 8c5f4b4..a10e8e8 100644
--- a/fs/nfs/pnfs.c
+++ b/fs/nfs/pnfs.c
@@ -441,17 +441,22 @@ should_free_lseg(struct pnfs_layout_range *lseg_range,
 	       lo_seg_intersecting(lseg_range, recall_range);
 }
 
-static void mark_lseg_invalid(struct pnfs_layout_segment *lseg,
+static bool mark_lseg_invalid(struct pnfs_layout_segment *lseg,
 			      struct list_head *tmp_list)
 {
+	bool rv;
+
 	assert_spin_locked(&lseg->layout->inode->i_lock);
-	if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
+	rv = test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags);
+	if (rv) {
 		/* Remove the reference keeping the lseg in the
 		 * list.  It will now be removed when all
 		 * outstanding io is finished.
 		 */
 		put_lseg_locked(lseg, tmp_list);
 	}
+
+	return rv;
 }
 
 /* Returns false if there was nothing to do, true otherwise */
@@ -652,8 +657,10 @@ bool nfs4_asynch_forget_layouts(struct pnfs_layout_hdr *lo,
 	dprintk("%s\n", __func__);
 	list_for_each_entry_safe(lseg, tmp, &lo->segs, fi_list)
 		if (should_free_lseg(&lseg->range, range)) {
-			lseg->pls_notify_mask |= (1 << notify_idx);
-			atomic_inc(notify_count);
+			if (lseg->pls_notify_mask & (1 << notify_idx)) {
+				lseg->pls_notify_mask |= (1 << notify_idx);
+				atomic_inc(notify_count);
+			}
 			mark_lseg_invalid(lseg, tmp_list);
 			rv = true;
 		}
-- 
1.7.3.4

[PATCH 9/9] pnfs: improve nfs4_layoutget_done debug printouts

[Benny Halevy]

Signed-off-by: Benny Halevy <bhalevy@panasas.com>
---
 fs/nfs/nfs4proc.c |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index 10d46c4..785c460 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -5446,7 +5446,7 @@ static void nfs4_layoutget_done(struct rpc_task *task, void *calldata)
 	struct nfs4_layoutget *lgp = calldata;
 	struct nfs_server *server = NFS_SERVER(lgp->args.inode);
 
-	dprintk("--> %s\n", __func__);
+	dprintk("--> %s: tk_status=%d\n", __func__, task->tk_status);
 
 	if (!nfs4_sequence_done(task, &lgp->res.seq_res)) {
 		/* layout code relies on fact that in this case
@@ -5467,6 +5467,7 @@ static void nfs4_layoutget_done(struct rpc_task *task, void *calldata)
 		if (nfs4_async_handle_error(task, server, NULL, NULL) == -EAGAIN) {
 			struct inode *ino = lgp->args.inode;
 
+			dprintk("<-- %s retrying\n", __func__);
 			spin_lock(&ino->i_lock);
 			atomic_dec(&NFS_I(ino)->layout->plh_outstanding);
 			spin_unlock(&ino->i_lock);
-- 
1.7.3.4