[PATCH 01/31] nfsd41: slots are freed with session

[PATCH 01/31] nfsd41: slots are freed with session

[andros]

From: Andy Adamson <andros@netapp.com>

The session and slots are allocated all in one piece.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c |    1 -
 1 files changed, 0 insertions(+), 1 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index e82a518..42ebcb3 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -582,7 +582,6 @@ free_session(struct kref *kref)
 		struct nfsd4_cache_entry *e = &ses->se_slots[i].sl_cache_entry;
 		nfsd4_release_respages(e->ce_respages, e->ce_resused);
 	}
-	kfree(ses->se_slots);
 	kfree(ses);
 }
 
-- 
1.5.4.3

[PATCH 02/31] nfsd41: change check_slot_seqid parameters

[andros]

From: Andy Adamson <andros@netapp.com>

For separation of session slot and clientid slot processing.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c |   24 +++++++++++++-----------
 1 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 42ebcb3..43c821f 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1310,26 +1310,26 @@ error:
 }
 
 static int
-check_slot_seqid(u32 seqid, struct nfsd4_slot *slot)
+check_slot_seqid(u32 seqid, u32 slot_seqid, int slot_inuse )
 {
-	dprintk("%s enter. seqid %d slot->sl_seqid %d\n", __func__, seqid,
-		slot->sl_seqid);
+	dprintk("%s enter. seqid %d slot_seqid %d\n", __func__, seqid,
+		slot_seqid);
 
 	/* The slot is in use, and no response has been sent. */
-	if (slot->sl_inuse) {
-		if (seqid == slot->sl_seqid)
+	if (slot_inuse) {
+		if (seqid == slot_seqid)
 			return nfserr_jukebox;
 		else
 			return nfserr_seq_misordered;
 	}
 	/* Normal */
-	if (likely(seqid == slot->sl_seqid + 1))
+	if (likely(seqid == slot_seqid + 1))
 		return nfs_ok;
 	/* Replay */
-	if (seqid == slot->sl_seqid)
+	if (seqid == slot_seqid)
 		return nfserr_replay_cache;
 	/* Wraparound */
-	if (seqid == 1 && (slot->sl_seqid + 1) == 0)
+	if (seqid == 1 && (slot_seqid + 1) == 0)
 		return nfs_ok;
 	/* Misordered replay or misordered new request */
 	return nfserr_seq_misordered;
@@ -1352,7 +1352,8 @@ nfsd4_create_session(struct svc_rqst *rqstp,
 
 	if (conf) {
 		slot = &conf->cl_slot;
-		status = check_slot_seqid(cr_ses->seqid, slot);
+		status = check_slot_seqid(cr_ses->seqid, slot->sl_seqid,
+					  slot->sl_inuse);
 		if (status == nfserr_replay_cache) {
 			dprintk("Got a create_session replay! seqid= %d\n",
 				slot->sl_seqid);
@@ -1377,7 +1378,8 @@ nfsd4_create_session(struct svc_rqst *rqstp,
 		}
 
 		slot = &unconf->cl_slot;
-		status = check_slot_seqid(cr_ses->seqid, slot);
+		status = check_slot_seqid(cr_ses->seqid, slot->sl_seqid,
+					  slot->sl_inuse);
 		if (status) {
 			/* an unconfirmed replay returns misordered */
 			status = nfserr_seq_misordered;
@@ -1487,7 +1489,7 @@ nfsd4_sequence(struct svc_rqst *rqstp,
 	slot = &session->se_slots[seq->slotid];
 	dprintk("%s: slotid %d\n", __func__, seq->slotid);
 
-	status = check_slot_seqid(seq->seqid, slot);
+	status = check_slot_seqid(seq->seqid, slot->sl_seqid, slot->sl_inuse);
 	if (status == nfserr_replay_cache) {
 		cstate->slot = slot;
 		cstate->session = session;
-- 
1.5.4.3

[PATCH 03/31] nfsd41: turn off create session caching

[andros]

From: Andy Adamson <andros@netapp.com>

In preparation for switching clientid cache strategies.
If cstate->slot is not set, nfsdsvc_encode_compoundres will not cache
the create session response.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c |    7 -------
 1 files changed, 0 insertions(+), 7 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 43c821f..e6dbf05 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1341,7 +1341,6 @@ nfsd4_create_session(struct svc_rqst *rqstp,
 		     struct nfsd4_create_session *cr_ses)
 {
 	u32 ip_addr = svc_addr_in(rqstp)->sin_addr.s_addr;
-	struct nfsd4_compoundres *resp = rqstp->rq_resp;
 	struct nfs4_client *conf, *unconf;
 	struct nfsd4_slot *slot = NULL;
 	int status = 0;
@@ -1357,11 +1356,6 @@ nfsd4_create_session(struct svc_rqst *rqstp,
 		if (status == nfserr_replay_cache) {
 			dprintk("Got a create_session replay! seqid= %d\n",
 				slot->sl_seqid);
-			cstate->slot = slot;
-			cstate->status = status;
-			/* Return the cached reply status */
-			status = nfsd4_replay_cache_entry(resp, NULL);
-			goto out;
 		} else if (cr_ses->seqid != conf->cl_slot.sl_seqid + 1) {
 			status = nfserr_seq_misordered;
 			dprintk("Sequence misordered!\n");
@@ -1419,7 +1413,6 @@ nfsd4_create_session(struct svc_rqst *rqstp,
 	cr_ses->seqid = slot->sl_seqid;
 
 	slot->sl_inuse = true;
-	cstate->slot = slot;
 	/* Ensure a page is used for the cache */
 	slot->sl_cache_entry.ce_cachethis = 1;
 out:
-- 
1.5.4.3

[PATCH 04/31] nfsd41: separate clientid slot from session slot

[andros]

From: Andy Adamson <andros@netapp.com>

The nfs41 single slot clientid cache holds an encoded create session response
which has a maximum size of 88 bytes. The slot does not need the inuse,
cachethis or other fields that the multiple slot session cache uses.

Declare a new structure with a static buffer to struct nfs4_client to cache
the encoded create session response.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c        |   28 ++++++++++++++++------------
 include/linux/nfsd/state.h |   16 +++++++++++++++-
 2 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index e6dbf05..fee6bf0 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -652,8 +652,6 @@ free_client(struct nfs4_client *clp)
 	shutdown_callback_client(clp);
 	if (clp->cl_cb_xprt)
 		svc_xprt_put(clp->cl_cb_xprt);
-	nfsd4_release_respages(clp->cl_slot.sl_cache_entry.ce_respages,
-			     clp->cl_slot.sl_cache_entry.ce_resused);
 	if (clp->cl_cred.cr_group_info)
 		put_group_info(clp->cl_cred.cr_group_info);
 	kfree(clp->cl_principal);
@@ -729,6 +727,18 @@ static struct nfs4_client *create_client(struct xdr_netobj name, char *recdir)
 	return clp;
 }
 
+static struct nfs4_client *create_client_session(struct xdr_netobj name,
+						  char *recdir)
+{
+	struct nfs4_client *clp;
+
+	clp = create_client(name, recdir);
+	if (clp)
+		/* the slot sl_seqid is 0 */
+		clp->cl_slot.sl_datalen = CS_MAX_ENC_SZ;
+	return clp;
+}
+
 static void copy_verf(struct nfs4_client *target, nfs4_verifier *source)
 {
 	memcpy(target->cl_verifier.data, source->data,
@@ -1277,7 +1287,7 @@ nfsd4_exchange_id(struct svc_rqst *rqstp,
 
 out_new:
 	/* Normal case */
-	new = create_client(exid->clname, dname);
+	new = create_client_session(exid->clname, dname);
 	if (new == NULL) {
 		status = nfserr_resource;
 		goto out;
@@ -1294,7 +1304,6 @@ out_copy:
 	exid->clientid.cl_boot = new->cl_clientid.cl_boot;
 	exid->clientid.cl_id = new->cl_clientid.cl_id;
 
-	new->cl_slot.sl_seqid = 0;
 	exid->seqid = 1;
 	nfsd4_set_ex_flags(new, exid);
 
@@ -1342,7 +1351,7 @@ nfsd4_create_session(struct svc_rqst *rqstp,
 {
 	u32 ip_addr = svc_addr_in(rqstp)->sin_addr.s_addr;
 	struct nfs4_client *conf, *unconf;
-	struct nfsd4_slot *slot = NULL;
+	struct nfsd4_clid_slot *slot = NULL;
 	int status = 0;
 
 	nfs4_lock_state();
@@ -1351,8 +1360,7 @@ nfsd4_create_session(struct svc_rqst *rqstp,
 
 	if (conf) {
 		slot = &conf->cl_slot;
-		status = check_slot_seqid(cr_ses->seqid, slot->sl_seqid,
-					  slot->sl_inuse);
+		status = check_slot_seqid(cr_ses->seqid, slot->sl_seqid, 0);
 		if (status == nfserr_replay_cache) {
 			dprintk("Got a create_session replay! seqid= %d\n",
 				slot->sl_seqid);
@@ -1372,8 +1380,7 @@ nfsd4_create_session(struct svc_rqst *rqstp,
 		}
 
 		slot = &unconf->cl_slot;
-		status = check_slot_seqid(cr_ses->seqid, slot->sl_seqid,
-					  slot->sl_inuse);
+		status = check_slot_seqid(cr_ses->seqid, slot->sl_seqid, 0);
 		if (status) {
 			/* an unconfirmed replay returns misordered */
 			status = nfserr_seq_misordered;
@@ -1412,9 +1419,6 @@ nfsd4_create_session(struct svc_rqst *rqstp,
 	       NFS4_MAX_SESSIONID_LEN);
 	cr_ses->seqid = slot->sl_seqid;
 
-	slot->sl_inuse = true;
-	/* Ensure a page is used for the cache */
-	slot->sl_cache_entry.ce_cachethis = 1;
 out:
 	nfs4_unlock_state();
 	dprintk("%s returns %d\n", __func__, ntohl(status));
diff --git a/include/linux/nfsd/state.h b/include/linux/nfsd/state.h
index 14da8f6..bc34876 100644
--- a/include/linux/nfsd/state.h
+++ b/include/linux/nfsd/state.h
@@ -128,6 +128,20 @@ struct nfsd4_slot {
 	struct nfsd4_cache_entry	sl_cache_entry;
 };
 
+/*
+ * maximum encoded size of create session response
+ * 16 - sessionid, 8 - sequence # and flags,
+ * 32 - fore channel attrs, 32 - back channel attrs
+ */
+#define CS_MAX_ENC_SZ  88
+
+struct nfsd4_clid_slot {
+	u32		sl_seqid;
+	__be32		sl_status;
+	u32		sl_datalen;
+	char		sl_data[CS_MAX_ENC_SZ];
+};
+
 struct nfsd4_channel_attrs {
 	u32		headerpadsz;
 	u32		maxreq_sz;
@@ -205,7 +219,7 @@ struct nfs4_client {
 
 	/* for nfs41 */
 	struct list_head	cl_sessions;
-	struct nfsd4_slot	cl_slot;	/* create_session slot */
+	struct nfsd4_clid_slot	cl_slot;	/* create_session slot */
 	u32			cl_exchange_flags;
 	struct nfs4_sessionid	cl_sessionid;
 
-- 
1.5.4.3

[PATCH 05/31] nfsd41: encode create_session result into cache

[andros]

From: Andy Adamson <andros@netapp.com>

CREATE_SESSION can be preceeded by a SEQUENCE operation and the
create session single slot cache must be maintained. Encode the results of
a create session call into the cache at the end of processing.

The create session result will also be encoded into the RPC response, and if
it is preceeded by a SEQUENCE operation, will also be encoded into the
session slot table cache.

Errors that do not change the create session cache:
A create session NFS4ERR_STALE_CLIENTID error means that a client record
(and associated create session slot) could not be found and therefore can't
be changed.  NFSERR_SEQ_MISORDERED errors do not change the slot cache.

All other errors get cached.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c       |    6 ++++--
 fs/nfsd/nfs4xdr.c         |   18 ++++++++++++++++++
 include/linux/nfsd/xdr4.h |    2 ++
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index fee6bf0..142c460 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1376,7 +1376,7 @@ nfsd4_create_session(struct svc_rqst *rqstp,
 		if (!same_creds(&unconf->cl_cred, &rqstp->rq_cred) ||
 		    (ip_addr != unconf->cl_addr)) {
 			status = nfserr_clid_inuse;
-			goto out;
+			goto out_cache;
 		}
 
 		slot = &unconf->cl_slot;
@@ -1418,7 +1418,9 @@ nfsd4_create_session(struct svc_rqst *rqstp,
 	memcpy(cr_ses->sessionid.data, conf->cl_sessionid.data,
 	       NFS4_MAX_SESSIONID_LEN);
 	cr_ses->seqid = slot->sl_seqid;
-
+out_cache:
+	/* cache solo and embedded create sessions under the state lock */
+	nfsd4_cache_create_session(cr_ses, slot, status);
 out:
 	nfs4_unlock_state();
 	dprintk("%s returns %d\n", __func__, ntohl(status));
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index b2f8d74..1dfec1d 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -3046,6 +3046,24 @@ nfsd4_encode_create_session(struct nfsd4_compoundres *resp, int nfserr,
 	return 0;
 }
 
+/*
+ * Encode the create_session result into the create session single DRC
+ * slot cache. Do this for solo or embedded create session operations.
+ */
+void
+nfsd4_cache_create_session(struct nfsd4_create_session *cr_ses,
+			   struct nfsd4_clid_slot *slot, int nfserr)
+{
+	__be32 *p = (__be32 *)slot->sl_data;
+	struct nfsd4_compoundres tmp = {
+		.p = p,
+		.end = p + XDR_QUADLEN(CS_MAX_ENC_SZ),
+	}, *resp = &tmp;
+
+	slot->sl_status = nfsd4_encode_create_session(resp, nfserr, cr_ses);
+	slot->sl_datalen = (char *)resp->p - (char *)p;
+}
+
 static __be32
 nfsd4_encode_destroy_session(struct nfsd4_compoundres *resp, int nfserr,
 			     struct nfsd4_destroy_session *destroy_session)
diff --git a/include/linux/nfsd/xdr4.h b/include/linux/nfsd/xdr4.h
index afd4464..21e0b73 100644
--- a/include/linux/nfsd/xdr4.h
+++ b/include/linux/nfsd/xdr4.h
@@ -517,6 +517,8 @@ extern __be32 nfsd4_setclientid_confirm(struct svc_rqst *rqstp,
 extern void nfsd4_store_cache_entry(struct nfsd4_compoundres *resp);
 extern __be32 nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
 		struct nfsd4_sequence *seq);
+extern void nfsd4_cache_create_session(struct nfsd4_create_session *cr_ses,
+		struct nfsd4_clid_slot *slot, int nfserr);
 extern __be32 nfsd4_exchange_id(struct svc_rqst *rqstp,
 		struct nfsd4_compound_state *,
 struct nfsd4_exchange_id *);
-- 
1.5.4.3

[PATCH 06/31] nfsd41: create_session check replay first

[andros]

From: Andy Adamson <andros@netapp.com>

Replay processing needs to preceed other error processing.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c |   12 ++++++------
 1 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 142c460..de7cc51 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1373,12 +1373,6 @@ nfsd4_create_session(struct svc_rqst *rqstp,
 		}
 		conf->cl_slot.sl_seqid++;
 	} else if (unconf) {
-		if (!same_creds(&unconf->cl_cred, &rqstp->rq_cred) ||
-		    (ip_addr != unconf->cl_addr)) {
-			status = nfserr_clid_inuse;
-			goto out_cache;
-		}
-
 		slot = &unconf->cl_slot;
 		status = check_slot_seqid(cr_ses->seqid, slot->sl_seqid, 0);
 		if (status) {
@@ -1387,6 +1381,12 @@ nfsd4_create_session(struct svc_rqst *rqstp,
 			goto out;
 		}
 
+		if (!same_creds(&unconf->cl_cred, &rqstp->rq_cred) ||
+		    (ip_addr != unconf->cl_addr)) {
+			status = nfserr_clid_inuse;
+			goto out_cache;
+		}
+
 		slot->sl_seqid++; /* from 0 to 1 */
 		move_to_confirmed(unconf);
 
-- 
1.5.4.3

[PATCH 07/31] nfsd41: replay solo and embedded create session

[andros]

From: Andy Adamson <andros@netapp.com>

CREATE_SESSION can be preceeded by a SEQUENCE operation (an embedded
CREATE_SESSION) and the create session single slot cache must be maintained.
Always replay a create session from nfsd4_replay_create_session(). Leave
nfsd4_store_cache_entry() for session (sequence operation) replays.

Set cstate->status to a new internal error, nfserr_replay_clientid_cache, and
change the logic in nfsd4_proc_compound so that CREATE_SESSION replays replace
encode_operation.
The new internal error is needed to differentiate between an embedded
CREATE_SESSION replay and a SEQUENCE replay.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4proc.c        |    9 +++++++--
 fs/nfsd/nfs4state.c       |    5 +++++
 fs/nfsd/nfs4xdr.c         |   17 +++++++++++++++++
 include/linux/nfsd/nfsd.h |    2 ++
 include/linux/nfsd/xdr4.h |    2 ++
 5 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index b2883e9..32d5866 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -996,7 +996,7 @@ nfsd4_proc_compound(struct svc_rqst *rqstp,
 			BUG_ON(op->status == nfs_ok);
 
 encode_op:
-		/* Only from SEQUENCE or CREATE_SESSION */
+		/* Only from SEQUENCE */
 		if (resp->cstate.status == nfserr_replay_cache) {
 			dprintk("%s NFS4.1 replay from cache\n", __func__);
 			if (nfsd4_not_cached(resp))
@@ -1005,7 +1005,12 @@ encode_op:
 				status = op->status;
 			goto out;
 		}
-		if (op->status == nfserr_replay_me) {
+		/* Only from CREATE_SESSION */
+		if (resp->cstate.status == nfserr_replay_clientid_cache) {
+			dprintk("%s NFS4.1 replay from clientid cache\n",
+				__func__);
+			status = op->status;
+		} else if (op->status == nfserr_replay_me) {
 			op->replay = &cstate->replay_owner->so_replay;
 			nfsd4_encode_replay(resp, op);
 			status = op->status = op->replay->rp_status;
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index de7cc51..e216169 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1350,6 +1350,7 @@ nfsd4_create_session(struct svc_rqst *rqstp,
 		     struct nfsd4_create_session *cr_ses)
 {
 	u32 ip_addr = svc_addr_in(rqstp)->sin_addr.s_addr;
+	struct nfsd4_compoundres *resp = rqstp->rq_resp;
 	struct nfs4_client *conf, *unconf;
 	struct nfsd4_clid_slot *slot = NULL;
 	int status = 0;
@@ -1364,6 +1365,10 @@ nfsd4_create_session(struct svc_rqst *rqstp,
 		if (status == nfserr_replay_cache) {
 			dprintk("Got a create_session replay! seqid= %d\n",
 				slot->sl_seqid);
+			cstate->status = nfserr_replay_clientid_cache;
+			/* Return the cached reply status */
+			status = nfsd4_replay_create_session(resp, slot);
+			goto out;
 		} else if (cr_ses->seqid != conf->cl_slot.sl_seqid + 1) {
 			status = nfserr_seq_misordered;
 			dprintk("Sequence misordered!\n");
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 1dfec1d..238612e 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -3064,6 +3064,23 @@ nfsd4_cache_create_session(struct nfsd4_create_session *cr_ses,
 	slot->sl_datalen = (char *)resp->p - (char *)p;
 }
 
+__be32
+nfsd4_replay_create_session(struct nfsd4_compoundres *resp,
+			    struct nfsd4_clid_slot *slot)
+{
+	__be32 *p;
+
+	RESERVE_SPACE(8);
+	WRITE32(OP_CREATE_SESSION);
+	*p++ = slot->sl_status; /* already in network byte order */
+	ADJUST_ARGS();
+
+	memcpy(resp->p, slot->sl_data, slot->sl_datalen);
+	p += XDR_QUADLEN(slot->sl_datalen);
+	ADJUST_ARGS();
+	return slot->sl_status;
+}
+
 static __be32
 nfsd4_encode_destroy_session(struct nfsd4_compoundres *resp, int nfserr,
 			     struct nfsd4_destroy_session *destroy_session)
diff --git a/include/linux/nfsd/nfsd.h b/include/linux/nfsd/nfsd.h
index 2b49d67..9cd6399 100644
--- a/include/linux/nfsd/nfsd.h
+++ b/include/linux/nfsd/nfsd.h
@@ -301,6 +301,8 @@ void		nfsd_lockd_shutdown(void);
 #define	nfserr_replay_me	cpu_to_be32(11001)
 /* nfs41 replay detected */
 #define	nfserr_replay_cache	cpu_to_be32(11002)
+/* nfs41 clientid cache replay detected */
+#define	nfserr_replay_clientid_cache	cpu_to_be32(11003)
 
 /* Check for dir entries '.' and '..' */
 #define isdotent(n, l)	(l < 3 && n[0] == '.' && (l == 1 || n[1] == '.'))
diff --git a/include/linux/nfsd/xdr4.h b/include/linux/nfsd/xdr4.h
index 21e0b73..ac00985 100644
--- a/include/linux/nfsd/xdr4.h
+++ b/include/linux/nfsd/xdr4.h
@@ -519,6 +519,8 @@ extern __be32 nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
 		struct nfsd4_sequence *seq);
 extern void nfsd4_cache_create_session(struct nfsd4_create_session *cr_ses,
 		struct nfsd4_clid_slot *slot, int nfserr);
+extern __be32 nfsd4_replay_create_session(struct nfsd4_compoundres *resp,
+		struct nfsd4_clid_slot *slot);
 extern __be32 nfsd4_exchange_id(struct svc_rqst *rqstp,
 		struct nfsd4_compound_state *,
 struct nfsd4_exchange_id *);
-- 
1.5.4.3

[PATCH 08/31] nfsd41: sanity check client drc maxreqs

[andros]

From: Andy Adamson <andros@netapp.com>

Ensure the client requested maximum requests are between 1 and
NFSD_MAX_SLOTS_PER_SESSION

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c |    5 +++++
 1 files changed, 5 insertions(+), 0 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index e216169..59b601b 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -427,6 +427,11 @@ static int set_forechannel_maxreqs(struct nfsd4_channel_attrs *fchan)
 {
 	int status = 0, np = fchan->maxreqs * NFSD_PAGES_PER_SLOT;
 
+	if (fchan->maxreqs < 1)
+		return nfserr_inval;
+	else if (fchan->maxreqs > NFSD_MAX_SLOTS_PER_SESSION)
+		fchan->maxreqs = NFSD_MAX_SLOTS_PER_SESSION;
+
 	spin_lock(&nfsd_serv->sv_lock);
 	if (np + nfsd_serv->sv_drc_pages_used > nfsd_serv->sv_drc_max_pages)
 		np = nfsd_serv->sv_drc_max_pages - nfsd_serv->sv_drc_pages_used;
-- 
1.5.4.3

[PATCH 09/31] nfsd41: change from page to memory based drc limits

[andros]

From: Andy Adamson <andros@netapp.com>

NFSD_SLOT_CACHE_SIZE is the size of all encoded operation responses (excluding
the sequence operation) that we want to cache.

Adjust NFSD_DRC_SIZE_SHIFT to reflect using 512 bytes instead of PAGE_SIZE.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c        |   29 +++++++++++++++--------------
 fs/nfsd/nfssvc.c           |   13 +++++++------
 include/linux/nfsd/state.h |    1 +
 include/linux/sunrpc/svc.h |    4 ++--
 4 files changed, 25 insertions(+), 22 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 59b601b..d12220c 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -418,33 +418,34 @@ gen_sessionid(struct nfsd4_session *ses)
  * Give the client the number of slots it requests bound by
  * NFSD_MAX_SLOTS_PER_SESSION and by sv_drc_max_pages.
  *
- * If we run out of pages (sv_drc_pages_used == sv_drc_max_pages) we
- * should (up to a point) re-negotiate active sessions and reduce their
- * slot usage to make rooom for new connections. For now we just fail the
- * create session.
+ * If we run out of reserved DRC memory we should (up to a point) re-negotiate
+ * active sessions and reduce their slot usage to make rooom for new
+ * connections. For now we just fail the create session.
  */
 static int set_forechannel_maxreqs(struct nfsd4_channel_attrs *fchan)
 {
-	int status = 0, np = fchan->maxreqs * NFSD_PAGES_PER_SLOT;
+	int mem;
 
 	if (fchan->maxreqs < 1)
 		return nfserr_inval;
 	else if (fchan->maxreqs > NFSD_MAX_SLOTS_PER_SESSION)
 		fchan->maxreqs = NFSD_MAX_SLOTS_PER_SESSION;
 
+	mem = fchan->maxreqs * NFSD_SLOT_CACHE_SIZE;
+
 	spin_lock(&nfsd_serv->sv_lock);
-	if (np + nfsd_serv->sv_drc_pages_used > nfsd_serv->sv_drc_max_pages)
-		np = nfsd_serv->sv_drc_max_pages - nfsd_serv->sv_drc_pages_used;
-	nfsd_serv->sv_drc_pages_used += np;
+	if (mem + nfsd_serv->sv_drc_mem_used > nfsd_serv->sv_drc_max_mem)
+		mem = nfsd_serv->sv_drc_max_mem - nfsd_serv->sv_drc_mem_used;
+	nfsd_serv->sv_drc_mem_used += mem;
 	spin_unlock(&nfsd_serv->sv_lock);
 
-	if (np <= 0) {
-		status = nfserr_resource;
+	if (mem < NFSD_SLOT_CACHE_SIZE) {
 		fchan->maxreqs = 0;
-	} else
-		fchan->maxreqs = np / NFSD_PAGES_PER_SLOT;
-
-	return status;
+		return nfserr_resource;
+	} else {
+		fchan->maxreqs = mem / NFSD_SLOT_CACHE_SIZE;
+		return 0;
+	}
 }
 
 /*
diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c
index cbba4a9..80588cc 100644
--- a/fs/nfsd/nfssvc.c
+++ b/fs/nfsd/nfssvc.c
@@ -237,12 +237,13 @@ void nfsd_reset_versions(void)
 static void set_max_drc(void)
 {
 	/* The percent of nr_free_buffer_pages used by the V4.1 server DRC */
-	#define NFSD_DRC_SIZE_SHIFT	7
-	nfsd_serv->sv_drc_max_pages = nr_free_buffer_pages()
-						>> NFSD_DRC_SIZE_SHIFT;
-	nfsd_serv->sv_drc_pages_used = 0;
-	dprintk("%s svc_drc_max_pages %u\n", __func__,
-		nfsd_serv->sv_drc_max_pages);
+	#define NFSD_DRC_SIZE_SHIFT	10
+	nfsd_serv->sv_drc_max_mem = (nr_free_buffer_pages()
+					>> NFSD_DRC_SIZE_SHIFT) * PAGE_SIZE;
+	nfsd_serv->sv_drc_mem_used = 0;
+	dprintk("%s svc_drc_max_mem %u [in pages %lu]\n", __func__,
+		nfsd_serv->sv_drc_max_mem,
+		nfsd_serv->sv_drc_max_mem / PAGE_SIZE);
 }
 
 int nfsd_create_serv(void)
diff --git a/include/linux/nfsd/state.h b/include/linux/nfsd/state.h
index bc34876..3580d55 100644
--- a/include/linux/nfsd/state.h
+++ b/include/linux/nfsd/state.h
@@ -109,6 +109,7 @@ struct nfs4_callback {
 #define NFSD_MAX_SLOTS_PER_SESSION	128
 /* Maximum number of pages per slot cache entry */
 #define NFSD_PAGES_PER_SLOT	1
+#define NFSD_SLOT_CACHE_SIZE		512
 /* Maximum number of operations per session compound */
 #define NFSD_MAX_OPS_PER_COMPOUND	16
 
diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
index 2a30775..243508e 100644
--- a/include/linux/sunrpc/svc.h
+++ b/include/linux/sunrpc/svc.h
@@ -94,8 +94,8 @@ struct svc_serv {
 	struct module *		sv_module;	/* optional module to count when
 						 * adding threads */
 	svc_thread_fn		sv_function;	/* main function for threads */
-	unsigned int		sv_drc_max_pages; /* Total pages for DRC */
-	unsigned int		sv_drc_pages_used;/* DRC pages used */
+	unsigned int		sv_drc_max_mem; /* Total pages for DRC */
+	unsigned int		sv_drc_mem_used;/* DRC pages used */
 };
 
 /*
-- 
1.5.4.3

[PATCH 10/31] nfsd41: use globals for DRC memory use management

[andros]

From: Andy Adamson <andros@netapp.com>

The version 4.1 DRC memory limit and tracking variables are server wide and
session specific.
Add a spinlock to serialize access to the management variables which change
on session creation and deletion (usage counter) or (future) administrative
action to adjust the total DRC memory limit.

Track DRC memory usage in free session.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c        |   11 +++++++----
 fs/nfsd/nfssvc.c           |   19 +++++++++++++++----
 include/linux/nfsd/nfsd.h  |    5 +++++
 include/linux/sunrpc/svc.h |    2 --
 4 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index d12220c..a9e722e 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -416,7 +416,7 @@ gen_sessionid(struct nfsd4_session *ses)
 
 /*
  * Give the client the number of slots it requests bound by
- * NFSD_MAX_SLOTS_PER_SESSION and by sv_drc_max_pages.
+ * NFSD_MAX_SLOTS_PER_SESSION and by nfsd_drc_max_mem.
  *
  * If we run out of reserved DRC memory we should (up to a point) re-negotiate
  * active sessions and reduce their slot usage to make rooom for new
@@ -434,9 +434,9 @@ static int set_forechannel_maxreqs(struct nfsd4_channel_attrs *fchan)
 	mem = fchan->maxreqs * NFSD_SLOT_CACHE_SIZE;
 
 	spin_lock(&nfsd_serv->sv_lock);
-	if (mem + nfsd_serv->sv_drc_mem_used > nfsd_serv->sv_drc_max_mem)
-		mem = nfsd_serv->sv_drc_max_mem - nfsd_serv->sv_drc_mem_used;
-	nfsd_serv->sv_drc_mem_used += mem;
+	if (mem + nfsd_drc_mem_used > nfsd_drc_max_mem)
+		mem = nfsd_drc_max_mem - nfsd_drc_mem_used;
+	nfsd_drc_mem_used += mem;
 	spin_unlock(&nfsd_serv->sv_lock);
 
 	if (mem < NFSD_SLOT_CACHE_SIZE) {
@@ -588,6 +588,9 @@ free_session(struct kref *kref)
 		struct nfsd4_cache_entry *e = &ses->se_slots[i].sl_cache_entry;
 		nfsd4_release_respages(e->ce_respages, e->ce_resused);
 	}
+	spin_lock(&nfsd_drc_lock);
+	nfsd_drc_mem_used -= ses->se_fchannel.maxreqs * NFSD_SLOT_CACHE_SIZE;
+	spin_unlock(&nfsd_drc_lock);
 	kfree(ses);
 }
 
diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c
index 80588cc..37633f5 100644
--- a/fs/nfsd/nfssvc.c
+++ b/fs/nfsd/nfssvc.c
@@ -67,6 +67,16 @@ struct timeval			nfssvc_boot;
 DEFINE_MUTEX(nfsd_mutex);
 struct svc_serv 		*nfsd_serv;
 
+/*
+ * nfsd_drc_lock protects nfsd_drc_max_pages and nfsd_drc_pages_used.
+ * nfsd_drc_max_pages limits the total amount of memory available for
+ * version 4.1 DRC caches.
+ * nfsd_drc_pages_used tracks the current version 4.1 DRC memory usage.
+ */
+spinlock_t	nfsd_drc_lock;
+unsigned int	nfsd_drc_max_mem;
+unsigned int	nfsd_drc_mem_used;
+
 #if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL)
 static struct svc_stat	nfsd_acl_svcstats;
 static struct svc_version *	nfsd_acl_version[] = {
@@ -238,12 +248,13 @@ static void set_max_drc(void)
 {
 	/* The percent of nr_free_buffer_pages used by the V4.1 server DRC */
 	#define NFSD_DRC_SIZE_SHIFT	10
-	nfsd_serv->sv_drc_max_mem = (nr_free_buffer_pages()
+	nfsd_drc_max_mem = (nr_free_buffer_pages()
 					>> NFSD_DRC_SIZE_SHIFT) * PAGE_SIZE;
-	nfsd_serv->sv_drc_mem_used = 0;
+	nfsd_drc_mem_used = 0;
+	spin_lock_init(&nfsd_drc_lock);
 	dprintk("%s svc_drc_max_mem %u [in pages %lu]\n", __func__,
-		nfsd_serv->sv_drc_max_mem,
-		nfsd_serv->sv_drc_max_mem / PAGE_SIZE);
+		nfsd_drc_max_mem,
+		nfsd_drc_max_mem / PAGE_SIZE);
 }
 
 int nfsd_create_serv(void)
diff --git a/include/linux/nfsd/nfsd.h b/include/linux/nfsd/nfsd.h
index 9cd6399..23ba7bd 100644
--- a/include/linux/nfsd/nfsd.h
+++ b/include/linux/nfsd/nfsd.h
@@ -56,6 +56,11 @@ extern struct svc_version	nfsd_version2, nfsd_version3,
 extern u32			nfsd_supported_minorversion;
 extern struct mutex		nfsd_mutex;
 extern struct svc_serv		*nfsd_serv;
+extern spinlock_t		nfsd_drc_lock;
+extern unsigned int		nfsd_drc_max_mem;
+extern unsigned int		nfsd_drc_mem_used;
+
+
 
 extern struct seq_operations nfs_exports_op;
 
diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
index 243508e..d0d8bf4 100644
--- a/include/linux/sunrpc/svc.h
+++ b/include/linux/sunrpc/svc.h
@@ -94,8 +94,6 @@ struct svc_serv {
 	struct module *		sv_module;	/* optional module to count when
 						 * adding threads */
 	svc_thread_fn		sv_function;	/* main function for threads */
-	unsigned int		sv_drc_max_mem; /* Total pages for DRC */
-	unsigned int		sv_drc_mem_used;/* DRC pages used */
 };
 
 /*
-- 
1.5.4.3

[PATCH 11/31] nfsd41: set the session maximum response size cached

[andros]

From: Andy Adamson <andros@netapp.com>

We won't cache the SEQUENCE operation because we can encode it from the session
slot values.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c |   10 +++++++---
 1 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index a9e722e..187e1da 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -448,6 +448,12 @@ static int set_forechannel_maxreqs(struct nfsd4_channel_attrs *fchan)
 	}
 }
 
+/* rpc header + encoded OP_SEQUENCE reply + NFSD_SLOT_CACHE_SIZE in bytes */
+#define NFSD_MAX_RESPONSE_CACHED  ((RPC_MAX_HEADER_WITH_AUTH + \
+				  2 + /* OP_SEQUENCE header */ \
+				  XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + 5 + \
+				  XDR_QUADLEN(NFSD_SLOT_CACHE_SIZE)) << 2)
+
 /*
  * fchan holds the client values on input, and the server values on output
  */
@@ -469,9 +475,7 @@ static int init_forechannel_attrs(struct svc_rqst *rqstp,
 		fchan->maxresp_sz = maxcount;
 	session_fchan->maxresp_sz = fchan->maxresp_sz;
 
-	/* Set the max response cached size our default which is
-	 * a multiple of PAGE_SIZE and small */
-	session_fchan->maxresp_cached = NFSD_PAGES_PER_SLOT * PAGE_SIZE;
+	session_fchan->maxresp_cached = NFSD_MAX_RESPONSE_CACHED;
 	fchan->maxresp_cached = session_fchan->maxresp_cached;
 
 	/* Use the client's maxops if possible */
-- 
1.5.4.3

[PATCH 12/31] nfsd41: use static buffers for sessions DRC

[andros]

From: Andy Adamson <andros@netapp.com>

Change from page based to memory based NFSv4.1 DRC.

We need to allocate memory for the session DRC in the CREATE_SESSION operation
because of the guarantee to the client that the memory resource is available
for caching responses.

In preparation to remove struct nfsd4_cache_entry, add static storage to
struct nfsd4_slot to hold up to NFSD_SLOT_CACH_SIZE of all encoded operation
data past the encoded sequence operation.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c        |    9 +++++++++
 fs/nfsd/nfs4xdr.c          |    1 +
 include/linux/nfsd/state.h |    2 ++
 include/linux/nfsd/xdr4.h  |    1 +
 4 files changed, 13 insertions(+), 0 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 187e1da..fecee33 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1066,6 +1066,7 @@ nfsd4_copy_pages(struct page **topages, struct page **frompages, short count)
 void
 nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
 {
+	struct nfsd4_slot *slot = resp->cstate.slot;
 	struct nfsd4_cache_entry *entry = &resp->cstate.slot->sl_cache_entry;
 	struct svc_rqst *rqstp = resp->rqstp;
 	struct nfsd4_compoundargs *args = rqstp->rq_argp;
@@ -1090,10 +1091,14 @@ nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
 	if (nfsd4_not_cached(resp)) {
 		entry->ce_resused = 0;
 		entry->ce_rpchdrlen = 0;
+		slot->sl_datalen = 0;
 		dprintk("%s Just cache SEQUENCE. ce_cachethis %d\n", __func__,
 			resp->cstate.slot->sl_cache_entry.ce_cachethis);
 		return;
 	}
+	slot->sl_datalen = (char *)resp->p - (char *)resp->cstate.datap;
+	memcpy(slot->sl_data, resp->cstate.datap, slot->sl_datalen);
+
 	entry->ce_resused = rqstp->rq_resused;
 	if (entry->ce_resused > NFSD_PAGES_PER_SLOT + 1)
 		entry->ce_resused = NFSD_PAGES_PER_SLOT + 1;
@@ -1141,6 +1146,7 @@ __be32
 nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
 			 struct nfsd4_sequence *seq)
 {
+	struct nfsd4_slot *slot = resp->cstate.slot;
 	struct nfsd4_cache_entry *entry = &resp->cstate.slot->sl_cache_entry;
 	__be32 status;
 
@@ -1161,6 +1167,9 @@ nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
 		return nfs_ok;
 	}
 
+	/* The sequence operation has been encoded, cstate->datap set. */
+	memcpy(resp->cstate.datap, slot->sl_data, slot->sl_datalen);
+
 	if (!nfsd41_copy_replay_data(resp, entry)) {
 		/*
 		 * Not enough room to use the replay rpc header, send the
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 238612e..41ac385 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -3111,6 +3111,7 @@ nfsd4_encode_sequence(struct nfsd4_compoundres *resp, int nfserr,
 	WRITE32(0);
 
 	ADJUST_ARGS();
+	resp->cstate.datap = p;	/* DRC cache data pointer */
 	return 0;
 }
 
diff --git a/include/linux/nfsd/state.h b/include/linux/nfsd/state.h
index 3580d55..362c684 100644
--- a/include/linux/nfsd/state.h
+++ b/include/linux/nfsd/state.h
@@ -126,6 +126,8 @@ struct nfsd4_cache_entry {
 struct nfsd4_slot {
 	bool				sl_inuse;
 	u32				sl_seqid;
+	u32				sl_datalen;
+	char				sl_data[NFSD_SLOT_CACHE_SIZE];
 	struct nfsd4_cache_entry	sl_cache_entry;
 };
 
diff --git a/include/linux/nfsd/xdr4.h b/include/linux/nfsd/xdr4.h
index ac00985..52589ba 100644
--- a/include/linux/nfsd/xdr4.h
+++ b/include/linux/nfsd/xdr4.h
@@ -52,6 +52,7 @@ struct nfsd4_compound_state {
 	struct nfsd4_session	*session;
 	struct nfsd4_slot	*slot;
 	__be32			*statp;
+	__be32			*datap;
 	size_t			iovlen;
 	u32			minorversion;
 	u32			status;
-- 
1.5.4.3

[PATCH 13/31] nfsd41: replace ce_cachethis with nfsd4_slot field

[andros]

From: Andy Adamson <andros@netapp.com>

In preparation to remove struct nfsd4_cache_entry

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4proc.c         |    6 +++---
 fs/nfsd/nfs4state.c        |   10 +++++-----
 fs/nfsd/nfs4xdr.c          |    2 +-
 include/linux/nfsd/state.h |    1 +
 include/linux/nfsd/xdr4.h  |    2 +-
 5 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 32d5866..d4c09a9 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -870,8 +870,8 @@ nfsd4_enc_uncached_replay(struct nfsd4_compoundargs *args,
 {
 	struct nfsd4_op *op;
 
-	dprintk("--> %s resp->opcnt %d ce_cachethis %u \n", __func__,
-		resp->opcnt, resp->cstate.slot->sl_cache_entry.ce_cachethis);
+	dprintk("--> %s resp->opcnt %d cachethis %u \n", __func__,
+		resp->opcnt, resp->cstate.slot->sl_cachethis);
 
 	/* Encode the replayed sequence operation */
 	BUG_ON(resp->opcnt != 1);
@@ -879,7 +879,7 @@ nfsd4_enc_uncached_replay(struct nfsd4_compoundargs *args,
 	nfsd4_encode_operation(resp, op);
 
 	/*return nfserr_retry_uncached_rep in next operation. */
-	if (resp->cstate.slot->sl_cache_entry.ce_cachethis == 0) {
+	if (resp->cstate.slot->sl_cachethis == 0) {
 		op = &args->ops[resp->opcnt++];
 		op->status = nfserr_retry_uncached_rep;
 		nfsd4_encode_operation(resp, op);
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index fecee33..ae1f316 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1092,8 +1092,8 @@ nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
 		entry->ce_resused = 0;
 		entry->ce_rpchdrlen = 0;
 		slot->sl_datalen = 0;
-		dprintk("%s Just cache SEQUENCE. ce_cachethis %d\n", __func__,
-			resp->cstate.slot->sl_cache_entry.ce_cachethis);
+		dprintk("%s Just cache SEQUENCE. cachethis %d\n", __func__,
+			resp->cstate.slot->sl_cachethis);
 		return;
 	}
 	slot->sl_datalen = (char *)resp->p - (char *)resp->cstate.datap;
@@ -1531,10 +1531,10 @@ nfsd4_sequence(struct svc_rqst *rqstp,
 	/* Success! bump slot seqid */
 	slot->sl_inuse = true;
 	slot->sl_seqid = seq->seqid;
-	slot->sl_cache_entry.ce_cachethis = seq->cachethis;
-	/* Always set the cache entry cachethis for solo sequence */
+	slot->sl_cachethis = seq->cachethis;
+	/* Always set the slot cachethis for solo sequence */
 	if (nfsd4_is_solo_sequence(resp))
-		slot->sl_cache_entry.ce_cachethis = 1;
+		slot->sl_cachethis = 1;
 
 	cstate->slot = slot;
 	cstate->session = session;
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 41ac385..827f6d4 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -3214,7 +3214,7 @@ static int nfsd4_check_drc_limit(struct nfsd4_compoundres *resp)
 		return status;
 
 	session = resp->cstate.session;
-	if (session == NULL || slot->sl_cache_entry.ce_cachethis == 0)
+	if (session == NULL || slot->sl_cachethis == 0)
 		return status;
 
 	if (resp->opcnt >= args->opcnt)
diff --git a/include/linux/nfsd/state.h b/include/linux/nfsd/state.h
index 362c684..76e456c 100644
--- a/include/linux/nfsd/state.h
+++ b/include/linux/nfsd/state.h
@@ -126,6 +126,7 @@ struct nfsd4_cache_entry {
 struct nfsd4_slot {
 	bool				sl_inuse;
 	u32				sl_seqid;
+	int				sl_cachethis;
 	u32				sl_datalen;
 	char				sl_data[NFSD_SLOT_CACHE_SIZE];
 	struct nfsd4_cache_entry	sl_cache_entry;
diff --git a/include/linux/nfsd/xdr4.h b/include/linux/nfsd/xdr4.h
index 52589ba..faf5f2e 100644
--- a/include/linux/nfsd/xdr4.h
+++ b/include/linux/nfsd/xdr4.h
@@ -482,7 +482,7 @@ static inline bool nfsd4_is_solo_sequence(struct nfsd4_compoundres *resp)
 
 static inline bool nfsd4_not_cached(struct nfsd4_compoundres *resp)
 {
-	return !resp->cstate.slot->sl_cache_entry.ce_cachethis ||
+	return !resp->cstate.slot->sl_cachethis ||
 			nfsd4_is_solo_sequence(resp);
 }
 
-- 
1.5.4.3

[PATCH 14/31] nfsd41: replace ce_opcnt with nfsd4_slot field

[andros]

From: Andy Adamson <andros@netapp.com>

In preparation to remove struct nfsd4_cache_entry

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c        |    4 ++--
 include/linux/nfsd/state.h |    1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index ae1f316..d237e62 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1080,7 +1080,7 @@ nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
 		return;
 
 	nfsd4_release_respages(entry->ce_respages, entry->ce_resused);
-	entry->ce_opcnt = resp->opcnt;
+	slot->sl_opcnt = resp->opcnt;
 	entry->ce_status = resp->cstate.status;
 
 	/*
@@ -1190,7 +1190,7 @@ nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
 	}
 
 	resp->rqstp->rq_resused = entry->ce_resused;
-	resp->opcnt = entry->ce_opcnt;
+	resp->opcnt = slot->sl_opcnt;
 	resp->cstate.iovlen = entry->ce_datav.iov_len + entry->ce_rpchdrlen;
 	status = entry->ce_status;
 
diff --git a/include/linux/nfsd/state.h b/include/linux/nfsd/state.h
index 76e456c..46aabb1 100644
--- a/include/linux/nfsd/state.h
+++ b/include/linux/nfsd/state.h
@@ -127,6 +127,7 @@ struct nfsd4_slot {
 	bool				sl_inuse;
 	u32				sl_seqid;
 	int				sl_cachethis;
+	int				sl_opcnt;
 	u32				sl_datalen;
 	char				sl_data[NFSD_SLOT_CACHE_SIZE];
 	struct nfsd4_cache_entry	sl_cache_entry;
-- 
1.5.4.3

[PATCH 15/31] nfsd41: nfsd41: replace ce_status with nfsd4_slot field

[andros]

From: Andy Adamson <andros@netapp.com>

In preparation to remove struct nfsd4_cache_entry

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c        |    4 ++--
 include/linux/nfsd/state.h |    1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index d237e62..05083d3 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1081,7 +1081,7 @@ nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
 
 	nfsd4_release_respages(entry->ce_respages, entry->ce_resused);
 	slot->sl_opcnt = resp->opcnt;
-	entry->ce_status = resp->cstate.status;
+	slot->sl_status = resp->cstate.status;
 
 	/*
 	 * Don't need a page to cache just the sequence operation - the slot
@@ -1192,7 +1192,7 @@ nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
 	resp->rqstp->rq_resused = entry->ce_resused;
 	resp->opcnt = slot->sl_opcnt;
 	resp->cstate.iovlen = entry->ce_datav.iov_len + entry->ce_rpchdrlen;
-	status = entry->ce_status;
+	status = slot->sl_status;
 
 	return status;
 }
diff --git a/include/linux/nfsd/state.h b/include/linux/nfsd/state.h
index 46aabb1..8800e18 100644
--- a/include/linux/nfsd/state.h
+++ b/include/linux/nfsd/state.h
@@ -128,6 +128,7 @@ struct nfsd4_slot {
 	u32				sl_seqid;
 	int				sl_cachethis;
 	int				sl_opcnt;
+	__be32				sl_status;
 	u32				sl_datalen;
 	char				sl_data[NFSD_SLOT_CACHE_SIZE];
 	struct nfsd4_cache_entry	sl_cache_entry;
-- 
1.5.4.3

[PATCH 16/31] nfsd41: obliterate nfsd4_copy_pages

[andros]

From: Andy Adamson <andros@netapp.com>

Replacing page based drc cache with buffer based drc cache.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c |   21 ---------------------
 1 files changed, 0 insertions(+), 21 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 05083d3..3d1e33a 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1040,19 +1040,6 @@ nfsd4_release_respages(struct page **respages, short resused)
 	}
 }
 
-static void
-nfsd4_copy_pages(struct page **topages, struct page **frompages, short count)
-{
-	int i;
-
-	for (i = 0; i < count; i++) {
-		topages[i] = frompages[i];
-		if (!topages[i])
-			continue;
-		get_page(topages[i]);
-	}
-}
-
 /*
  * Cache the reply pages up to NFSD_PAGES_PER_SLOT + 1, clearing the previous
  * pages. We add a page to NFSD_PAGES_PER_SLOT for the case where the total
@@ -1102,8 +1089,6 @@ nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
 	entry->ce_resused = rqstp->rq_resused;
 	if (entry->ce_resused > NFSD_PAGES_PER_SLOT + 1)
 		entry->ce_resused = NFSD_PAGES_PER_SLOT + 1;
-	nfsd4_copy_pages(entry->ce_respages, rqstp->rq_respages,
-			 entry->ce_resused);
 	entry->ce_datav.iov_base = resp->cstate.statp;
 	entry->ce_datav.iov_len = resv->iov_len - ((char *)resp->cstate.statp -
 				(char *)page_address(rqstp->rq_respages[0]));
@@ -1176,17 +1161,11 @@ nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
 		 * cached header. Release all the allocated result pages.
 		 */
 		svc_free_res_pages(resp->rqstp);
-		nfsd4_copy_pages(resp->rqstp->rq_respages, entry->ce_respages,
-			entry->ce_resused);
 	} else {
 		/* Release all but the first allocated result page */
 
 		resp->rqstp->rq_resused--;
 		svc_free_res_pages(resp->rqstp);
-
-		nfsd4_copy_pages(&resp->rqstp->rq_respages[1],
-				 &entry->ce_respages[1],
-				 entry->ce_resused - 1);
 	}
 
 	resp->rqstp->rq_resused = entry->ce_resused;
-- 
1.5.4.3

[PATCH 17/31] nfsd41: obliterate nfsd41_copy_replay_data

[andros]

From: Andy Adamson <andros@netapp.com>

Replacing page based drc cache with buffer based drc cache.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c |   39 ---------------------------------------
 1 files changed, 0 insertions(+), 39 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 3d1e33a..bed5944 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1098,32 +1098,6 @@ nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
 }
 
 /*
- * We keep the rpc header, but take the nfs reply from the replycache.
- */
-static int
-nfsd41_copy_replay_data(struct nfsd4_compoundres *resp,
-			struct nfsd4_cache_entry *entry)
-{
-	struct svc_rqst *rqstp = resp->rqstp;
-	struct kvec *resv = &resp->rqstp->rq_res.head[0];
-	int len;
-
-	/* Current request rpc header length*/
-	len = (char *)resp->cstate.statp -
-			(char *)page_address(rqstp->rq_respages[0]);
-	if (entry->ce_datav.iov_len + len > PAGE_SIZE) {
-		dprintk("%s v41 cached reply too large (%Zd).\n", __func__,
-			entry->ce_datav.iov_len);
-		return 0;
-	}
-	/* copy the cached reply nfsd data past the current rpc header */
-	memcpy((char *)resv->iov_base + len, entry->ce_datav.iov_base,
-		entry->ce_datav.iov_len);
-	resv->iov_len = len + entry->ce_datav.iov_len;
-	return 1;
-}
-
-/*
  * Keep the first page of the replay. Copy the NFSv4.1 data from the first
  * cached page.  Replace any futher replay pages from the cache.
  */
@@ -1155,19 +1129,6 @@ nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
 	/* The sequence operation has been encoded, cstate->datap set. */
 	memcpy(resp->cstate.datap, slot->sl_data, slot->sl_datalen);
 
-	if (!nfsd41_copy_replay_data(resp, entry)) {
-		/*
-		 * Not enough room to use the replay rpc header, send the
-		 * cached header. Release all the allocated result pages.
-		 */
-		svc_free_res_pages(resp->rqstp);
-	} else {
-		/* Release all but the first allocated result page */
-
-		resp->rqstp->rq_resused--;
-		svc_free_res_pages(resp->rqstp);
-	}
-
 	resp->rqstp->rq_resused = entry->ce_resused;
 	resp->opcnt = slot->sl_opcnt;
 	resp->cstate.iovlen = entry->ce_datav.iov_len + entry->ce_rpchdrlen;
-- 
1.5.4.3

[PATCH 18/31] nfsd41: obliterate nfsd4_release_respages

[andros]

From: Andy Adamson <andros@netapp.com>

Replacing page based drc cache with buffer based drc cache.

Signed-off-by: Andy Adamson <andros@netapp.com
---
 fs/nfsd/nfs4state.c |   25 -------------------------
 1 files changed, 0 insertions(+), 25 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index bed5944..9ed99f4 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -579,19 +579,12 @@ release_session(struct nfsd4_session *ses)
 	nfsd4_put_session(ses);
 }
 
-static void nfsd4_release_respages(struct page **respages, short resused);
-
 void
 free_session(struct kref *kref)
 {
 	struct nfsd4_session *ses;
-	int i;
 
 	ses = container_of(kref, struct nfsd4_session, se_ref);
-	for (i = 0; i < ses->se_fchannel.maxreqs; i++) {
-		struct nfsd4_cache_entry *e = &ses->se_slots[i].sl_cache_entry;
-		nfsd4_release_respages(e->ce_respages, e->ce_resused);
-	}
 	spin_lock(&nfsd_drc_lock);
 	nfsd_drc_mem_used -= ses->se_fchannel.maxreqs * NFSD_SLOT_CACHE_SIZE;
 	spin_unlock(&nfsd_drc_lock);
@@ -1024,23 +1017,6 @@ nfsd4_set_statp(struct svc_rqst *rqstp, __be32 *statp)
 }
 
 /*
- * Dereference the result pages.
- */
-static void
-nfsd4_release_respages(struct page **respages, short resused)
-{
-	int i;
-
-	dprintk("--> %s\n", __func__);
-	for (i = 0; i < resused; i++) {
-		if (!respages[i])
-			continue;
-		put_page(respages[i]);
-		respages[i] = NULL;
-	}
-}
-
-/*
  * Cache the reply pages up to NFSD_PAGES_PER_SLOT + 1, clearing the previous
  * pages. We add a page to NFSD_PAGES_PER_SLOT for the case where the total
  * length of the XDR response is less than se_fmaxresp_cached
@@ -1066,7 +1042,6 @@ nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
 	if (resp->opcnt == 1 && op->opnum == OP_SEQUENCE && resp->cstate.status)
 		return;
 
-	nfsd4_release_respages(entry->ce_respages, entry->ce_resused);
 	slot->sl_opcnt = resp->opcnt;
 	slot->sl_status = resp->cstate.status;
 
-- 
1.5.4.3

[PATCH 19/31] nfsd41: remove iovlen field from nfsd4_compound_state

[andros]

From: Andy Adamson <andros@netapp.com>

Set resp->p in nfsd4_replay_cache_entry so that the nfs4svc_encode_compoundres
iov_len calculation is correct.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c       |    2 +-
 fs/nfsd/nfs4xdr.c         |    5 +----
 include/linux/nfsd/xdr4.h |    1 -
 3 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 9ed99f4..616ec0f 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1106,7 +1106,7 @@ nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
 
 	resp->rqstp->rq_resused = entry->ce_resused;
 	resp->opcnt = slot->sl_opcnt;
-	resp->cstate.iovlen = entry->ce_datav.iov_len + entry->ce_rpchdrlen;
+	resp->p = resp->cstate.datap + XDR_QUADLEN(slot->sl_datalen);
 	status = slot->sl_status;
 
 	return status;
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 827f6d4..9e8464c 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -3357,10 +3357,7 @@ nfs4svc_encode_compoundres(struct svc_rqst *rqstp, __be32 *p, struct nfsd4_compo
 	iov->iov_len = ((char*)resp->p) - (char*)iov->iov_base;
 	BUG_ON(iov->iov_len > PAGE_SIZE);
 	if (nfsd4_has_session(&resp->cstate)) {
-		if (resp->cstate.status == nfserr_replay_cache &&
-				!nfsd4_not_cached(resp)) {
-			iov->iov_len = resp->cstate.iovlen;
-		} else {
+		if (resp->cstate.status != nfserr_replay_cache) {
 			nfsd4_store_cache_entry(resp);
 			dprintk("%s: SET SLOT STATE TO AVAILABLE\n", __func__);
 			resp->cstate.slot->sl_inuse = 0;
diff --git a/include/linux/nfsd/xdr4.h b/include/linux/nfsd/xdr4.h
index faf5f2e..967129e 100644
--- a/include/linux/nfsd/xdr4.h
+++ b/include/linux/nfsd/xdr4.h
@@ -53,7 +53,6 @@ struct nfsd4_compound_state {
 	struct nfsd4_slot	*slot;
 	__be32			*statp;
 	__be32			*datap;
-	size_t			iovlen;
 	u32			minorversion;
 	u32			status;
 };
-- 
1.5.4.3

[PATCH 20/31] nfsd41: remove struct nfsd4_cache_entry

[andros]

From: Andy Adamson <andros@netapp.com>

Replaced by fields in struct nfsd4_slot

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c        |   20 ++------------------
 include/linux/nfsd/state.h |   27 +++++++--------------------
 2 files changed, 9 insertions(+), 38 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 616ec0f..ff43793 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1030,13 +1030,11 @@ void
 nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
 {
 	struct nfsd4_slot *slot = resp->cstate.slot;
-	struct nfsd4_cache_entry *entry = &resp->cstate.slot->sl_cache_entry;
 	struct svc_rqst *rqstp = resp->rqstp;
 	struct nfsd4_compoundargs *args = rqstp->rq_argp;
 	struct nfsd4_op *op = &args->ops[resp->opcnt];
-	struct kvec *resv = &rqstp->rq_res.head[0];
 
-	dprintk("--> %s entry %p\n", __func__, entry);
+	dprintk("--> %s\n", __func__);
 
 	/* Don't cache a failed OP_SEQUENCE. */
 	if (resp->opcnt == 1 && op->opnum == OP_SEQUENCE && resp->cstate.status)
@@ -1051,8 +1049,6 @@ nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
 	 */
 
 	if (nfsd4_not_cached(resp)) {
-		entry->ce_resused = 0;
-		entry->ce_rpchdrlen = 0;
 		slot->sl_datalen = 0;
 		dprintk("%s Just cache SEQUENCE. cachethis %d\n", __func__,
 			resp->cstate.slot->sl_cachethis);
@@ -1060,16 +1056,6 @@ nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
 	}
 	slot->sl_datalen = (char *)resp->p - (char *)resp->cstate.datap;
 	memcpy(slot->sl_data, resp->cstate.datap, slot->sl_datalen);
-
-	entry->ce_resused = rqstp->rq_resused;
-	if (entry->ce_resused > NFSD_PAGES_PER_SLOT + 1)
-		entry->ce_resused = NFSD_PAGES_PER_SLOT + 1;
-	entry->ce_datav.iov_base = resp->cstate.statp;
-	entry->ce_datav.iov_len = resv->iov_len - ((char *)resp->cstate.statp -
-				(char *)page_address(rqstp->rq_respages[0]));
-	/* Current request rpc header length*/
-	entry->ce_rpchdrlen = (char *)resp->cstate.statp -
-				(char *)page_address(rqstp->rq_respages[0]);
 }
 
 /*
@@ -1081,10 +1067,9 @@ nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
 			 struct nfsd4_sequence *seq)
 {
 	struct nfsd4_slot *slot = resp->cstate.slot;
-	struct nfsd4_cache_entry *entry = &resp->cstate.slot->sl_cache_entry;
 	__be32 status;
 
-	dprintk("--> %s entry %p\n", __func__, entry);
+	dprintk("--> %s datalen %d\n", __func__, slot->sl_datalen);
 
 	/*
 	 * If this is just the sequence operation, we did not keep
@@ -1104,7 +1089,6 @@ nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
 	/* The sequence operation has been encoded, cstate->datap set. */
 	memcpy(resp->cstate.datap, slot->sl_data, slot->sl_datalen);
 
-	resp->rqstp->rq_resused = entry->ce_resused;
 	resp->opcnt = slot->sl_opcnt;
 	resp->p = resp->cstate.datap + XDR_QUADLEN(slot->sl_datalen);
 	status = slot->sl_status;
diff --git a/include/linux/nfsd/state.h b/include/linux/nfsd/state.h
index 8800e18..20d9958 100644
--- a/include/linux/nfsd/state.h
+++ b/include/linux/nfsd/state.h
@@ -107,31 +107,18 @@ struct nfs4_callback {
 
 /* Maximum number of slots per session. 128 is useful for long haul TCP */
 #define NFSD_MAX_SLOTS_PER_SESSION	128
-/* Maximum number of pages per slot cache entry */
-#define NFSD_PAGES_PER_SLOT	1
 #define NFSD_SLOT_CACHE_SIZE		512
 /* Maximum number of operations per session compound */
 #define NFSD_MAX_OPS_PER_COMPOUND	16
 
-struct nfsd4_cache_entry {
-	__be32		ce_status;
-	struct kvec	ce_datav; /* encoded NFSv4.1 data in rq_res.head[0] */
-	struct page	*ce_respages[NFSD_PAGES_PER_SLOT + 1];
-	int		ce_cachethis;
-	short		ce_resused;
-	int		ce_opcnt;
-	int		ce_rpchdrlen;
-};
-
 struct nfsd4_slot {
-	bool				sl_inuse;
-	u32				sl_seqid;
-	int				sl_cachethis;
-	int				sl_opcnt;
-	__be32				sl_status;
-	u32				sl_datalen;
-	char				sl_data[NFSD_SLOT_CACHE_SIZE];
-	struct nfsd4_cache_entry	sl_cache_entry;
+	bool	sl_inuse;
+	u32	sl_seqid;
+	int	sl_cachethis;
+	int	sl_opcnt;
+	__be32	sl_status;
+	u32	sl_datalen;
+	char	sl_data[NFSD_SLOT_CACHE_SIZE];
 };
 
 /*
-- 
1.5.4.3

[PATCH 21/31] nfsd41: obliterate nfsd4_set_statp

[andros]

From: Andy Adamson <andros@netapp.com>

Replacing page based drc cache with buffer based drc cache.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c       |    8 --------
 fs/nfsd/nfssvc.c          |    4 ----
 include/linux/nfsd/xdr4.h |    1 -
 3 files changed, 0 insertions(+), 13 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index ff43793..b852cab 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1008,14 +1008,6 @@ out_err:
 	return;
 }
 
-void
-nfsd4_set_statp(struct svc_rqst *rqstp, __be32 *statp)
-{
-	struct nfsd4_compoundres *resp = rqstp->rq_resp;
-
-	resp->cstate.statp = statp;
-}
-
 /*
  * Cache the reply pages up to NFSD_PAGES_PER_SLOT + 1, clearing the previous
  * pages. We add a page to NFSD_PAGES_PER_SLOT for the case where the total
diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c
index 37633f5..d14adea 100644
--- a/fs/nfsd/nfssvc.c
+++ b/fs/nfsd/nfssvc.c
@@ -572,10 +572,6 @@ nfsd_dispatch(struct svc_rqst *rqstp, __be32 *statp)
 		+ rqstp->rq_res.head[0].iov_len;
 	rqstp->rq_res.head[0].iov_len += sizeof(__be32);
 
-	/* NFSv4.1 DRC requires statp */
-	if (rqstp->rq_vers == 4)
-		nfsd4_set_statp(rqstp, statp);
-
 	/* Now call the procedure handler, and encode NFS status. */
 	nfserr = proc->pc_func(rqstp, rqstp->rq_argp, rqstp->rq_resp);
 	nfserr = map_new_errors(rqstp->rq_vers, nfserr);
diff --git a/include/linux/nfsd/xdr4.h b/include/linux/nfsd/xdr4.h
index 967129e..cd51fa8 100644
--- a/include/linux/nfsd/xdr4.h
+++ b/include/linux/nfsd/xdr4.h
@@ -51,7 +51,6 @@ struct nfsd4_compound_state {
 	/* For sessions DRC */
 	struct nfsd4_session	*session;
 	struct nfsd4_slot	*slot;
-	__be32			*statp;
 	__be32			*datap;
 	u32			minorversion;
 	u32			status;
-- 
1.5.4.3

[PATCH 22/31] nfsd41: rename nfsd4_enc_uncached_replay

[andros]

From: Andy Adamson <andros@netapp.com>

This function is only used for SEQUENCE replay.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4proc.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index d4c09a9..d277e42 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -865,7 +865,7 @@ static const char *nfsd4_op_name(unsigned opnum);
  * encode the uncache rep error on the next operation.
  */
 static __be32
-nfsd4_enc_uncached_replay(struct nfsd4_compoundargs *args,
+nfsd4_enc_sequence_replay(struct nfsd4_compoundargs *args,
 			 struct nfsd4_compoundres *resp)
 {
 	struct nfsd4_op *op;
@@ -1000,7 +1000,7 @@ encode_op:
 		if (resp->cstate.status == nfserr_replay_cache) {
 			dprintk("%s NFS4.1 replay from cache\n", __func__);
 			if (nfsd4_not_cached(resp))
-				status = nfsd4_enc_uncached_replay(args, resp);
+				status = nfsd4_enc_sequence_replay(args, resp);
 			else
 				status = op->status;
 			goto out;
-- 
1.5.4.3

[PATCH 23/31] nfsd41: encode replay sequence from the slot values

[andros]

From: Andy Adamson <andros@netapp.com>

The sequence operation is not cached; always encode the sequence operation on
a replay from the slot table and session values.

If this is a replay of a compound that was specified not to be cached, return
NFS4ERR_RETRY_UNCACHED_REP.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4proc.c  |   33 +--------------------------------
 fs/nfsd/nfs4state.c |   40 ++++++++++++++++++++++++++++++++++++----
 2 files changed, 37 insertions(+), 36 deletions(-)

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index d277e42..d8e3be9 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -860,34 +860,6 @@ static struct nfsd4_operation nfsd4_ops[];
 static const char *nfsd4_op_name(unsigned opnum);
 
 /*
- * This is a replay of a compound for which no cache entry pages
- * were used. Encode the sequence operation, and if cachethis is FALSE
- * encode the uncache rep error on the next operation.
- */
-static __be32
-nfsd4_enc_sequence_replay(struct nfsd4_compoundargs *args,
-			 struct nfsd4_compoundres *resp)
-{
-	struct nfsd4_op *op;
-
-	dprintk("--> %s resp->opcnt %d cachethis %u \n", __func__,
-		resp->opcnt, resp->cstate.slot->sl_cachethis);
-
-	/* Encode the replayed sequence operation */
-	BUG_ON(resp->opcnt != 1);
-	op = &args->ops[resp->opcnt - 1];
-	nfsd4_encode_operation(resp, op);
-
-	/*return nfserr_retry_uncached_rep in next operation. */
-	if (resp->cstate.slot->sl_cachethis == 0) {
-		op = &args->ops[resp->opcnt++];
-		op->status = nfserr_retry_uncached_rep;
-		nfsd4_encode_operation(resp, op);
-	}
-	return op->status;
-}
-
-/*
  * Enforce NFSv4.1 COMPOUND ordering rules.
  *
  * TODO:
@@ -999,10 +971,7 @@ encode_op:
 		/* Only from SEQUENCE */
 		if (resp->cstate.status == nfserr_replay_cache) {
 			dprintk("%s NFS4.1 replay from cache\n", __func__);
-			if (nfsd4_not_cached(resp))
-				status = nfsd4_enc_sequence_replay(args, resp);
-			else
-				status = op->status;
+			status = op->status;
 			goto out;
 		}
 		/* Only from CREATE_SESSION */
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index b852cab..61a25cf 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1051,6 +1051,36 @@ nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
 }
 
 /*
+ * Encode the replay sequence operation from the slot values.
+ * If cachethis is FALSE encode the uncached rep error on the next
+ * operation which sets resp->p and increments resp->opcnt for
+ * nfs4svc_encode_compoundres.
+ *
+ */
+static __be32
+nfsd4_enc_sequence_replay(struct nfsd4_compoundargs *args,
+			  struct nfsd4_compoundres *resp)
+{
+	struct nfsd4_op *op;
+	struct nfsd4_slot *slot = resp->cstate.slot;
+
+	dprintk("--> %s resp->opcnt %d cachethis %u \n", __func__,
+		resp->opcnt, resp->cstate.slot->sl_cachethis);
+
+	/* Encode the replayed sequence operation */
+	op = &args->ops[resp->opcnt - 1];
+	nfsd4_encode_operation(resp, op);
+
+	/* Return nfserr_retry_uncached_rep in next operation. */
+	if (args->opcnt > 1 && slot->sl_cachethis == 0) {
+		op = &args->ops[resp->opcnt++];
+		op->status = nfserr_retry_uncached_rep;
+		nfsd4_encode_operation(resp, op);
+	}
+	return op->status;
+}
+
+/*
  * Keep the first page of the replay. Copy the NFSv4.1 data from the first
  * cached page.  Replace any futher replay pages from the cache.
  */
@@ -1073,10 +1103,12 @@ nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
 	 * session inactivity timer fires and a solo sequence operation
 	 * is sent (lease renewal).
 	 */
-	if (seq && nfsd4_not_cached(resp)) {
-		seq->maxslots = resp->cstate.session->se_fchannel.maxreqs;
-		return nfs_ok;
-	}
+	seq->maxslots = resp->cstate.session->se_fchannel.maxreqs;
+
+	/* Either returns 0 or nfserr_retry_uncached */
+	status = nfsd4_enc_sequence_replay(resp->rqstp->rq_argp, resp);
+	if (status == nfserr_retry_uncached_rep)
+		return status;
 
 	/* The sequence operation has been encoded, cstate->datap set. */
 	memcpy(resp->cstate.datap, slot->sl_data, slot->sl_datalen);
-- 
1.5.4.3

[PATCH 24/31] nfsd41: fix nfsd4_replay_cache_entry comments

[andros]

From: Andy Adamson <andros@netapp.com>

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c |   15 +++------------
 1 files changed, 3 insertions(+), 12 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 61a25cf..d38a48f 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1081,8 +1081,8 @@ nfsd4_enc_sequence_replay(struct nfsd4_compoundargs *args,
 }
 
 /*
- * Keep the first page of the replay. Copy the NFSv4.1 data from the first
- * cached page.  Replace any futher replay pages from the cache.
+ * The sequence operation is not cached because we can use the slot and
+ * session values.
  */
 __be32
 nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
@@ -1093,16 +1093,7 @@ nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
 
 	dprintk("--> %s datalen %d\n", __func__, slot->sl_datalen);
 
-	/*
-	 * If this is just the sequence operation, we did not keep
-	 * a page in the cache entry because we can just use the
-	 * slot info stored in struct nfsd4_sequence that was checked
-	 * against the slot in nfsd4_sequence().
-	 *
-	 * This occurs when seq->cachethis is FALSE, or when the client
-	 * session inactivity timer fires and a solo sequence operation
-	 * is sent (lease renewal).
-	 */
+	/* Target max slot and flags will be set once they are implemented */
 	seq->maxslots = resp->cstate.session->se_fchannel.maxreqs;
 
 	/* Either returns 0 or nfserr_retry_uncached */
-- 
1.5.4.3

[PATCH 25/31] nfsd41: fix nfsd4_store_cache_entry comments

[andros]

From: Andy Adamson <andros@netapp.com>

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c |   15 +++------------
 1 files changed, 3 insertions(+), 12 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index d38a48f..56114fd 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1009,14 +1009,8 @@ out_err:
 }
 
 /*
- * Cache the reply pages up to NFSD_PAGES_PER_SLOT + 1, clearing the previous
- * pages. We add a page to NFSD_PAGES_PER_SLOT for the case where the total
- * length of the XDR response is less than se_fmaxresp_cached
- * (NFSD_PAGES_PER_SLOT * PAGE_SIZE) but the xdr_buf pages is used for a
- * of the reply (e.g. readdir).
- *
- * Store the base and length of the rq_req.head[0] page
- * of the NFSv4.1 data, just past the rpc header.
+ *  Copy all encoded operation responses past the sequence response into the
+ *  slot's cache.
  */
 void
 nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
@@ -1035,11 +1029,8 @@ nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
 	slot->sl_opcnt = resp->opcnt;
 	slot->sl_status = resp->cstate.status;
 
-	/*
-	 * Don't need a page to cache just the sequence operation - the slot
-	 * does this for us!
-	 */
 
+	/* Don't cache the sequence operation, use the slot values on replay */
 	if (nfsd4_not_cached(resp)) {
 		slot->sl_datalen = 0;
 		dprintk("%s Just cache SEQUENCE. cachethis %d\n", __func__,
-- 
1.5.4.3

[PATCH 26/31] nfsd41: support 16 slots per session

[andros]

From: Andy Adamson <andros@netapp.com>

128 slots is a bit much without proof that they are needed.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 include/linux/nfsd/state.h |    3 +--
 1 files changed, 1 insertions(+), 2 deletions(-)

diff --git a/include/linux/nfsd/state.h b/include/linux/nfsd/state.h
index 20d9958..0b07d40 100644
--- a/include/linux/nfsd/state.h
+++ b/include/linux/nfsd/state.h
@@ -105,8 +105,7 @@ struct nfs4_callback {
 	struct rpc_clnt *       cb_client;
 };
 
-/* Maximum number of slots per session. 128 is useful for long haul TCP */
-#define NFSD_MAX_SLOTS_PER_SESSION	128
+#define NFSD_MAX_SLOTS_PER_SESSION	16
 #define NFSD_SLOT_CACHE_SIZE		512
 /* Maximum number of operations per session compound */
 #define NFSD_MAX_OPS_PER_COMPOUND	16
-- 
1.5.4.3

[PATCH 27/31] nfsd41: use the maximum operations per compound in nfsd4_compoundargs

[andros]

From: Andy Adamson <andros@netapp.com>

The size of the nfsd4_op array in nfsd4_compoundargs determines the
supported maximum number of operations.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 include/linux/nfsd/state.h |    3 +--
 include/linux/nfsd/xdr4.h  |    2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/include/linux/nfsd/state.h b/include/linux/nfsd/state.h
index 0b07d40..49e3f60 100644
--- a/include/linux/nfsd/state.h
+++ b/include/linux/nfsd/state.h
@@ -107,8 +107,7 @@ struct nfs4_callback {
 
 #define NFSD_MAX_SLOTS_PER_SESSION	16
 #define NFSD_SLOT_CACHE_SIZE		512
-/* Maximum number of operations per session compound */
-#define NFSD_MAX_OPS_PER_COMPOUND	16
+#define NFSD_MAX_OPS_PER_COMPOUND	8
 
 struct nfsd4_slot {
 	bool	sl_inuse;
diff --git a/include/linux/nfsd/xdr4.h b/include/linux/nfsd/xdr4.h
index cd51fa8..bc8fab3 100644
--- a/include/linux/nfsd/xdr4.h
+++ b/include/linux/nfsd/xdr4.h
@@ -455,7 +455,7 @@ struct nfsd4_compoundargs {
 	u32				minorversion;
 	u32				opcnt;
 	struct nfsd4_op			*ops;
-	struct nfsd4_op			iops[8];
+	struct nfsd4_op			iops[NFSD_MAX_OPS_PER_COMPOUND];
 };
 
 struct nfsd4_compoundres {
-- 
1.5.4.3

[PATCH 28/31] nfsd41: fix nfsd4_store_cache_entry dprintk

[andros]

From: Andy Adamson <andros@netapp.com>

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c |    4 +---
 1 files changed, 1 insertions(+), 3 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 56114fd..c7a78e2 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1020,7 +1020,7 @@ nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
 	struct nfsd4_compoundargs *args = rqstp->rq_argp;
 	struct nfsd4_op *op = &args->ops[resp->opcnt];
 
-	dprintk("--> %s\n", __func__);
+	dprintk("--> %s cachethis %d\n", __func__, slot->sl_cachethis);
 
 	/* Don't cache a failed OP_SEQUENCE. */
 	if (resp->opcnt == 1 && op->opnum == OP_SEQUENCE && resp->cstate.status)
@@ -1033,8 +1033,6 @@ nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
 	/* Don't cache the sequence operation, use the slot values on replay */
 	if (nfsd4_not_cached(resp)) {
 		slot->sl_datalen = 0;
-		dprintk("%s Just cache SEQUENCE. cachethis %d\n", __func__,
-			resp->cstate.slot->sl_cachethis);
 		return;
 	}
 	slot->sl_datalen = (char *)resp->p - (char *)resp->cstate.datap;
-- 
1.5.4.3

[PATCH 29/31] nfsd41: add test for failed sequence operation

[andros]

From: Andy Adamson <andros@netapp.com>

Failed sequence operations are not cached.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 include/linux/nfsd/xdr4.h |   10 +++++++++-
 1 files changed, 9 insertions(+), 1 deletions(-)

diff --git a/include/linux/nfsd/xdr4.h b/include/linux/nfsd/xdr4.h
index bc8fab3..7cbb5e7 100644
--- a/include/linux/nfsd/xdr4.h
+++ b/include/linux/nfsd/xdr4.h
@@ -472,6 +472,13 @@ struct nfsd4_compoundres {
 	struct nfsd4_compound_state	cstate;
 };
 
+static inline bool nfsd4_is_failed_sequence(struct nfsd4_compoundres *resp)
+{
+	struct nfsd4_compoundargs *args = resp->rqstp->rq_argp;
+	return resp->opcnt == 1 && args->ops[0].opnum == OP_SEQUENCE &&
+		resp->cstate.status;
+}
+
 static inline bool nfsd4_is_solo_sequence(struct nfsd4_compoundres *resp)
 {
 	struct nfsd4_compoundargs *args = resp->rqstp->rq_argp;
@@ -481,7 +488,8 @@ static inline bool nfsd4_is_solo_sequence(struct nfsd4_compoundres *resp)
 static inline bool nfsd4_not_cached(struct nfsd4_compoundres *resp)
 {
 	return !resp->cstate.slot->sl_cachethis ||
-			nfsd4_is_solo_sequence(resp);
+			nfsd4_is_solo_sequence(resp) ||
+			nfsd4_is_failed_sequence(resp);
 }
 
 #define NFS4_SVC_XDRSIZE		sizeof(struct nfsd4_compoundargs)
-- 
1.5.4.3

[PATCH 30/31] nfsd41: remove redundant failed sequence check

[andros]

From: Andy Adamson <andros@netapp.com>

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c |    7 -------
 1 files changed, 0 insertions(+), 7 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index c7a78e2..1ea62cd 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1016,16 +1016,9 @@ void
 nfsd4_store_cache_entry(struct nfsd4_compoundres *resp)
 {
 	struct nfsd4_slot *slot = resp->cstate.slot;
-	struct svc_rqst *rqstp = resp->rqstp;
-	struct nfsd4_compoundargs *args = rqstp->rq_argp;
-	struct nfsd4_op *op = &args->ops[resp->opcnt];
 
 	dprintk("--> %s cachethis %d\n", __func__, slot->sl_cachethis);
 
-	/* Don't cache a failed OP_SEQUENCE. */
-	if (resp->opcnt == 1 && op->opnum == OP_SEQUENCE && resp->cstate.status)
-		return;
-
 	slot->sl_opcnt = resp->opcnt;
 	slot->sl_status = resp->cstate.status;
 
-- 
1.5.4.3

[PATCH 31/31] nfsd41: only reference the session on non-replay sequence

[andros]

From: Andy Adamson <andros@netapp.com>

Solo CREATE_SESSION replays used to share the logic in nfsdsvc_enc_compound_res
and use nfsd4_store_cache_entry, but not set the cstate session nor take a
reference on the session. CREATE_SESSION no longer uses nfsd4_store_cache_entry.

The SEQUENCE operation only needs to reference the session to call
nfsd4_store_cache_entry.

Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4state.c |    9 ++++-----
 fs/nfsd/nfs4xdr.c   |   14 ++++++--------
 2 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 1ea62cd..1407848 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1435,13 +1435,12 @@ nfsd4_sequence(struct svc_rqst *rqstp,
 	cstate->slot = slot;
 	cstate->session = session;
 
+	/* Hold a session reference until done caching the response */
+	nfsd4_get_session(session);
+
 replay_cache:
-	/* Renew the clientid on success and on replay.
-	 * Hold a session reference until done processing the compound:
-	 * nfsd4_put_session called only if the cstate slot is set.
-	 */
+	/* Renew the clientid on success and on replay */
 	renew_client(session->se_client);
-	nfsd4_get_session(session);
 out:
 	spin_unlock(&sessionid_lock);
 	dprintk("%s: return %d\n", __func__, ntohl(status));
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 9e8464c..9f98618 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -3343,6 +3343,7 @@ nfs4svc_encode_compoundres(struct svc_rqst *rqstp, __be32 *p, struct nfsd4_compo
 	/*
 	 * All that remains is to write the tag and operation count...
 	 */
+	struct nfsd4_compound_state *cs = &resp->cstate;
 	struct kvec *iov;
 	p = resp->tagp;
 	*p++ = htonl(resp->taglen);
@@ -3356,14 +3357,11 @@ nfs4svc_encode_compoundres(struct svc_rqst *rqstp, __be32 *p, struct nfsd4_compo
 		iov = &rqstp->rq_res.head[0];
 	iov->iov_len = ((char*)resp->p) - (char*)iov->iov_base;
 	BUG_ON(iov->iov_len > PAGE_SIZE);
-	if (nfsd4_has_session(&resp->cstate)) {
-		if (resp->cstate.status != nfserr_replay_cache) {
-			nfsd4_store_cache_entry(resp);
-			dprintk("%s: SET SLOT STATE TO AVAILABLE\n", __func__);
-			resp->cstate.slot->sl_inuse = 0;
-		}
-		if (resp->cstate.session)
-			nfsd4_put_session(resp->cstate.session);
+	if (nfsd4_has_session(cs) && cs->status != nfserr_replay_cache) {
+		nfsd4_store_cache_entry(resp);
+		dprintk("%s: SET SLOT STATE TO AVAILABLE\n", __func__);
+		cs->slot->sl_inuse = 0;
+		nfsd4_put_session(cs->session);
 	}
 	return 1;
 }
-- 
1.5.4.3

Re: [PATCH 10/31] nfsd41: use globals for DRC memory use management

[Benny Halevy]

On Apr. 28, 2009, 19:59 +0300, andros@netapp.com wrote:
> From: Andy Adamson <andros@netapp.com>
> 
> The version 4.1 DRC memory limit and tracking variables are server wide and
> session specific.
> Add a spinlock to serialize access to the management variables which change
> on session creation and deletion (usage counter) or (future) administrative
> action to adjust the total DRC memory limit.
> 
> Track DRC memory usage in free session.
> 
> Signed-off-by: Andy Adamson <andros@netapp.com>
> ---
>  fs/nfsd/nfs4state.c        |   11 +++++++----
>  fs/nfsd/nfssvc.c           |   19 +++++++++++++++----
>  include/linux/nfsd/nfsd.h  |    5 +++++
>  include/linux/sunrpc/svc.h |    2 --
>  4 files changed, 27 insertions(+), 10 deletions(-)
> 
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index d12220c..a9e722e 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -416,7 +416,7 @@ gen_sessionid(struct nfsd4_session *ses)
>  
>  /*
>   * Give the client the number of slots it requests bound by
> - * NFSD_MAX_SLOTS_PER_SESSION and by sv_drc_max_pages.
> + * NFSD_MAX_SLOTS_PER_SESSION and by nfsd_drc_max_mem.
>   *
>   * If we run out of reserved DRC memory we should (up to a point) re-negotiate
>   * active sessions and reduce their slot usage to make rooom for new
> @@ -434,9 +434,9 @@ static int set_forechannel_maxreqs(struct nfsd4_channel_attrs *fchan)
>  	mem = fchan->maxreqs * NFSD_SLOT_CACHE_SIZE;
>  
>  	spin_lock(&nfsd_serv->sv_lock);

Hmm, shouldn't you grab nfsd_drc_lock instead?

> -	if (mem + nfsd_serv->sv_drc_mem_used > nfsd_serv->sv_drc_max_mem)
> -		mem = nfsd_serv->sv_drc_max_mem - nfsd_serv->sv_drc_mem_used;
> -	nfsd_serv->sv_drc_mem_used += mem;
> +	if (mem + nfsd_drc_mem_used > nfsd_drc_max_mem)
> +		mem = nfsd_drc_max_mem - nfsd_drc_mem_used;
> +	nfsd_drc_mem_used += mem;
>  	spin_unlock(&nfsd_serv->sv_lock);

and unlock nfsd_drc_lock

Benny

>  
>  	if (mem < NFSD_SLOT_CACHE_SIZE) {
> @@ -588,6 +588,9 @@ free_session(struct kref *kref)
>  		struct nfsd4_cache_entry *e = &ses->se_slots[i].sl_cache_entry;
>  		nfsd4_release_respages(e->ce_respages, e->ce_resused);
>  	}
> +	spin_lock(&nfsd_drc_lock);
> +	nfsd_drc_mem_used -= ses->se_fchannel.maxreqs * NFSD_SLOT_CACHE_SIZE;
> +	spin_unlock(&nfsd_drc_lock);
>  	kfree(ses);
>  }
>  
> diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c
> index 80588cc..37633f5 100644
> --- a/fs/nfsd/nfssvc.c
> +++ b/fs/nfsd/nfssvc.c
> @@ -67,6 +67,16 @@ struct timeval			nfssvc_boot;
>  DEFINE_MUTEX(nfsd_mutex);
>  struct svc_serv 		*nfsd_serv;
>  
> +/*
> + * nfsd_drc_lock protects nfsd_drc_max_pages and nfsd_drc_pages_used.
> + * nfsd_drc_max_pages limits the total amount of memory available for
> + * version 4.1 DRC caches.
> + * nfsd_drc_pages_used tracks the current version 4.1 DRC memory usage.
> + */
> +spinlock_t	nfsd_drc_lock;
> +unsigned int	nfsd_drc_max_mem;
> +unsigned int	nfsd_drc_mem_used;
> +
>  #if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL)
>  static struct svc_stat	nfsd_acl_svcstats;
>  static struct svc_version *	nfsd_acl_version[] = {
> @@ -238,12 +248,13 @@ static void set_max_drc(void)
>  {
>  	/* The percent of nr_free_buffer_pages used by the V4.1 server DRC */
>  	#define NFSD_DRC_SIZE_SHIFT	10
> -	nfsd_serv->sv_drc_max_mem = (nr_free_buffer_pages()
> +	nfsd_drc_max_mem = (nr_free_buffer_pages()
>  					>> NFSD_DRC_SIZE_SHIFT) * PAGE_SIZE;
> -	nfsd_serv->sv_drc_mem_used = 0;
> +	nfsd_drc_mem_used = 0;
> +	spin_lock_init(&nfsd_drc_lock);
>  	dprintk("%s svc_drc_max_mem %u [in pages %lu]\n", __func__,
> -		nfsd_serv->sv_drc_max_mem,
> -		nfsd_serv->sv_drc_max_mem / PAGE_SIZE);
> +		nfsd_drc_max_mem,
> +		nfsd_drc_max_mem / PAGE_SIZE);
>  }
>  
>  int nfsd_create_serv(void)
> diff --git a/include/linux/nfsd/nfsd.h b/include/linux/nfsd/nfsd.h
> index 9cd6399..23ba7bd 100644
> --- a/include/linux/nfsd/nfsd.h
> +++ b/include/linux/nfsd/nfsd.h
> @@ -56,6 +56,11 @@ extern struct svc_version	nfsd_version2, nfsd_version3,
>  extern u32			nfsd_supported_minorversion;
>  extern struct mutex		nfsd_mutex;
>  extern struct svc_serv		*nfsd_serv;
> +extern spinlock_t		nfsd_drc_lock;
> +extern unsigned int		nfsd_drc_max_mem;
> +extern unsigned int		nfsd_drc_mem_used;
> +
> +
>  
>  extern struct seq_operations nfs_exports_op;
>  
> diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
> index 243508e..d0d8bf4 100644
> --- a/include/linux/sunrpc/svc.h
> +++ b/include/linux/sunrpc/svc.h
> @@ -94,8 +94,6 @@ struct svc_serv {
>  	struct module *		sv_module;	/* optional module to count when
>  						 * adding threads */
>  	svc_thread_fn		sv_function;	/* main function for threads */
> -	unsigned int		sv_drc_max_mem; /* Total pages for DRC */
> -	unsigned int		sv_drc_mem_used;/* DRC pages used */
>  };
>  
>  /*

Re: [PATCH 10/31] nfsd41: use globals for DRC memory use management

[Andy Adamson]

On Apr 30, 2009, at 7:08 PM, Benny Halevy wrote:

> On Apr. 28, 2009, 19:59 +0300, andros@netapp.com wrote:
>> From: Andy Adamson <andros@netapp.com>
>>
>> The version 4.1 DRC memory limit and tracking variables are server  
>> wide and
>> session specific.
>> Add a spinlock to serialize access to the management variables  
>> which change
>> on session creation and deletion (usage counter) or (future)  
>> administrative
>> action to adjust the total DRC memory limit.
>>
>> Track DRC memory usage in free session.
>>
>> Signed-off-by: Andy Adamson <andros@netapp.com>
>> ---
>> fs/nfsd/nfs4state.c        |   11 +++++++----
>> fs/nfsd/nfssvc.c           |   19 +++++++++++++++----
>> include/linux/nfsd/nfsd.h  |    5 +++++
>> include/linux/sunrpc/svc.h |    2 --
>> 4 files changed, 27 insertions(+), 10 deletions(-)
>>
>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
>> index d12220c..a9e722e 100644
>> --- a/fs/nfsd/nfs4state.c
>> +++ b/fs/nfsd/nfs4state.c
>> @@ -416,7 +416,7 @@ gen_sessionid(struct nfsd4_session *ses)
>>
>> /*
>>  * Give the client the number of slots it requests bound by
>> - * NFSD_MAX_SLOTS_PER_SESSION and by sv_drc_max_pages.
>> + * NFSD_MAX_SLOTS_PER_SESSION and by nfsd_drc_max_mem.
>>  *
>>  * If we run out of reserved DRC memory we should (up to a point)  
>> re-negotiate
>>  * active sessions and reduce their slot usage to make rooom for new
>> @@ -434,9 +434,9 @@ static int set_forechannel_maxreqs(struct  
>> nfsd4_channel_attrs *fchan)
>> 	mem = fchan->maxreqs * NFSD_SLOT_CACHE_SIZE;
>>
>> 	spin_lock(&nfsd_serv->sv_lock);
>
> Hmm, shouldn't you grab nfsd_drc_lock instead?
>
>> -	if (mem + nfsd_serv->sv_drc_mem_used > nfsd_serv->sv_drc_max_mem)
>> -		mem = nfsd_serv->sv_drc_max_mem - nfsd_serv->sv_drc_mem_used;
>> -	nfsd_serv->sv_drc_mem_used += mem;
>> +	if (mem + nfsd_drc_mem_used > nfsd_drc_max_mem)
>> +		mem = nfsd_drc_max_mem - nfsd_drc_mem_used;
>> +	nfsd_drc_mem_used += mem;
>> 	spin_unlock(&nfsd_serv->sv_lock);
>
> and unlock nfsd_drc_lock


Yes, somehow that got lost the last time I refactored the patch set.

Thanks for catching this.

-->Andy
>
>
> Benny
>
>>
>> 	if (mem < NFSD_SLOT_CACHE_SIZE) {
>> @@ -588,6 +588,9 @@ free_session(struct kref *kref)
>> 		struct nfsd4_cache_entry *e = &ses->se_slots[i].sl_cache_entry;
>> 		nfsd4_release_respages(e->ce_respages, e->ce_resused);
>> 	}
>> +	spin_lock(&nfsd_drc_lock);
>> +	nfsd_drc_mem_used -= ses->se_fchannel.maxreqs *  
>> NFSD_SLOT_CACHE_SIZE;
>> +	spin_unlock(&nfsd_drc_lock);
>> 	kfree(ses);
>> }
>>
>> diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c
>> index 80588cc..37633f5 100644
>> --- a/fs/nfsd/nfssvc.c
>> +++ b/fs/nfsd/nfssvc.c
>> @@ -67,6 +67,16 @@ struct timeval			nfssvc_boot;
>> DEFINE_MUTEX(nfsd_mutex);
>> struct svc_serv 		*nfsd_serv;
>>
>> +/*
>> + * nfsd_drc_lock protects nfsd_drc_max_pages and  
>> nfsd_drc_pages_used.
>> + * nfsd_drc_max_pages limits the total amount of memory available  
>> for
>> + * version 4.1 DRC caches.
>> + * nfsd_drc_pages_used tracks the current version 4.1 DRC memory  
>> usage.
>> + */
>> +spinlock_t	nfsd_drc_lock;
>> +unsigned int	nfsd_drc_max_mem;
>> +unsigned int	nfsd_drc_mem_used;
>> +
>> #if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL)
>> static struct svc_stat	nfsd_acl_svcstats;
>> static struct svc_version *	nfsd_acl_version[] = {
>> @@ -238,12 +248,13 @@ static void set_max_drc(void)
>> {
>> 	/* The percent of nr_free_buffer_pages used by the V4.1 server DRC  
>> */
>> 	#define NFSD_DRC_SIZE_SHIFT	10
>> -	nfsd_serv->sv_drc_max_mem = (nr_free_buffer_pages()
>> +	nfsd_drc_max_mem = (nr_free_buffer_pages()
>> 					>> NFSD_DRC_SIZE_SHIFT) * PAGE_SIZE;
>> -	nfsd_serv->sv_drc_mem_used = 0;
>> +	nfsd_drc_mem_used = 0;
>> +	spin_lock_init(&nfsd_drc_lock);
>> 	dprintk("%s svc_drc_max_mem %u [in pages %lu]\n", __func__,
>> -		nfsd_serv->sv_drc_max_mem,
>> -		nfsd_serv->sv_drc_max_mem / PAGE_SIZE);
>> +		nfsd_drc_max_mem,
>> +		nfsd_drc_max_mem / PAGE_SIZE);
>> }
>>
>> int nfsd_create_serv(void)
>> diff --git a/include/linux/nfsd/nfsd.h b/include/linux/nfsd/nfsd.h
>> index 9cd6399..23ba7bd 100644
>> --- a/include/linux/nfsd/nfsd.h
>> +++ b/include/linux/nfsd/nfsd.h
>> @@ -56,6 +56,11 @@ extern struct svc_version	nfsd_version2,  
>> nfsd_version3,
>> extern u32			nfsd_supported_minorversion;
>> extern struct mutex		nfsd_mutex;
>> extern struct svc_serv		*nfsd_serv;
>> +extern spinlock_t		nfsd_drc_lock;
>> +extern unsigned int		nfsd_drc_max_mem;
>> +extern unsigned int		nfsd_drc_mem_used;
>> +
>> +
>>
>> extern struct seq_operations nfs_exports_op;
>>
>> diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
>> index 243508e..d0d8bf4 100644
>> --- a/include/linux/sunrpc/svc.h
>> +++ b/include/linux/sunrpc/svc.h
>> @@ -94,8 +94,6 @@ struct svc_serv {
>> 	struct module *		sv_module;	/* optional module to count when
>> 						 * adding threads */
>> 	svc_thread_fn		sv_function;	/* main function for threads */
>> -	unsigned int		sv_drc_max_mem; /* Total pages for DRC */
>> -	unsigned int		sv_drc_mem_used;/* DRC pages used */
>> };
>>
>> /*

Re: [PATCH 08/31] nfsd41: sanity check client drc maxreqs

[Benny Halevy]

On Apr. 28, 2009, 19:59 +0300, andros@netapp.com wrote:
> From: Andy Adamson <andros@netapp.com>
> 
> Ensure the client requested maximum requests are between 1 and
> NFSD_MAX_SLOTS_PER_SESSION
> 
> Signed-off-by: Andy Adamson <andros@netapp.com>
> ---
>  fs/nfsd/nfs4state.c |    5 +++++
>  1 files changed, 5 insertions(+), 0 deletions(-)
> 
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index e216169..59b601b 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -427,6 +427,11 @@ static int set_forechannel_maxreqs(struct nfsd4_channel_attrs *fchan)
>  {
>  	int status = 0, np = fchan->maxreqs * NFSD_PAGES_PER_SLOT;
>  
> +	if (fchan->maxreqs < 1)
> +		return nfserr_inval;

Is 0 prohibited by the protocol?
The server can set it to whatever value it wants,
or if we can live with it, it actually provides a
nice way to test the server end-cases.

Benny

> +	else if (fchan->maxreqs > NFSD_MAX_SLOTS_PER_SESSION)
> +		fchan->maxreqs = NFSD_MAX_SLOTS_PER_SESSION;
> +
>  	spin_lock(&nfsd_serv->sv_lock);
>  	if (np + nfsd_serv->sv_drc_pages_used > nfsd_serv->sv_drc_max_pages)
>  		np = nfsd_serv->sv_drc_max_pages - nfsd_serv->sv_drc_pages_used;

Re: [PATCH 08/31] nfsd41: sanity check client drc maxreqs

[Trond Myklebust]

On Thu, 2009-04-30 at 20:34 +0300, Benny Halevy wrote:
> On Apr. 28, 2009, 19:59 +0300, andros@netapp.com wrote:
> > From: Andy Adamson <andros@netapp.com>
> > 
> > Ensure the client requested maximum requests are between 1 and
> > NFSD_MAX_SLOTS_PER_SESSION
> > 
> > Signed-off-by: Andy Adamson <andros@netapp.com>
> > ---
> >  fs/nfsd/nfs4state.c |    5 +++++
> >  1 files changed, 5 insertions(+), 0 deletions(-)
> > 
> > diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> > index e216169..59b601b 100644
> > --- a/fs/nfsd/nfs4state.c
> > +++ b/fs/nfsd/nfs4state.c
> > @@ -427,6 +427,11 @@ static int set_forechannel_maxreqs(struct nfsd4_channel_attrs *fchan)
> >  {
> >  	int status = 0, np = fchan->maxreqs * NFSD_PAGES_PER_SLOT;
> >  
> > +	if (fchan->maxreqs < 1)
> > +		return nfserr_inval;
> 
> Is 0 prohibited by the protocol?
> The server can set it to whatever value it wants,
> or if we can live with it, it actually provides a
> nice way to test the server end-cases.
> 
> Benny

The draft spec doesn't appear to explicitly exclude the value 0 for
ca_maxrequests.

I suppose a client might be able to request a session with a back
channel, but no fore channel if it wants to. The problem is that it
wouldn't be able to send SEQUENCE or BACKCHANNEL_CTL requests, and so
managing that back channel would be tough. There would be no way for the
server to tell the client of a back channel fault, or of the GSS context
expiring, and no way for the client to change that context.

In practice, therefore, it probably isn't a bad idea to return
NFS4ERR_INVAL...

Cheers
  Trond

Re: [PATCH 07/31] nfsd41: replay solo and embedded create session

[J. Bruce Fields]

On Tue, Apr 28, 2009 at 12:59:41PM -0400, andros@netapp.com wrote:
> From: Andy Adamson <andros@netapp.com>
> 
> CREATE_SESSION can be preceeded by a SEQUENCE operation (an embedded
> CREATE_SESSION) and the create session single slot cache must be maintained.
> Always replay a create session from nfsd4_replay_create_session(). Leave
> nfsd4_store_cache_entry() for session (sequence operation) replays.
> 
> Set cstate->status to a new internal error, nfserr_replay_clientid_cache, and
> change the logic in nfsd4_proc_compound so that CREATE_SESSION replays replace
> encode_operation.
> The new internal error is needed to differentiate between an embedded
> CREATE_SESSION replay and a SEQUENCE replay.

And making the clientid_cache cache just the unencoded
nfsd4_create_session would avoid the need for this special case, since
it would look to nfsd4_proc_compound() like any other create_session.

--b.

> 
> Signed-off-by: Andy Adamson <andros@netapp.com>
> ---
>  fs/nfsd/nfs4proc.c        |    9 +++++++--
>  fs/nfsd/nfs4state.c       |    5 +++++
>  fs/nfsd/nfs4xdr.c         |   17 +++++++++++++++++
>  include/linux/nfsd/nfsd.h |    2 ++
>  include/linux/nfsd/xdr4.h |    2 ++
>  5 files changed, 33 insertions(+), 2 deletions(-)
> 
> diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
> index b2883e9..32d5866 100644
> --- a/fs/nfsd/nfs4proc.c
> +++ b/fs/nfsd/nfs4proc.c
> @@ -996,7 +996,7 @@ nfsd4_proc_compound(struct svc_rqst *rqstp,
>  			BUG_ON(op->status == nfs_ok);
>  
>  encode_op:
> -		/* Only from SEQUENCE or CREATE_SESSION */
> +		/* Only from SEQUENCE */
>  		if (resp->cstate.status == nfserr_replay_cache) {
>  			dprintk("%s NFS4.1 replay from cache\n", __func__);
>  			if (nfsd4_not_cached(resp))
> @@ -1005,7 +1005,12 @@ encode_op:
>  				status = op->status;
>  			goto out;
>  		}
> -		if (op->status == nfserr_replay_me) {
> +		/* Only from CREATE_SESSION */
> +		if (resp->cstate.status == nfserr_replay_clientid_cache) {
> +			dprintk("%s NFS4.1 replay from clientid cache\n",
> +				__func__);
> +			status = op->status;
> +		} else if (op->status == nfserr_replay_me) {
>  			op->replay = &cstate->replay_owner->so_replay;
>  			nfsd4_encode_replay(resp, op);
>  			status = op->status = op->replay->rp_status;
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index de7cc51..e216169 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -1350,6 +1350,7 @@ nfsd4_create_session(struct svc_rqst *rqstp,
>  		     struct nfsd4_create_session *cr_ses)
>  {
>  	u32 ip_addr = svc_addr_in(rqstp)->sin_addr.s_addr;
> +	struct nfsd4_compoundres *resp = rqstp->rq_resp;
>  	struct nfs4_client *conf, *unconf;
>  	struct nfsd4_clid_slot *slot = NULL;
>  	int status = 0;
> @@ -1364,6 +1365,10 @@ nfsd4_create_session(struct svc_rqst *rqstp,
>  		if (status == nfserr_replay_cache) {
>  			dprintk("Got a create_session replay! seqid= %d\n",
>  				slot->sl_seqid);
> +			cstate->status = nfserr_replay_clientid_cache;
> +			/* Return the cached reply status */
> +			status = nfsd4_replay_create_session(resp, slot);
> +			goto out;
>  		} else if (cr_ses->seqid != conf->cl_slot.sl_seqid + 1) {
>  			status = nfserr_seq_misordered;
>  			dprintk("Sequence misordered!\n");
> diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
> index 1dfec1d..238612e 100644
> --- a/fs/nfsd/nfs4xdr.c
> +++ b/fs/nfsd/nfs4xdr.c
> @@ -3064,6 +3064,23 @@ nfsd4_cache_create_session(struct nfsd4_create_session *cr_ses,
>  	slot->sl_datalen = (char *)resp->p - (char *)p;
>  }
>  
> +__be32
> +nfsd4_replay_create_session(struct nfsd4_compoundres *resp,
> +			    struct nfsd4_clid_slot *slot)
> +{
> +	__be32 *p;
> +
> +	RESERVE_SPACE(8);
> +	WRITE32(OP_CREATE_SESSION);
> +	*p++ = slot->sl_status; /* already in network byte order */
> +	ADJUST_ARGS();
> +
> +	memcpy(resp->p, slot->sl_data, slot->sl_datalen);
> +	p += XDR_QUADLEN(slot->sl_datalen);
> +	ADJUST_ARGS();
> +	return slot->sl_status;
> +}
> +
>  static __be32
>  nfsd4_encode_destroy_session(struct nfsd4_compoundres *resp, int nfserr,
>  			     struct nfsd4_destroy_session *destroy_session)
> diff --git a/include/linux/nfsd/nfsd.h b/include/linux/nfsd/nfsd.h
> index 2b49d67..9cd6399 100644
> --- a/include/linux/nfsd/nfsd.h
> +++ b/include/linux/nfsd/nfsd.h
> @@ -301,6 +301,8 @@ void		nfsd_lockd_shutdown(void);
>  #define	nfserr_replay_me	cpu_to_be32(11001)
>  /* nfs41 replay detected */
>  #define	nfserr_replay_cache	cpu_to_be32(11002)
> +/* nfs41 clientid cache replay detected */
> +#define	nfserr_replay_clientid_cache	cpu_to_be32(11003)
>  
>  /* Check for dir entries '.' and '..' */
>  #define isdotent(n, l)	(l < 3 && n[0] == '.' && (l == 1 || n[1] == '.'))
> diff --git a/include/linux/nfsd/xdr4.h b/include/linux/nfsd/xdr4.h
> index 21e0b73..ac00985 100644
> --- a/include/linux/nfsd/xdr4.h
> +++ b/include/linux/nfsd/xdr4.h
> @@ -519,6 +519,8 @@ extern __be32 nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
>  		struct nfsd4_sequence *seq);
>  extern void nfsd4_cache_create_session(struct nfsd4_create_session *cr_ses,
>  		struct nfsd4_clid_slot *slot, int nfserr);
> +extern __be32 nfsd4_replay_create_session(struct nfsd4_compoundres *resp,
> +		struct nfsd4_clid_slot *slot);
>  extern __be32 nfsd4_exchange_id(struct svc_rqst *rqstp,
>  		struct nfsd4_compound_state *,
>  struct nfsd4_exchange_id *);
> -- 
> 1.5.4.3
> 

Re: [pnfs] [PATCH 07/31] nfsd41: replay solo and embedded create session

[William A. (Andy) Adamson]

On Fri, May 15, 2009 at 7:08 PM, J. Bruce Fields <bfields@fieldses.org> wrote:
> On Tue, Apr 28, 2009 at 12:59:41PM -0400, andros@netapp.com wrote:
>> From: Andy Adamson <andros@netapp.com>
>>
>> CREATE_SESSION can be preceeded by a SEQUENCE operation (an embedded
>> CREATE_SESSION) and the create session single slot cache must be maintained.
>> Always replay a create session from nfsd4_replay_create_session(). Leave
>> nfsd4_store_cache_entry() for session (sequence operation) replays.
>>
>> Set cstate->status to a new internal error, nfserr_replay_clientid_cache, and
>> change the logic in nfsd4_proc_compound so that CREATE_SESSION replays replace
>> encode_operation.
>> The new internal error is needed to differentiate between an embedded
>> CREATE_SESSION replay and a SEQUENCE replay.
>
> And making the clientid_cache cache just the unencoded
> nfsd4_create_session would avoid the need for this special case, since
> it would look to nfsd4_proc_compound() like any other create_session.

I believe we will still need the new internal case.
When the sequence operation replay case gets hit in
nfsd4_proc_compound, the entire reply has been constructed from the
cache, so the sequence operation replay exits nfsd4_proc_compound
processing.

In the create session replay case, only the create session operation
will be reconstructed from the cache, and it might be an embedded
create session (part of another sessions sequence compound) and so the
rest of the operations in that compound would need to be processed.
So, the create session replay does not exit nfsd4_proc_compound
processing.

-->Andy
>
> --b.
>
>>
>> Signed-off-by: Andy Adamson <andros@netapp.com>
>> ---
>>  fs/nfsd/nfs4proc.c        |    9 +++++++--
>>  fs/nfsd/nfs4state.c       |    5 +++++
>>  fs/nfsd/nfs4xdr.c         |   17 +++++++++++++++++
>>  include/linux/nfsd/nfsd.h |    2 ++
>>  include/linux/nfsd/xdr4.h |    2 ++
>>  5 files changed, 33 insertions(+), 2 deletions(-)
>>
>> diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
>> index b2883e9..32d5866 100644
>> --- a/fs/nfsd/nfs4proc.c
>> +++ b/fs/nfsd/nfs4proc.c
>> @@ -996,7 +996,7 @@ nfsd4_proc_compound(struct svc_rqst *rqstp,
>>                       BUG_ON(op->status == nfs_ok);
>>
>>  encode_op:
>> -             /* Only from SEQUENCE or CREATE_SESSION */
>> +             /* Only from SEQUENCE */
>>               if (resp->cstate.status == nfserr_replay_cache) {
>>                       dprintk("%s NFS4.1 replay from cache\n", __func__);
>>                       if (nfsd4_not_cached(resp))
>> @@ -1005,7 +1005,12 @@ encode_op:
>>                               status = op->status;
>>                       goto out;
>>               }
>> -             if (op->status == nfserr_replay_me) {
>> +             /* Only from CREATE_SESSION */
>> +             if (resp->cstate.status == nfserr_replay_clientid_cache) {
>> +                     dprintk("%s NFS4.1 replay from clientid cache\n",
>> +                             __func__);
>> +                     status = op->status;
>> +             } else if (op->status == nfserr_replay_me) {
>>                       op->replay = &cstate->replay_owner->so_replay;
>>                       nfsd4_encode_replay(resp, op);
>>                       status = op->status = op->replay->rp_status;
>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
>> index de7cc51..e216169 100644
>> --- a/fs/nfsd/nfs4state.c
>> +++ b/fs/nfsd/nfs4state.c
>> @@ -1350,6 +1350,7 @@ nfsd4_create_session(struct svc_rqst *rqstp,
>>                    struct nfsd4_create_session *cr_ses)
>>  {
>>       u32 ip_addr = svc_addr_in(rqstp)->sin_addr.s_addr;
>> +     struct nfsd4_compoundres *resp = rqstp->rq_resp;
>>       struct nfs4_client *conf, *unconf;
>>       struct nfsd4_clid_slot *slot = NULL;
>>       int status = 0;
>> @@ -1364,6 +1365,10 @@ nfsd4_create_session(struct svc_rqst *rqstp,
>>               if (status == nfserr_replay_cache) {
>>                       dprintk("Got a create_session replay! seqid= %d\n",
>>                               slot->sl_seqid);
>> +                     cstate->status = nfserr_replay_clientid_cache;
>> +                     /* Return the cached reply status */
>> +                     status = nfsd4_replay_create_session(resp, slot);
>> +                     goto out;
>>               } else if (cr_ses->seqid != conf->cl_slot.sl_seqid + 1) {
>>                       status = nfserr_seq_misordered;
>>                       dprintk("Sequence misordered!\n");
>> diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
>> index 1dfec1d..238612e 100644
>> --- a/fs/nfsd/nfs4xdr.c
>> +++ b/fs/nfsd/nfs4xdr.c
>> @@ -3064,6 +3064,23 @@ nfsd4_cache_create_session(struct nfsd4_create_session *cr_ses,
>>       slot->sl_datalen = (char *)resp->p - (char *)p;
>>  }
>>
>> +__be32
>> +nfsd4_replay_create_session(struct nfsd4_compoundres *resp,
>> +                         struct nfsd4_clid_slot *slot)
>> +{
>> +     __be32 *p;
>> +
>> +     RESERVE_SPACE(8);
>> +     WRITE32(OP_CREATE_SESSION);
>> +     *p++ = slot->sl_status; /* already in network byte order */
>> +     ADJUST_ARGS();
>> +
>> +     memcpy(resp->p, slot->sl_data, slot->sl_datalen);
>> +     p += XDR_QUADLEN(slot->sl_datalen);
>> +     ADJUST_ARGS();
>> +     return slot->sl_status;
>> +}
>> +
>>  static __be32
>>  nfsd4_encode_destroy_session(struct nfsd4_compoundres *resp, int nfserr,
>>                            struct nfsd4_destroy_session *destroy_session)
>> diff --git a/include/linux/nfsd/nfsd.h b/include/linux/nfsd/nfsd.h
>> index 2b49d67..9cd6399 100644
>> --- a/include/linux/nfsd/nfsd.h
>> +++ b/include/linux/nfsd/nfsd.h
>> @@ -301,6 +301,8 @@ void              nfsd_lockd_shutdown(void);
>>  #define      nfserr_replay_me        cpu_to_be32(11001)
>>  /* nfs41 replay detected */
>>  #define      nfserr_replay_cache     cpu_to_be32(11002)
>> +/* nfs41 clientid cache replay detected */
>> +#define      nfserr_replay_clientid_cache    cpu_to_be32(11003)
>>
>>  /* Check for dir entries '.' and '..' */
>>  #define isdotent(n, l)       (l < 3 && n[0] == '.' && (l == 1 || n[1] == '.'))
>> diff --git a/include/linux/nfsd/xdr4.h b/include/linux/nfsd/xdr4.h
>> index 21e0b73..ac00985 100644
>> --- a/include/linux/nfsd/xdr4.h
>> +++ b/include/linux/nfsd/xdr4.h
>> @@ -519,6 +519,8 @@ extern __be32 nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
>>               struct nfsd4_sequence *seq);
>>  extern void nfsd4_cache_create_session(struct nfsd4_create_session *cr_ses,
>>               struct nfsd4_clid_slot *slot, int nfserr);
>> +extern __be32 nfsd4_replay_create_session(struct nfsd4_compoundres *resp,
>> +             struct nfsd4_clid_slot *slot);
>>  extern __be32 nfsd4_exchange_id(struct svc_rqst *rqstp,
>>               struct nfsd4_compound_state *,
>>  struct nfsd4_exchange_id *);
>> --
>> 1.5.4.3
>>
> _______________________________________________
> pNFS mailing list
> pNFS@linux-nfs.org
> http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs
>

Re: [pnfs] [PATCH 07/31] nfsd41: replay solo and embedded create session

[J. Bruce Fields]

On Mon, May 18, 2009 at 10:02:25AM -0400, William A. (Andy) Adamson wrote:
> On Fri, May 15, 2009 at 7:08 PM, J. Bruce Fields <bfields@fieldses.org> wrote:
> > On Tue, Apr 28, 2009 at 12:59:41PM -0400, andros@netapp.com wrote:
> >> From: Andy Adamson <andros@netapp.com>
> >>
> >> CREATE_SESSION can be preceeded by a SEQUENCE operation (an embedded
> >> CREATE_SESSION) and the create session single slot cache must be maintained.
> >> Always replay a create session from nfsd4_replay_create_session(). Leave
> >> nfsd4_store_cache_entry() for session (sequence operation) replays.
> >>
> >> Set cstate->status to a new internal error, nfserr_replay_clientid_cache, and
> >> change the logic in nfsd4_proc_compound so that CREATE_SESSION replays replace
> >> encode_operation.
> >> The new internal error is needed to differentiate between an embedded
> >> CREATE_SESSION replay and a SEQUENCE replay.
> >
> > And making the clientid_cache cache just the unencoded
> > nfsd4_create_session would avoid the need for this special case, since
> > it would look to nfsd4_proc_compound() like any other create_session.
> 
> I believe we will still need the new internal case.
> When the sequence operation replay case gets hit in
> nfsd4_proc_compound, the entire reply has been constructed from the
> cache, so the sequence operation replay exits nfsd4_proc_compound
> processing.
> 
> In the create session replay case, only the create session operation
> will be reconstructed from the cache, and it might be an embedded
> create session (part of another sessions sequence compound) and so the
> rest of the operations in that compound would need to be processed.
> So, the create session replay does not exit nfsd4_proc_compound
> processing.

Right--and in that respect it's just like any other (non-replay) case.

So you'd still need the sequence case, but you wouldn't need the
replay_clientid_cache case, which would be encoded normally--the only
"replay" would be done inside create_session's proc function, which
would restore the saved values to the struct nfsd4_create_session.

Where possible, I'm happier keeping special cases inside the individual
op functions rather than in nfsd_proc_compound.

--b.

> 
> -->Andy
> >
> > --b.
> >
> >>
> >> Signed-off-by: Andy Adamson <andros@netapp.com>
> >> ---
> >>  fs/nfsd/nfs4proc.c        |    9 +++++++--
> >>  fs/nfsd/nfs4state.c       |    5 +++++
> >>  fs/nfsd/nfs4xdr.c         |   17 +++++++++++++++++
> >>  include/linux/nfsd/nfsd.h |    2 ++
> >>  include/linux/nfsd/xdr4.h |    2 ++
> >>  5 files changed, 33 insertions(+), 2 deletions(-)
> >>
> >> diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
> >> index b2883e9..32d5866 100644
> >> --- a/fs/nfsd/nfs4proc.c
> >> +++ b/fs/nfsd/nfs4proc.c
> >> @@ -996,7 +996,7 @@ nfsd4_proc_compound(struct svc_rqst *rqstp,
> >>                       BUG_ON(op->status == nfs_ok);
> >>
> >>  encode_op:
> >> -             /* Only from SEQUENCE or CREATE_SESSION */
> >> +             /* Only from SEQUENCE */
> >>               if (resp->cstate.status == nfserr_replay_cache) {
> >>                       dprintk("%s NFS4.1 replay from cache\n", __func__);
> >>                       if (nfsd4_not_cached(resp))
> >> @@ -1005,7 +1005,12 @@ encode_op:
> >>                               status = op->status;
> >>                       goto out;
> >>               }
> >> -             if (op->status == nfserr_replay_me) {
> >> +             /* Only from CREATE_SESSION */
> >> +             if (resp->cstate.status == nfserr_replay_clientid_cache) {
> >> +                     dprintk("%s NFS4.1 replay from clientid cache\n",
> >> +                             __func__);
> >> +                     status = op->status;
> >> +             } else if (op->status == nfserr_replay_me) {
> >>                       op->replay = &cstate->replay_owner->so_replay;
> >>                       nfsd4_encode_replay(resp, op);
> >>                       status = op->status = op->replay->rp_status;
> >> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> >> index de7cc51..e216169 100644
> >> --- a/fs/nfsd/nfs4state.c
> >> +++ b/fs/nfsd/nfs4state.c
> >> @@ -1350,6 +1350,7 @@ nfsd4_create_session(struct svc_rqst *rqstp,
> >>                    struct nfsd4_create_session *cr_ses)
> >>  {
> >>       u32 ip_addr = svc_addr_in(rqstp)->sin_addr.s_addr;
> >> +     struct nfsd4_compoundres *resp = rqstp->rq_resp;
> >>       struct nfs4_client *conf, *unconf;
> >>       struct nfsd4_clid_slot *slot = NULL;
> >>       int status = 0;
> >> @@ -1364,6 +1365,10 @@ nfsd4_create_session(struct svc_rqst *rqstp,
> >>               if (status == nfserr_replay_cache) {
> >>                       dprintk("Got a create_session replay! seqid= %d\n",
> >>                               slot->sl_seqid);
> >> +                     cstate->status = nfserr_replay_clientid_cache;
> >> +                     /* Return the cached reply status */
> >> +                     status = nfsd4_replay_create_session(resp, slot);
> >> +                     goto out;
> >>               } else if (cr_ses->seqid != conf->cl_slot.sl_seqid + 1) {
> >>                       status = nfserr_seq_misordered;
> >>                       dprintk("Sequence misordered!\n");
> >> diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
> >> index 1dfec1d..238612e 100644
> >> --- a/fs/nfsd/nfs4xdr.c
> >> +++ b/fs/nfsd/nfs4xdr.c
> >> @@ -3064,6 +3064,23 @@ nfsd4_cache_create_session(struct nfsd4_create_session *cr_ses,
> >>       slot->sl_datalen = (char *)resp->p - (char *)p;
> >>  }
> >>
> >> +__be32
> >> +nfsd4_replay_create_session(struct nfsd4_compoundres *resp,
> >> +                         struct nfsd4_clid_slot *slot)
> >> +{
> >> +     __be32 *p;
> >> +
> >> +     RESERVE_SPACE(8);
> >> +     WRITE32(OP_CREATE_SESSION);
> >> +     *p++ = slot->sl_status; /* already in network byte order */
> >> +     ADJUST_ARGS();
> >> +
> >> +     memcpy(resp->p, slot->sl_data, slot->sl_datalen);
> >> +     p += XDR_QUADLEN(slot->sl_datalen);
> >> +     ADJUST_ARGS();
> >> +     return slot->sl_status;
> >> +}
> >> +
> >>  static __be32
> >>  nfsd4_encode_destroy_session(struct nfsd4_compoundres *resp, int nfserr,
> >>                            struct nfsd4_destroy_session *destroy_session)
> >> diff --git a/include/linux/nfsd/nfsd.h b/include/linux/nfsd/nfsd.h
> >> index 2b49d67..9cd6399 100644
> >> --- a/include/linux/nfsd/nfsd.h
> >> +++ b/include/linux/nfsd/nfsd.h
> >> @@ -301,6 +301,8 @@ void              nfsd_lockd_shutdown(void);
> >>  #define      nfserr_replay_me        cpu_to_be32(11001)
> >>  /* nfs41 replay detected */
> >>  #define      nfserr_replay_cache     cpu_to_be32(11002)
> >> +/* nfs41 clientid cache replay detected */
> >> +#define      nfserr_replay_clientid_cache    cpu_to_be32(11003)
> >>
> >>  /* Check for dir entries '.' and '..' */
> >>  #define isdotent(n, l)       (l < 3 && n[0] == '.' && (l == 1 || n[1] == '.'))
> >> diff --git a/include/linux/nfsd/xdr4.h b/include/linux/nfsd/xdr4.h
> >> index 21e0b73..ac00985 100644
> >> --- a/include/linux/nfsd/xdr4.h
> >> +++ b/include/linux/nfsd/xdr4.h
> >> @@ -519,6 +519,8 @@ extern __be32 nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
> >>               struct nfsd4_sequence *seq);
> >>  extern void nfsd4_cache_create_session(struct nfsd4_create_session *cr_ses,
> >>               struct nfsd4_clid_slot *slot, int nfserr);
> >> +extern __be32 nfsd4_replay_create_session(struct nfsd4_compoundres *resp,
> >> +             struct nfsd4_clid_slot *slot);
> >>  extern __be32 nfsd4_exchange_id(struct svc_rqst *rqstp,
> >>               struct nfsd4_compound_state *,
> >>  struct nfsd4_exchange_id *);
> >> --
> >> 1.5.4.3
> >>
> > _______________________________________________
> > pNFS mailing list
> > pNFS@linux-nfs.org
> > http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs
> >

Re: [pnfs] [PATCH 07/31] nfsd41: replay solo and embedded create session

[William A. (Andy) Adamson]

On Mon, May 18, 2009 at 10:23 AM, J. Bruce Fields <bfields@fieldses.org> wrote:
> On Mon, May 18, 2009 at 10:02:25AM -0400, William A. (Andy) Adamson wrote:
>> On Fri, May 15, 2009 at 7:08 PM, J. Bruce Fields <bfields@fieldses.org> wrote:
>> > On Tue, Apr 28, 2009 at 12:59:41PM -0400, andros@netapp.com wrote:
>> >> From: Andy Adamson <andros@netapp.com>
>> >>
>> >> CREATE_SESSION can be preceeded by a SEQUENCE operation (an embedded
>> >> CREATE_SESSION) and the create session single slot cache must be maintained.
>> >> Always replay a create session from nfsd4_replay_create_session(). Leave
>> >> nfsd4_store_cache_entry() for session (sequence operation) replays.
>> >>
>> >> Set cstate->status to a new internal error, nfserr_replay_clientid_cache, and
>> >> change the logic in nfsd4_proc_compound so that CREATE_SESSION replays replace
>> >> encode_operation.
>> >> The new internal error is needed to differentiate between an embedded
>> >> CREATE_SESSION replay and a SEQUENCE replay.
>> >
>> > And making the clientid_cache cache just the unencoded
>> > nfsd4_create_session would avoid the need for this special case, since
>> > it would look to nfsd4_proc_compound() like any other create_session.
>>
>> I believe we will still need the new internal case.
>> When the sequence operation replay case gets hit in
>> nfsd4_proc_compound, the entire reply has been constructed from the
>> cache, so the sequence operation replay exits nfsd4_proc_compound
>> processing.
>>
>> In the create session replay case, only the create session operation
>> will be reconstructed from the cache, and it might be an embedded
>> create session (part of another sessions sequence compound) and so the
>> rest of the operations in that compound would need to be processed.
>> So, the create session replay does not exit nfsd4_proc_compound
>> processing.
>
> Right--and in that respect it's just like any other (non-replay) case.
>
> So you'd still need the sequence case, but you wouldn't need the
> replay_clientid_cache case, which would be encoded normally--the only
> "replay" would be done inside create_session's proc function, which
> would restore the saved values to the struct nfsd4_create_session.

Got it.

-->Andy

>
> Where possible, I'm happier keeping special cases inside the individual
> op functions rather than in nfsd_proc_compound.
>
> --b.
>
>>
>> -->Andy
>> >
>> > --b.
>> >
>> >>
>> >> Signed-off-by: Andy Adamson <andros@netapp.com>
>> >> ---
>> >>  fs/nfsd/nfs4proc.c        |    9 +++++++--
>> >>  fs/nfsd/nfs4state.c       |    5 +++++
>> >>  fs/nfsd/nfs4xdr.c         |   17 +++++++++++++++++
>> >>  include/linux/nfsd/nfsd.h |    2 ++
>> >>  include/linux/nfsd/xdr4.h |    2 ++
>> >>  5 files changed, 33 insertions(+), 2 deletions(-)
>> >>
>> >> diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
>> >> index b2883e9..32d5866 100644
>> >> --- a/fs/nfsd/nfs4proc.c
>> >> +++ b/fs/nfsd/nfs4proc.c
>> >> @@ -996,7 +996,7 @@ nfsd4_proc_compound(struct svc_rqst *rqstp,
>> >>                       BUG_ON(op->status == nfs_ok);
>> >>
>> >>  encode_op:
>> >> -             /* Only from SEQUENCE or CREATE_SESSION */
>> >> +             /* Only from SEQUENCE */
>> >>               if (resp->cstate.status == nfserr_replay_cache) {
>> >>                       dprintk("%s NFS4.1 replay from cache\n", __func__);
>> >>                       if (nfsd4_not_cached(resp))
>> >> @@ -1005,7 +1005,12 @@ encode_op:
>> >>                               status = op->status;
>> >>                       goto out;
>> >>               }
>> >> -             if (op->status == nfserr_replay_me) {
>> >> +             /* Only from CREATE_SESSION */
>> >> +             if (resp->cstate.status == nfserr_replay_clientid_cache) {
>> >> +                     dprintk("%s NFS4.1 replay from clientid cache\n",
>> >> +                             __func__);
>> >> +                     status = op->status;
>> >> +             } else if (op->status == nfserr_replay_me) {
>> >>                       op->replay = &cstate->replay_owner->so_replay;
>> >>                       nfsd4_encode_replay(resp, op);
>> >>                       status = op->status = op->replay->rp_status;
>> >> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
>> >> index de7cc51..e216169 100644
>> >> --- a/fs/nfsd/nfs4state.c
>> >> +++ b/fs/nfsd/nfs4state.c
>> >> @@ -1350,6 +1350,7 @@ nfsd4_create_session(struct svc_rqst *rqstp,
>> >>                    struct nfsd4_create_session *cr_ses)
>> >>  {
>> >>       u32 ip_addr = svc_addr_in(rqstp)->sin_addr.s_addr;
>> >> +     struct nfsd4_compoundres *resp = rqstp->rq_resp;
>> >>       struct nfs4_client *conf, *unconf;
>> >>       struct nfsd4_clid_slot *slot = NULL;
>> >>       int status = 0;
>> >> @@ -1364,6 +1365,10 @@ nfsd4_create_session(struct svc_rqst *rqstp,
>> >>               if (status == nfserr_replay_cache) {
>> >>                       dprintk("Got a create_session replay! seqid= %d\n",
>> >>                               slot->sl_seqid);
>> >> +                     cstate->status = nfserr_replay_clientid_cache;
>> >> +                     /* Return the cached reply status */
>> >> +                     status = nfsd4_replay_create_session(resp, slot);
>> >> +                     goto out;
>> >>               } else if (cr_ses->seqid != conf->cl_slot.sl_seqid + 1) {
>> >>                       status = nfserr_seq_misordered;
>> >>                       dprintk("Sequence misordered!\n");
>> >> diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
>> >> index 1dfec1d..238612e 100644
>> >> --- a/fs/nfsd/nfs4xdr.c
>> >> +++ b/fs/nfsd/nfs4xdr.c
>> >> @@ -3064,6 +3064,23 @@ nfsd4_cache_create_session(struct nfsd4_create_session *cr_ses,
>> >>       slot->sl_datalen = (char *)resp->p - (char *)p;
>> >>  }
>> >>
>> >> +__be32
>> >> +nfsd4_replay_create_session(struct nfsd4_compoundres *resp,
>> >> +                         struct nfsd4_clid_slot *slot)
>> >> +{
>> >> +     __be32 *p;
>> >> +
>> >> +     RESERVE_SPACE(8);
>> >> +     WRITE32(OP_CREATE_SESSION);
>> >> +     *p++ = slot->sl_status; /* already in network byte order */
>> >> +     ADJUST_ARGS();
>> >> +
>> >> +     memcpy(resp->p, slot->sl_data, slot->sl_datalen);
>> >> +     p += XDR_QUADLEN(slot->sl_datalen);
>> >> +     ADJUST_ARGS();
>> >> +     return slot->sl_status;
>> >> +}
>> >> +
>> >>  static __be32
>> >>  nfsd4_encode_destroy_session(struct nfsd4_compoundres *resp, int nfserr,
>> >>                            struct nfsd4_destroy_session *destroy_session)
>> >> diff --git a/include/linux/nfsd/nfsd.h b/include/linux/nfsd/nfsd.h
>> >> index 2b49d67..9cd6399 100644
>> >> --- a/include/linux/nfsd/nfsd.h
>> >> +++ b/include/linux/nfsd/nfsd.h
>> >> @@ -301,6 +301,8 @@ void              nfsd_lockd_shutdown(void);
>> >>  #define      nfserr_replay_me        cpu_to_be32(11001)
>> >>  /* nfs41 replay detected */
>> >>  #define      nfserr_replay_cache     cpu_to_be32(11002)
>> >> +/* nfs41 clientid cache replay detected */
>> >> +#define      nfserr_replay_clientid_cache    cpu_to_be32(11003)
>> >>
>> >>  /* Check for dir entries '.' and '..' */
>> >>  #define isdotent(n, l)       (l < 3 && n[0] == '.' && (l == 1 || n[1] == '.'))
>> >> diff --git a/include/linux/nfsd/xdr4.h b/include/linux/nfsd/xdr4.h
>> >> index 21e0b73..ac00985 100644
>> >> --- a/include/linux/nfsd/xdr4.h
>> >> +++ b/include/linux/nfsd/xdr4.h
>> >> @@ -519,6 +519,8 @@ extern __be32 nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
>> >>               struct nfsd4_sequence *seq);
>> >>  extern void nfsd4_cache_create_session(struct nfsd4_create_session *cr_ses,
>> >>               struct nfsd4_clid_slot *slot, int nfserr);
>> >> +extern __be32 nfsd4_replay_create_session(struct nfsd4_compoundres *resp,
>> >> +             struct nfsd4_clid_slot *slot);
>> >>  extern __be32 nfsd4_exchange_id(struct svc_rqst *rqstp,
>> >>               struct nfsd4_compound_state *,
>> >>  struct nfsd4_exchange_id *);
>> >> --
>> >> 1.5.4.3
>> >>
>> > _______________________________________________
>> > pNFS mailing list
>> > pNFS@linux-nfs.org
>> > http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs
>> >
>

Re: [PATCH 05/31] nfsd41: encode create_session result into cache

[J. Bruce Fields]

On Tue, Apr 28, 2009 at 12:59:39PM -0400, andros@netapp.com wrote:
> From: Andy Adamson <andros@netapp.com>
> 
> CREATE_SESSION can be preceeded by a SEQUENCE operation and the
> create session single slot cache must be maintained. Encode the results of
> a create session call into the cache at the end of processing.
> 
> The create session result will also be encoded into the RPC response, and if
> it is preceeded by a SEQUENCE operation, will also be encoded into the
> session slot table cache.
> 
> Errors that do not change the create session cache:
> A create session NFS4ERR_STALE_CLIENTID error means that a client record
> (and associated create session slot) could not be found and therefore can't
> be changed.  NFSERR_SEQ_MISORDERED errors do not change the slot cache.
> 
> All other errors get cached.
> 
> Signed-off-by: Andy Adamson <andros@netapp.com>
> ---
>  fs/nfsd/nfs4state.c       |    6 ++++--
>  fs/nfsd/nfs4xdr.c         |   18 ++++++++++++++++++
>  include/linux/nfsd/xdr4.h |    2 ++
>  3 files changed, 24 insertions(+), 2 deletions(-)
> 
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index fee6bf0..142c460 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -1376,7 +1376,7 @@ nfsd4_create_session(struct svc_rqst *rqstp,
>  		if (!same_creds(&unconf->cl_cred, &rqstp->rq_cred) ||
>  		    (ip_addr != unconf->cl_addr)) {
>  			status = nfserr_clid_inuse;
> -			goto out;
> +			goto out_cache;
>  		}
>  
>  		slot = &unconf->cl_slot;
> @@ -1418,7 +1418,9 @@ nfsd4_create_session(struct svc_rqst *rqstp,
>  	memcpy(cr_ses->sessionid.data, conf->cl_sessionid.data,
>  	       NFS4_MAX_SESSIONID_LEN);
>  	cr_ses->seqid = slot->sl_seqid;
> -
> +out_cache:
> +	/* cache solo and embedded create sessions under the state lock */
> +	nfsd4_cache_create_session(cr_ses, slot, status);
>  out:
>  	nfs4_unlock_state();
>  	dprintk("%s returns %d\n", __func__, ntohl(status));
> diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
> index b2f8d74..1dfec1d 100644
> --- a/fs/nfsd/nfs4xdr.c
> +++ b/fs/nfsd/nfs4xdr.c
> @@ -3046,6 +3046,24 @@ nfsd4_encode_create_session(struct nfsd4_compoundres *resp, int nfserr,
>  	return 0;
>  }
>  
> +/*
> + * Encode the create_session result into the create session single DRC
> + * slot cache. Do this for solo or embedded create session operations.
> + */
> +void
> +nfsd4_cache_create_session(struct nfsd4_create_session *cr_ses,
> +			   struct nfsd4_clid_slot *slot, int nfserr)
> +{
> +	__be32 *p = (__be32 *)slot->sl_data;
> +	struct nfsd4_compoundres tmp = {
> +		.p = p,
> +		.end = p + XDR_QUADLEN(CS_MAX_ENC_SZ),
> +	}, *resp = &tmp;

bfields@pig:~/local/linux-2.6$ gdb vmlinux 
GNU gdb 6.8-debian
...
This GDB was configured as "i486-linux-gnu"...
(gdb) p sizeof(struct nfsd4_compoundres)
$1 = 588

So that's a 588 byte structure on my machine--better not to put it on
the stack if we can avoid it.

This trick of faking up a temporary nfsd4_compoundres to be able to use
nfsd4_encode_create_session() unmodified is sort of brilliant but
possibly also sort of nuts?  I don't know.  Anyway, maybe:

	- rewrite nfsd4_encode_create_session() to call a common
	  encode_create_session that takes just the start and end
	  pointers, then call that?  Or
	- forget the xdr encoding entirely and just save the values we
	  care about from nfsd4_create_session?  (Heck, there's no
	  pointers in there or anything--you could just make a copy of
	  the whole struct.)

I'm leaning towards the latter--the xdr encoding seems unnecessary, and
introduces error-handling for a case (overflowing the CS_MAX_ENC_SZ-size
buffer) that we'll never hit.  And if we're making the clientid slot
such a special case then I suppose there's no need to xdr-encode for
uniform handling with the regular sessions slot case.

--b.

> +
> +	slot->sl_status = nfsd4_encode_create_session(resp, nfserr, cr_ses);
> +	slot->sl_datalen = (char *)resp->p - (char *)p;
> +}
> +
>  static __be32
>  nfsd4_encode_destroy_session(struct nfsd4_compoundres *resp, int nfserr,
>  			     struct nfsd4_destroy_session *destroy_session)
> diff --git a/include/linux/nfsd/xdr4.h b/include/linux/nfsd/xdr4.h
> index afd4464..21e0b73 100644
> --- a/include/linux/nfsd/xdr4.h
> +++ b/include/linux/nfsd/xdr4.h
> @@ -517,6 +517,8 @@ extern __be32 nfsd4_setclientid_confirm(struct svc_rqst *rqstp,
>  extern void nfsd4_store_cache_entry(struct nfsd4_compoundres *resp);
>  extern __be32 nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
>  		struct nfsd4_sequence *seq);
> +extern void nfsd4_cache_create_session(struct nfsd4_create_session *cr_ses,
> +		struct nfsd4_clid_slot *slot, int nfserr);
>  extern __be32 nfsd4_exchange_id(struct svc_rqst *rqstp,
>  		struct nfsd4_compound_state *,
>  struct nfsd4_exchange_id *);
> -- 
> 1.5.4.3
> 

Re: [pnfs] [PATCH 05/31] nfsd41: encode create_session result into cache

[William A. (Andy) Adamson]

On Fri, May 15, 2009 at 7:05 PM, J. Bruce Fields <bfields@fieldses.org> wrote:
> On Tue, Apr 28, 2009 at 12:59:39PM -0400, andros@netapp.com wrote:
>> From: Andy Adamson <andros@netapp.com>
>>
>> CREATE_SESSION can be preceeded by a SEQUENCE operation and the
>> create session single slot cache must be maintained. Encode the results of
>> a create session call into the cache at the end of processing.
>>
>> The create session result will also be encoded into the RPC response, and if
>> it is preceeded by a SEQUENCE operation, will also be encoded into the
>> session slot table cache.
>>
>> Errors that do not change the create session cache:
>> A create session NFS4ERR_STALE_CLIENTID error means that a client record
>> (and associated create session slot) could not be found and therefore can't
>> be changed.  NFSERR_SEQ_MISORDERED errors do not change the slot cache.
>>
>> All other errors get cached.
>>
>> Signed-off-by: Andy Adamson <andros@netapp.com>
>> ---
>>  fs/nfsd/nfs4state.c       |    6 ++++--
>>  fs/nfsd/nfs4xdr.c         |   18 ++++++++++++++++++
>>  include/linux/nfsd/xdr4.h |    2 ++
>>  3 files changed, 24 insertions(+), 2 deletions(-)
>>
>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
>> index fee6bf0..142c460 100644
>> --- a/fs/nfsd/nfs4state.c
>> +++ b/fs/nfsd/nfs4state.c
>> @@ -1376,7 +1376,7 @@ nfsd4_create_session(struct svc_rqst *rqstp,
>>               if (!same_creds(&unconf->cl_cred, &rqstp->rq_cred) ||
>>                   (ip_addr != unconf->cl_addr)) {
>>                       status = nfserr_clid_inuse;
>> -                     goto out;
>> +                     goto out_cache;
>>               }
>>
>>               slot = &unconf->cl_slot;
>> @@ -1418,7 +1418,9 @@ nfsd4_create_session(struct svc_rqst *rqstp,
>>       memcpy(cr_ses->sessionid.data, conf->cl_sessionid.data,
>>              NFS4_MAX_SESSIONID_LEN);
>>       cr_ses->seqid = slot->sl_seqid;
>> -
>> +out_cache:
>> +     /* cache solo and embedded create sessions under the state lock */
>> +     nfsd4_cache_create_session(cr_ses, slot, status);
>>  out:
>>       nfs4_unlock_state();
>>       dprintk("%s returns %d\n", __func__, ntohl(status));
>> diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
>> index b2f8d74..1dfec1d 100644
>> --- a/fs/nfsd/nfs4xdr.c
>> +++ b/fs/nfsd/nfs4xdr.c
>> @@ -3046,6 +3046,24 @@ nfsd4_encode_create_session(struct nfsd4_compoundres *resp, int nfserr,
>>       return 0;
>>  }
>>
>> +/*
>> + * Encode the create_session result into the create session single DRC
>> + * slot cache. Do this for solo or embedded create session operations.
>> + */
>> +void
>> +nfsd4_cache_create_session(struct nfsd4_create_session *cr_ses,
>> +                        struct nfsd4_clid_slot *slot, int nfserr)
>> +{
>> +     __be32 *p = (__be32 *)slot->sl_data;
>> +     struct nfsd4_compoundres tmp = {
>> +             .p = p,
>> +             .end = p + XDR_QUADLEN(CS_MAX_ENC_SZ),
>> +     }, *resp = &tmp;
>
> bfields@pig:~/local/linux-2.6$ gdb vmlinux
> GNU gdb 6.8-debian
> ...
> This GDB was configured as "i486-linux-gnu"...
> (gdb) p sizeof(struct nfsd4_compoundres)
> $1 = 588
>
> So that's a 588 byte structure on my machine--better not to put it on
> the stack if we can avoid it.

How did Benny put it -- oh yeah.

Ouch.

Should not be on the stack.

>
> This trick of faking up a temporary nfsd4_compoundres to be able to use
> nfsd4_encode_create_session() unmodified is sort of brilliant but
> possibly also sort of nuts?  I don't know.  Anyway, maybe:
>
>        - rewrite nfsd4_encode_create_session() to call a common
>          encode_create_session that takes just the start and end
>          pointers, then call that?  Or
>        - forget the xdr encoding entirely and just save the values we
>          care about from nfsd4_create_session?  (Heck, there's no
>          pointers in there or anything--you could just make a copy of
>          the whole struct.)
>
> I'm leaning towards the latter--the xdr encoding seems unnecessary, and
> introduces error-handling for a case (overflowing the CS_MAX_ENC_SZ-size
> buffer) that we'll never hit.  And if we're making the clientid slot
> such a special case then I suppose there's no need to xdr-encode for
> uniform handling with the regular sessions slot case.

I like #2 as well - the same method used for the sessions operation. Good idea.
I'll call nfsd4_encode_create_session on replay.

-->Andy

>
> --b.
>
>> +
>> +     slot->sl_status = nfsd4_encode_create_session(resp, nfserr, cr_ses);
>> +     slot->sl_datalen = (char *)resp->p - (char *)p;
>> +}
>> +
>>  static __be32
>>  nfsd4_encode_destroy_session(struct nfsd4_compoundres *resp, int nfserr,
>>                            struct nfsd4_destroy_session *destroy_session)
>> diff --git a/include/linux/nfsd/xdr4.h b/include/linux/nfsd/xdr4.h
>> index afd4464..21e0b73 100644
>> --- a/include/linux/nfsd/xdr4.h
>> +++ b/include/linux/nfsd/xdr4.h
>> @@ -517,6 +517,8 @@ extern __be32 nfsd4_setclientid_confirm(struct svc_rqst *rqstp,
>>  extern void nfsd4_store_cache_entry(struct nfsd4_compoundres *resp);
>>  extern __be32 nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp,
>>               struct nfsd4_sequence *seq);
>> +extern void nfsd4_cache_create_session(struct nfsd4_create_session *cr_ses,
>> +             struct nfsd4_clid_slot *slot, int nfserr);
>>  extern __be32 nfsd4_exchange_id(struct svc_rqst *rqstp,
>>               struct nfsd4_compound_state *,
>>  struct nfsd4_exchange_id *);
>> --
>> 1.5.4.3
>>
> _______________________________________________
> pNFS mailing list
> pNFS@linux-nfs.org
> http://linux-nfs.org/cgi-bin/mailman/listinfo/pnfs
>

Re: [PATCH 02/31] nfsd41: change check_slot_seqid parameters

[Benny Halevy]

On Apr. 28, 2009, 19:59 +0300, andros@netapp.com wrote:
> From: Andy Adamson <andros@netapp.com>
> 
> For separation of session slot and clientid slot processing.
> 
> Signed-off-by: Andy Adamson <andros@netapp.com>
> ---
>  fs/nfsd/nfs4state.c |   24 +++++++++++++-----------
>  1 files changed, 13 insertions(+), 11 deletions(-)
> 
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index 42ebcb3..43c821f 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -1310,26 +1310,26 @@ error:
>  }
>  
>  static int
> -check_slot_seqid(u32 seqid, struct nfsd4_slot *slot)
> +check_slot_seqid(u32 seqid, u32 slot_seqid, int slot_inuse )

nit: extra space before closing parentheses.
(I'll remove it when merging into the pnfs tree)

Benny

>  {
> -	dprintk("%s enter. seqid %d slot->sl_seqid %d\n", __func__, seqid,
> -		slot->sl_seqid);
> +	dprintk("%s enter. seqid %d slot_seqid %d\n", __func__, seqid,
> +		slot_seqid);
>  
>  	/* The slot is in use, and no response has been sent. */
> -	if (slot->sl_inuse) {
> -		if (seqid == slot->sl_seqid)
> +	if (slot_inuse) {
> +		if (seqid == slot_seqid)
>  			return nfserr_jukebox;
>  		else
>  			return nfserr_seq_misordered;
>  	}
>  	/* Normal */
> -	if (likely(seqid == slot->sl_seqid + 1))
> +	if (likely(seqid == slot_seqid + 1))
>  		return nfs_ok;
>  	/* Replay */
> -	if (seqid == slot->sl_seqid)
> +	if (seqid == slot_seqid)
>  		return nfserr_replay_cache;
>  	/* Wraparound */
> -	if (seqid == 1 && (slot->sl_seqid + 1) == 0)
> +	if (seqid == 1 && (slot_seqid + 1) == 0)
>  		return nfs_ok;
>  	/* Misordered replay or misordered new request */
>  	return nfserr_seq_misordered;
> @@ -1352,7 +1352,8 @@ nfsd4_create_session(struct svc_rqst *rqstp,
>  
>  	if (conf) {
>  		slot = &conf->cl_slot;
> -		status = check_slot_seqid(cr_ses->seqid, slot);
> +		status = check_slot_seqid(cr_ses->seqid, slot->sl_seqid,
> +					  slot->sl_inuse);
>  		if (status == nfserr_replay_cache) {
>  			dprintk("Got a create_session replay! seqid= %d\n",
>  				slot->sl_seqid);
> @@ -1377,7 +1378,8 @@ nfsd4_create_session(struct svc_rqst *rqstp,
>  		}
>  
>  		slot = &unconf->cl_slot;
> -		status = check_slot_seqid(cr_ses->seqid, slot);
> +		status = check_slot_seqid(cr_ses->seqid, slot->sl_seqid,
> +					  slot->sl_inuse);
>  		if (status) {
>  			/* an unconfirmed replay returns misordered */
>  			status = nfserr_seq_misordered;
> @@ -1487,7 +1489,7 @@ nfsd4_sequence(struct svc_rqst *rqstp,
>  	slot = &session->se_slots[seq->slotid];
>  	dprintk("%s: slotid %d\n", __func__, seq->slotid);
>  
> -	status = check_slot_seqid(seq->seqid, slot);
> +	status = check_slot_seqid(seq->seqid, slot->sl_seqid, slot->sl_inuse);
>  	if (status == nfserr_replay_cache) {
>  		cstate->slot = slot;
>  		cstate->session = session;

Re: [PATCH 02/31] nfsd41: change check_slot_seqid parameters

[Andy Adamson]

On Apr 30, 2009, at 1:35 PM, Benny Halevy wrote:

> On Apr. 28, 2009, 19:59 +0300, andros@netapp.com wrote:
>> From: Andy Adamson <andros@netapp.com>
>>
>> For separation of session slot and clientid slot processing.
>>
>> Signed-off-by: Andy Adamson <andros@netapp.com>
>> ---
>> fs/nfsd/nfs4state.c |   24 +++++++++++++-----------
>> 1 files changed, 13 insertions(+), 11 deletions(-)
>>
>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
>> index 42ebcb3..43c821f 100644
>> --- a/fs/nfsd/nfs4state.c
>> +++ b/fs/nfsd/nfs4state.c
>> @@ -1310,26 +1310,26 @@ error:
>> }
>>
>> static int
>> -check_slot_seqid(u32 seqid, struct nfsd4_slot *slot)
>> +check_slot_seqid(u32 seqid, u32 slot_seqid, int slot_inuse )
>
> nit: extra space before closing parentheses.
> (I'll remove it when merging into the pnfs tree)
>
Thanks - I thought I ran checkpatch on all of these....


> Benny
>
>> {
>> -	dprintk("%s enter. seqid %d slot->sl_seqid %d\n", __func__, seqid,
>> -		slot->sl_seqid);
>> +	dprintk("%s enter. seqid %d slot_seqid %d\n", __func__, seqid,
>> +		slot_seqid);
>>
>> 	/* The slot is in use, and no response has been sent. */
>> -	if (slot->sl_inuse) {
>> -		if (seqid == slot->sl_seqid)
>> +	if (slot_inuse) {
>> +		if (seqid == slot_seqid)
>> 			return nfserr_jukebox;
>> 		else
>> 			return nfserr_seq_misordered;
>> 	}
>> 	/* Normal */
>> -	if (likely(seqid == slot->sl_seqid + 1))
>> +	if (likely(seqid == slot_seqid + 1))
>> 		return nfs_ok;
>> 	/* Replay */
>> -	if (seqid == slot->sl_seqid)
>> +	if (seqid == slot_seqid)
>> 		return nfserr_replay_cache;
>> 	/* Wraparound */
>> -	if (seqid == 1 && (slot->sl_seqid + 1) == 0)
>> +	if (seqid == 1 && (slot_seqid + 1) == 0)
>> 		return nfs_ok;
>> 	/* Misordered replay or misordered new request */
>> 	return nfserr_seq_misordered;
>> @@ -1352,7 +1352,8 @@ nfsd4_create_session(struct svc_rqst *rqstp,
>>
>> 	if (conf) {
>> 		slot = &conf->cl_slot;
>> -		status = check_slot_seqid(cr_ses->seqid, slot);
>> +		status = check_slot_seqid(cr_ses->seqid, slot->sl_seqid,
>> +					  slot->sl_inuse);
>> 		if (status == nfserr_replay_cache) {
>> 			dprintk("Got a create_session replay! seqid= %d\n",
>> 				slot->sl_seqid);
>> @@ -1377,7 +1378,8 @@ nfsd4_create_session(struct svc_rqst *rqstp,
>> 		}
>>
>> 		slot = &unconf->cl_slot;
>> -		status = check_slot_seqid(cr_ses->seqid, slot);
>> +		status = check_slot_seqid(cr_ses->seqid, slot->sl_seqid,
>> +					  slot->sl_inuse);
>> 		if (status) {
>> 			/* an unconfirmed replay returns misordered */
>> 			status = nfserr_seq_misordered;
>> @@ -1487,7 +1489,7 @@ nfsd4_sequence(struct svc_rqst *rqstp,
>> 	slot = &session->se_slots[seq->slotid];
>> 	dprintk("%s: slotid %d\n", __func__, seq->slotid);
>>
>> -	status = check_slot_seqid(seq->seqid, slot);
>> +	status = check_slot_seqid(seq->seqid, slot->sl_seqid, slot- 
>> >sl_inuse);
>> 	if (status == nfserr_replay_cache) {
>> 		cstate->slot = slot;
>> 		cstate->session = session;

Re: [PATCH 01/31] nfsd41: slots are freed with session

[Benny Halevy]

On Apr. 28, 2009, 19:59 +0300, andros@netapp.com wrote:
> From: Andy Adamson <andros@netapp.com>
> 
> The session and slots are allocated all in one piece.

This patch should go into 2.6.30, right?

Benny

> 
> Signed-off-by: Andy Adamson <andros@netapp.com>
> ---
>  fs/nfsd/nfs4state.c |    1 -
>  1 files changed, 0 insertions(+), 1 deletions(-)
> 
> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
> index e82a518..42ebcb3 100644
> --- a/fs/nfsd/nfs4state.c
> +++ b/fs/nfsd/nfs4state.c
> @@ -582,7 +582,6 @@ free_session(struct kref *kref)
>  		struct nfsd4_cache_entry *e = &ses->se_slots[i].sl_cache_entry;
>  		nfsd4_release_respages(e->ce_respages, e->ce_resused);
>  	}
> -	kfree(ses->se_slots);
>  	kfree(ses);
>  }
>  

Re: [PATCH 01/31] nfsd41: slots are freed with session

[Andy Adamson]

yes, you're right - it prevents an oops on server shutdown.

-->Andy

On Apr 30, 2009, at 5:12 AM, Benny Halevy wrote:

> On Apr. 28, 2009, 19:59 +0300, andros@netapp.com wrote:
>> From: Andy Adamson <andros@netapp.com>
>>
>> The session and slots are allocated all in one piece.
>
> This patch should go into 2.6.30, right?
>
> Benny
>
>>
>> Signed-off-by: Andy Adamson <andros@netapp.com>
>> ---
>> fs/nfsd/nfs4state.c |    1 -
>> 1 files changed, 0 insertions(+), 1 deletions(-)
>>
>> diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
>> index e82a518..42ebcb3 100644
>> --- a/fs/nfsd/nfs4state.c
>> +++ b/fs/nfsd/nfs4state.c
>> @@ -582,7 +582,6 @@ free_session(struct kref *kref)
>> 		struct nfsd4_cache_entry *e = &ses->se_slots[i].sl_cache_entry;
>> 		nfsd4_release_respages(e->ce_respages, e->ce_resused);
>> 	}
>> -	kfree(ses->se_slots);
>> 	kfree(ses);
>> }
>>